blob: bf51abb78deed14392a4c3e80d180c41d1e76546 [file] [log] [blame]
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +02001/*
Joerg Roedel5d0d7152010-10-13 11:13:21 +02002 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +02003 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/pci.h>
21#include <linux/acpi.h>
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020022#include <linux/list.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/slab.h>
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +010024#include <linux/syscore_ops.h>
Joerg Roedela80dc3e2008-09-11 16:51:41 +020025#include <linux/interrupt.h>
26#include <linux/msi.h>
Joerg Roedel403f81d2011-06-14 16:44:25 +020027#include <linux/amd-iommu.h>
Joerg Roedel400a28a2011-11-28 15:11:02 +010028#include <linux/export.h>
Joerg Roedel02f3b3f2012-06-11 17:45:25 +020029#include <acpi/acpi.h>
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020030#include <asm/pci-direct.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090031#include <asm/iommu.h>
Joerg Roedel1d9b16d2008-11-27 18:39:15 +010032#include <asm/gart.h>
FUJITA Tomonoriea1b0d32009-11-10 19:46:15 +090033#include <asm/x86_init.h>
Konrad Rzeszutek Wilk22e6daf2010-08-26 13:58:03 -040034#include <asm/iommu_table.h>
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +020035#include <asm/io_apic.h>
Joerg Roedel6b474b82012-06-26 16:46:04 +020036#include <asm/irq_remapping.h>
Joerg Roedel403f81d2011-06-14 16:44:25 +020037
38#include "amd_iommu_proto.h"
39#include "amd_iommu_types.h"
Joerg Roedel05152a02012-06-15 16:53:51 +020040#include "irq_remapping.h"
Joerg Roedel403f81d2011-06-14 16:44:25 +020041
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020042/*
43 * definitions for the ACPI scanning code
44 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020045#define IVRS_HEADER_LENGTH 48
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020046
47#define ACPI_IVHD_TYPE 0x10
48#define ACPI_IVMD_TYPE_ALL 0x20
49#define ACPI_IVMD_TYPE 0x21
50#define ACPI_IVMD_TYPE_RANGE 0x22
51
52#define IVHD_DEV_ALL 0x01
53#define IVHD_DEV_SELECT 0x02
54#define IVHD_DEV_SELECT_RANGE_START 0x03
55#define IVHD_DEV_RANGE_END 0x04
56#define IVHD_DEV_ALIAS 0x42
57#define IVHD_DEV_ALIAS_RANGE 0x43
58#define IVHD_DEV_EXT_SELECT 0x46
59#define IVHD_DEV_EXT_SELECT_RANGE 0x47
Joerg Roedel6efed632012-06-14 15:52:58 +020060#define IVHD_DEV_SPECIAL 0x48
61
62#define IVHD_SPECIAL_IOAPIC 1
63#define IVHD_SPECIAL_HPET 2
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020064
Joerg Roedel6da73422009-05-04 11:44:38 +020065#define IVHD_FLAG_HT_TUN_EN_MASK 0x01
66#define IVHD_FLAG_PASSPW_EN_MASK 0x02
67#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
68#define IVHD_FLAG_ISOC_EN_MASK 0x08
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020069
70#define IVMD_FLAG_EXCL_RANGE 0x08
71#define IVMD_FLAG_UNITY_MAP 0x01
72
73#define ACPI_DEVFLAG_INITPASS 0x01
74#define ACPI_DEVFLAG_EXTINT 0x02
75#define ACPI_DEVFLAG_NMI 0x04
76#define ACPI_DEVFLAG_SYSMGT1 0x10
77#define ACPI_DEVFLAG_SYSMGT2 0x20
78#define ACPI_DEVFLAG_LINT0 0x40
79#define ACPI_DEVFLAG_LINT1 0x80
80#define ACPI_DEVFLAG_ATSDIS 0x10000000
81
Joerg Roedelb65233a2008-07-11 17:14:21 +020082/*
83 * ACPI table definitions
84 *
85 * These data structures are laid over the table to parse the important values
86 * out of it.
87 */
88
89/*
90 * structure describing one IOMMU in the ACPI table. Typically followed by one
91 * or more ivhd_entrys.
92 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020093struct ivhd_header {
94 u8 type;
95 u8 flags;
96 u16 length;
97 u16 devid;
98 u16 cap_ptr;
99 u64 mmio_phys;
100 u16 pci_seg;
101 u16 info;
102 u32 reserved;
103} __attribute__((packed));
104
Joerg Roedelb65233a2008-07-11 17:14:21 +0200105/*
106 * A device entry describing which devices a specific IOMMU translates and
107 * which requestor ids they use.
108 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +0200109struct ivhd_entry {
110 u8 type;
111 u16 devid;
112 u8 flags;
113 u32 ext;
114} __attribute__((packed));
115
Joerg Roedelb65233a2008-07-11 17:14:21 +0200116/*
117 * An AMD IOMMU memory definition structure. It defines things like exclusion
118 * ranges for devices and regions that should be unity mapped.
119 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +0200120struct ivmd_header {
121 u8 type;
122 u8 flags;
123 u16 length;
124 u16 devid;
125 u16 aux;
126 u64 resv;
127 u64 range_start;
128 u64 range_length;
129} __attribute__((packed));
130
Joerg Roedelfefda112009-05-20 12:21:42 +0200131bool amd_iommu_dump;
Joerg Roedel05152a02012-06-15 16:53:51 +0200132bool amd_iommu_irq_remap __read_mostly;
Joerg Roedelfefda112009-05-20 12:21:42 +0200133
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200134static bool amd_iommu_detected;
Joerg Roedela5235722010-05-11 17:12:33 +0200135static bool __initdata amd_iommu_disabled;
Joerg Roedelc1cbebe2008-07-03 19:35:10 +0200136
Joerg Roedelb65233a2008-07-11 17:14:21 +0200137u16 amd_iommu_last_bdf; /* largest PCI device id we have
138 to handle */
Joerg Roedel2e228472008-07-11 17:14:31 +0200139LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
Joerg Roedelb65233a2008-07-11 17:14:21 +0200140 we find in ACPI */
Dan Carpenter3775d482012-06-27 12:09:18 +0300141u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */
Joerg Roedel928abd22008-06-26 21:27:40 +0200142
Joerg Roedel2e228472008-07-11 17:14:31 +0200143LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
Joerg Roedelb65233a2008-07-11 17:14:21 +0200144 system */
145
Joerg Roedelbb527772009-11-20 14:31:51 +0100146/* Array to assign indices to IOMMUs*/
147struct amd_iommu *amd_iommus[MAX_IOMMUS];
148int amd_iommus_present;
149
Joerg Roedel318afd42009-11-23 18:32:38 +0100150/* IOMMUs have a non-present cache? */
151bool amd_iommu_np_cache __read_mostly;
Joerg Roedel60f723b2011-04-05 12:50:24 +0200152bool amd_iommu_iotlb_sup __read_mostly = true;
Joerg Roedel318afd42009-11-23 18:32:38 +0100153
Joerg Roedel62f71ab2011-11-10 14:41:57 +0100154u32 amd_iommu_max_pasids __read_mostly = ~0;
155
Joerg Roedel400a28a2011-11-28 15:11:02 +0100156bool amd_iommu_v2_present __read_mostly;
157
Joerg Roedel5abcdba2011-12-01 15:49:45 +0100158bool amd_iommu_force_isolation __read_mostly;
159
Joerg Roedelb65233a2008-07-11 17:14:21 +0200160/*
Joerg Roedelaeb26f52009-11-20 16:44:01 +0100161 * List of protection domains - used during resume
162 */
163LIST_HEAD(amd_iommu_pd_list);
164spinlock_t amd_iommu_pd_lock;
165
166/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200167 * Pointer to the device table which is shared by all AMD IOMMUs
168 * it is indexed by the PCI device id or the HT unit id and contains
169 * information about the domain the device belongs to as well as the
170 * page table root pointer.
171 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200172struct dev_table_entry *amd_iommu_dev_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200173
174/*
175 * The alias table is a driver specific data structure which contains the
176 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
177 * More than one device can share the same requestor id.
178 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200179u16 *amd_iommu_alias_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200180
181/*
182 * The rlookup table is used to find the IOMMU which is responsible
183 * for a specific device. It is also indexed by the PCI device id.
184 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200185struct amd_iommu **amd_iommu_rlookup_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200186
187/*
Joerg Roedel0ea2c422012-06-15 18:05:20 +0200188 * This table is used to find the irq remapping table for a given device id
189 * quickly.
190 */
191struct irq_remap_table **irq_lookup_table;
192
193/*
Frank Arnolddf805ab2012-08-27 19:21:04 +0200194 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
Joerg Roedelb65233a2008-07-11 17:14:21 +0200195 * to know which ones are already in use.
196 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200197unsigned long *amd_iommu_pd_alloc_bitmap;
198
Joerg Roedelb65233a2008-07-11 17:14:21 +0200199static u32 dev_table_size; /* size of the device table */
200static u32 alias_table_size; /* size of the alias table */
201static u32 rlookup_table_size; /* size if the rlookup table */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200202
Joerg Roedel2c0ae172012-06-12 15:59:30 +0200203enum iommu_init_state {
204 IOMMU_START_STATE,
205 IOMMU_IVRS_DETECTED,
206 IOMMU_ACPI_FINISHED,
207 IOMMU_ENABLED,
208 IOMMU_PCI_INIT,
209 IOMMU_INTERRUPTS_EN,
210 IOMMU_DMA_OPS,
211 IOMMU_INITIALIZED,
212 IOMMU_NOT_FOUND,
213 IOMMU_INIT_ERROR,
214};
215
Joerg Roedel235dacb2013-04-09 17:53:14 +0200216/* Early ioapic and hpet maps from kernel command line */
217#define EARLY_MAP_SIZE 4
218static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
219static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
220static int __initdata early_ioapic_map_size;
221static int __initdata early_hpet_map_size;
Joerg Roedeldfbb6d42013-04-09 19:06:18 +0200222static bool __initdata cmdline_maps;
Joerg Roedel235dacb2013-04-09 17:53:14 +0200223
Joerg Roedel2c0ae172012-06-12 15:59:30 +0200224static enum iommu_init_state init_state = IOMMU_START_STATE;
225
Gerard Snitselaarae295142012-03-16 11:38:22 -0700226static int amd_iommu_enable_interrupts(void);
Joerg Roedel2c0ae172012-06-12 15:59:30 +0200227static int __init iommu_go_to_state(enum iommu_init_state state);
Joerg Roedel3d9761e2012-03-15 16:39:21 +0100228
Joerg Roedel208ec8c2008-07-11 17:14:24 +0200229static inline void update_last_devid(u16 devid)
230{
231 if (devid > amd_iommu_last_bdf)
232 amd_iommu_last_bdf = devid;
233}
234
Joerg Roedelc5714842008-07-11 17:14:25 +0200235static inline unsigned long tbl_size(int entry_size)
236{
237 unsigned shift = PAGE_SHIFT +
Neil Turton421f9092009-05-14 14:00:35 +0100238 get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
Joerg Roedelc5714842008-07-11 17:14:25 +0200239
240 return 1UL << shift;
241}
242
Matthew Garrett5bcd7572010-10-04 14:59:31 -0400243/* Access to l1 and l2 indexed register spaces */
244
245static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
246{
247 u32 val;
248
249 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
250 pci_read_config_dword(iommu->dev, 0xfc, &val);
251 return val;
252}
253
254static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
255{
256 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
257 pci_write_config_dword(iommu->dev, 0xfc, val);
258 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
259}
260
261static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
262{
263 u32 val;
264
265 pci_write_config_dword(iommu->dev, 0xf0, address);
266 pci_read_config_dword(iommu->dev, 0xf4, &val);
267 return val;
268}
269
270static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
271{
272 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
273 pci_write_config_dword(iommu->dev, 0xf4, val);
274}
275
Joerg Roedelb65233a2008-07-11 17:14:21 +0200276/****************************************************************************
277 *
278 * AMD IOMMU MMIO register space handling functions
279 *
280 * These functions are used to program the IOMMU device registers in
281 * MMIO space required for that driver.
282 *
283 ****************************************************************************/
284
285/*
286 * This function set the exclusion range in the IOMMU. DMA accesses to the
287 * exclusion range are passed through untranslated
288 */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200289static void iommu_set_exclusion_range(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200290{
291 u64 start = iommu->exclusion_start & PAGE_MASK;
292 u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
293 u64 entry;
294
295 if (!iommu->exclusion_start)
296 return;
297
298 entry = start | MMIO_EXCL_ENABLE_MASK;
299 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
300 &entry, sizeof(entry));
301
302 entry = limit;
303 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
304 &entry, sizeof(entry));
305}
306
Joerg Roedelb65233a2008-07-11 17:14:21 +0200307/* Programs the physical address of the device table into the IOMMU hardware */
Jan Beulich6b7f0002012-03-08 08:58:13 +0000308static void iommu_set_device_table(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200309{
Andreas Herrmannf6098912008-10-16 16:27:36 +0200310 u64 entry;
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200311
312 BUG_ON(iommu->mmio_base == NULL);
313
314 entry = virt_to_phys(amd_iommu_dev_table);
315 entry |= (dev_table_size >> 12) - 1;
316 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
317 &entry, sizeof(entry));
318}
319
Joerg Roedelb65233a2008-07-11 17:14:21 +0200320/* Generic functions to enable/disable certain features of the IOMMU. */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200321static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200322{
323 u32 ctrl;
324
325 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
326 ctrl |= (1 << bit);
327 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
328}
329
Joerg Roedelca0207112009-10-28 18:02:26 +0100330static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200331{
332 u32 ctrl;
333
Joerg Roedel199d0d52008-09-17 16:45:59 +0200334 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200335 ctrl &= ~(1 << bit);
336 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
337}
338
Joerg Roedel1456e9d2011-12-22 14:51:53 +0100339static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
340{
341 u32 ctrl;
342
343 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
344 ctrl &= ~CTRL_INV_TO_MASK;
345 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
346 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
347}
348
Joerg Roedelb65233a2008-07-11 17:14:21 +0200349/* Function to enable the hardware */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200350static void iommu_enable(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200351{
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200352 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200353}
354
Joerg Roedel92ac4322009-05-19 19:06:27 +0200355static void iommu_disable(struct amd_iommu *iommu)
Joerg Roedel126c52b2008-09-09 16:47:35 +0200356{
Chris Wrighta8c485b2009-06-15 15:53:45 +0200357 /* Disable command buffer */
358 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
359
360 /* Disable event logging and event interrupts */
361 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
362 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
363
364 /* Disable IOMMU hardware itself */
Joerg Roedel92ac4322009-05-19 19:06:27 +0200365 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
Joerg Roedel126c52b2008-09-09 16:47:35 +0200366}
367
Joerg Roedelb65233a2008-07-11 17:14:21 +0200368/*
369 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
370 * the system has one.
371 */
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200372static u8 __iomem * __init iommu_map_mmio_space(u64 address)
Joerg Roedel6c567472008-06-26 21:27:43 +0200373{
Joerg Roedele82752d2010-05-28 14:26:48 +0200374 if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {
375 pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",
376 address);
377 pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
Joerg Roedel6c567472008-06-26 21:27:43 +0200378 return NULL;
Joerg Roedele82752d2010-05-28 14:26:48 +0200379 }
Joerg Roedel6c567472008-06-26 21:27:43 +0200380
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200381 return (u8 __iomem *)ioremap_nocache(address, MMIO_REGION_LENGTH);
Joerg Roedel6c567472008-06-26 21:27:43 +0200382}
383
384static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
385{
386 if (iommu->mmio_base)
387 iounmap(iommu->mmio_base);
388 release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
389}
390
Joerg Roedelb65233a2008-07-11 17:14:21 +0200391/****************************************************************************
392 *
393 * The functions below belong to the first pass of AMD IOMMU ACPI table
394 * parsing. In this pass we try to find out the highest device id this
395 * code has to handle. Upon this information the size of the shared data
396 * structures is determined later.
397 *
398 ****************************************************************************/
399
400/*
Joerg Roedelb514e552008-09-17 17:14:27 +0200401 * This function calculates the length of a given IVHD entry
402 */
403static inline int ivhd_entry_length(u8 *ivhd)
404{
405 return 0x04 << (*ivhd >> 6);
406}
407
408/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200409 * This function reads the last device id the IOMMU has to handle from the PCI
410 * capability header for this IOMMU
411 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200412static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
413{
414 u32 cap;
415
416 cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
Shuah Khan6f2729b2013-02-27 17:07:30 -0700417 update_last_devid(PCI_DEVID(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200418
419 return 0;
420}
421
Joerg Roedelb65233a2008-07-11 17:14:21 +0200422/*
423 * After reading the highest device id from the IOMMU PCI capability header
424 * this function looks if there is a higher device id defined in the ACPI table
425 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200426static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
427{
428 u8 *p = (void *)h, *end = (void *)h;
429 struct ivhd_entry *dev;
430
431 p += sizeof(*h);
432 end += h->length;
433
Shuah Khanc5081cd2013-02-27 17:07:19 -0700434 find_last_devid_on_pci(PCI_BUS_NUM(h->devid),
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200435 PCI_SLOT(h->devid),
436 PCI_FUNC(h->devid),
437 h->cap_ptr);
438
439 while (p < end) {
440 dev = (struct ivhd_entry *)p;
441 switch (dev->type) {
442 case IVHD_DEV_SELECT:
443 case IVHD_DEV_RANGE_END:
444 case IVHD_DEV_ALIAS:
445 case IVHD_DEV_EXT_SELECT:
Joerg Roedelb65233a2008-07-11 17:14:21 +0200446 /* all the above subfield types refer to device ids */
Joerg Roedel208ec8c2008-07-11 17:14:24 +0200447 update_last_devid(dev->devid);
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200448 break;
449 default:
450 break;
451 }
Joerg Roedelb514e552008-09-17 17:14:27 +0200452 p += ivhd_entry_length(p);
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200453 }
454
455 WARN_ON(p != end);
456
457 return 0;
458}
459
Joerg Roedelb65233a2008-07-11 17:14:21 +0200460/*
461 * Iterate over all IVHD entries in the ACPI table and find the highest device
462 * id which we need to handle. This is the first of three functions which parse
463 * the ACPI table. So we check the checksum here.
464 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200465static int __init find_last_devid_acpi(struct acpi_table_header *table)
466{
467 int i;
468 u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
469 struct ivhd_header *h;
470
471 /*
472 * Validate checksum here so we don't need to do it when
473 * we actually parse the table
474 */
475 for (i = 0; i < table->length; ++i)
476 checksum += p[i];
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200477 if (checksum != 0)
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200478 /* ACPI table corrupt */
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200479 return -ENODEV;
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200480
481 p += IVRS_HEADER_LENGTH;
482
483 end += table->length;
484 while (p < end) {
485 h = (struct ivhd_header *)p;
486 switch (h->type) {
487 case ACPI_IVHD_TYPE:
488 find_last_devid_from_ivhd(h);
489 break;
490 default:
491 break;
492 }
493 p += h->length;
494 }
495 WARN_ON(p != end);
496
497 return 0;
498}
499
Joerg Roedelb65233a2008-07-11 17:14:21 +0200500/****************************************************************************
501 *
Frank Arnolddf805ab2012-08-27 19:21:04 +0200502 * The following functions belong to the code path which parses the ACPI table
Joerg Roedelb65233a2008-07-11 17:14:21 +0200503 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
504 * data structures, initialize the device/alias/rlookup table and also
505 * basically initialize the hardware.
506 *
507 ****************************************************************************/
508
509/*
510 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
511 * write commands to that buffer later and the IOMMU will execute them
512 * asynchronously
513 */
Joerg Roedelb36ca912008-06-26 21:27:45 +0200514static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
515{
Joerg Roedeld0312b22008-07-11 17:14:29 +0200516 u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
Joerg Roedelb36ca912008-06-26 21:27:45 +0200517 get_order(CMD_BUFFER_SIZE));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200518
519 if (cmd_buf == NULL)
520 return NULL;
521
Chris Wright549c90d2010-04-02 18:27:53 -0700522 iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;
Joerg Roedelb36ca912008-06-26 21:27:45 +0200523
Joerg Roedel58492e12009-05-04 18:41:16 +0200524 return cmd_buf;
525}
526
527/*
Joerg Roedel93f1cc672009-09-03 14:50:20 +0200528 * This function resets the command buffer if the IOMMU stopped fetching
529 * commands from it.
530 */
531void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
532{
533 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
534
535 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
536 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
537
538 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
539}
540
541/*
Joerg Roedel58492e12009-05-04 18:41:16 +0200542 * This function writes the command buffer address to the hardware and
543 * enables it.
544 */
545static void iommu_enable_command_buffer(struct amd_iommu *iommu)
546{
547 u64 entry;
548
549 BUG_ON(iommu->cmd_buf == NULL);
550
551 entry = (u64)virt_to_phys(iommu->cmd_buf);
Joerg Roedelb36ca912008-06-26 21:27:45 +0200552 entry |= MMIO_CMD_SIZE_512;
Joerg Roedel58492e12009-05-04 18:41:16 +0200553
Joerg Roedelb36ca912008-06-26 21:27:45 +0200554 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
Joerg Roedel58492e12009-05-04 18:41:16 +0200555 &entry, sizeof(entry));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200556
Joerg Roedel93f1cc672009-09-03 14:50:20 +0200557 amd_iommu_reset_cmd_buffer(iommu);
Chris Wright549c90d2010-04-02 18:27:53 -0700558 iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);
Joerg Roedelb36ca912008-06-26 21:27:45 +0200559}
560
561static void __init free_command_buffer(struct amd_iommu *iommu)
562{
Joerg Roedel23c17132008-09-17 17:18:17 +0200563 free_pages((unsigned long)iommu->cmd_buf,
Chris Wright549c90d2010-04-02 18:27:53 -0700564 get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200565}
566
Joerg Roedel335503e2008-09-05 14:29:07 +0200567/* allocates the memory where the IOMMU will log its events to */
568static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
569{
Joerg Roedel335503e2008-09-05 14:29:07 +0200570 iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
571 get_order(EVT_BUFFER_SIZE));
572
573 if (iommu->evt_buf == NULL)
574 return NULL;
575
Joerg Roedel1bc6f832009-07-02 18:32:05 +0200576 iommu->evt_buf_size = EVT_BUFFER_SIZE;
577
Joerg Roedel58492e12009-05-04 18:41:16 +0200578 return iommu->evt_buf;
579}
580
581static void iommu_enable_event_buffer(struct amd_iommu *iommu)
582{
583 u64 entry;
584
585 BUG_ON(iommu->evt_buf == NULL);
586
Joerg Roedel335503e2008-09-05 14:29:07 +0200587 entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
Joerg Roedel58492e12009-05-04 18:41:16 +0200588
Joerg Roedel335503e2008-09-05 14:29:07 +0200589 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
590 &entry, sizeof(entry));
591
Joerg Roedel090672072009-06-15 16:06:48 +0200592 /* set head and tail to zero manually */
593 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
594 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
595
Joerg Roedel58492e12009-05-04 18:41:16 +0200596 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
Joerg Roedel335503e2008-09-05 14:29:07 +0200597}
598
599static void __init free_event_buffer(struct amd_iommu *iommu)
600{
601 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
602}
603
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100604/* allocates the memory where the IOMMU will log its events to */
605static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
606{
607 iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
608 get_order(PPR_LOG_SIZE));
609
610 if (iommu->ppr_log == NULL)
611 return NULL;
612
613 return iommu->ppr_log;
614}
615
616static void iommu_enable_ppr_log(struct amd_iommu *iommu)
617{
618 u64 entry;
619
620 if (iommu->ppr_log == NULL)
621 return;
622
623 entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
624
625 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
626 &entry, sizeof(entry));
627
628 /* set head and tail to zero manually */
629 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
630 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
631
632 iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
633 iommu_feature_enable(iommu, CONTROL_PPR_EN);
634}
635
636static void __init free_ppr_log(struct amd_iommu *iommu)
637{
638 if (iommu->ppr_log == NULL)
639 return;
640
641 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
642}
643
Joerg Roedelcbc33a92011-11-25 11:41:31 +0100644static void iommu_enable_gt(struct amd_iommu *iommu)
645{
646 if (!iommu_feature(iommu, FEATURE_GT))
647 return;
648
649 iommu_feature_enable(iommu, CONTROL_GT_EN);
650}
651
Joerg Roedelb65233a2008-07-11 17:14:21 +0200652/* sets a specific bit in the device table entry. */
Joerg Roedel3566b772008-06-26 21:27:46 +0200653static void set_dev_entry_bit(u16 devid, u8 bit)
654{
Joerg Roedelee6c2862011-11-09 12:06:03 +0100655 int i = (bit >> 6) & 0x03;
656 int _bit = bit & 0x3f;
Joerg Roedel3566b772008-06-26 21:27:46 +0200657
Joerg Roedelee6c2862011-11-09 12:06:03 +0100658 amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);
Joerg Roedel3566b772008-06-26 21:27:46 +0200659}
660
Joerg Roedelc5cca142009-10-09 18:31:20 +0200661static int get_dev_entry_bit(u16 devid, u8 bit)
662{
Joerg Roedelee6c2862011-11-09 12:06:03 +0100663 int i = (bit >> 6) & 0x03;
664 int _bit = bit & 0x3f;
Joerg Roedelc5cca142009-10-09 18:31:20 +0200665
Joerg Roedelee6c2862011-11-09 12:06:03 +0100666 return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
Joerg Roedelc5cca142009-10-09 18:31:20 +0200667}
668
669
670void amd_iommu_apply_erratum_63(u16 devid)
671{
672 int sysmgt;
673
674 sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |
675 (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
676
677 if (sysmgt == 0x01)
678 set_dev_entry_bit(devid, DEV_ENTRY_IW);
679}
680
Joerg Roedel5ff47892008-07-14 20:11:18 +0200681/* Writes the specific IOMMU for a device into the rlookup table */
682static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
683{
684 amd_iommu_rlookup_table[devid] = iommu;
685}
686
Joerg Roedelb65233a2008-07-11 17:14:21 +0200687/*
688 * This function takes the device specific flags read from the ACPI
689 * table and sets up the device table entry with that information
690 */
Joerg Roedel5ff47892008-07-14 20:11:18 +0200691static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
692 u16 devid, u32 flags, u32 ext_flags)
Joerg Roedel3566b772008-06-26 21:27:46 +0200693{
694 if (flags & ACPI_DEVFLAG_INITPASS)
695 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
696 if (flags & ACPI_DEVFLAG_EXTINT)
697 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
698 if (flags & ACPI_DEVFLAG_NMI)
699 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
700 if (flags & ACPI_DEVFLAG_SYSMGT1)
701 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
702 if (flags & ACPI_DEVFLAG_SYSMGT2)
703 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
704 if (flags & ACPI_DEVFLAG_LINT0)
705 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
706 if (flags & ACPI_DEVFLAG_LINT1)
707 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
Joerg Roedel3566b772008-06-26 21:27:46 +0200708
Joerg Roedelc5cca142009-10-09 18:31:20 +0200709 amd_iommu_apply_erratum_63(devid);
710
Joerg Roedel5ff47892008-07-14 20:11:18 +0200711 set_iommu_for_device(iommu, devid);
Joerg Roedel3566b772008-06-26 21:27:46 +0200712}
713
Joerg Roedel31cff672013-04-09 16:53:58 +0200714static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line)
Joerg Roedel6efed632012-06-14 15:52:58 +0200715{
716 struct devid_map *entry;
717 struct list_head *list;
718
Joerg Roedel31cff672013-04-09 16:53:58 +0200719 if (type == IVHD_SPECIAL_IOAPIC)
720 list = &ioapic_map;
721 else if (type == IVHD_SPECIAL_HPET)
722 list = &hpet_map;
723 else
Joerg Roedel6efed632012-06-14 15:52:58 +0200724 return -EINVAL;
725
Joerg Roedel31cff672013-04-09 16:53:58 +0200726 list_for_each_entry(entry, list, list) {
727 if (!(entry->id == id && entry->cmd_line))
728 continue;
729
730 pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n",
731 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
732
733 return 0;
734 }
735
Joerg Roedel6efed632012-06-14 15:52:58 +0200736 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
737 if (!entry)
738 return -ENOMEM;
739
Joerg Roedel31cff672013-04-09 16:53:58 +0200740 entry->id = id;
741 entry->devid = devid;
742 entry->cmd_line = cmd_line;
Joerg Roedel6efed632012-06-14 15:52:58 +0200743
744 list_add_tail(&entry->list, list);
745
746 return 0;
747}
748
Joerg Roedel235dacb2013-04-09 17:53:14 +0200749static int __init add_early_maps(void)
750{
751 int i, ret;
752
753 for (i = 0; i < early_ioapic_map_size; ++i) {
754 ret = add_special_device(IVHD_SPECIAL_IOAPIC,
755 early_ioapic_map[i].id,
756 early_ioapic_map[i].devid,
757 early_ioapic_map[i].cmd_line);
758 if (ret)
759 return ret;
760 }
761
762 for (i = 0; i < early_hpet_map_size; ++i) {
763 ret = add_special_device(IVHD_SPECIAL_HPET,
764 early_hpet_map[i].id,
765 early_hpet_map[i].devid,
766 early_hpet_map[i].cmd_line);
767 if (ret)
768 return ret;
769 }
770
771 return 0;
772}
773
Joerg Roedelb65233a2008-07-11 17:14:21 +0200774/*
Frank Arnolddf805ab2012-08-27 19:21:04 +0200775 * Reads the device exclusion range from ACPI and initializes the IOMMU with
Joerg Roedelb65233a2008-07-11 17:14:21 +0200776 * it
777 */
Joerg Roedel3566b772008-06-26 21:27:46 +0200778static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
779{
780 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
781
782 if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
783 return;
784
785 if (iommu) {
Joerg Roedelb65233a2008-07-11 17:14:21 +0200786 /*
787 * We only can configure exclusion ranges per IOMMU, not
788 * per device. But we can enable the exclusion range per
789 * device. This is done here
790 */
Joerg Roedel3566b772008-06-26 21:27:46 +0200791 set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
792 iommu->exclusion_start = m->range_start;
793 iommu->exclusion_length = m->range_length;
794 }
795}
796
Joerg Roedelb65233a2008-07-11 17:14:21 +0200797/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200798 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
799 * initializes the hardware and our data structures with it.
800 */
Joerg Roedel6efed632012-06-14 15:52:58 +0200801static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200802 struct ivhd_header *h)
803{
804 u8 *p = (u8 *)h;
805 u8 *end = p, flags = 0;
Joerg Roedel0de66d52011-06-06 16:04:02 +0200806 u16 devid = 0, devid_start = 0, devid_to = 0;
807 u32 dev_i, ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200808 bool alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200809 struct ivhd_entry *e;
Joerg Roedel235dacb2013-04-09 17:53:14 +0200810 int ret;
811
812
813 ret = add_early_maps();
814 if (ret)
815 return ret;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200816
817 /*
Joerg Roedele9bf5192010-09-20 14:33:07 +0200818 * First save the recommended feature enable bits from ACPI
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200819 */
Joerg Roedele9bf5192010-09-20 14:33:07 +0200820 iommu->acpi_flags = h->flags;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200821
822 /*
823 * Done. Now parse the device entries
824 */
825 p += sizeof(struct ivhd_header);
826 end += h->length;
827
Joerg Roedel42a698f2009-05-20 15:41:28 +0200828
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200829 while (p < end) {
830 e = (struct ivhd_entry *)p;
831 switch (e->type) {
832 case IVHD_DEV_ALL:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200833
834 DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
835 " last device %02x:%02x.%x flags: %02x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700836 PCI_BUS_NUM(iommu->first_device),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200837 PCI_SLOT(iommu->first_device),
838 PCI_FUNC(iommu->first_device),
Shuah Khanc5081cd2013-02-27 17:07:19 -0700839 PCI_BUS_NUM(iommu->last_device),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200840 PCI_SLOT(iommu->last_device),
841 PCI_FUNC(iommu->last_device),
842 e->flags);
843
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200844 for (dev_i = iommu->first_device;
845 dev_i <= iommu->last_device; ++dev_i)
Joerg Roedel5ff47892008-07-14 20:11:18 +0200846 set_dev_entry_from_acpi(iommu, dev_i,
847 e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200848 break;
849 case IVHD_DEV_SELECT:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200850
851 DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
852 "flags: %02x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700853 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200854 PCI_SLOT(e->devid),
855 PCI_FUNC(e->devid),
856 e->flags);
857
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200858 devid = e->devid;
Joerg Roedel5ff47892008-07-14 20:11:18 +0200859 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200860 break;
861 case IVHD_DEV_SELECT_RANGE_START:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200862
863 DUMP_printk(" DEV_SELECT_RANGE_START\t "
864 "devid: %02x:%02x.%x flags: %02x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700865 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200866 PCI_SLOT(e->devid),
867 PCI_FUNC(e->devid),
868 e->flags);
869
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200870 devid_start = e->devid;
871 flags = e->flags;
872 ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200873 alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200874 break;
875 case IVHD_DEV_ALIAS:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200876
877 DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
878 "flags: %02x devid_to: %02x:%02x.%x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700879 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200880 PCI_SLOT(e->devid),
881 PCI_FUNC(e->devid),
882 e->flags,
Shuah Khanc5081cd2013-02-27 17:07:19 -0700883 PCI_BUS_NUM(e->ext >> 8),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200884 PCI_SLOT(e->ext >> 8),
885 PCI_FUNC(e->ext >> 8));
886
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200887 devid = e->devid;
888 devid_to = e->ext >> 8;
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200889 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
Neil Turton7455aab2009-05-14 14:08:11 +0100890 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200891 amd_iommu_alias_table[devid] = devid_to;
892 break;
893 case IVHD_DEV_ALIAS_RANGE:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200894
895 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
896 "devid: %02x:%02x.%x flags: %02x "
897 "devid_to: %02x:%02x.%x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700898 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200899 PCI_SLOT(e->devid),
900 PCI_FUNC(e->devid),
901 e->flags,
Shuah Khanc5081cd2013-02-27 17:07:19 -0700902 PCI_BUS_NUM(e->ext >> 8),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200903 PCI_SLOT(e->ext >> 8),
904 PCI_FUNC(e->ext >> 8));
905
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200906 devid_start = e->devid;
907 flags = e->flags;
908 devid_to = e->ext >> 8;
909 ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200910 alias = true;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200911 break;
912 case IVHD_DEV_EXT_SELECT:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200913
914 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
915 "flags: %02x ext: %08x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700916 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200917 PCI_SLOT(e->devid),
918 PCI_FUNC(e->devid),
919 e->flags, e->ext);
920
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200921 devid = e->devid;
Joerg Roedel5ff47892008-07-14 20:11:18 +0200922 set_dev_entry_from_acpi(iommu, devid, e->flags,
923 e->ext);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200924 break;
925 case IVHD_DEV_EXT_SELECT_RANGE:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200926
927 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
928 "%02x:%02x.%x flags: %02x ext: %08x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700929 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200930 PCI_SLOT(e->devid),
931 PCI_FUNC(e->devid),
932 e->flags, e->ext);
933
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200934 devid_start = e->devid;
935 flags = e->flags;
936 ext_flags = e->ext;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200937 alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200938 break;
939 case IVHD_DEV_RANGE_END:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200940
941 DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -0700942 PCI_BUS_NUM(e->devid),
Joerg Roedel42a698f2009-05-20 15:41:28 +0200943 PCI_SLOT(e->devid),
944 PCI_FUNC(e->devid));
945
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200946 devid = e->devid;
947 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200948 if (alias) {
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200949 amd_iommu_alias_table[dev_i] = devid_to;
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200950 set_dev_entry_from_acpi(iommu,
951 devid_to, flags, ext_flags);
952 }
953 set_dev_entry_from_acpi(iommu, dev_i,
954 flags, ext_flags);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200955 }
956 break;
Joerg Roedel6efed632012-06-14 15:52:58 +0200957 case IVHD_DEV_SPECIAL: {
958 u8 handle, type;
959 const char *var;
960 u16 devid;
961 int ret;
962
963 handle = e->ext & 0xff;
964 devid = (e->ext >> 8) & 0xffff;
965 type = (e->ext >> 24) & 0xff;
966
967 if (type == IVHD_SPECIAL_IOAPIC)
968 var = "IOAPIC";
969 else if (type == IVHD_SPECIAL_HPET)
970 var = "HPET";
971 else
972 var = "UNKNOWN";
973
974 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
975 var, (int)handle,
Shuah Khanc5081cd2013-02-27 17:07:19 -0700976 PCI_BUS_NUM(devid),
Joerg Roedel6efed632012-06-14 15:52:58 +0200977 PCI_SLOT(devid),
978 PCI_FUNC(devid));
979
980 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
Joerg Roedel31cff672013-04-09 16:53:58 +0200981 ret = add_special_device(type, handle, devid, false);
Joerg Roedel6efed632012-06-14 15:52:58 +0200982 if (ret)
983 return ret;
984 break;
985 }
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200986 default:
987 break;
988 }
989
Joerg Roedelb514e552008-09-17 17:14:27 +0200990 p += ivhd_entry_length(p);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200991 }
Joerg Roedel6efed632012-06-14 15:52:58 +0200992
993 return 0;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200994}
995
Joerg Roedelb65233a2008-07-11 17:14:21 +0200996/* Initializes the device->iommu mapping for the driver */
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200997static int __init init_iommu_devices(struct amd_iommu *iommu)
998{
Joerg Roedel0de66d52011-06-06 16:04:02 +0200999 u32 i;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +02001000
1001 for (i = iommu->first_device; i <= iommu->last_device; ++i)
1002 set_iommu_for_device(iommu, i);
1003
1004 return 0;
1005}
1006
Joerg Roedele47d4022008-06-26 21:27:48 +02001007static void __init free_iommu_one(struct amd_iommu *iommu)
1008{
1009 free_command_buffer(iommu);
Joerg Roedel335503e2008-09-05 14:29:07 +02001010 free_event_buffer(iommu);
Joerg Roedel1a29ac02011-11-10 15:41:40 +01001011 free_ppr_log(iommu);
Joerg Roedele47d4022008-06-26 21:27:48 +02001012 iommu_unmap_mmio_space(iommu);
1013}
1014
1015static void __init free_iommu_all(void)
1016{
1017 struct amd_iommu *iommu, *next;
1018
Joerg Roedel3bd22172009-05-04 15:06:20 +02001019 for_each_iommu_safe(iommu, next) {
Joerg Roedele47d4022008-06-26 21:27:48 +02001020 list_del(&iommu->list);
1021 free_iommu_one(iommu);
1022 kfree(iommu);
1023 }
1024}
1025
Joerg Roedelb65233a2008-07-11 17:14:21 +02001026/*
Suravee Suthikulpanit318fe782013-01-24 13:17:53 -06001027 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1028 * Workaround:
1029 * BIOS should disable L2B micellaneous clock gating by setting
1030 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1031 */
Nikola Pajkovskye2f1a3b2013-02-26 16:12:05 +01001032static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
Suravee Suthikulpanit318fe782013-01-24 13:17:53 -06001033{
1034 u32 value;
1035
1036 if ((boot_cpu_data.x86 != 0x15) ||
1037 (boot_cpu_data.x86_model < 0x10) ||
1038 (boot_cpu_data.x86_model > 0x1f))
1039 return;
1040
1041 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1042 pci_read_config_dword(iommu->dev, 0xf4, &value);
1043
1044 if (value & BIT(2))
1045 return;
1046
1047 /* Select NB indirect register 0x90 and enable writing */
1048 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1049
1050 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1051 pr_info("AMD-Vi: Applying erratum 746 workaround for IOMMU at %s\n",
1052 dev_name(&iommu->dev->dev));
1053
1054 /* Clear the enable writing bit */
1055 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1056}
1057
1058/*
Joerg Roedelb65233a2008-07-11 17:14:21 +02001059 * This function clues the initialization function for one IOMMU
1060 * together and also allocates the command buffer and programs the
1061 * hardware. It does NOT enable the IOMMU. This is done afterwards.
1062 */
Joerg Roedele47d4022008-06-26 21:27:48 +02001063static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
1064{
Joerg Roedel6efed632012-06-14 15:52:58 +02001065 int ret;
1066
Joerg Roedele47d4022008-06-26 21:27:48 +02001067 spin_lock_init(&iommu->lock);
Joerg Roedelbb527772009-11-20 14:31:51 +01001068
1069 /* Add IOMMU to internal data structures */
Joerg Roedele47d4022008-06-26 21:27:48 +02001070 list_add_tail(&iommu->list, &amd_iommu_list);
Joerg Roedelbb527772009-11-20 14:31:51 +01001071 iommu->index = amd_iommus_present++;
1072
1073 if (unlikely(iommu->index >= MAX_IOMMUS)) {
1074 WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
1075 return -ENOSYS;
1076 }
1077
1078 /* Index is fine - add IOMMU to the array */
1079 amd_iommus[iommu->index] = iommu;
Joerg Roedele47d4022008-06-26 21:27:48 +02001080
1081 /*
1082 * Copy data from ACPI table entry to the iommu struct
1083 */
Joerg Roedel23c742d2012-06-12 11:47:34 +02001084 iommu->devid = h->devid;
Joerg Roedele47d4022008-06-26 21:27:48 +02001085 iommu->cap_ptr = h->cap_ptr;
Joerg Roedelee893c22008-09-08 14:48:04 +02001086 iommu->pci_seg = h->pci_seg;
Joerg Roedele47d4022008-06-26 21:27:48 +02001087 iommu->mmio_phys = h->mmio_phys;
1088 iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
1089 if (!iommu->mmio_base)
1090 return -ENOMEM;
1091
Joerg Roedele47d4022008-06-26 21:27:48 +02001092 iommu->cmd_buf = alloc_command_buffer(iommu);
1093 if (!iommu->cmd_buf)
1094 return -ENOMEM;
1095
Joerg Roedel335503e2008-09-05 14:29:07 +02001096 iommu->evt_buf = alloc_event_buffer(iommu);
1097 if (!iommu->evt_buf)
1098 return -ENOMEM;
1099
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001100 iommu->int_enabled = false;
1101
Joerg Roedel6efed632012-06-14 15:52:58 +02001102 ret = init_iommu_from_acpi(iommu, h);
1103 if (ret)
1104 return ret;
Joerg Roedelf6fec002012-06-21 16:51:25 +02001105
1106 /*
1107 * Make sure IOMMU is not considered to translate itself. The IVRS
1108 * table tells us so, but this is a lie!
1109 */
1110 amd_iommu_rlookup_table[iommu->devid] = NULL;
1111
Joerg Roedele47d4022008-06-26 21:27:48 +02001112 init_iommu_devices(iommu);
1113
Joerg Roedel23c742d2012-06-12 11:47:34 +02001114 return 0;
Joerg Roedele47d4022008-06-26 21:27:48 +02001115}
1116
Joerg Roedelb65233a2008-07-11 17:14:21 +02001117/*
1118 * Iterates over all IOMMU entries in the ACPI table, allocates the
1119 * IOMMU structure and initializes it with init_iommu_one()
1120 */
Joerg Roedele47d4022008-06-26 21:27:48 +02001121static int __init init_iommu_all(struct acpi_table_header *table)
1122{
1123 u8 *p = (u8 *)table, *end = (u8 *)table;
1124 struct ivhd_header *h;
1125 struct amd_iommu *iommu;
1126 int ret;
1127
Joerg Roedele47d4022008-06-26 21:27:48 +02001128 end += table->length;
1129 p += IVRS_HEADER_LENGTH;
1130
1131 while (p < end) {
1132 h = (struct ivhd_header *)p;
1133 switch (*p) {
1134 case ACPI_IVHD_TYPE:
Joerg Roedel9c720412009-05-20 13:53:57 +02001135
Joerg Roedelae908c22009-09-01 16:52:16 +02001136 DUMP_printk("device: %02x:%02x.%01x cap: %04x "
Joerg Roedel9c720412009-05-20 13:53:57 +02001137 "seg: %d flags: %01x info %04x\n",
Shuah Khanc5081cd2013-02-27 17:07:19 -07001138 PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid),
Joerg Roedel9c720412009-05-20 13:53:57 +02001139 PCI_FUNC(h->devid), h->cap_ptr,
1140 h->pci_seg, h->flags, h->info);
1141 DUMP_printk(" mmio-addr: %016llx\n",
1142 h->mmio_phys);
1143
Joerg Roedele47d4022008-06-26 21:27:48 +02001144 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001145 if (iommu == NULL)
1146 return -ENOMEM;
Joerg Roedel3551a702010-03-01 13:52:19 +01001147
Joerg Roedele47d4022008-06-26 21:27:48 +02001148 ret = init_iommu_one(iommu, h);
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001149 if (ret)
1150 return ret;
Joerg Roedele47d4022008-06-26 21:27:48 +02001151 break;
1152 default:
1153 break;
1154 }
1155 p += h->length;
1156
1157 }
1158 WARN_ON(p != end);
1159
1160 return 0;
1161}
1162
Joerg Roedel23c742d2012-06-12 11:47:34 +02001163static int iommu_init_pci(struct amd_iommu *iommu)
1164{
1165 int cap_ptr = iommu->cap_ptr;
1166 u32 range, misc, low, high;
1167
Shuah Khanc5081cd2013-02-27 17:07:19 -07001168 iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),
Joerg Roedel23c742d2012-06-12 11:47:34 +02001169 iommu->devid & 0xff);
1170 if (!iommu->dev)
1171 return -ENODEV;
1172
1173 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
1174 &iommu->cap);
1175 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
1176 &range);
1177 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
1178 &misc);
1179
Shuah Khan6f2729b2013-02-27 17:07:30 -07001180 iommu->first_device = PCI_DEVID(MMIO_GET_BUS(range),
Joerg Roedel23c742d2012-06-12 11:47:34 +02001181 MMIO_GET_FD(range));
Shuah Khan6f2729b2013-02-27 17:07:30 -07001182 iommu->last_device = PCI_DEVID(MMIO_GET_BUS(range),
Joerg Roedel23c742d2012-06-12 11:47:34 +02001183 MMIO_GET_LD(range));
1184
1185 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
1186 amd_iommu_iotlb_sup = false;
1187
1188 /* read extended feature bits */
1189 low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
1190 high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
1191
1192 iommu->features = ((u64)high << 32) | low;
1193
1194 if (iommu_feature(iommu, FEATURE_GT)) {
1195 int glxval;
1196 u32 pasids;
1197 u64 shift;
1198
1199 shift = iommu->features & FEATURE_PASID_MASK;
1200 shift >>= FEATURE_PASID_SHIFT;
1201 pasids = (1 << shift);
1202
1203 amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
1204
1205 glxval = iommu->features & FEATURE_GLXVAL_MASK;
1206 glxval >>= FEATURE_GLXVAL_SHIFT;
1207
1208 if (amd_iommu_max_glx_val == -1)
1209 amd_iommu_max_glx_val = glxval;
1210 else
1211 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
1212 }
1213
1214 if (iommu_feature(iommu, FEATURE_GT) &&
1215 iommu_feature(iommu, FEATURE_PPR)) {
1216 iommu->is_iommu_v2 = true;
1217 amd_iommu_v2_present = true;
1218 }
1219
1220 if (iommu_feature(iommu, FEATURE_PPR)) {
1221 iommu->ppr_log = alloc_ppr_log(iommu);
1222 if (!iommu->ppr_log)
1223 return -ENOMEM;
1224 }
1225
1226 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
1227 amd_iommu_np_cache = true;
1228
1229 if (is_rd890_iommu(iommu->dev)) {
1230 int i, j;
1231
1232 iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
1233 PCI_DEVFN(0, 0));
1234
1235 /*
1236 * Some rd890 systems may not be fully reconfigured by the
1237 * BIOS, so it's necessary for us to store this information so
1238 * it can be reprogrammed on resume
1239 */
1240 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
1241 &iommu->stored_addr_lo);
1242 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
1243 &iommu->stored_addr_hi);
1244
1245 /* Low bit locks writes to configuration space */
1246 iommu->stored_addr_lo &= ~1;
1247
1248 for (i = 0; i < 6; i++)
1249 for (j = 0; j < 0x12; j++)
1250 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
1251
1252 for (i = 0; i < 0x83; i++)
1253 iommu->stored_l2[i] = iommu_read_l2(iommu, i);
1254 }
1255
Suravee Suthikulpanit318fe782013-01-24 13:17:53 -06001256 amd_iommu_erratum_746_workaround(iommu);
1257
Joerg Roedel23c742d2012-06-12 11:47:34 +02001258 return pci_enable_device(iommu->dev);
1259}
1260
Joerg Roedel4d121c32012-06-14 12:21:55 +02001261static void print_iommu_info(void)
1262{
1263 static const char * const feat_str[] = {
1264 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
1265 "IA", "GA", "HE", "PC"
1266 };
1267 struct amd_iommu *iommu;
1268
1269 for_each_iommu(iommu) {
1270 int i;
1271
1272 pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
1273 dev_name(&iommu->dev->dev), iommu->cap_ptr);
1274
1275 if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
1276 pr_info("AMD-Vi: Extended features: ");
Joerg Roedel2bd5ed02012-08-10 11:34:08 +02001277 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
Joerg Roedel4d121c32012-06-14 12:21:55 +02001278 if (iommu_feature(iommu, (1ULL << i)))
1279 pr_cont(" %s", feat_str[i]);
1280 }
Joerg Roedel4d121c32012-06-14 12:21:55 +02001281 pr_cont("\n");
Borislav Petkov500c25e2012-09-28 16:22:26 +02001282 }
Joerg Roedel4d121c32012-06-14 12:21:55 +02001283 }
Joerg Roedelebe60bb2012-07-02 18:36:03 +02001284 if (irq_remapping_enabled)
1285 pr_info("AMD-Vi: Interrupt remapping enabled\n");
Joerg Roedel4d121c32012-06-14 12:21:55 +02001286}
1287
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001288static int __init amd_iommu_init_pci(void)
Joerg Roedel23c742d2012-06-12 11:47:34 +02001289{
1290 struct amd_iommu *iommu;
1291 int ret = 0;
1292
1293 for_each_iommu(iommu) {
1294 ret = iommu_init_pci(iommu);
1295 if (ret)
1296 break;
1297 }
1298
Joerg Roedel23c742d2012-06-12 11:47:34 +02001299 ret = amd_iommu_init_devices();
1300
Joerg Roedel4d121c32012-06-14 12:21:55 +02001301 print_iommu_info();
1302
Joerg Roedel23c742d2012-06-12 11:47:34 +02001303 return ret;
1304}
1305
Joerg Roedelb65233a2008-07-11 17:14:21 +02001306/****************************************************************************
1307 *
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001308 * The following functions initialize the MSI interrupts for all IOMMUs
Frank Arnolddf805ab2012-08-27 19:21:04 +02001309 * in the system. It's a bit challenging because there could be multiple
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001310 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
1311 * pci_dev.
1312 *
1313 ****************************************************************************/
1314
Joerg Roedel9f800de2009-11-23 12:45:25 +01001315static int iommu_setup_msi(struct amd_iommu *iommu)
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001316{
1317 int r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001318
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001319 r = pci_enable_msi(iommu->dev);
1320 if (r)
1321 return r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001322
Joerg Roedel72fe00f2011-05-10 10:50:42 +02001323 r = request_threaded_irq(iommu->dev->irq,
1324 amd_iommu_int_handler,
1325 amd_iommu_int_thread,
1326 0, "AMD-Vi",
Suravee Suthikulpanit3f398bc2013-04-22 16:32:34 -05001327 iommu);
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001328
1329 if (r) {
1330 pci_disable_msi(iommu->dev);
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001331 return r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001332 }
1333
Joerg Roedelfab6afa2009-05-04 18:46:34 +02001334 iommu->int_enabled = true;
Joerg Roedel1a29ac02011-11-10 15:41:40 +01001335
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001336 return 0;
1337}
1338
Joerg Roedel05f92db2009-05-12 09:52:46 +02001339static int iommu_init_msi(struct amd_iommu *iommu)
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001340{
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001341 int ret;
1342
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001343 if (iommu->int_enabled)
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001344 goto enable_faults;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001345
Joerg Roedeld91cecd2009-05-04 18:51:00 +02001346 if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001347 ret = iommu_setup_msi(iommu);
1348 else
1349 ret = -ENODEV;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001350
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001351 if (ret)
1352 return ret;
1353
1354enable_faults:
1355 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
1356
1357 if (iommu->ppr_log != NULL)
1358 iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
1359
1360 return 0;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001361}
1362
1363/****************************************************************************
1364 *
Joerg Roedelb65233a2008-07-11 17:14:21 +02001365 * The next functions belong to the third pass of parsing the ACPI
1366 * table. In this last pass the memory mapping requirements are
Frank Arnolddf805ab2012-08-27 19:21:04 +02001367 * gathered (like exclusion and unity mapping ranges).
Joerg Roedelb65233a2008-07-11 17:14:21 +02001368 *
1369 ****************************************************************************/
1370
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001371static void __init free_unity_maps(void)
1372{
1373 struct unity_map_entry *entry, *next;
1374
1375 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
1376 list_del(&entry->list);
1377 kfree(entry);
1378 }
1379}
1380
Joerg Roedelb65233a2008-07-11 17:14:21 +02001381/* called when we find an exclusion range definition in ACPI */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001382static int __init init_exclusion_range(struct ivmd_header *m)
1383{
1384 int i;
1385
1386 switch (m->type) {
1387 case ACPI_IVMD_TYPE:
1388 set_device_exclusion_range(m->devid, m);
1389 break;
1390 case ACPI_IVMD_TYPE_ALL:
Joerg Roedel3a61ec32008-07-25 13:07:50 +02001391 for (i = 0; i <= amd_iommu_last_bdf; ++i)
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001392 set_device_exclusion_range(i, m);
1393 break;
1394 case ACPI_IVMD_TYPE_RANGE:
1395 for (i = m->devid; i <= m->aux; ++i)
1396 set_device_exclusion_range(i, m);
1397 break;
1398 default:
1399 break;
1400 }
1401
1402 return 0;
1403}
1404
Joerg Roedelb65233a2008-07-11 17:14:21 +02001405/* called for unity map ACPI definition */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001406static int __init init_unity_map_range(struct ivmd_header *m)
1407{
Joerg Roedel98f1ad22012-07-06 13:28:37 +02001408 struct unity_map_entry *e = NULL;
Joerg Roedel02acc432009-05-20 16:24:21 +02001409 char *s;
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001410
1411 e = kzalloc(sizeof(*e), GFP_KERNEL);
1412 if (e == NULL)
1413 return -ENOMEM;
1414
1415 switch (m->type) {
1416 default:
Joerg Roedel0bc252f2009-05-22 12:48:05 +02001417 kfree(e);
1418 return 0;
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001419 case ACPI_IVMD_TYPE:
Joerg Roedel02acc432009-05-20 16:24:21 +02001420 s = "IVMD_TYPEi\t\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001421 e->devid_start = e->devid_end = m->devid;
1422 break;
1423 case ACPI_IVMD_TYPE_ALL:
Joerg Roedel02acc432009-05-20 16:24:21 +02001424 s = "IVMD_TYPE_ALL\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001425 e->devid_start = 0;
1426 e->devid_end = amd_iommu_last_bdf;
1427 break;
1428 case ACPI_IVMD_TYPE_RANGE:
Joerg Roedel02acc432009-05-20 16:24:21 +02001429 s = "IVMD_TYPE_RANGE\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001430 e->devid_start = m->devid;
1431 e->devid_end = m->aux;
1432 break;
1433 }
1434 e->address_start = PAGE_ALIGN(m->range_start);
1435 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
1436 e->prot = m->flags >> 1;
1437
Joerg Roedel02acc432009-05-20 16:24:21 +02001438 DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
1439 " range_start: %016llx range_end: %016llx flags: %x\n", s,
Shuah Khanc5081cd2013-02-27 17:07:19 -07001440 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
1441 PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end),
Joerg Roedel02acc432009-05-20 16:24:21 +02001442 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
1443 e->address_start, e->address_end, m->flags);
1444
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001445 list_add_tail(&e->list, &amd_iommu_unity_map);
1446
1447 return 0;
1448}
1449
Joerg Roedelb65233a2008-07-11 17:14:21 +02001450/* iterates over all memory definitions we find in the ACPI table */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001451static int __init init_memory_definitions(struct acpi_table_header *table)
1452{
1453 u8 *p = (u8 *)table, *end = (u8 *)table;
1454 struct ivmd_header *m;
1455
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001456 end += table->length;
1457 p += IVRS_HEADER_LENGTH;
1458
1459 while (p < end) {
1460 m = (struct ivmd_header *)p;
1461 if (m->flags & IVMD_FLAG_EXCL_RANGE)
1462 init_exclusion_range(m);
1463 else if (m->flags & IVMD_FLAG_UNITY_MAP)
1464 init_unity_map_range(m);
1465
1466 p += m->length;
1467 }
1468
1469 return 0;
1470}
1471
Joerg Roedelb65233a2008-07-11 17:14:21 +02001472/*
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001473 * Init the device table to not allow DMA access for devices and
1474 * suppress all page faults
1475 */
Joerg Roedel33f28c52012-06-15 18:03:31 +02001476static void init_device_table_dma(void)
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001477{
Joerg Roedel0de66d52011-06-06 16:04:02 +02001478 u32 devid;
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001479
1480 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1481 set_dev_entry_bit(devid, DEV_ENTRY_VALID);
1482 set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001483 }
1484}
1485
Joerg Roedeld04e0ba2012-07-02 16:02:20 +02001486static void __init uninit_device_table_dma(void)
1487{
1488 u32 devid;
1489
1490 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1491 amd_iommu_dev_table[devid].data[0] = 0ULL;
1492 amd_iommu_dev_table[devid].data[1] = 0ULL;
1493 }
1494}
1495
Joerg Roedel33f28c52012-06-15 18:03:31 +02001496static void init_device_table(void)
1497{
1498 u32 devid;
1499
1500 if (!amd_iommu_irq_remap)
1501 return;
1502
1503 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
1504 set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN);
1505}
1506
Joerg Roedele9bf5192010-09-20 14:33:07 +02001507static void iommu_init_flags(struct amd_iommu *iommu)
1508{
1509 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
1510 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
1511 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
1512
1513 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
1514 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
1515 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
1516
1517 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
1518 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
1519 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
1520
1521 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
1522 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
1523 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
1524
1525 /*
1526 * make IOMMU memory accesses cache coherent
1527 */
1528 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
Joerg Roedel1456e9d2011-12-22 14:51:53 +01001529
1530 /* Set IOTLB invalidation timeout to 1s */
1531 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
Joerg Roedele9bf5192010-09-20 14:33:07 +02001532}
1533
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001534static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
Joerg Roedel4c894f42010-09-23 15:15:19 +02001535{
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001536 int i, j;
1537 u32 ioc_feature_control;
Joerg Roedelc1bf94e2012-05-31 17:38:11 +02001538 struct pci_dev *pdev = iommu->root_pdev;
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001539
1540 /* RD890 BIOSes may not have completely reconfigured the iommu */
Joerg Roedelc1bf94e2012-05-31 17:38:11 +02001541 if (!is_rd890_iommu(iommu->dev) || !pdev)
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001542 return;
1543
1544 /*
1545 * First, we need to ensure that the iommu is enabled. This is
1546 * controlled by a register in the northbridge
1547 */
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001548
1549 /* Select Northbridge indirect register 0x75 and enable writing */
1550 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
1551 pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
1552
1553 /* Enable the iommu */
1554 if (!(ioc_feature_control & 0x1))
1555 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
1556
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001557 /* Restore the iommu BAR */
1558 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1559 iommu->stored_addr_lo);
1560 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
1561 iommu->stored_addr_hi);
1562
1563 /* Restore the l1 indirect regs for each of the 6 l1s */
1564 for (i = 0; i < 6; i++)
1565 for (j = 0; j < 0x12; j++)
1566 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
1567
1568 /* Restore the l2 indirect regs */
1569 for (i = 0; i < 0x83; i++)
1570 iommu_write_l2(iommu, i, iommu->stored_l2[i]);
1571
1572 /* Lock PCI setup registers */
1573 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1574 iommu->stored_addr_lo | 1);
Joerg Roedel4c894f42010-09-23 15:15:19 +02001575}
1576
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001577/*
Joerg Roedelb65233a2008-07-11 17:14:21 +02001578 * This function finally enables all IOMMUs found in the system after
1579 * they have been initialized
1580 */
Joerg Roedel11ee5ac2012-06-12 16:30:06 +02001581static void early_enable_iommus(void)
Joerg Roedel87361972008-06-26 21:28:07 +02001582{
1583 struct amd_iommu *iommu;
1584
Joerg Roedel3bd22172009-05-04 15:06:20 +02001585 for_each_iommu(iommu) {
Chris Wrighta8c485b2009-06-15 15:53:45 +02001586 iommu_disable(iommu);
Joerg Roedele9bf5192010-09-20 14:33:07 +02001587 iommu_init_flags(iommu);
Joerg Roedel58492e12009-05-04 18:41:16 +02001588 iommu_set_device_table(iommu);
1589 iommu_enable_command_buffer(iommu);
1590 iommu_enable_event_buffer(iommu);
Joerg Roedel87361972008-06-26 21:28:07 +02001591 iommu_set_exclusion_range(iommu);
1592 iommu_enable(iommu);
Joerg Roedel7d0c5cc2011-04-07 08:16:10 +02001593 iommu_flush_all_caches(iommu);
Joerg Roedel87361972008-06-26 21:28:07 +02001594 }
1595}
1596
Joerg Roedel11ee5ac2012-06-12 16:30:06 +02001597static void enable_iommus_v2(void)
1598{
1599 struct amd_iommu *iommu;
1600
1601 for_each_iommu(iommu) {
1602 iommu_enable_ppr_log(iommu);
1603 iommu_enable_gt(iommu);
1604 }
1605}
1606
1607static void enable_iommus(void)
1608{
1609 early_enable_iommus();
1610
1611 enable_iommus_v2();
1612}
1613
Joerg Roedel92ac4322009-05-19 19:06:27 +02001614static void disable_iommus(void)
1615{
1616 struct amd_iommu *iommu;
1617
1618 for_each_iommu(iommu)
1619 iommu_disable(iommu);
1620}
1621
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001622/*
1623 * Suspend/Resume support
1624 * disable suspend until real resume implemented
1625 */
1626
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001627static void amd_iommu_resume(void)
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001628{
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001629 struct amd_iommu *iommu;
1630
1631 for_each_iommu(iommu)
1632 iommu_apply_resume_quirks(iommu);
1633
Joerg Roedel736501e2009-05-12 09:56:12 +02001634 /* re-load the hardware */
1635 enable_iommus();
Joerg Roedel3d9761e2012-03-15 16:39:21 +01001636
1637 amd_iommu_enable_interrupts();
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001638}
1639
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001640static int amd_iommu_suspend(void)
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001641{
Joerg Roedel736501e2009-05-12 09:56:12 +02001642 /* disable IOMMUs to go out of the way for BIOS */
1643 disable_iommus();
1644
1645 return 0;
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001646}
1647
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001648static struct syscore_ops amd_iommu_syscore_ops = {
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001649 .suspend = amd_iommu_suspend,
1650 .resume = amd_iommu_resume,
1651};
1652
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001653static void __init free_on_init_error(void)
1654{
Joerg Roedel0ea2c422012-06-15 18:05:20 +02001655 free_pages((unsigned long)irq_lookup_table,
1656 get_order(rlookup_table_size));
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001657
Joerg Roedel05152a02012-06-15 16:53:51 +02001658 if (amd_iommu_irq_cache) {
1659 kmem_cache_destroy(amd_iommu_irq_cache);
1660 amd_iommu_irq_cache = NULL;
Joerg Roedel0ea2c422012-06-15 18:05:20 +02001661
Joerg Roedel05152a02012-06-15 16:53:51 +02001662 }
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001663
1664 free_pages((unsigned long)amd_iommu_rlookup_table,
1665 get_order(rlookup_table_size));
1666
1667 free_pages((unsigned long)amd_iommu_alias_table,
1668 get_order(alias_table_size));
1669
1670 free_pages((unsigned long)amd_iommu_dev_table,
1671 get_order(dev_table_size));
1672
1673 free_iommu_all();
1674
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001675#ifdef CONFIG_GART_IOMMU
1676 /*
1677 * We failed to initialize the AMD IOMMU - try fallback to GART
1678 * if possible.
1679 */
1680 gart_iommu_init();
1681
1682#endif
1683}
1684
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001685/* SB IOAPIC is always on this device in AMD systems */
1686#define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
1687
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001688static bool __init check_ioapic_information(void)
1689{
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02001690 const char *fw_bug = FW_BUG;
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001691 bool ret, has_sb_ioapic;
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001692 int idx;
1693
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001694 has_sb_ioapic = false;
1695 ret = false;
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001696
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02001697 /*
1698 * If we have map overrides on the kernel command line the
1699 * messages in this function might not describe firmware bugs
1700 * anymore - so be careful
1701 */
1702 if (cmdline_maps)
1703 fw_bug = "";
1704
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001705 for (idx = 0; idx < nr_ioapics; idx++) {
1706 int devid, id = mpc_ioapic_id(idx);
1707
1708 devid = get_ioapic_devid(id);
1709 if (devid < 0) {
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02001710 pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n",
1711 fw_bug, id);
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001712 ret = false;
1713 } else if (devid == IOAPIC_SB_DEVID) {
1714 has_sb_ioapic = true;
1715 ret = true;
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001716 }
1717 }
1718
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001719 if (!has_sb_ioapic) {
1720 /*
1721 * We expect the SB IOAPIC to be listed in the IVRS
1722 * table. The system timer is connected to the SB IOAPIC
1723 * and if we don't have it in the list the system will
1724 * panic at boot time. This situation usually happens
1725 * when the BIOS is buggy and provides us the wrong
1726 * device id for the IOAPIC in the system.
1727 */
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02001728 pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug);
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001729 }
1730
1731 if (!ret)
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02001732 pr_err("AMD-Vi: Disabling interrupt remapping\n");
Joerg Roedelc2ff5cf52012-10-16 14:52:51 +02001733
1734 return ret;
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001735}
1736
Joerg Roedeld04e0ba2012-07-02 16:02:20 +02001737static void __init free_dma_resources(void)
1738{
1739 amd_iommu_uninit_devices();
1740
1741 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1742 get_order(MAX_DOMAIN_ID/8));
1743
1744 free_unity_maps();
1745}
1746
Joerg Roedelb65233a2008-07-11 17:14:21 +02001747/*
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001748 * This is the hardware init function for AMD IOMMU in the system.
1749 * This function is called either from amd_iommu_init or from the interrupt
1750 * remapping setup code.
Joerg Roedelb65233a2008-07-11 17:14:21 +02001751 *
1752 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
1753 * three times:
1754 *
1755 * 1 pass) Find the highest PCI device id the driver has to handle.
1756 * Upon this information the size of the data structures is
1757 * determined that needs to be allocated.
1758 *
1759 * 2 pass) Initialize the data structures just allocated with the
1760 * information in the ACPI table about available AMD IOMMUs
1761 * in the system. It also maps the PCI devices in the
1762 * system to specific IOMMUs
1763 *
1764 * 3 pass) After the basic data structures are allocated and
1765 * initialized we update them with information about memory
1766 * remapping requirements parsed out of the ACPI table in
1767 * this last pass.
1768 *
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001769 * After everything is set up the IOMMUs are enabled and the necessary
1770 * hotplug and suspend notifiers are registered.
Joerg Roedelb65233a2008-07-11 17:14:21 +02001771 */
Joerg Roedel643511b2012-06-12 12:09:35 +02001772static int __init early_amd_iommu_init(void)
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001773{
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001774 struct acpi_table_header *ivrs_base;
1775 acpi_size ivrs_size;
1776 acpi_status status;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001777 int i, ret = 0;
1778
Joerg Roedel643511b2012-06-12 12:09:35 +02001779 if (!amd_iommu_detected)
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001780 return -ENODEV;
1781
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001782 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1783 if (status == AE_NOT_FOUND)
1784 return -ENODEV;
1785 else if (ACPI_FAILURE(status)) {
1786 const char *err = acpi_format_exception(status);
1787 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1788 return -EINVAL;
1789 }
1790
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001791 /*
1792 * First parse ACPI tables to find the largest Bus/Dev/Func
1793 * we need to handle. Upon this information the shared data
1794 * structures for the IOMMUs in the system will be allocated
1795 */
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001796 ret = find_last_devid_acpi(ivrs_base);
1797 if (ret)
Joerg Roedel3551a702010-03-01 13:52:19 +01001798 goto out;
1799
Joerg Roedelc5714842008-07-11 17:14:25 +02001800 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
1801 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
1802 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001803
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001804 /* Device table - directly used by all IOMMUs */
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001805 ret = -ENOMEM;
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001806 amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001807 get_order(dev_table_size));
1808 if (amd_iommu_dev_table == NULL)
1809 goto out;
1810
1811 /*
1812 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
1813 * IOMMU see for that device
1814 */
1815 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
1816 get_order(alias_table_size));
1817 if (amd_iommu_alias_table == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001818 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001819
1820 /* IOMMU rlookup table - find the IOMMU for a specific device */
Joerg Roedel83fd5cc2008-12-16 19:17:11 +01001821 amd_iommu_rlookup_table = (void *)__get_free_pages(
1822 GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001823 get_order(rlookup_table_size));
1824 if (amd_iommu_rlookup_table == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001825 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001826
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001827 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
1828 GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001829 get_order(MAX_DOMAIN_ID/8));
1830 if (amd_iommu_pd_alloc_bitmap == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001831 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001832
1833 /*
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001834 * let all alias entries point to itself
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001835 */
Joerg Roedel3a61ec32008-07-25 13:07:50 +02001836 for (i = 0; i <= amd_iommu_last_bdf; ++i)
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001837 amd_iommu_alias_table[i] = i;
1838
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001839 /*
1840 * never allocate domain 0 because its used as the non-allocated and
1841 * error value placeholder
1842 */
1843 amd_iommu_pd_alloc_bitmap[0] = 1;
1844
Joerg Roedelaeb26f52009-11-20 16:44:01 +01001845 spin_lock_init(&amd_iommu_pd_lock);
1846
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001847 /*
1848 * now the data structures are allocated and basically initialized
1849 * start the real acpi table scan
1850 */
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001851 ret = init_iommu_all(ivrs_base);
1852 if (ret)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001853 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001854
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001855 if (amd_iommu_irq_remap)
1856 amd_iommu_irq_remap = check_ioapic_information();
1857
Joerg Roedel05152a02012-06-15 16:53:51 +02001858 if (amd_iommu_irq_remap) {
1859 /*
1860 * Interrupt remapping enabled, create kmem_cache for the
1861 * remapping tables.
1862 */
Wei Yongjun83ed9c12013-04-23 10:47:44 +08001863 ret = -ENOMEM;
Joerg Roedel05152a02012-06-15 16:53:51 +02001864 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
1865 MAX_IRQS_PER_TABLE * sizeof(u32),
1866 IRQ_TABLE_ALIGNMENT,
1867 0, NULL);
1868 if (!amd_iommu_irq_cache)
1869 goto out;
Joerg Roedel0ea2c422012-06-15 18:05:20 +02001870
1871 irq_lookup_table = (void *)__get_free_pages(
1872 GFP_KERNEL | __GFP_ZERO,
1873 get_order(rlookup_table_size));
1874 if (!irq_lookup_table)
1875 goto out;
Joerg Roedel05152a02012-06-15 16:53:51 +02001876 }
1877
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001878 ret = init_memory_definitions(ivrs_base);
1879 if (ret)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001880 goto out;
Joerg Roedel3551a702010-03-01 13:52:19 +01001881
Joerg Roedeleb1eb7a2012-07-05 11:58:02 +02001882 /* init the device table */
1883 init_device_table();
1884
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001885out:
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001886 /* Don't leak any ACPI memory */
1887 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1888 ivrs_base = NULL;
1889
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001890 return ret;
Joerg Roedel643511b2012-06-12 12:09:35 +02001891}
1892
Gerard Snitselaarae295142012-03-16 11:38:22 -07001893static int amd_iommu_enable_interrupts(void)
Joerg Roedel3d9761e2012-03-15 16:39:21 +01001894{
1895 struct amd_iommu *iommu;
1896 int ret = 0;
1897
1898 for_each_iommu(iommu) {
1899 ret = iommu_init_msi(iommu);
1900 if (ret)
1901 goto out;
1902 }
1903
1904out:
1905 return ret;
1906}
1907
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001908static bool detect_ivrs(void)
1909{
1910 struct acpi_table_header *ivrs_base;
1911 acpi_size ivrs_size;
1912 acpi_status status;
1913
1914 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1915 if (status == AE_NOT_FOUND)
1916 return false;
1917 else if (ACPI_FAILURE(status)) {
1918 const char *err = acpi_format_exception(status);
1919 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1920 return false;
1921 }
1922
1923 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1924
Joerg Roedel1adb7d32012-08-06 14:18:42 +02001925 /* Make sure ACS will be enabled during PCI probe */
1926 pci_request_acs();
1927
Joerg Roedel05152a02012-06-15 16:53:51 +02001928 if (!disable_irq_remap)
1929 amd_iommu_irq_remap = true;
1930
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001931 return true;
1932}
1933
Joerg Roedelb9b1ce702012-06-12 16:51:12 +02001934static int amd_iommu_init_dma(void)
1935{
Joerg Roedel33f28c52012-06-15 18:03:31 +02001936 struct amd_iommu *iommu;
Joerg Roedelb9b1ce702012-06-12 16:51:12 +02001937 int ret;
1938
1939 if (iommu_pass_through)
1940 ret = amd_iommu_init_passthrough();
1941 else
1942 ret = amd_iommu_init_dma_ops();
1943
1944 if (ret)
1945 return ret;
1946
Joerg Roedelf528d982013-02-06 12:55:23 +01001947 init_device_table_dma();
1948
1949 for_each_iommu(iommu)
1950 iommu_flush_all_caches(iommu);
1951
Joerg Roedelb9b1ce702012-06-12 16:51:12 +02001952 amd_iommu_init_api();
1953
1954 amd_iommu_init_notifier();
1955
1956 return 0;
1957}
1958
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001959/****************************************************************************
1960 *
1961 * AMD IOMMU Initialization State Machine
1962 *
1963 ****************************************************************************/
1964
1965static int __init state_next(void)
1966{
1967 int ret = 0;
1968
1969 switch (init_state) {
1970 case IOMMU_START_STATE:
1971 if (!detect_ivrs()) {
1972 init_state = IOMMU_NOT_FOUND;
1973 ret = -ENODEV;
1974 } else {
1975 init_state = IOMMU_IVRS_DETECTED;
1976 }
1977 break;
1978 case IOMMU_IVRS_DETECTED:
1979 ret = early_amd_iommu_init();
1980 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
1981 break;
1982 case IOMMU_ACPI_FINISHED:
1983 early_enable_iommus();
1984 register_syscore_ops(&amd_iommu_syscore_ops);
1985 x86_platform.iommu_shutdown = disable_iommus;
1986 init_state = IOMMU_ENABLED;
1987 break;
1988 case IOMMU_ENABLED:
1989 ret = amd_iommu_init_pci();
1990 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
1991 enable_iommus_v2();
1992 break;
1993 case IOMMU_PCI_INIT:
1994 ret = amd_iommu_enable_interrupts();
1995 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
1996 break;
1997 case IOMMU_INTERRUPTS_EN:
1998 ret = amd_iommu_init_dma();
1999 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
2000 break;
2001 case IOMMU_DMA_OPS:
2002 init_state = IOMMU_INITIALIZED;
2003 break;
2004 case IOMMU_INITIALIZED:
2005 /* Nothing to do */
2006 break;
2007 case IOMMU_NOT_FOUND:
2008 case IOMMU_INIT_ERROR:
2009 /* Error states => do nothing */
2010 ret = -EINVAL;
2011 break;
2012 default:
2013 /* Unknown state */
2014 BUG();
2015 }
2016
2017 return ret;
2018}
2019
2020static int __init iommu_go_to_state(enum iommu_init_state state)
2021{
2022 int ret = 0;
2023
2024 while (init_state != state) {
2025 ret = state_next();
2026 if (init_state == IOMMU_NOT_FOUND ||
2027 init_state == IOMMU_INIT_ERROR)
2028 break;
2029 }
2030
2031 return ret;
2032}
2033
Joerg Roedel6b474b82012-06-26 16:46:04 +02002034#ifdef CONFIG_IRQ_REMAP
2035int __init amd_iommu_prepare(void)
2036{
2037 return iommu_go_to_state(IOMMU_ACPI_FINISHED);
2038}
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002039
Joerg Roedel6b474b82012-06-26 16:46:04 +02002040int __init amd_iommu_supported(void)
2041{
2042 return amd_iommu_irq_remap ? 1 : 0;
2043}
2044
2045int __init amd_iommu_enable(void)
2046{
2047 int ret;
2048
2049 ret = iommu_go_to_state(IOMMU_ENABLED);
2050 if (ret)
2051 return ret;
2052
2053 irq_remapping_enabled = 1;
2054
2055 return 0;
2056}
2057
2058void amd_iommu_disable(void)
2059{
2060 amd_iommu_suspend();
2061}
2062
2063int amd_iommu_reenable(int mode)
2064{
2065 amd_iommu_resume();
2066
2067 return 0;
2068}
2069
2070int __init amd_iommu_enable_faulting(void)
2071{
2072 /* We enable MSI later when PCI is initialized */
2073 return 0;
2074}
2075#endif
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002076
Joerg Roedel8704a1b2012-03-01 15:57:53 +01002077/*
2078 * This is the core init function for AMD IOMMU hardware in the system.
2079 * This function is called from the generic x86 DMA layer initialization
2080 * code.
Joerg Roedel8704a1b2012-03-01 15:57:53 +01002081 */
2082static int __init amd_iommu_init(void)
2083{
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002084 int ret;
Joerg Roedel8704a1b2012-03-01 15:57:53 +01002085
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002086 ret = iommu_go_to_state(IOMMU_INITIALIZED);
2087 if (ret) {
Joerg Roedeld04e0ba2012-07-02 16:02:20 +02002088 free_dma_resources();
2089 if (!irq_remapping_enabled) {
2090 disable_iommus();
2091 free_on_init_error();
2092 } else {
2093 struct amd_iommu *iommu;
2094
2095 uninit_device_table_dma();
2096 for_each_iommu(iommu)
2097 iommu_flush_all_caches(iommu);
2098 }
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002099 }
Joerg Roedel8704a1b2012-03-01 15:57:53 +01002100
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02002101 return ret;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02002102}
2103
Joerg Roedelb65233a2008-07-11 17:14:21 +02002104/****************************************************************************
2105 *
2106 * Early detect code. This code runs at IOMMU detection time in the DMA
2107 * layer. It just looks if there is an IVRS ACPI table to detect AMD
2108 * IOMMUs
2109 *
2110 ****************************************************************************/
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04002111int __init amd_iommu_detect(void)
Joerg Roedelae7877d2008-06-26 21:27:51 +02002112{
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002113 int ret;
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02002114
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09002115 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04002116 return -ENODEV;
Joerg Roedelae7877d2008-06-26 21:27:51 +02002117
Joerg Roedela5235722010-05-11 17:12:33 +02002118 if (amd_iommu_disabled)
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04002119 return -ENODEV;
Joerg Roedela5235722010-05-11 17:12:33 +02002120
Joerg Roedel2c0ae172012-06-12 15:59:30 +02002121 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
2122 if (ret)
2123 return ret;
Linus Torvalds11bd04f2009-12-11 12:18:16 -08002124
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02002125 amd_iommu_detected = true;
2126 iommu_detected = 1;
2127 x86_init.iommu.iommu_init = amd_iommu_init;
2128
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02002129 return 0;
Joerg Roedelae7877d2008-06-26 21:27:51 +02002130}
2131
Joerg Roedelb65233a2008-07-11 17:14:21 +02002132/****************************************************************************
2133 *
2134 * Parsing functions for the AMD IOMMU specific kernel command line
2135 * options.
2136 *
2137 ****************************************************************************/
2138
Joerg Roedelfefda112009-05-20 12:21:42 +02002139static int __init parse_amd_iommu_dump(char *str)
2140{
2141 amd_iommu_dump = true;
2142
2143 return 1;
2144}
2145
Joerg Roedel918ad6c2008-06-26 21:27:52 +02002146static int __init parse_amd_iommu_options(char *str)
2147{
2148 for (; *str; ++str) {
Joerg Roedel695b5672008-11-17 15:16:43 +01002149 if (strncmp(str, "fullflush", 9) == 0)
FUJITA Tomonoriafa9fdc2008-09-20 01:23:30 +09002150 amd_iommu_unmap_flush = true;
Joerg Roedela5235722010-05-11 17:12:33 +02002151 if (strncmp(str, "off", 3) == 0)
2152 amd_iommu_disabled = true;
Joerg Roedel5abcdba2011-12-01 15:49:45 +01002153 if (strncmp(str, "force_isolation", 15) == 0)
2154 amd_iommu_force_isolation = true;
Joerg Roedel918ad6c2008-06-26 21:27:52 +02002155 }
2156
2157 return 1;
2158}
2159
Joerg Roedel440e89982013-04-09 16:35:28 +02002160static int __init parse_ivrs_ioapic(char *str)
2161{
2162 unsigned int bus, dev, fn;
2163 int ret, id, i;
2164 u16 devid;
2165
2166 ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
2167
2168 if (ret != 4) {
2169 pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str);
2170 return 1;
2171 }
2172
2173 if (early_ioapic_map_size == EARLY_MAP_SIZE) {
2174 pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
2175 str);
2176 return 1;
2177 }
2178
2179 devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7);
2180
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02002181 cmdline_maps = true;
Joerg Roedel440e89982013-04-09 16:35:28 +02002182 i = early_ioapic_map_size++;
2183 early_ioapic_map[i].id = id;
2184 early_ioapic_map[i].devid = devid;
2185 early_ioapic_map[i].cmd_line = true;
2186
2187 return 1;
2188}
2189
2190static int __init parse_ivrs_hpet(char *str)
2191{
2192 unsigned int bus, dev, fn;
2193 int ret, id, i;
2194 u16 devid;
2195
2196 ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
2197
2198 if (ret != 4) {
2199 pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str);
2200 return 1;
2201 }
2202
2203 if (early_hpet_map_size == EARLY_MAP_SIZE) {
2204 pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n",
2205 str);
2206 return 1;
2207 }
2208
2209 devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7);
2210
Joerg Roedeldfbb6d42013-04-09 19:06:18 +02002211 cmdline_maps = true;
Joerg Roedel440e89982013-04-09 16:35:28 +02002212 i = early_hpet_map_size++;
2213 early_hpet_map[i].id = id;
2214 early_hpet_map[i].devid = devid;
2215 early_hpet_map[i].cmd_line = true;
2216
2217 return 1;
2218}
2219
2220__setup("amd_iommu_dump", parse_amd_iommu_dump);
2221__setup("amd_iommu=", parse_amd_iommu_options);
2222__setup("ivrs_ioapic", parse_ivrs_ioapic);
2223__setup("ivrs_hpet", parse_ivrs_hpet);
Konrad Rzeszutek Wilk22e6daf2010-08-26 13:58:03 -04002224
2225IOMMU_INIT_FINISH(amd_iommu_detect,
2226 gart_iommu_hole_init,
Joerg Roedel98f1ad22012-07-06 13:28:37 +02002227 NULL,
2228 NULL);
Joerg Roedel400a28a2011-11-28 15:11:02 +01002229
2230bool amd_iommu_v2_supported(void)
2231{
2232 return amd_iommu_v2_present;
2233}
2234EXPORT_SYMBOL(amd_iommu_v2_supported);