blob: 073ed5d1e0532c268f4d6c4f2b6ba10b7a2a35a8 [file] [log] [blame]
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +02001/*
Joerg Roedel5d0d7152010-10-13 11:13:21 +02002 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +02003 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/pci.h>
21#include <linux/acpi.h>
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020022#include <linux/list.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/slab.h>
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +010024#include <linux/syscore_ops.h>
Joerg Roedela80dc3e2008-09-11 16:51:41 +020025#include <linux/interrupt.h>
26#include <linux/msi.h>
Joerg Roedel403f81d2011-06-14 16:44:25 +020027#include <linux/amd-iommu.h>
Joerg Roedel400a28a2011-11-28 15:11:02 +010028#include <linux/export.h>
Joerg Roedel02f3b3f2012-06-11 17:45:25 +020029#include <linux/acpi.h>
30#include <acpi/acpi.h>
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020031#include <asm/pci-direct.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090032#include <asm/iommu.h>
Joerg Roedel1d9b16d2008-11-27 18:39:15 +010033#include <asm/gart.h>
FUJITA Tomonoriea1b0d32009-11-10 19:46:15 +090034#include <asm/x86_init.h>
Konrad Rzeszutek Wilk22e6daf2010-08-26 13:58:03 -040035#include <asm/iommu_table.h>
Joerg Roedel403f81d2011-06-14 16:44:25 +020036
37#include "amd_iommu_proto.h"
38#include "amd_iommu_types.h"
Joerg Roedel05152a02012-06-15 16:53:51 +020039#include "irq_remapping.h"
Joerg Roedel403f81d2011-06-14 16:44:25 +020040
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020041/*
42 * definitions for the ACPI scanning code
43 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020044#define IVRS_HEADER_LENGTH 48
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020045
46#define ACPI_IVHD_TYPE 0x10
47#define ACPI_IVMD_TYPE_ALL 0x20
48#define ACPI_IVMD_TYPE 0x21
49#define ACPI_IVMD_TYPE_RANGE 0x22
50
51#define IVHD_DEV_ALL 0x01
52#define IVHD_DEV_SELECT 0x02
53#define IVHD_DEV_SELECT_RANGE_START 0x03
54#define IVHD_DEV_RANGE_END 0x04
55#define IVHD_DEV_ALIAS 0x42
56#define IVHD_DEV_ALIAS_RANGE 0x43
57#define IVHD_DEV_EXT_SELECT 0x46
58#define IVHD_DEV_EXT_SELECT_RANGE 0x47
Joerg Roedel6efed632012-06-14 15:52:58 +020059#define IVHD_DEV_SPECIAL 0x48
60
61#define IVHD_SPECIAL_IOAPIC 1
62#define IVHD_SPECIAL_HPET 2
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020063
Joerg Roedel6da73422009-05-04 11:44:38 +020064#define IVHD_FLAG_HT_TUN_EN_MASK 0x01
65#define IVHD_FLAG_PASSPW_EN_MASK 0x02
66#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
67#define IVHD_FLAG_ISOC_EN_MASK 0x08
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020068
69#define IVMD_FLAG_EXCL_RANGE 0x08
70#define IVMD_FLAG_UNITY_MAP 0x01
71
72#define ACPI_DEVFLAG_INITPASS 0x01
73#define ACPI_DEVFLAG_EXTINT 0x02
74#define ACPI_DEVFLAG_NMI 0x04
75#define ACPI_DEVFLAG_SYSMGT1 0x10
76#define ACPI_DEVFLAG_SYSMGT2 0x20
77#define ACPI_DEVFLAG_LINT0 0x40
78#define ACPI_DEVFLAG_LINT1 0x80
79#define ACPI_DEVFLAG_ATSDIS 0x10000000
80
Joerg Roedelb65233a2008-07-11 17:14:21 +020081/*
82 * ACPI table definitions
83 *
84 * These data structures are laid over the table to parse the important values
85 * out of it.
86 */
87
88/*
89 * structure describing one IOMMU in the ACPI table. Typically followed by one
90 * or more ivhd_entrys.
91 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +020092struct ivhd_header {
93 u8 type;
94 u8 flags;
95 u16 length;
96 u16 devid;
97 u16 cap_ptr;
98 u64 mmio_phys;
99 u16 pci_seg;
100 u16 info;
101 u32 reserved;
102} __attribute__((packed));
103
Joerg Roedelb65233a2008-07-11 17:14:21 +0200104/*
105 * A device entry describing which devices a specific IOMMU translates and
106 * which requestor ids they use.
107 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +0200108struct ivhd_entry {
109 u8 type;
110 u16 devid;
111 u8 flags;
112 u32 ext;
113} __attribute__((packed));
114
Joerg Roedelb65233a2008-07-11 17:14:21 +0200115/*
116 * An AMD IOMMU memory definition structure. It defines things like exclusion
117 * ranges for devices and regions that should be unity mapped.
118 */
Joerg Roedelf6e2e6b2008-06-26 21:27:39 +0200119struct ivmd_header {
120 u8 type;
121 u8 flags;
122 u16 length;
123 u16 devid;
124 u16 aux;
125 u64 resv;
126 u64 range_start;
127 u64 range_length;
128} __attribute__((packed));
129
Joerg Roedelfefda112009-05-20 12:21:42 +0200130bool amd_iommu_dump;
Joerg Roedel05152a02012-06-15 16:53:51 +0200131bool amd_iommu_irq_remap __read_mostly;
Joerg Roedelfefda112009-05-20 12:21:42 +0200132
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200133static bool amd_iommu_detected;
Joerg Roedela5235722010-05-11 17:12:33 +0200134static bool __initdata amd_iommu_disabled;
Joerg Roedelc1cbebe2008-07-03 19:35:10 +0200135
Joerg Roedelb65233a2008-07-11 17:14:21 +0200136u16 amd_iommu_last_bdf; /* largest PCI device id we have
137 to handle */
Joerg Roedel2e228472008-07-11 17:14:31 +0200138LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
Joerg Roedelb65233a2008-07-11 17:14:21 +0200139 we find in ACPI */
Dan Carpenter3775d482012-06-27 12:09:18 +0300140u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */
Joerg Roedel928abd22008-06-26 21:27:40 +0200141
Joerg Roedel2e228472008-07-11 17:14:31 +0200142LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
Joerg Roedelb65233a2008-07-11 17:14:21 +0200143 system */
144
Joerg Roedelbb527772009-11-20 14:31:51 +0100145/* Array to assign indices to IOMMUs*/
146struct amd_iommu *amd_iommus[MAX_IOMMUS];
147int amd_iommus_present;
148
Joerg Roedel318afd42009-11-23 18:32:38 +0100149/* IOMMUs have a non-present cache? */
150bool amd_iommu_np_cache __read_mostly;
Joerg Roedel60f723b2011-04-05 12:50:24 +0200151bool amd_iommu_iotlb_sup __read_mostly = true;
Joerg Roedel318afd42009-11-23 18:32:38 +0100152
Joerg Roedel62f71ab2011-11-10 14:41:57 +0100153u32 amd_iommu_max_pasids __read_mostly = ~0;
154
Joerg Roedel400a28a2011-11-28 15:11:02 +0100155bool amd_iommu_v2_present __read_mostly;
156
Joerg Roedel5abcdba2011-12-01 15:49:45 +0100157bool amd_iommu_force_isolation __read_mostly;
158
Joerg Roedelb65233a2008-07-11 17:14:21 +0200159/*
Joerg Roedelaeb26f52009-11-20 16:44:01 +0100160 * List of protection domains - used during resume
161 */
162LIST_HEAD(amd_iommu_pd_list);
163spinlock_t amd_iommu_pd_lock;
164
165/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200166 * Pointer to the device table which is shared by all AMD IOMMUs
167 * it is indexed by the PCI device id or the HT unit id and contains
168 * information about the domain the device belongs to as well as the
169 * page table root pointer.
170 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200171struct dev_table_entry *amd_iommu_dev_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200172
173/*
174 * The alias table is a driver specific data structure which contains the
175 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
176 * More than one device can share the same requestor id.
177 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200178u16 *amd_iommu_alias_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200179
180/*
181 * The rlookup table is used to find the IOMMU which is responsible
182 * for a specific device. It is also indexed by the PCI device id.
183 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200184struct amd_iommu **amd_iommu_rlookup_table;
Joerg Roedelb65233a2008-07-11 17:14:21 +0200185
186/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200187 * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap
188 * to know which ones are already in use.
189 */
Joerg Roedel928abd22008-06-26 21:27:40 +0200190unsigned long *amd_iommu_pd_alloc_bitmap;
191
Joerg Roedelb65233a2008-07-11 17:14:21 +0200192static u32 dev_table_size; /* size of the device table */
193static u32 alias_table_size; /* size of the alias table */
194static u32 rlookup_table_size; /* size if the rlookup table */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200195
Joerg Roedel2c0ae172012-06-12 15:59:30 +0200196enum iommu_init_state {
197 IOMMU_START_STATE,
198 IOMMU_IVRS_DETECTED,
199 IOMMU_ACPI_FINISHED,
200 IOMMU_ENABLED,
201 IOMMU_PCI_INIT,
202 IOMMU_INTERRUPTS_EN,
203 IOMMU_DMA_OPS,
204 IOMMU_INITIALIZED,
205 IOMMU_NOT_FOUND,
206 IOMMU_INIT_ERROR,
207};
208
209static enum iommu_init_state init_state = IOMMU_START_STATE;
210
Gerard Snitselaarae295142012-03-16 11:38:22 -0700211static int amd_iommu_enable_interrupts(void);
Joerg Roedel2c0ae172012-06-12 15:59:30 +0200212static int __init iommu_go_to_state(enum iommu_init_state state);
Joerg Roedel3d9761e2012-03-15 16:39:21 +0100213
Joerg Roedel208ec8c2008-07-11 17:14:24 +0200214static inline void update_last_devid(u16 devid)
215{
216 if (devid > amd_iommu_last_bdf)
217 amd_iommu_last_bdf = devid;
218}
219
Joerg Roedelc5714842008-07-11 17:14:25 +0200220static inline unsigned long tbl_size(int entry_size)
221{
222 unsigned shift = PAGE_SHIFT +
Neil Turton421f9092009-05-14 14:00:35 +0100223 get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
Joerg Roedelc5714842008-07-11 17:14:25 +0200224
225 return 1UL << shift;
226}
227
Matthew Garrett5bcd7572010-10-04 14:59:31 -0400228/* Access to l1 and l2 indexed register spaces */
229
230static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
231{
232 u32 val;
233
234 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
235 pci_read_config_dword(iommu->dev, 0xfc, &val);
236 return val;
237}
238
239static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
240{
241 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
242 pci_write_config_dword(iommu->dev, 0xfc, val);
243 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
244}
245
246static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
247{
248 u32 val;
249
250 pci_write_config_dword(iommu->dev, 0xf0, address);
251 pci_read_config_dword(iommu->dev, 0xf4, &val);
252 return val;
253}
254
255static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
256{
257 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
258 pci_write_config_dword(iommu->dev, 0xf4, val);
259}
260
Joerg Roedelb65233a2008-07-11 17:14:21 +0200261/****************************************************************************
262 *
263 * AMD IOMMU MMIO register space handling functions
264 *
265 * These functions are used to program the IOMMU device registers in
266 * MMIO space required for that driver.
267 *
268 ****************************************************************************/
269
270/*
271 * This function set the exclusion range in the IOMMU. DMA accesses to the
272 * exclusion range are passed through untranslated
273 */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200274static void iommu_set_exclusion_range(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200275{
276 u64 start = iommu->exclusion_start & PAGE_MASK;
277 u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
278 u64 entry;
279
280 if (!iommu->exclusion_start)
281 return;
282
283 entry = start | MMIO_EXCL_ENABLE_MASK;
284 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
285 &entry, sizeof(entry));
286
287 entry = limit;
288 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
289 &entry, sizeof(entry));
290}
291
Joerg Roedelb65233a2008-07-11 17:14:21 +0200292/* Programs the physical address of the device table into the IOMMU hardware */
Jan Beulich6b7f0002012-03-08 08:58:13 +0000293static void iommu_set_device_table(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200294{
Andreas Herrmannf6098912008-10-16 16:27:36 +0200295 u64 entry;
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200296
297 BUG_ON(iommu->mmio_base == NULL);
298
299 entry = virt_to_phys(amd_iommu_dev_table);
300 entry |= (dev_table_size >> 12) - 1;
301 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
302 &entry, sizeof(entry));
303}
304
Joerg Roedelb65233a2008-07-11 17:14:21 +0200305/* Generic functions to enable/disable certain features of the IOMMU. */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200306static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200307{
308 u32 ctrl;
309
310 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
311 ctrl |= (1 << bit);
312 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
313}
314
Joerg Roedelca0207112009-10-28 18:02:26 +0100315static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200316{
317 u32 ctrl;
318
Joerg Roedel199d0d52008-09-17 16:45:59 +0200319 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200320 ctrl &= ~(1 << bit);
321 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
322}
323
Joerg Roedel1456e9d2011-12-22 14:51:53 +0100324static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
325{
326 u32 ctrl;
327
328 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
329 ctrl &= ~CTRL_INV_TO_MASK;
330 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
331 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
332}
333
Joerg Roedelb65233a2008-07-11 17:14:21 +0200334/* Function to enable the hardware */
Joerg Roedel05f92db2009-05-12 09:52:46 +0200335static void iommu_enable(struct amd_iommu *iommu)
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200336{
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200337 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
Joerg Roedelb2026aa2008-06-26 21:27:44 +0200338}
339
Joerg Roedel92ac4322009-05-19 19:06:27 +0200340static void iommu_disable(struct amd_iommu *iommu)
Joerg Roedel126c52b2008-09-09 16:47:35 +0200341{
Chris Wrighta8c485b2009-06-15 15:53:45 +0200342 /* Disable command buffer */
343 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
344
345 /* Disable event logging and event interrupts */
346 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
347 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
348
349 /* Disable IOMMU hardware itself */
Joerg Roedel92ac4322009-05-19 19:06:27 +0200350 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
Joerg Roedel126c52b2008-09-09 16:47:35 +0200351}
352
Joerg Roedelb65233a2008-07-11 17:14:21 +0200353/*
354 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
355 * the system has one.
356 */
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200357static u8 __iomem * __init iommu_map_mmio_space(u64 address)
Joerg Roedel6c567472008-06-26 21:27:43 +0200358{
Joerg Roedele82752d2010-05-28 14:26:48 +0200359 if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {
360 pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",
361 address);
362 pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
Joerg Roedel6c567472008-06-26 21:27:43 +0200363 return NULL;
Joerg Roedele82752d2010-05-28 14:26:48 +0200364 }
Joerg Roedel6c567472008-06-26 21:27:43 +0200365
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200366 return (u8 __iomem *)ioremap_nocache(address, MMIO_REGION_LENGTH);
Joerg Roedel6c567472008-06-26 21:27:43 +0200367}
368
369static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
370{
371 if (iommu->mmio_base)
372 iounmap(iommu->mmio_base);
373 release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
374}
375
Joerg Roedelb65233a2008-07-11 17:14:21 +0200376/****************************************************************************
377 *
378 * The functions below belong to the first pass of AMD IOMMU ACPI table
379 * parsing. In this pass we try to find out the highest device id this
380 * code has to handle. Upon this information the size of the shared data
381 * structures is determined later.
382 *
383 ****************************************************************************/
384
385/*
Joerg Roedelb514e552008-09-17 17:14:27 +0200386 * This function calculates the length of a given IVHD entry
387 */
388static inline int ivhd_entry_length(u8 *ivhd)
389{
390 return 0x04 << (*ivhd >> 6);
391}
392
393/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200394 * This function reads the last device id the IOMMU has to handle from the PCI
395 * capability header for this IOMMU
396 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200397static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
398{
399 u32 cap;
400
401 cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
Joerg Roedeld591b0a2008-07-11 17:14:35 +0200402 update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200403
404 return 0;
405}
406
Joerg Roedelb65233a2008-07-11 17:14:21 +0200407/*
408 * After reading the highest device id from the IOMMU PCI capability header
409 * this function looks if there is a higher device id defined in the ACPI table
410 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200411static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
412{
413 u8 *p = (void *)h, *end = (void *)h;
414 struct ivhd_entry *dev;
415
416 p += sizeof(*h);
417 end += h->length;
418
419 find_last_devid_on_pci(PCI_BUS(h->devid),
420 PCI_SLOT(h->devid),
421 PCI_FUNC(h->devid),
422 h->cap_ptr);
423
424 while (p < end) {
425 dev = (struct ivhd_entry *)p;
426 switch (dev->type) {
427 case IVHD_DEV_SELECT:
428 case IVHD_DEV_RANGE_END:
429 case IVHD_DEV_ALIAS:
430 case IVHD_DEV_EXT_SELECT:
Joerg Roedelb65233a2008-07-11 17:14:21 +0200431 /* all the above subfield types refer to device ids */
Joerg Roedel208ec8c2008-07-11 17:14:24 +0200432 update_last_devid(dev->devid);
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200433 break;
434 default:
435 break;
436 }
Joerg Roedelb514e552008-09-17 17:14:27 +0200437 p += ivhd_entry_length(p);
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200438 }
439
440 WARN_ON(p != end);
441
442 return 0;
443}
444
Joerg Roedelb65233a2008-07-11 17:14:21 +0200445/*
446 * Iterate over all IVHD entries in the ACPI table and find the highest device
447 * id which we need to handle. This is the first of three functions which parse
448 * the ACPI table. So we check the checksum here.
449 */
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200450static int __init find_last_devid_acpi(struct acpi_table_header *table)
451{
452 int i;
453 u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
454 struct ivhd_header *h;
455
456 /*
457 * Validate checksum here so we don't need to do it when
458 * we actually parse the table
459 */
460 for (i = 0; i < table->length; ++i)
461 checksum += p[i];
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200462 if (checksum != 0)
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200463 /* ACPI table corrupt */
Joerg Roedel02f3b3f2012-06-11 17:45:25 +0200464 return -ENODEV;
Joerg Roedel3e8064b2008-06-26 21:27:41 +0200465
466 p += IVRS_HEADER_LENGTH;
467
468 end += table->length;
469 while (p < end) {
470 h = (struct ivhd_header *)p;
471 switch (h->type) {
472 case ACPI_IVHD_TYPE:
473 find_last_devid_from_ivhd(h);
474 break;
475 default:
476 break;
477 }
478 p += h->length;
479 }
480 WARN_ON(p != end);
481
482 return 0;
483}
484
Joerg Roedelb65233a2008-07-11 17:14:21 +0200485/****************************************************************************
486 *
487 * The following functions belong the the code path which parses the ACPI table
488 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
489 * data structures, initialize the device/alias/rlookup table and also
490 * basically initialize the hardware.
491 *
492 ****************************************************************************/
493
494/*
495 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
496 * write commands to that buffer later and the IOMMU will execute them
497 * asynchronously
498 */
Joerg Roedelb36ca912008-06-26 21:27:45 +0200499static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
500{
Joerg Roedeld0312b22008-07-11 17:14:29 +0200501 u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
Joerg Roedelb36ca912008-06-26 21:27:45 +0200502 get_order(CMD_BUFFER_SIZE));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200503
504 if (cmd_buf == NULL)
505 return NULL;
506
Chris Wright549c90dc2010-04-02 18:27:53 -0700507 iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;
Joerg Roedelb36ca912008-06-26 21:27:45 +0200508
Joerg Roedel58492e12009-05-04 18:41:16 +0200509 return cmd_buf;
510}
511
512/*
Joerg Roedel93f1cc672009-09-03 14:50:20 +0200513 * This function resets the command buffer if the IOMMU stopped fetching
514 * commands from it.
515 */
516void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
517{
518 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
519
520 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
521 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
522
523 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
524}
525
526/*
Joerg Roedel58492e12009-05-04 18:41:16 +0200527 * This function writes the command buffer address to the hardware and
528 * enables it.
529 */
530static void iommu_enable_command_buffer(struct amd_iommu *iommu)
531{
532 u64 entry;
533
534 BUG_ON(iommu->cmd_buf == NULL);
535
536 entry = (u64)virt_to_phys(iommu->cmd_buf);
Joerg Roedelb36ca912008-06-26 21:27:45 +0200537 entry |= MMIO_CMD_SIZE_512;
Joerg Roedel58492e12009-05-04 18:41:16 +0200538
Joerg Roedelb36ca912008-06-26 21:27:45 +0200539 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
Joerg Roedel58492e12009-05-04 18:41:16 +0200540 &entry, sizeof(entry));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200541
Joerg Roedel93f1cc672009-09-03 14:50:20 +0200542 amd_iommu_reset_cmd_buffer(iommu);
Chris Wright549c90dc2010-04-02 18:27:53 -0700543 iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);
Joerg Roedelb36ca912008-06-26 21:27:45 +0200544}
545
546static void __init free_command_buffer(struct amd_iommu *iommu)
547{
Joerg Roedel23c17132008-09-17 17:18:17 +0200548 free_pages((unsigned long)iommu->cmd_buf,
Chris Wright549c90dc2010-04-02 18:27:53 -0700549 get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));
Joerg Roedelb36ca912008-06-26 21:27:45 +0200550}
551
Joerg Roedel335503e2008-09-05 14:29:07 +0200552/* allocates the memory where the IOMMU will log its events to */
553static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
554{
Joerg Roedel335503e2008-09-05 14:29:07 +0200555 iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
556 get_order(EVT_BUFFER_SIZE));
557
558 if (iommu->evt_buf == NULL)
559 return NULL;
560
Joerg Roedel1bc6f832009-07-02 18:32:05 +0200561 iommu->evt_buf_size = EVT_BUFFER_SIZE;
562
Joerg Roedel58492e12009-05-04 18:41:16 +0200563 return iommu->evt_buf;
564}
565
566static void iommu_enable_event_buffer(struct amd_iommu *iommu)
567{
568 u64 entry;
569
570 BUG_ON(iommu->evt_buf == NULL);
571
Joerg Roedel335503e2008-09-05 14:29:07 +0200572 entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
Joerg Roedel58492e12009-05-04 18:41:16 +0200573
Joerg Roedel335503e2008-09-05 14:29:07 +0200574 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
575 &entry, sizeof(entry));
576
Joerg Roedel090672072009-06-15 16:06:48 +0200577 /* set head and tail to zero manually */
578 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
579 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
580
Joerg Roedel58492e12009-05-04 18:41:16 +0200581 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
Joerg Roedel335503e2008-09-05 14:29:07 +0200582}
583
584static void __init free_event_buffer(struct amd_iommu *iommu)
585{
586 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
587}
588
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100589/* allocates the memory where the IOMMU will log its events to */
590static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
591{
592 iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
593 get_order(PPR_LOG_SIZE));
594
595 if (iommu->ppr_log == NULL)
596 return NULL;
597
598 return iommu->ppr_log;
599}
600
601static void iommu_enable_ppr_log(struct amd_iommu *iommu)
602{
603 u64 entry;
604
605 if (iommu->ppr_log == NULL)
606 return;
607
608 entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
609
610 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
611 &entry, sizeof(entry));
612
613 /* set head and tail to zero manually */
614 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
615 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
616
617 iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
618 iommu_feature_enable(iommu, CONTROL_PPR_EN);
619}
620
621static void __init free_ppr_log(struct amd_iommu *iommu)
622{
623 if (iommu->ppr_log == NULL)
624 return;
625
626 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
627}
628
Joerg Roedelcbc33a92011-11-25 11:41:31 +0100629static void iommu_enable_gt(struct amd_iommu *iommu)
630{
631 if (!iommu_feature(iommu, FEATURE_GT))
632 return;
633
634 iommu_feature_enable(iommu, CONTROL_GT_EN);
635}
636
Joerg Roedelb65233a2008-07-11 17:14:21 +0200637/* sets a specific bit in the device table entry. */
Joerg Roedel3566b772008-06-26 21:27:46 +0200638static void set_dev_entry_bit(u16 devid, u8 bit)
639{
Joerg Roedelee6c2862011-11-09 12:06:03 +0100640 int i = (bit >> 6) & 0x03;
641 int _bit = bit & 0x3f;
Joerg Roedel3566b772008-06-26 21:27:46 +0200642
Joerg Roedelee6c2862011-11-09 12:06:03 +0100643 amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);
Joerg Roedel3566b772008-06-26 21:27:46 +0200644}
645
Joerg Roedelc5cca142009-10-09 18:31:20 +0200646static int get_dev_entry_bit(u16 devid, u8 bit)
647{
Joerg Roedelee6c2862011-11-09 12:06:03 +0100648 int i = (bit >> 6) & 0x03;
649 int _bit = bit & 0x3f;
Joerg Roedelc5cca142009-10-09 18:31:20 +0200650
Joerg Roedelee6c2862011-11-09 12:06:03 +0100651 return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
Joerg Roedelc5cca142009-10-09 18:31:20 +0200652}
653
654
655void amd_iommu_apply_erratum_63(u16 devid)
656{
657 int sysmgt;
658
659 sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |
660 (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
661
662 if (sysmgt == 0x01)
663 set_dev_entry_bit(devid, DEV_ENTRY_IW);
664}
665
Joerg Roedel5ff47892008-07-14 20:11:18 +0200666/* Writes the specific IOMMU for a device into the rlookup table */
667static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
668{
669 amd_iommu_rlookup_table[devid] = iommu;
670}
671
Joerg Roedelb65233a2008-07-11 17:14:21 +0200672/*
673 * This function takes the device specific flags read from the ACPI
674 * table and sets up the device table entry with that information
675 */
Joerg Roedel5ff47892008-07-14 20:11:18 +0200676static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
677 u16 devid, u32 flags, u32 ext_flags)
Joerg Roedel3566b772008-06-26 21:27:46 +0200678{
679 if (flags & ACPI_DEVFLAG_INITPASS)
680 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
681 if (flags & ACPI_DEVFLAG_EXTINT)
682 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
683 if (flags & ACPI_DEVFLAG_NMI)
684 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
685 if (flags & ACPI_DEVFLAG_SYSMGT1)
686 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
687 if (flags & ACPI_DEVFLAG_SYSMGT2)
688 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
689 if (flags & ACPI_DEVFLAG_LINT0)
690 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
691 if (flags & ACPI_DEVFLAG_LINT1)
692 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
Joerg Roedel3566b772008-06-26 21:27:46 +0200693
Joerg Roedelc5cca142009-10-09 18:31:20 +0200694 amd_iommu_apply_erratum_63(devid);
695
Joerg Roedel5ff47892008-07-14 20:11:18 +0200696 set_iommu_for_device(iommu, devid);
Joerg Roedel3566b772008-06-26 21:27:46 +0200697}
698
Joerg Roedel6efed632012-06-14 15:52:58 +0200699static int add_special_device(u8 type, u8 id, u16 devid)
700{
701 struct devid_map *entry;
702 struct list_head *list;
703
704 if (type != IVHD_SPECIAL_IOAPIC && type != IVHD_SPECIAL_HPET)
705 return -EINVAL;
706
707 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
708 if (!entry)
709 return -ENOMEM;
710
711 entry->id = id;
712 entry->devid = devid;
713
714 if (type == IVHD_SPECIAL_IOAPIC)
715 list = &ioapic_map;
716 else
717 list = &hpet_map;
718
719 list_add_tail(&entry->list, list);
720
721 return 0;
722}
723
Joerg Roedelb65233a2008-07-11 17:14:21 +0200724/*
725 * Reads the device exclusion range from ACPI and initialize IOMMU with
726 * it
727 */
Joerg Roedel3566b772008-06-26 21:27:46 +0200728static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
729{
730 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
731
732 if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
733 return;
734
735 if (iommu) {
Joerg Roedelb65233a2008-07-11 17:14:21 +0200736 /*
737 * We only can configure exclusion ranges per IOMMU, not
738 * per device. But we can enable the exclusion range per
739 * device. This is done here
740 */
Joerg Roedel3566b772008-06-26 21:27:46 +0200741 set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
742 iommu->exclusion_start = m->range_start;
743 iommu->exclusion_length = m->range_length;
744 }
745}
746
Joerg Roedelb65233a2008-07-11 17:14:21 +0200747/*
Joerg Roedelb65233a2008-07-11 17:14:21 +0200748 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
749 * initializes the hardware and our data structures with it.
750 */
Joerg Roedel6efed632012-06-14 15:52:58 +0200751static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200752 struct ivhd_header *h)
753{
754 u8 *p = (u8 *)h;
755 u8 *end = p, flags = 0;
Joerg Roedel0de66d52011-06-06 16:04:02 +0200756 u16 devid = 0, devid_start = 0, devid_to = 0;
757 u32 dev_i, ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200758 bool alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200759 struct ivhd_entry *e;
760
761 /*
Joerg Roedele9bf5192010-09-20 14:33:07 +0200762 * First save the recommended feature enable bits from ACPI
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200763 */
Joerg Roedele9bf5192010-09-20 14:33:07 +0200764 iommu->acpi_flags = h->flags;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200765
766 /*
767 * Done. Now parse the device entries
768 */
769 p += sizeof(struct ivhd_header);
770 end += h->length;
771
Joerg Roedel42a698f2009-05-20 15:41:28 +0200772
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200773 while (p < end) {
774 e = (struct ivhd_entry *)p;
775 switch (e->type) {
776 case IVHD_DEV_ALL:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200777
778 DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
779 " last device %02x:%02x.%x flags: %02x\n",
780 PCI_BUS(iommu->first_device),
781 PCI_SLOT(iommu->first_device),
782 PCI_FUNC(iommu->first_device),
783 PCI_BUS(iommu->last_device),
784 PCI_SLOT(iommu->last_device),
785 PCI_FUNC(iommu->last_device),
786 e->flags);
787
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200788 for (dev_i = iommu->first_device;
789 dev_i <= iommu->last_device; ++dev_i)
Joerg Roedel5ff47892008-07-14 20:11:18 +0200790 set_dev_entry_from_acpi(iommu, dev_i,
791 e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200792 break;
793 case IVHD_DEV_SELECT:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200794
795 DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
796 "flags: %02x\n",
797 PCI_BUS(e->devid),
798 PCI_SLOT(e->devid),
799 PCI_FUNC(e->devid),
800 e->flags);
801
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200802 devid = e->devid;
Joerg Roedel5ff47892008-07-14 20:11:18 +0200803 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200804 break;
805 case IVHD_DEV_SELECT_RANGE_START:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200806
807 DUMP_printk(" DEV_SELECT_RANGE_START\t "
808 "devid: %02x:%02x.%x flags: %02x\n",
809 PCI_BUS(e->devid),
810 PCI_SLOT(e->devid),
811 PCI_FUNC(e->devid),
812 e->flags);
813
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200814 devid_start = e->devid;
815 flags = e->flags;
816 ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200817 alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200818 break;
819 case IVHD_DEV_ALIAS:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200820
821 DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
822 "flags: %02x devid_to: %02x:%02x.%x\n",
823 PCI_BUS(e->devid),
824 PCI_SLOT(e->devid),
825 PCI_FUNC(e->devid),
826 e->flags,
827 PCI_BUS(e->ext >> 8),
828 PCI_SLOT(e->ext >> 8),
829 PCI_FUNC(e->ext >> 8));
830
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200831 devid = e->devid;
832 devid_to = e->ext >> 8;
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200833 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
Neil Turton7455aab2009-05-14 14:08:11 +0100834 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200835 amd_iommu_alias_table[devid] = devid_to;
836 break;
837 case IVHD_DEV_ALIAS_RANGE:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200838
839 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
840 "devid: %02x:%02x.%x flags: %02x "
841 "devid_to: %02x:%02x.%x\n",
842 PCI_BUS(e->devid),
843 PCI_SLOT(e->devid),
844 PCI_FUNC(e->devid),
845 e->flags,
846 PCI_BUS(e->ext >> 8),
847 PCI_SLOT(e->ext >> 8),
848 PCI_FUNC(e->ext >> 8));
849
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200850 devid_start = e->devid;
851 flags = e->flags;
852 devid_to = e->ext >> 8;
853 ext_flags = 0;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200854 alias = true;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200855 break;
856 case IVHD_DEV_EXT_SELECT:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200857
858 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
859 "flags: %02x ext: %08x\n",
860 PCI_BUS(e->devid),
861 PCI_SLOT(e->devid),
862 PCI_FUNC(e->devid),
863 e->flags, e->ext);
864
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200865 devid = e->devid;
Joerg Roedel5ff47892008-07-14 20:11:18 +0200866 set_dev_entry_from_acpi(iommu, devid, e->flags,
867 e->ext);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200868 break;
869 case IVHD_DEV_EXT_SELECT_RANGE:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200870
871 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
872 "%02x:%02x.%x flags: %02x ext: %08x\n",
873 PCI_BUS(e->devid),
874 PCI_SLOT(e->devid),
875 PCI_FUNC(e->devid),
876 e->flags, e->ext);
877
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200878 devid_start = e->devid;
879 flags = e->flags;
880 ext_flags = e->ext;
Joerg Roedel58a3bee2008-07-11 17:14:30 +0200881 alias = false;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200882 break;
883 case IVHD_DEV_RANGE_END:
Joerg Roedel42a698f2009-05-20 15:41:28 +0200884
885 DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
886 PCI_BUS(e->devid),
887 PCI_SLOT(e->devid),
888 PCI_FUNC(e->devid));
889
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200890 devid = e->devid;
891 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200892 if (alias) {
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200893 amd_iommu_alias_table[dev_i] = devid_to;
Joerg Roedel7a6a3a02009-07-02 12:23:23 +0200894 set_dev_entry_from_acpi(iommu,
895 devid_to, flags, ext_flags);
896 }
897 set_dev_entry_from_acpi(iommu, dev_i,
898 flags, ext_flags);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200899 }
900 break;
Joerg Roedel6efed632012-06-14 15:52:58 +0200901 case IVHD_DEV_SPECIAL: {
902 u8 handle, type;
903 const char *var;
904 u16 devid;
905 int ret;
906
907 handle = e->ext & 0xff;
908 devid = (e->ext >> 8) & 0xffff;
909 type = (e->ext >> 24) & 0xff;
910
911 if (type == IVHD_SPECIAL_IOAPIC)
912 var = "IOAPIC";
913 else if (type == IVHD_SPECIAL_HPET)
914 var = "HPET";
915 else
916 var = "UNKNOWN";
917
918 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
919 var, (int)handle,
920 PCI_BUS(devid),
921 PCI_SLOT(devid),
922 PCI_FUNC(devid));
923
924 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
925 ret = add_special_device(type, handle, devid);
926 if (ret)
927 return ret;
928 break;
929 }
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200930 default:
931 break;
932 }
933
Joerg Roedelb514e552008-09-17 17:14:27 +0200934 p += ivhd_entry_length(p);
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200935 }
Joerg Roedel6efed632012-06-14 15:52:58 +0200936
937 return 0;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200938}
939
Joerg Roedelb65233a2008-07-11 17:14:21 +0200940/* Initializes the device->iommu mapping for the driver */
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200941static int __init init_iommu_devices(struct amd_iommu *iommu)
942{
Joerg Roedel0de66d52011-06-06 16:04:02 +0200943 u32 i;
Joerg Roedel5d0c8e42008-06-26 21:27:47 +0200944
945 for (i = iommu->first_device; i <= iommu->last_device; ++i)
946 set_iommu_for_device(iommu, i);
947
948 return 0;
949}
950
Joerg Roedele47d4022008-06-26 21:27:48 +0200951static void __init free_iommu_one(struct amd_iommu *iommu)
952{
953 free_command_buffer(iommu);
Joerg Roedel335503e2008-09-05 14:29:07 +0200954 free_event_buffer(iommu);
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100955 free_ppr_log(iommu);
Joerg Roedele47d4022008-06-26 21:27:48 +0200956 iommu_unmap_mmio_space(iommu);
957}
958
959static void __init free_iommu_all(void)
960{
961 struct amd_iommu *iommu, *next;
962
Joerg Roedel3bd22172009-05-04 15:06:20 +0200963 for_each_iommu_safe(iommu, next) {
Joerg Roedele47d4022008-06-26 21:27:48 +0200964 list_del(&iommu->list);
965 free_iommu_one(iommu);
966 kfree(iommu);
967 }
968}
969
Joerg Roedelb65233a2008-07-11 17:14:21 +0200970/*
971 * This function clues the initialization function for one IOMMU
972 * together and also allocates the command buffer and programs the
973 * hardware. It does NOT enable the IOMMU. This is done afterwards.
974 */
Joerg Roedele47d4022008-06-26 21:27:48 +0200975static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
976{
Joerg Roedel6efed632012-06-14 15:52:58 +0200977 int ret;
978
Joerg Roedele47d4022008-06-26 21:27:48 +0200979 spin_lock_init(&iommu->lock);
Joerg Roedelbb527772009-11-20 14:31:51 +0100980
981 /* Add IOMMU to internal data structures */
Joerg Roedele47d4022008-06-26 21:27:48 +0200982 list_add_tail(&iommu->list, &amd_iommu_list);
Joerg Roedelbb527772009-11-20 14:31:51 +0100983 iommu->index = amd_iommus_present++;
984
985 if (unlikely(iommu->index >= MAX_IOMMUS)) {
986 WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
987 return -ENOSYS;
988 }
989
990 /* Index is fine - add IOMMU to the array */
991 amd_iommus[iommu->index] = iommu;
Joerg Roedele47d4022008-06-26 21:27:48 +0200992
993 /*
994 * Copy data from ACPI table entry to the iommu struct
995 */
Joerg Roedel23c742d2012-06-12 11:47:34 +0200996 iommu->devid = h->devid;
Joerg Roedele47d4022008-06-26 21:27:48 +0200997 iommu->cap_ptr = h->cap_ptr;
Joerg Roedelee893c22008-09-08 14:48:04 +0200998 iommu->pci_seg = h->pci_seg;
Joerg Roedele47d4022008-06-26 21:27:48 +0200999 iommu->mmio_phys = h->mmio_phys;
1000 iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
1001 if (!iommu->mmio_base)
1002 return -ENOMEM;
1003
Joerg Roedele47d4022008-06-26 21:27:48 +02001004 iommu->cmd_buf = alloc_command_buffer(iommu);
1005 if (!iommu->cmd_buf)
1006 return -ENOMEM;
1007
Joerg Roedel335503e2008-09-05 14:29:07 +02001008 iommu->evt_buf = alloc_event_buffer(iommu);
1009 if (!iommu->evt_buf)
1010 return -ENOMEM;
1011
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001012 iommu->int_enabled = false;
1013
Joerg Roedel6efed632012-06-14 15:52:58 +02001014 ret = init_iommu_from_acpi(iommu, h);
1015 if (ret)
1016 return ret;
Joerg Roedele47d4022008-06-26 21:27:48 +02001017 init_iommu_devices(iommu);
1018
Joerg Roedel23c742d2012-06-12 11:47:34 +02001019 return 0;
Joerg Roedele47d4022008-06-26 21:27:48 +02001020}
1021
Joerg Roedelb65233a2008-07-11 17:14:21 +02001022/*
1023 * Iterates over all IOMMU entries in the ACPI table, allocates the
1024 * IOMMU structure and initializes it with init_iommu_one()
1025 */
Joerg Roedele47d4022008-06-26 21:27:48 +02001026static int __init init_iommu_all(struct acpi_table_header *table)
1027{
1028 u8 *p = (u8 *)table, *end = (u8 *)table;
1029 struct ivhd_header *h;
1030 struct amd_iommu *iommu;
1031 int ret;
1032
Joerg Roedele47d4022008-06-26 21:27:48 +02001033 end += table->length;
1034 p += IVRS_HEADER_LENGTH;
1035
1036 while (p < end) {
1037 h = (struct ivhd_header *)p;
1038 switch (*p) {
1039 case ACPI_IVHD_TYPE:
Joerg Roedel9c720412009-05-20 13:53:57 +02001040
Joerg Roedelae908c22009-09-01 16:52:16 +02001041 DUMP_printk("device: %02x:%02x.%01x cap: %04x "
Joerg Roedel9c720412009-05-20 13:53:57 +02001042 "seg: %d flags: %01x info %04x\n",
1043 PCI_BUS(h->devid), PCI_SLOT(h->devid),
1044 PCI_FUNC(h->devid), h->cap_ptr,
1045 h->pci_seg, h->flags, h->info);
1046 DUMP_printk(" mmio-addr: %016llx\n",
1047 h->mmio_phys);
1048
Joerg Roedele47d4022008-06-26 21:27:48 +02001049 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001050 if (iommu == NULL)
1051 return -ENOMEM;
Joerg Roedel3551a702010-03-01 13:52:19 +01001052
Joerg Roedele47d4022008-06-26 21:27:48 +02001053 ret = init_iommu_one(iommu, h);
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001054 if (ret)
1055 return ret;
Joerg Roedele47d4022008-06-26 21:27:48 +02001056 break;
1057 default:
1058 break;
1059 }
1060 p += h->length;
1061
1062 }
1063 WARN_ON(p != end);
1064
1065 return 0;
1066}
1067
Joerg Roedel23c742d2012-06-12 11:47:34 +02001068static int iommu_init_pci(struct amd_iommu *iommu)
1069{
1070 int cap_ptr = iommu->cap_ptr;
1071 u32 range, misc, low, high;
1072
1073 iommu->dev = pci_get_bus_and_slot(PCI_BUS(iommu->devid),
1074 iommu->devid & 0xff);
1075 if (!iommu->dev)
1076 return -ENODEV;
1077
1078 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
1079 &iommu->cap);
1080 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
1081 &range);
1082 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
1083 &misc);
1084
1085 iommu->first_device = calc_devid(MMIO_GET_BUS(range),
1086 MMIO_GET_FD(range));
1087 iommu->last_device = calc_devid(MMIO_GET_BUS(range),
1088 MMIO_GET_LD(range));
1089
1090 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
1091 amd_iommu_iotlb_sup = false;
1092
1093 /* read extended feature bits */
1094 low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
1095 high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
1096
1097 iommu->features = ((u64)high << 32) | low;
1098
1099 if (iommu_feature(iommu, FEATURE_GT)) {
1100 int glxval;
1101 u32 pasids;
1102 u64 shift;
1103
1104 shift = iommu->features & FEATURE_PASID_MASK;
1105 shift >>= FEATURE_PASID_SHIFT;
1106 pasids = (1 << shift);
1107
1108 amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
1109
1110 glxval = iommu->features & FEATURE_GLXVAL_MASK;
1111 glxval >>= FEATURE_GLXVAL_SHIFT;
1112
1113 if (amd_iommu_max_glx_val == -1)
1114 amd_iommu_max_glx_val = glxval;
1115 else
1116 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
1117 }
1118
1119 if (iommu_feature(iommu, FEATURE_GT) &&
1120 iommu_feature(iommu, FEATURE_PPR)) {
1121 iommu->is_iommu_v2 = true;
1122 amd_iommu_v2_present = true;
1123 }
1124
1125 if (iommu_feature(iommu, FEATURE_PPR)) {
1126 iommu->ppr_log = alloc_ppr_log(iommu);
1127 if (!iommu->ppr_log)
1128 return -ENOMEM;
1129 }
1130
1131 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
1132 amd_iommu_np_cache = true;
1133
1134 if (is_rd890_iommu(iommu->dev)) {
1135 int i, j;
1136
1137 iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
1138 PCI_DEVFN(0, 0));
1139
1140 /*
1141 * Some rd890 systems may not be fully reconfigured by the
1142 * BIOS, so it's necessary for us to store this information so
1143 * it can be reprogrammed on resume
1144 */
1145 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
1146 &iommu->stored_addr_lo);
1147 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
1148 &iommu->stored_addr_hi);
1149
1150 /* Low bit locks writes to configuration space */
1151 iommu->stored_addr_lo &= ~1;
1152
1153 for (i = 0; i < 6; i++)
1154 for (j = 0; j < 0x12; j++)
1155 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
1156
1157 for (i = 0; i < 0x83; i++)
1158 iommu->stored_l2[i] = iommu_read_l2(iommu, i);
1159 }
1160
1161 return pci_enable_device(iommu->dev);
1162}
1163
Joerg Roedel4d121c32012-06-14 12:21:55 +02001164static void print_iommu_info(void)
1165{
1166 static const char * const feat_str[] = {
1167 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
1168 "IA", "GA", "HE", "PC"
1169 };
1170 struct amd_iommu *iommu;
1171
1172 for_each_iommu(iommu) {
1173 int i;
1174
1175 pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
1176 dev_name(&iommu->dev->dev), iommu->cap_ptr);
1177
1178 if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
1179 pr_info("AMD-Vi: Extended features: ");
Joerg Roedel2bd5ed02012-08-10 11:34:08 +02001180 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
Joerg Roedel4d121c32012-06-14 12:21:55 +02001181 if (iommu_feature(iommu, (1ULL << i)))
1182 pr_cont(" %s", feat_str[i]);
1183 }
1184 }
1185 pr_cont("\n");
1186 }
1187}
1188
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001189static int __init amd_iommu_init_pci(void)
Joerg Roedel23c742d2012-06-12 11:47:34 +02001190{
1191 struct amd_iommu *iommu;
1192 int ret = 0;
1193
1194 for_each_iommu(iommu) {
1195 ret = iommu_init_pci(iommu);
1196 if (ret)
1197 break;
1198 }
1199
Joerg Roedel23c742d2012-06-12 11:47:34 +02001200 ret = amd_iommu_init_devices();
1201
Joerg Roedel4d121c32012-06-14 12:21:55 +02001202 print_iommu_info();
1203
Joerg Roedel23c742d2012-06-12 11:47:34 +02001204 return ret;
1205}
1206
Joerg Roedelb65233a2008-07-11 17:14:21 +02001207/****************************************************************************
1208 *
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001209 * The following functions initialize the MSI interrupts for all IOMMUs
1210 * in the system. Its a bit challenging because there could be multiple
1211 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
1212 * pci_dev.
1213 *
1214 ****************************************************************************/
1215
Joerg Roedel9f800de2009-11-23 12:45:25 +01001216static int iommu_setup_msi(struct amd_iommu *iommu)
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001217{
1218 int r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001219
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001220 r = pci_enable_msi(iommu->dev);
1221 if (r)
1222 return r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001223
Joerg Roedel72fe00f2011-05-10 10:50:42 +02001224 r = request_threaded_irq(iommu->dev->irq,
1225 amd_iommu_int_handler,
1226 amd_iommu_int_thread,
1227 0, "AMD-Vi",
1228 iommu->dev);
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001229
1230 if (r) {
1231 pci_disable_msi(iommu->dev);
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001232 return r;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001233 }
1234
Joerg Roedelfab6afa2009-05-04 18:46:34 +02001235 iommu->int_enabled = true;
Joerg Roedel1a29ac02011-11-10 15:41:40 +01001236
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001237 return 0;
1238}
1239
Joerg Roedel05f92db2009-05-12 09:52:46 +02001240static int iommu_init_msi(struct amd_iommu *iommu)
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001241{
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001242 int ret;
1243
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001244 if (iommu->int_enabled)
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001245 goto enable_faults;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001246
Joerg Roedeld91cecd2009-05-04 18:51:00 +02001247 if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001248 ret = iommu_setup_msi(iommu);
1249 else
1250 ret = -ENODEV;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001251
Joerg Roedel9ddd5922012-03-15 16:29:47 +01001252 if (ret)
1253 return ret;
1254
1255enable_faults:
1256 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
1257
1258 if (iommu->ppr_log != NULL)
1259 iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
1260
1261 return 0;
Joerg Roedela80dc3e2008-09-11 16:51:41 +02001262}
1263
1264/****************************************************************************
1265 *
Joerg Roedelb65233a2008-07-11 17:14:21 +02001266 * The next functions belong to the third pass of parsing the ACPI
1267 * table. In this last pass the memory mapping requirements are
1268 * gathered (like exclusion and unity mapping reanges).
1269 *
1270 ****************************************************************************/
1271
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001272static void __init free_unity_maps(void)
1273{
1274 struct unity_map_entry *entry, *next;
1275
1276 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
1277 list_del(&entry->list);
1278 kfree(entry);
1279 }
1280}
1281
Joerg Roedelb65233a2008-07-11 17:14:21 +02001282/* called when we find an exclusion range definition in ACPI */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001283static int __init init_exclusion_range(struct ivmd_header *m)
1284{
1285 int i;
1286
1287 switch (m->type) {
1288 case ACPI_IVMD_TYPE:
1289 set_device_exclusion_range(m->devid, m);
1290 break;
1291 case ACPI_IVMD_TYPE_ALL:
Joerg Roedel3a61ec32008-07-25 13:07:50 +02001292 for (i = 0; i <= amd_iommu_last_bdf; ++i)
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001293 set_device_exclusion_range(i, m);
1294 break;
1295 case ACPI_IVMD_TYPE_RANGE:
1296 for (i = m->devid; i <= m->aux; ++i)
1297 set_device_exclusion_range(i, m);
1298 break;
1299 default:
1300 break;
1301 }
1302
1303 return 0;
1304}
1305
Joerg Roedelb65233a2008-07-11 17:14:21 +02001306/* called for unity map ACPI definition */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001307static int __init init_unity_map_range(struct ivmd_header *m)
1308{
Joerg Roedel98f1ad22012-07-06 13:28:37 +02001309 struct unity_map_entry *e = NULL;
Joerg Roedel02acc432009-05-20 16:24:21 +02001310 char *s;
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001311
1312 e = kzalloc(sizeof(*e), GFP_KERNEL);
1313 if (e == NULL)
1314 return -ENOMEM;
1315
1316 switch (m->type) {
1317 default:
Joerg Roedel0bc252f2009-05-22 12:48:05 +02001318 kfree(e);
1319 return 0;
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001320 case ACPI_IVMD_TYPE:
Joerg Roedel02acc432009-05-20 16:24:21 +02001321 s = "IVMD_TYPEi\t\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001322 e->devid_start = e->devid_end = m->devid;
1323 break;
1324 case ACPI_IVMD_TYPE_ALL:
Joerg Roedel02acc432009-05-20 16:24:21 +02001325 s = "IVMD_TYPE_ALL\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001326 e->devid_start = 0;
1327 e->devid_end = amd_iommu_last_bdf;
1328 break;
1329 case ACPI_IVMD_TYPE_RANGE:
Joerg Roedel02acc432009-05-20 16:24:21 +02001330 s = "IVMD_TYPE_RANGE\t\t";
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001331 e->devid_start = m->devid;
1332 e->devid_end = m->aux;
1333 break;
1334 }
1335 e->address_start = PAGE_ALIGN(m->range_start);
1336 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
1337 e->prot = m->flags >> 1;
1338
Joerg Roedel02acc432009-05-20 16:24:21 +02001339 DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
1340 " range_start: %016llx range_end: %016llx flags: %x\n", s,
1341 PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),
1342 PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),
1343 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
1344 e->address_start, e->address_end, m->flags);
1345
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001346 list_add_tail(&e->list, &amd_iommu_unity_map);
1347
1348 return 0;
1349}
1350
Joerg Roedelb65233a2008-07-11 17:14:21 +02001351/* iterates over all memory definitions we find in the ACPI table */
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001352static int __init init_memory_definitions(struct acpi_table_header *table)
1353{
1354 u8 *p = (u8 *)table, *end = (u8 *)table;
1355 struct ivmd_header *m;
1356
Joerg Roedelbe2a0222008-06-26 21:27:49 +02001357 end += table->length;
1358 p += IVRS_HEADER_LENGTH;
1359
1360 while (p < end) {
1361 m = (struct ivmd_header *)p;
1362 if (m->flags & IVMD_FLAG_EXCL_RANGE)
1363 init_exclusion_range(m);
1364 else if (m->flags & IVMD_FLAG_UNITY_MAP)
1365 init_unity_map_range(m);
1366
1367 p += m->length;
1368 }
1369
1370 return 0;
1371}
1372
Joerg Roedelb65233a2008-07-11 17:14:21 +02001373/*
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001374 * Init the device table to not allow DMA access for devices and
1375 * suppress all page faults
1376 */
1377static void init_device_table(void)
1378{
Joerg Roedel0de66d52011-06-06 16:04:02 +02001379 u32 devid;
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001380
1381 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1382 set_dev_entry_bit(devid, DEV_ENTRY_VALID);
1383 set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001384 }
1385}
1386
Joerg Roedele9bf5192010-09-20 14:33:07 +02001387static void iommu_init_flags(struct amd_iommu *iommu)
1388{
1389 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
1390 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
1391 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
1392
1393 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
1394 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
1395 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
1396
1397 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
1398 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
1399 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
1400
1401 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
1402 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
1403 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
1404
1405 /*
1406 * make IOMMU memory accesses cache coherent
1407 */
1408 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
Joerg Roedel1456e9d2011-12-22 14:51:53 +01001409
1410 /* Set IOTLB invalidation timeout to 1s */
1411 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
Joerg Roedele9bf5192010-09-20 14:33:07 +02001412}
1413
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001414static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
Joerg Roedel4c894f42010-09-23 15:15:19 +02001415{
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001416 int i, j;
1417 u32 ioc_feature_control;
Joerg Roedelc1bf94e2012-05-31 17:38:11 +02001418 struct pci_dev *pdev = iommu->root_pdev;
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001419
1420 /* RD890 BIOSes may not have completely reconfigured the iommu */
Joerg Roedelc1bf94e2012-05-31 17:38:11 +02001421 if (!is_rd890_iommu(iommu->dev) || !pdev)
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001422 return;
1423
1424 /*
1425 * First, we need to ensure that the iommu is enabled. This is
1426 * controlled by a register in the northbridge
1427 */
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001428
1429 /* Select Northbridge indirect register 0x75 and enable writing */
1430 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
1431 pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
1432
1433 /* Enable the iommu */
1434 if (!(ioc_feature_control & 0x1))
1435 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
1436
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001437 /* Restore the iommu BAR */
1438 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1439 iommu->stored_addr_lo);
1440 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
1441 iommu->stored_addr_hi);
1442
1443 /* Restore the l1 indirect regs for each of the 6 l1s */
1444 for (i = 0; i < 6; i++)
1445 for (j = 0; j < 0x12; j++)
1446 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
1447
1448 /* Restore the l2 indirect regs */
1449 for (i = 0; i < 0x83; i++)
1450 iommu_write_l2(iommu, i, iommu->stored_l2[i]);
1451
1452 /* Lock PCI setup registers */
1453 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1454 iommu->stored_addr_lo | 1);
Joerg Roedel4c894f42010-09-23 15:15:19 +02001455}
1456
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001457/*
Joerg Roedelb65233a2008-07-11 17:14:21 +02001458 * This function finally enables all IOMMUs found in the system after
1459 * they have been initialized
1460 */
Joerg Roedel11ee5ac2012-06-12 16:30:06 +02001461static void early_enable_iommus(void)
Joerg Roedel87361972008-06-26 21:28:07 +02001462{
1463 struct amd_iommu *iommu;
1464
Joerg Roedel3bd22172009-05-04 15:06:20 +02001465 for_each_iommu(iommu) {
Chris Wrighta8c485b2009-06-15 15:53:45 +02001466 iommu_disable(iommu);
Joerg Roedele9bf5192010-09-20 14:33:07 +02001467 iommu_init_flags(iommu);
Joerg Roedel58492e12009-05-04 18:41:16 +02001468 iommu_set_device_table(iommu);
1469 iommu_enable_command_buffer(iommu);
1470 iommu_enable_event_buffer(iommu);
Joerg Roedel87361972008-06-26 21:28:07 +02001471 iommu_set_exclusion_range(iommu);
1472 iommu_enable(iommu);
Joerg Roedel7d0c5cc2011-04-07 08:16:10 +02001473 iommu_flush_all_caches(iommu);
Joerg Roedel87361972008-06-26 21:28:07 +02001474 }
1475}
1476
Joerg Roedel11ee5ac2012-06-12 16:30:06 +02001477static void enable_iommus_v2(void)
1478{
1479 struct amd_iommu *iommu;
1480
1481 for_each_iommu(iommu) {
1482 iommu_enable_ppr_log(iommu);
1483 iommu_enable_gt(iommu);
1484 }
1485}
1486
1487static void enable_iommus(void)
1488{
1489 early_enable_iommus();
1490
1491 enable_iommus_v2();
1492}
1493
Joerg Roedel92ac4322009-05-19 19:06:27 +02001494static void disable_iommus(void)
1495{
1496 struct amd_iommu *iommu;
1497
1498 for_each_iommu(iommu)
1499 iommu_disable(iommu);
1500}
1501
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001502/*
1503 * Suspend/Resume support
1504 * disable suspend until real resume implemented
1505 */
1506
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001507static void amd_iommu_resume(void)
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001508{
Matthew Garrett5bcd7572010-10-04 14:59:31 -04001509 struct amd_iommu *iommu;
1510
1511 for_each_iommu(iommu)
1512 iommu_apply_resume_quirks(iommu);
1513
Joerg Roedel736501e2009-05-12 09:56:12 +02001514 /* re-load the hardware */
1515 enable_iommus();
Joerg Roedel3d9761e2012-03-15 16:39:21 +01001516
1517 amd_iommu_enable_interrupts();
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001518}
1519
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001520static int amd_iommu_suspend(void)
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001521{
Joerg Roedel736501e2009-05-12 09:56:12 +02001522 /* disable IOMMUs to go out of the way for BIOS */
1523 disable_iommus();
1524
1525 return 0;
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001526}
1527
Rafael J. Wysockif3c6ea12011-03-23 22:15:54 +01001528static struct syscore_ops amd_iommu_syscore_ops = {
Joerg Roedel7441e9c2008-06-30 20:18:02 +02001529 .suspend = amd_iommu_suspend,
1530 .resume = amd_iommu_resume,
1531};
1532
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001533static void __init free_on_init_error(void)
1534{
Joerg Roedel05152a02012-06-15 16:53:51 +02001535 if (amd_iommu_irq_cache) {
1536 kmem_cache_destroy(amd_iommu_irq_cache);
1537 amd_iommu_irq_cache = NULL;
1538 }
1539
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001540 amd_iommu_uninit_devices();
1541
1542 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1543 get_order(MAX_DOMAIN_ID/8));
1544
1545 free_pages((unsigned long)amd_iommu_rlookup_table,
1546 get_order(rlookup_table_size));
1547
1548 free_pages((unsigned long)amd_iommu_alias_table,
1549 get_order(alias_table_size));
1550
1551 free_pages((unsigned long)amd_iommu_dev_table,
1552 get_order(dev_table_size));
1553
1554 free_iommu_all();
1555
1556 free_unity_maps();
1557
1558#ifdef CONFIG_GART_IOMMU
1559 /*
1560 * We failed to initialize the AMD IOMMU - try fallback to GART
1561 * if possible.
1562 */
1563 gart_iommu_init();
1564
1565#endif
1566}
1567
Joerg Roedelb65233a2008-07-11 17:14:21 +02001568/*
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001569 * This is the hardware init function for AMD IOMMU in the system.
1570 * This function is called either from amd_iommu_init or from the interrupt
1571 * remapping setup code.
Joerg Roedelb65233a2008-07-11 17:14:21 +02001572 *
1573 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
1574 * three times:
1575 *
1576 * 1 pass) Find the highest PCI device id the driver has to handle.
1577 * Upon this information the size of the data structures is
1578 * determined that needs to be allocated.
1579 *
1580 * 2 pass) Initialize the data structures just allocated with the
1581 * information in the ACPI table about available AMD IOMMUs
1582 * in the system. It also maps the PCI devices in the
1583 * system to specific IOMMUs
1584 *
1585 * 3 pass) After the basic data structures are allocated and
1586 * initialized we update them with information about memory
1587 * remapping requirements parsed out of the ACPI table in
1588 * this last pass.
1589 *
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001590 * After everything is set up the IOMMUs are enabled and the necessary
1591 * hotplug and suspend notifiers are registered.
Joerg Roedelb65233a2008-07-11 17:14:21 +02001592 */
Joerg Roedel643511b2012-06-12 12:09:35 +02001593static int __init early_amd_iommu_init(void)
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001594{
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001595 struct acpi_table_header *ivrs_base;
1596 acpi_size ivrs_size;
1597 acpi_status status;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001598 int i, ret = 0;
1599
Joerg Roedel643511b2012-06-12 12:09:35 +02001600 if (!amd_iommu_detected)
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001601 return -ENODEV;
1602
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001603 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1604 if (status == AE_NOT_FOUND)
1605 return -ENODEV;
1606 else if (ACPI_FAILURE(status)) {
1607 const char *err = acpi_format_exception(status);
1608 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1609 return -EINVAL;
1610 }
1611
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001612 /*
1613 * First parse ACPI tables to find the largest Bus/Dev/Func
1614 * we need to handle. Upon this information the shared data
1615 * structures for the IOMMUs in the system will be allocated
1616 */
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001617 ret = find_last_devid_acpi(ivrs_base);
1618 if (ret)
Joerg Roedel3551a702010-03-01 13:52:19 +01001619 goto out;
1620
Joerg Roedelc5714842008-07-11 17:14:25 +02001621 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
1622 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
1623 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001624
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001625 /* Device table - directly used by all IOMMUs */
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001626 ret = -ENOMEM;
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001627 amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001628 get_order(dev_table_size));
1629 if (amd_iommu_dev_table == NULL)
1630 goto out;
1631
1632 /*
1633 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
1634 * IOMMU see for that device
1635 */
1636 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
1637 get_order(alias_table_size));
1638 if (amd_iommu_alias_table == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001639 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001640
1641 /* IOMMU rlookup table - find the IOMMU for a specific device */
Joerg Roedel83fd5cc2008-12-16 19:17:11 +01001642 amd_iommu_rlookup_table = (void *)__get_free_pages(
1643 GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001644 get_order(rlookup_table_size));
1645 if (amd_iommu_rlookup_table == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001646 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001647
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001648 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
1649 GFP_KERNEL | __GFP_ZERO,
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001650 get_order(MAX_DOMAIN_ID/8));
1651 if (amd_iommu_pd_alloc_bitmap == NULL)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001652 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001653
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +02001654 /* init the device table */
1655 init_device_table();
1656
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001657 /*
Joerg Roedel5dc8bff2008-07-11 17:14:32 +02001658 * let all alias entries point to itself
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001659 */
Joerg Roedel3a61ec32008-07-25 13:07:50 +02001660 for (i = 0; i <= amd_iommu_last_bdf; ++i)
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001661 amd_iommu_alias_table[i] = i;
1662
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001663 /*
1664 * never allocate domain 0 because its used as the non-allocated and
1665 * error value placeholder
1666 */
1667 amd_iommu_pd_alloc_bitmap[0] = 1;
1668
Joerg Roedelaeb26f52009-11-20 16:44:01 +01001669 spin_lock_init(&amd_iommu_pd_lock);
1670
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001671 /*
1672 * now the data structures are allocated and basically initialized
1673 * start the real acpi table scan
1674 */
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001675 ret = init_iommu_all(ivrs_base);
1676 if (ret)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001677 goto out;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001678
Joerg Roedel05152a02012-06-15 16:53:51 +02001679 if (amd_iommu_irq_remap) {
1680 /*
1681 * Interrupt remapping enabled, create kmem_cache for the
1682 * remapping tables.
1683 */
1684 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
1685 MAX_IRQS_PER_TABLE * sizeof(u32),
1686 IRQ_TABLE_ALIGNMENT,
1687 0, NULL);
1688 if (!amd_iommu_irq_cache)
1689 goto out;
1690 }
1691
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001692 ret = init_memory_definitions(ivrs_base);
1693 if (ret)
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001694 goto out;
Joerg Roedel3551a702010-03-01 13:52:19 +01001695
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001696out:
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001697 /* Don't leak any ACPI memory */
1698 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1699 ivrs_base = NULL;
1700
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001701 return ret;
Joerg Roedel643511b2012-06-12 12:09:35 +02001702}
1703
Gerard Snitselaarae295142012-03-16 11:38:22 -07001704static int amd_iommu_enable_interrupts(void)
Joerg Roedel3d9761e2012-03-15 16:39:21 +01001705{
1706 struct amd_iommu *iommu;
1707 int ret = 0;
1708
1709 for_each_iommu(iommu) {
1710 ret = iommu_init_msi(iommu);
1711 if (ret)
1712 goto out;
1713 }
1714
1715out:
1716 return ret;
1717}
1718
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001719static bool detect_ivrs(void)
1720{
1721 struct acpi_table_header *ivrs_base;
1722 acpi_size ivrs_size;
1723 acpi_status status;
1724
1725 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1726 if (status == AE_NOT_FOUND)
1727 return false;
1728 else if (ACPI_FAILURE(status)) {
1729 const char *err = acpi_format_exception(status);
1730 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1731 return false;
1732 }
1733
1734 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1735
Joerg Roedel1adb7d32012-08-06 14:18:42 +02001736 /* Make sure ACS will be enabled during PCI probe */
1737 pci_request_acs();
1738
Joerg Roedel05152a02012-06-15 16:53:51 +02001739 if (!disable_irq_remap)
1740 amd_iommu_irq_remap = true;
1741
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001742 return true;
1743}
1744
Joerg Roedelb9b1ce702012-06-12 16:51:12 +02001745static int amd_iommu_init_dma(void)
1746{
1747 int ret;
1748
1749 if (iommu_pass_through)
1750 ret = amd_iommu_init_passthrough();
1751 else
1752 ret = amd_iommu_init_dma_ops();
1753
1754 if (ret)
1755 return ret;
1756
1757 amd_iommu_init_api();
1758
1759 amd_iommu_init_notifier();
1760
1761 return 0;
1762}
1763
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001764/****************************************************************************
1765 *
1766 * AMD IOMMU Initialization State Machine
1767 *
1768 ****************************************************************************/
1769
1770static int __init state_next(void)
1771{
1772 int ret = 0;
1773
1774 switch (init_state) {
1775 case IOMMU_START_STATE:
1776 if (!detect_ivrs()) {
1777 init_state = IOMMU_NOT_FOUND;
1778 ret = -ENODEV;
1779 } else {
1780 init_state = IOMMU_IVRS_DETECTED;
1781 }
1782 break;
1783 case IOMMU_IVRS_DETECTED:
1784 ret = early_amd_iommu_init();
1785 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
1786 break;
1787 case IOMMU_ACPI_FINISHED:
1788 early_enable_iommus();
1789 register_syscore_ops(&amd_iommu_syscore_ops);
1790 x86_platform.iommu_shutdown = disable_iommus;
1791 init_state = IOMMU_ENABLED;
1792 break;
1793 case IOMMU_ENABLED:
1794 ret = amd_iommu_init_pci();
1795 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
1796 enable_iommus_v2();
1797 break;
1798 case IOMMU_PCI_INIT:
1799 ret = amd_iommu_enable_interrupts();
1800 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
1801 break;
1802 case IOMMU_INTERRUPTS_EN:
1803 ret = amd_iommu_init_dma();
1804 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
1805 break;
1806 case IOMMU_DMA_OPS:
1807 init_state = IOMMU_INITIALIZED;
1808 break;
1809 case IOMMU_INITIALIZED:
1810 /* Nothing to do */
1811 break;
1812 case IOMMU_NOT_FOUND:
1813 case IOMMU_INIT_ERROR:
1814 /* Error states => do nothing */
1815 ret = -EINVAL;
1816 break;
1817 default:
1818 /* Unknown state */
1819 BUG();
1820 }
1821
1822 return ret;
1823}
1824
1825static int __init iommu_go_to_state(enum iommu_init_state state)
1826{
1827 int ret = 0;
1828
1829 while (init_state != state) {
1830 ret = state_next();
1831 if (init_state == IOMMU_NOT_FOUND ||
1832 init_state == IOMMU_INIT_ERROR)
1833 break;
1834 }
1835
1836 return ret;
1837}
1838
1839
1840
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001841/*
1842 * This is the core init function for AMD IOMMU hardware in the system.
1843 * This function is called from the generic x86 DMA layer initialization
1844 * code.
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001845 */
1846static int __init amd_iommu_init(void)
1847{
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001848 int ret;
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001849
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001850 ret = iommu_go_to_state(IOMMU_INITIALIZED);
1851 if (ret) {
1852 disable_iommus();
1853 free_on_init_error();
1854 }
Joerg Roedel8704a1b2012-03-01 15:57:53 +01001855
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001856 return ret;
Joerg Roedelfe74c9c2008-06-26 21:27:50 +02001857}
1858
Joerg Roedelb65233a2008-07-11 17:14:21 +02001859/****************************************************************************
1860 *
1861 * Early detect code. This code runs at IOMMU detection time in the DMA
1862 * layer. It just looks if there is an IVRS ACPI table to detect AMD
1863 * IOMMUs
1864 *
1865 ****************************************************************************/
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04001866int __init amd_iommu_detect(void)
Joerg Roedelae7877d2008-06-26 21:27:51 +02001867{
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001868 int ret;
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001869
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09001870 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04001871 return -ENODEV;
Joerg Roedelae7877d2008-06-26 21:27:51 +02001872
Joerg Roedela5235722010-05-11 17:12:33 +02001873 if (amd_iommu_disabled)
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -04001874 return -ENODEV;
Joerg Roedela5235722010-05-11 17:12:33 +02001875
Joerg Roedel2c0ae172012-06-12 15:59:30 +02001876 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
1877 if (ret)
1878 return ret;
Linus Torvalds11bd04f2009-12-11 12:18:16 -08001879
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001880 amd_iommu_detected = true;
1881 iommu_detected = 1;
1882 x86_init.iommu.iommu_init = amd_iommu_init;
1883
Joerg Roedel02f3b3f2012-06-11 17:45:25 +02001884 return 0;
Joerg Roedelae7877d2008-06-26 21:27:51 +02001885}
1886
Joerg Roedelb65233a2008-07-11 17:14:21 +02001887/****************************************************************************
1888 *
1889 * Parsing functions for the AMD IOMMU specific kernel command line
1890 * options.
1891 *
1892 ****************************************************************************/
1893
Joerg Roedelfefda112009-05-20 12:21:42 +02001894static int __init parse_amd_iommu_dump(char *str)
1895{
1896 amd_iommu_dump = true;
1897
1898 return 1;
1899}
1900
Joerg Roedel918ad6c2008-06-26 21:27:52 +02001901static int __init parse_amd_iommu_options(char *str)
1902{
1903 for (; *str; ++str) {
Joerg Roedel695b5672008-11-17 15:16:43 +01001904 if (strncmp(str, "fullflush", 9) == 0)
FUJITA Tomonoriafa9fdc2008-09-20 01:23:30 +09001905 amd_iommu_unmap_flush = true;
Joerg Roedela5235722010-05-11 17:12:33 +02001906 if (strncmp(str, "off", 3) == 0)
1907 amd_iommu_disabled = true;
Joerg Roedel5abcdba2011-12-01 15:49:45 +01001908 if (strncmp(str, "force_isolation", 15) == 0)
1909 amd_iommu_force_isolation = true;
Joerg Roedel918ad6c2008-06-26 21:27:52 +02001910 }
1911
1912 return 1;
1913}
1914
Joerg Roedelfefda112009-05-20 12:21:42 +02001915__setup("amd_iommu_dump", parse_amd_iommu_dump);
Joerg Roedel918ad6c2008-06-26 21:27:52 +02001916__setup("amd_iommu=", parse_amd_iommu_options);
Konrad Rzeszutek Wilk22e6daf2010-08-26 13:58:03 -04001917
1918IOMMU_INIT_FINISH(amd_iommu_detect,
1919 gart_iommu_hole_init,
Joerg Roedel98f1ad22012-07-06 13:28:37 +02001920 NULL,
1921 NULL);
Joerg Roedel400a28a2011-11-28 15:11:02 +01001922
1923bool amd_iommu_v2_supported(void)
1924{
1925 return amd_iommu_v2_present;
1926}
1927EXPORT_SYMBOL(amd_iommu_v2_supported);