blob: b952ebc7a78b67776d985f006fb33633be43b80b [file] [log] [blame]
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -07001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
mark gross98bcef52008-02-23 15:23:35 -080017 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070021 *
Suresh Siddhae61d98d2008-07-10 11:16:35 -070022 * This file implements early detection/parsing of Remapping Devices
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070023 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
Suresh Siddhae61d98d2008-07-10 11:16:35 -070025 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070027 */
28
29#include <linux/pci.h>
30#include <linux/dmar.h>
Kay, Allen M38717942008-09-09 18:37:29 +030031#include <linux/iova.h>
32#include <linux/intel-iommu.h>
Suresh Siddhafe962e92008-07-10 11:16:42 -070033#include <linux/timer.h>
Suresh Siddha0ac24912009-03-16 17:04:54 -070034#include <linux/irq.h>
35#include <linux/interrupt.h>
Shane Wang69575d32009-09-01 18:25:07 -070036#include <linux/tboot.h>
Len Browneb27cae2009-07-06 23:40:19 -040037#include <linux/dmi.h>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070038
Len Browna192a952009-07-28 16:45:54 -040039#define PREFIX "DMAR: "
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070040
41/* No locks are needed as DMA remapping hardware unit
42 * list is constructed at boot time and hotplug of
43 * these units are not supported by the architecture.
44 */
45LIST_HEAD(dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070046
47static struct acpi_table_header * __initdata dmar_tbl;
Yinghai Lu8e1568f2009-02-11 01:06:59 -080048static acpi_size dmar_tbl_size;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070049
50static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
51{
52 /*
53 * add INCLUDE_ALL at the tail, so scan the list will find it at
54 * the very end.
55 */
56 if (drhd->include_all)
57 list_add_tail(&drhd->list, &dmar_drhd_units);
58 else
59 list_add(&drhd->list, &dmar_drhd_units);
60}
61
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070062static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
63 struct pci_dev **dev, u16 segment)
64{
65 struct pci_bus *bus;
66 struct pci_dev *pdev = NULL;
67 struct acpi_dmar_pci_path *path;
68 int count;
69
70 bus = pci_find_bus(segment, scope->bus);
71 path = (struct acpi_dmar_pci_path *)(scope + 1);
72 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
73 / sizeof(struct acpi_dmar_pci_path);
74
75 while (count) {
76 if (pdev)
77 pci_dev_put(pdev);
78 /*
79 * Some BIOSes list non-exist devices in DMAR table, just
80 * ignore it
81 */
82 if (!bus) {
83 printk(KERN_WARNING
84 PREFIX "Device scope bus [%d] not found\n",
85 scope->bus);
86 break;
87 }
88 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
89 if (!pdev) {
90 printk(KERN_WARNING PREFIX
91 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
92 segment, bus->number, path->dev, path->fn);
93 break;
94 }
95 path ++;
96 count --;
97 bus = pdev->subordinate;
98 }
99 if (!pdev) {
100 printk(KERN_WARNING PREFIX
101 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
103 *dev = NULL;
104 return 0;
105 }
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
109 pci_dev_put(pdev);
110 printk(KERN_WARNING PREFIX
111 "Device scope type does not match for %s\n",
112 pci_name(pdev));
113 return -EINVAL;
114 }
115 *dev = pdev;
116 return 0;
117}
118
119static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
120 struct pci_dev ***devices, u16 segment)
121{
122 struct acpi_dmar_device_scope *scope;
123 void * tmp = start;
124 int index;
125 int ret;
126
127 *cnt = 0;
128 while (start < end) {
129 scope = start;
130 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
131 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
132 (*cnt)++;
133 else
134 printk(KERN_WARNING PREFIX
135 "Unsupported device scope\n");
136 start += scope->length;
137 }
138 if (*cnt == 0)
139 return 0;
140
141 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
142 if (!*devices)
143 return -ENOMEM;
144
145 start = tmp;
146 index = 0;
147 while (start < end) {
148 scope = start;
149 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
150 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
151 ret = dmar_parse_one_dev_scope(scope,
152 &(*devices)[index], segment);
153 if (ret) {
154 kfree(*devices);
155 return ret;
156 }
157 index ++;
158 }
159 start += scope->length;
160 }
161
162 return 0;
163}
164
165/**
166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
167 * structure which uniquely represent one DMA remapping hardware unit
168 * present in the platform
169 */
170static int __init
171dmar_parse_one_drhd(struct acpi_dmar_header *header)
172{
173 struct acpi_dmar_hardware_unit *drhd;
174 struct dmar_drhd_unit *dmaru;
175 int ret = 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700176
David Woodhousee523b382009-04-10 22:27:48 -0700177 drhd = (struct acpi_dmar_hardware_unit *)header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700178 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
179 if (!dmaru)
180 return -ENOMEM;
181
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700182 dmaru->hdr = header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700183 dmaru->reg_base_addr = drhd->address;
David Woodhouse276dbf92009-04-04 01:45:37 +0100184 dmaru->segment = drhd->segment;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700185 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
186
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700187 ret = alloc_iommu(dmaru);
188 if (ret) {
189 kfree(dmaru);
190 return ret;
191 }
192 dmar_register_drhd_unit(dmaru);
193 return 0;
194}
195
David Woodhousef82851a2008-10-18 15:43:14 +0100196static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700197{
198 struct acpi_dmar_hardware_unit *drhd;
David Woodhousef82851a2008-10-18 15:43:14 +0100199 int ret = 0;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700200
201 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
202
Yu Zhao2e824f72008-12-22 16:54:58 +0800203 if (dmaru->include_all)
204 return 0;
205
206 ret = dmar_parse_dev_scope((void *)(drhd + 1),
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700207 ((void *)drhd) + drhd->header.length,
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700208 &dmaru->devices_cnt, &dmaru->devices,
209 drhd->segment);
Suresh Siddha1c7d1bc2008-09-03 16:58:35 -0700210 if (ret) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700211 list_del(&dmaru->list);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700212 kfree(dmaru);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700213 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700214 return ret;
215}
216
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700217#ifdef CONFIG_DMAR
218LIST_HEAD(dmar_rmrr_units);
219
220static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
221{
222 list_add(&rmrr->list, &dmar_rmrr_units);
223}
224
225
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700226static int __init
227dmar_parse_one_rmrr(struct acpi_dmar_header *header)
228{
229 struct acpi_dmar_reserved_memory *rmrr;
230 struct dmar_rmrr_unit *rmrru;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700231
232 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
233 if (!rmrru)
234 return -ENOMEM;
235
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700236 rmrru->hdr = header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700237 rmrr = (struct acpi_dmar_reserved_memory *)header;
238 rmrru->base_address = rmrr->base_address;
239 rmrru->end_address = rmrr->end_address;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700240
241 dmar_register_rmrr_unit(rmrru);
242 return 0;
243}
244
245static int __init
246rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
247{
248 struct acpi_dmar_reserved_memory *rmrr;
249 int ret;
250
251 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700252 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700253 ((void *)rmrr) + rmrr->header.length,
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700254 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
255
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700256 if (ret || (rmrru->devices_cnt == 0)) {
257 list_del(&rmrru->list);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700258 kfree(rmrru);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700259 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700260 return ret;
261}
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800262
263static LIST_HEAD(dmar_atsr_units);
264
265static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
266{
267 struct acpi_dmar_atsr *atsr;
268 struct dmar_atsr_unit *atsru;
269
270 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
271 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
272 if (!atsru)
273 return -ENOMEM;
274
275 atsru->hdr = hdr;
276 atsru->include_all = atsr->flags & 0x1;
277
278 list_add(&atsru->list, &dmar_atsr_units);
279
280 return 0;
281}
282
283static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
284{
285 int rc;
286 struct acpi_dmar_atsr *atsr;
287
288 if (atsru->include_all)
289 return 0;
290
291 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
292 rc = dmar_parse_dev_scope((void *)(atsr + 1),
293 (void *)atsr + atsr->header.length,
294 &atsru->devices_cnt, &atsru->devices,
295 atsr->segment);
296 if (rc || !atsru->devices_cnt) {
297 list_del(&atsru->list);
298 kfree(atsru);
299 }
300
301 return rc;
302}
303
304int dmar_find_matched_atsr_unit(struct pci_dev *dev)
305{
306 int i;
307 struct pci_bus *bus;
308 struct acpi_dmar_atsr *atsr;
309 struct dmar_atsr_unit *atsru;
310
311 list_for_each_entry(atsru, &dmar_atsr_units, list) {
312 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
313 if (atsr->segment == pci_domain_nr(dev->bus))
314 goto found;
315 }
316
317 return 0;
318
319found:
320 for (bus = dev->bus; bus; bus = bus->parent) {
321 struct pci_dev *bridge = bus->self;
322
323 if (!bridge || !bridge->is_pcie ||
324 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
325 return 0;
326
327 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
328 for (i = 0; i < atsru->devices_cnt; i++)
329 if (atsru->devices[i] == bridge)
330 return 1;
331 break;
332 }
333 }
334
335 if (atsru->include_all)
336 return 1;
337
338 return 0;
339}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700340#endif
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700341
342static void __init
343dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
344{
345 struct acpi_dmar_hardware_unit *drhd;
346 struct acpi_dmar_reserved_memory *rmrr;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800347 struct acpi_dmar_atsr *atsr;
Roland Dreier17b60972009-09-24 12:14:00 -0700348 struct acpi_dmar_rhsa *rhsa;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700349
350 switch (header->type) {
351 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800352 drhd = container_of(header, struct acpi_dmar_hardware_unit,
353 header);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700354 printk (KERN_INFO PREFIX
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800355 "DRHD base: %#016Lx flags: %#x\n",
356 (unsigned long long)drhd->address, drhd->flags);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700357 break;
358 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800359 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
360 header);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700361 printk (KERN_INFO PREFIX
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800362 "RMRR base: %#016Lx end: %#016Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700363 (unsigned long long)rmrr->base_address,
364 (unsigned long long)rmrr->end_address);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700365 break;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800366 case ACPI_DMAR_TYPE_ATSR:
367 atsr = container_of(header, struct acpi_dmar_atsr, header);
368 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
369 break;
Roland Dreier17b60972009-09-24 12:14:00 -0700370 case ACPI_DMAR_HARDWARE_AFFINITY:
371 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
372 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
373 (unsigned long long)rhsa->base_address,
374 rhsa->proximity_domain);
375 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700376 }
377}
378
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700379/**
380 * dmar_table_detect - checks to see if the platform supports DMAR devices
381 */
382static int __init dmar_table_detect(void)
383{
384 acpi_status status = AE_OK;
385
386 /* if we could find DMAR table, then there are DMAR devices */
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800387 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
388 (struct acpi_table_header **)&dmar_tbl,
389 &dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700390
391 if (ACPI_SUCCESS(status) && !dmar_tbl) {
392 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
393 status = AE_NOT_FOUND;
394 }
395
396 return (ACPI_SUCCESS(status) ? 1 : 0);
397}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700398
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700399/**
400 * parse_dmar_table - parses the DMA reporting table
401 */
402static int __init
403parse_dmar_table(void)
404{
405 struct acpi_table_dmar *dmar;
406 struct acpi_dmar_header *entry_header;
407 int ret = 0;
408
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700409 /*
410 * Do it again, earlier dmar_tbl mapping could be mapped with
411 * fixed map.
412 */
413 dmar_table_detect();
414
Joseph Cihulaa59b50e2009-06-30 19:31:10 -0700415 /*
416 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
417 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
418 */
419 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
420
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700421 dmar = (struct acpi_table_dmar *)dmar_tbl;
422 if (!dmar)
423 return -ENODEV;
424
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700425 if (dmar->width < PAGE_SHIFT - 1) {
Fenghua Yu093f87d2007-11-21 15:07:14 -0800426 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700427 return -EINVAL;
428 }
429
430 printk (KERN_INFO PREFIX "Host address width %d\n",
431 dmar->width + 1);
432
433 entry_header = (struct acpi_dmar_header *)(dmar + 1);
434 while (((unsigned long)entry_header) <
435 (((unsigned long)dmar) + dmar_tbl->length)) {
Tony Battersby084eb962009-02-11 13:24:19 -0800436 /* Avoid looping forever on bad ACPI tables */
437 if (entry_header->length == 0) {
438 printk(KERN_WARNING PREFIX
439 "Invalid 0-length structure\n");
440 ret = -EINVAL;
441 break;
442 }
443
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700444 dmar_table_print_dmar_entry(entry_header);
445
446 switch (entry_header->type) {
447 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
448 ret = dmar_parse_one_drhd(entry_header);
449 break;
450 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700451#ifdef CONFIG_DMAR
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700452 ret = dmar_parse_one_rmrr(entry_header);
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700453#endif
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700454 break;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800455 case ACPI_DMAR_TYPE_ATSR:
456#ifdef CONFIG_DMAR
457 ret = dmar_parse_one_atsr(entry_header);
458#endif
459 break;
Roland Dreier17b60972009-09-24 12:14:00 -0700460 case ACPI_DMAR_HARDWARE_AFFINITY:
461 /* We don't do anything with RHSA (yet?) */
462 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700463 default:
464 printk(KERN_WARNING PREFIX
Roland Dreier4de75cf2009-09-24 01:01:29 +0100465 "Unknown DMAR structure type %d\n",
466 entry_header->type);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700467 ret = 0; /* for forward compatibility */
468 break;
469 }
470 if (ret)
471 break;
472
473 entry_header = ((void *)entry_header + entry_header->length);
474 }
475 return ret;
476}
477
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700478int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
479 struct pci_dev *dev)
480{
481 int index;
482
483 while (dev) {
484 for (index = 0; index < cnt; index++)
485 if (dev == devices[index])
486 return 1;
487
488 /* Check our parent */
489 dev = dev->bus->self;
490 }
491
492 return 0;
493}
494
495struct dmar_drhd_unit *
496dmar_find_matched_drhd_unit(struct pci_dev *dev)
497{
Yu Zhao2e824f72008-12-22 16:54:58 +0800498 struct dmar_drhd_unit *dmaru = NULL;
499 struct acpi_dmar_hardware_unit *drhd;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700500
Yu Zhao2e824f72008-12-22 16:54:58 +0800501 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
502 drhd = container_of(dmaru->hdr,
503 struct acpi_dmar_hardware_unit,
504 header);
505
506 if (dmaru->include_all &&
507 drhd->segment == pci_domain_nr(dev->bus))
508 return dmaru;
509
510 if (dmar_pci_device_match(dmaru->devices,
511 dmaru->devices_cnt, dev))
512 return dmaru;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700513 }
514
515 return NULL;
516}
517
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700518int __init dmar_dev_scope_init(void)
519{
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700520 struct dmar_drhd_unit *drhd, *drhd_n;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700521 int ret = -ENODEV;
522
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700523 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700524 ret = dmar_parse_dev(drhd);
525 if (ret)
526 return ret;
527 }
528
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700529#ifdef CONFIG_DMAR
530 {
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700531 struct dmar_rmrr_unit *rmrr, *rmrr_n;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800532 struct dmar_atsr_unit *atsr, *atsr_n;
533
Suresh Siddha04e2ea62008-09-03 16:58:34 -0700534 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700535 ret = rmrr_parse_dev(rmrr);
536 if (ret)
537 return ret;
538 }
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800539
540 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
541 ret = atsr_parse_dev(atsr);
542 if (ret)
543 return ret;
544 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700545 }
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700546#endif
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700547
548 return ret;
549}
550
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700551
552int __init dmar_table_init(void)
553{
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700554 static int dmar_table_initialized;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800555 int ret;
556
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700557 if (dmar_table_initialized)
558 return 0;
559
560 dmar_table_initialized = 1;
561
Fenghua Yu093f87d2007-11-21 15:07:14 -0800562 ret = parse_dmar_table();
563 if (ret) {
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700564 if (ret != -ENODEV)
565 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
Fenghua Yu093f87d2007-11-21 15:07:14 -0800566 return ret;
567 }
568
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700569 if (list_empty(&dmar_drhd_units)) {
570 printk(KERN_INFO PREFIX "No DMAR devices found\n");
571 return -ENODEV;
572 }
Fenghua Yu093f87d2007-11-21 15:07:14 -0800573
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700574#ifdef CONFIG_DMAR
Suresh Siddha2d6b5f82008-07-10 11:16:39 -0700575 if (list_empty(&dmar_rmrr_units))
Fenghua Yu093f87d2007-11-21 15:07:14 -0800576 printk(KERN_INFO PREFIX "No RMRR found\n");
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800577
578 if (list_empty(&dmar_atsr_units))
579 printk(KERN_INFO PREFIX "No ATSR found\n");
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700580#endif
Fenghua Yu093f87d2007-11-21 15:07:14 -0800581
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700582 return 0;
583}
584
David Woodhouse86cf8982009-11-09 22:15:15 +0000585int __init check_zero_address(void)
586{
587 struct acpi_table_dmar *dmar;
588 struct acpi_dmar_header *entry_header;
589 struct acpi_dmar_hardware_unit *drhd;
590
591 dmar = (struct acpi_table_dmar *)dmar_tbl;
592 entry_header = (struct acpi_dmar_header *)(dmar + 1);
593
594 while (((unsigned long)entry_header) <
595 (((unsigned long)dmar) + dmar_tbl->length)) {
596 /* Avoid looping forever on bad ACPI tables */
597 if (entry_header->length == 0) {
598 printk(KERN_WARNING PREFIX
599 "Invalid 0-length structure\n");
600 return 0;
601 }
602
603 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
604 drhd = (void *)entry_header;
605 if (!drhd->address) {
606 /* Promote an attitude of violence to a BIOS engineer today */
607 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
608 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
609 dmi_get_system_info(DMI_BIOS_VENDOR),
610 dmi_get_system_info(DMI_BIOS_VERSION),
611 dmi_get_system_info(DMI_PRODUCT_VERSION));
David Woodhouse5854d9c2009-11-19 02:18:44 +0000612#ifdef CONFIG_DMAR
613 dmar_disabled = 1;
614#endif
David Woodhouse86cf8982009-11-09 22:15:15 +0000615 return 0;
616 }
617 break;
618 }
619
620 entry_header = ((void *)entry_header + entry_header->length);
621 }
622 return 1;
623}
624
Suresh Siddha2ae21012008-07-10 11:16:43 -0700625void __init detect_intel_iommu(void)
626{
627 int ret;
628
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700629 ret = dmar_table_detect();
David Woodhouse86cf8982009-11-09 22:15:15 +0000630 if (ret)
631 ret = check_zero_address();
Suresh Siddha2ae21012008-07-10 11:16:43 -0700632 {
Youquan Songcacd4212008-10-16 16:31:57 -0700633#ifdef CONFIG_INTR_REMAP
Suresh Siddha1cb11582008-07-10 11:16:51 -0700634 struct acpi_table_dmar *dmar;
635 /*
636 * for now we will disable dma-remapping when interrupt
637 * remapping is enabled.
638 * When support for queued invalidation for IOTLB invalidation
639 * is added, we will not need this any more.
640 */
641 dmar = (struct acpi_table_dmar *) dmar_tbl;
Youquan Songcacd4212008-10-16 16:31:57 -0700642 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
Suresh Siddha1cb11582008-07-10 11:16:51 -0700643 printk(KERN_INFO
644 "Queued invalidation will be enabled to support "
645 "x2apic and Intr-remapping.\n");
Youquan Songcacd4212008-10-16 16:31:57 -0700646#endif
Youquan Songcacd4212008-10-16 16:31:57 -0700647#ifdef CONFIG_DMAR
Suresh Siddha2ae21012008-07-10 11:16:43 -0700648 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
649 !dmar_disabled)
650 iommu_detected = 1;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700651#endif
Youquan Songcacd4212008-10-16 16:31:57 -0700652 }
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800653 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700654 dmar_tbl = NULL;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700655}
656
657
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700658int alloc_iommu(struct dmar_drhd_unit *drhd)
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700659{
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700660 struct intel_iommu *iommu;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700661 int map_size;
662 u32 ver;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700663 static int iommu_allocated = 0;
Joerg Roedel43f73922009-01-03 23:56:27 +0100664 int agaw = 0;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700665 int msagaw = 0;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700666
667 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
668 if (!iommu)
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700669 return -ENOMEM;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700670
671 iommu->seq_id = iommu_allocated++;
Suresh Siddha9d783ba2009-03-16 17:04:55 -0700672 sprintf (iommu->name, "dmar%d", iommu->seq_id);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700673
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700674 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700675 if (!iommu->reg) {
676 printk(KERN_ERR "IOMMU: can't map the region\n");
677 goto error;
678 }
679 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
680 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
681
David Woodhouse08155652009-08-04 09:17:20 +0100682 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
683 /* Promote an attitude of violence to a BIOS engineer today */
684 WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
685 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
686 drhd->reg_base_addr,
687 dmi_get_system_info(DMI_BIOS_VENDOR),
688 dmi_get_system_info(DMI_BIOS_VERSION),
689 dmi_get_system_info(DMI_PRODUCT_VERSION));
690 goto err_unmap;
691 }
692
Joerg Roedel43f73922009-01-03 23:56:27 +0100693#ifdef CONFIG_DMAR
Weidong Han1b573682008-12-08 15:34:06 +0800694 agaw = iommu_calculate_agaw(iommu);
695 if (agaw < 0) {
696 printk(KERN_ERR
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700697 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
698 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100699 goto err_unmap;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700700 }
701 msagaw = iommu_calculate_max_sagaw(iommu);
702 if (msagaw < 0) {
703 printk(KERN_ERR
704 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
Weidong Han1b573682008-12-08 15:34:06 +0800705 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100706 goto err_unmap;
Weidong Han1b573682008-12-08 15:34:06 +0800707 }
Joerg Roedel43f73922009-01-03 23:56:27 +0100708#endif
Weidong Han1b573682008-12-08 15:34:06 +0800709 iommu->agaw = agaw;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700710 iommu->msagaw = msagaw;
Weidong Han1b573682008-12-08 15:34:06 +0800711
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700712 /* the registers might be more than one page */
713 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
714 cap_max_fault_reg_offset(iommu->cap));
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700715 map_size = VTD_PAGE_ALIGN(map_size);
716 if (map_size > VTD_PAGE_SIZE) {
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700717 iounmap(iommu->reg);
718 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
719 if (!iommu->reg) {
720 printk(KERN_ERR "IOMMU: can't map the region\n");
721 goto error;
722 }
723 }
724
725 ver = readl(iommu->reg + DMAR_VER_REG);
David Woodhouse08155652009-08-04 09:17:20 +0100726 pr_info("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700727 (unsigned long long)drhd->reg_base_addr,
728 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
729 (unsigned long long)iommu->cap,
730 (unsigned long long)iommu->ecap);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700731
732 spin_lock_init(&iommu->register_lock);
733
734 drhd->iommu = iommu;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700735 return 0;
David Woodhouse08155652009-08-04 09:17:20 +0100736
737 err_unmap:
738 iounmap(iommu->reg);
739 error:
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700740 kfree(iommu);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700741 return -1;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700742}
743
744void free_iommu(struct intel_iommu *iommu)
745{
746 if (!iommu)
747 return;
748
749#ifdef CONFIG_DMAR
750 free_dmar_iommu(iommu);
751#endif
752
753 if (iommu->reg)
754 iounmap(iommu->reg);
755 kfree(iommu);
756}
Suresh Siddhafe962e92008-07-10 11:16:42 -0700757
758/*
759 * Reclaim all the submitted descriptors which have completed its work.
760 */
761static inline void reclaim_free_desc(struct q_inval *qi)
762{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800763 while (qi->desc_status[qi->free_tail] == QI_DONE ||
764 qi->desc_status[qi->free_tail] == QI_ABORT) {
Suresh Siddhafe962e92008-07-10 11:16:42 -0700765 qi->desc_status[qi->free_tail] = QI_FREE;
766 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
767 qi->free_cnt++;
768 }
769}
770
Yu Zhao704126a2009-01-04 16:28:52 +0800771static int qi_check_fault(struct intel_iommu *iommu, int index)
772{
773 u32 fault;
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800774 int head, tail;
Yu Zhao704126a2009-01-04 16:28:52 +0800775 struct q_inval *qi = iommu->qi;
776 int wait_index = (index + 1) % QI_LENGTH;
777
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800778 if (qi->desc_status[wait_index] == QI_ABORT)
779 return -EAGAIN;
780
Yu Zhao704126a2009-01-04 16:28:52 +0800781 fault = readl(iommu->reg + DMAR_FSTS_REG);
782
783 /*
784 * If IQE happens, the head points to the descriptor associated
785 * with the error. No new descriptors are fetched until the IQE
786 * is cleared.
787 */
788 if (fault & DMA_FSTS_IQE) {
789 head = readl(iommu->reg + DMAR_IQH_REG);
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800790 if ((head >> DMAR_IQ_SHIFT) == index) {
791 printk(KERN_ERR "VT-d detected invalid descriptor: "
792 "low=%llx, high=%llx\n",
793 (unsigned long long)qi->desc[index].low,
794 (unsigned long long)qi->desc[index].high);
Yu Zhao704126a2009-01-04 16:28:52 +0800795 memcpy(&qi->desc[index], &qi->desc[wait_index],
796 sizeof(struct qi_desc));
797 __iommu_flush_cache(iommu, &qi->desc[index],
798 sizeof(struct qi_desc));
799 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
800 return -EINVAL;
801 }
802 }
803
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800804 /*
805 * If ITE happens, all pending wait_desc commands are aborted.
806 * No new descriptors are fetched until the ITE is cleared.
807 */
808 if (fault & DMA_FSTS_ITE) {
809 head = readl(iommu->reg + DMAR_IQH_REG);
810 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
811 head |= 1;
812 tail = readl(iommu->reg + DMAR_IQT_REG);
813 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
814
815 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
816
817 do {
818 if (qi->desc_status[head] == QI_IN_USE)
819 qi->desc_status[head] = QI_ABORT;
820 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
821 } while (head != tail);
822
823 if (qi->desc_status[wait_index] == QI_ABORT)
824 return -EAGAIN;
825 }
826
827 if (fault & DMA_FSTS_ICE)
828 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
829
Yu Zhao704126a2009-01-04 16:28:52 +0800830 return 0;
831}
832
Suresh Siddhafe962e92008-07-10 11:16:42 -0700833/*
834 * Submit the queued invalidation descriptor to the remapping
835 * hardware unit and wait for its completion.
836 */
Yu Zhao704126a2009-01-04 16:28:52 +0800837int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
Suresh Siddhafe962e92008-07-10 11:16:42 -0700838{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800839 int rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700840 struct q_inval *qi = iommu->qi;
841 struct qi_desc *hw, wait_desc;
842 int wait_index, index;
843 unsigned long flags;
844
845 if (!qi)
Yu Zhao704126a2009-01-04 16:28:52 +0800846 return 0;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700847
848 hw = qi->desc;
849
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800850restart:
851 rc = 0;
852
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700853 spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700854 while (qi->free_cnt < 3) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700855 spin_unlock_irqrestore(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700856 cpu_relax();
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700857 spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700858 }
859
860 index = qi->free_head;
861 wait_index = (index + 1) % QI_LENGTH;
862
863 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
864
865 hw[index] = *desc;
866
Yu Zhao704126a2009-01-04 16:28:52 +0800867 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
868 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700869 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
870
871 hw[wait_index] = wait_desc;
872
873 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
874 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
875
876 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
877 qi->free_cnt -= 2;
878
Suresh Siddhafe962e92008-07-10 11:16:42 -0700879 /*
880 * update the HW tail register indicating the presence of
881 * new descriptors.
882 */
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800883 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
Suresh Siddhafe962e92008-07-10 11:16:42 -0700884
885 while (qi->desc_status[wait_index] != QI_DONE) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700886 /*
887 * We will leave the interrupts disabled, to prevent interrupt
888 * context to queue another cmd while a cmd is already submitted
889 * and waiting for completion on this cpu. This is to avoid
890 * a deadlock where the interrupt context can wait indefinitely
891 * for free slots in the queue.
892 */
Yu Zhao704126a2009-01-04 16:28:52 +0800893 rc = qi_check_fault(iommu, index);
894 if (rc)
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800895 break;
Yu Zhao704126a2009-01-04 16:28:52 +0800896
Suresh Siddhafe962e92008-07-10 11:16:42 -0700897 spin_unlock(&qi->q_lock);
898 cpu_relax();
899 spin_lock(&qi->q_lock);
900 }
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800901
902 qi->desc_status[index] = QI_DONE;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700903
904 reclaim_free_desc(qi);
Suresh Siddhaf05810c2008-10-16 16:31:54 -0700905 spin_unlock_irqrestore(&qi->q_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +0800906
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800907 if (rc == -EAGAIN)
908 goto restart;
909
Yu Zhao704126a2009-01-04 16:28:52 +0800910 return rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -0700911}
912
913/*
914 * Flush the global interrupt entry cache.
915 */
916void qi_global_iec(struct intel_iommu *iommu)
917{
918 struct qi_desc desc;
919
920 desc.low = QI_IEC_TYPE;
921 desc.high = 0;
922
Yu Zhao704126a2009-01-04 16:28:52 +0800923 /* should never fail */
Suresh Siddhafe962e92008-07-10 11:16:42 -0700924 qi_submit_sync(&desc, iommu);
925}
926
David Woodhouse4c25a2c2009-05-10 17:16:06 +0100927void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
928 u64 type)
Youquan Song3481f212008-10-16 16:31:55 -0700929{
Youquan Song3481f212008-10-16 16:31:55 -0700930 struct qi_desc desc;
931
Youquan Song3481f212008-10-16 16:31:55 -0700932 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
933 | QI_CC_GRAN(type) | QI_CC_TYPE;
934 desc.high = 0;
935
David Woodhouse4c25a2c2009-05-10 17:16:06 +0100936 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -0700937}
938
David Woodhouse1f0ef2a2009-05-10 19:58:49 +0100939void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
940 unsigned int size_order, u64 type)
Youquan Song3481f212008-10-16 16:31:55 -0700941{
942 u8 dw = 0, dr = 0;
943
944 struct qi_desc desc;
945 int ih = 0;
946
Youquan Song3481f212008-10-16 16:31:55 -0700947 if (cap_write_drain(iommu->cap))
948 dw = 1;
949
950 if (cap_read_drain(iommu->cap))
951 dr = 1;
952
953 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
954 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
955 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
956 | QI_IOTLB_AM(size_order);
957
David Woodhouse1f0ef2a2009-05-10 19:58:49 +0100958 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -0700959}
960
Yu Zhao6ba6c3a2009-05-18 13:51:35 +0800961void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
962 u64 addr, unsigned mask)
963{
964 struct qi_desc desc;
965
966 if (mask) {
967 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
968 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
969 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
970 } else
971 desc.high = QI_DEV_IOTLB_ADDR(addr);
972
973 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
974 qdep = 0;
975
976 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
977 QI_DIOTLB_TYPE;
978
979 qi_submit_sync(&desc, iommu);
980}
981
Suresh Siddhafe962e92008-07-10 11:16:42 -0700982/*
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700983 * Disable Queued Invalidation interface.
984 */
985void dmar_disable_qi(struct intel_iommu *iommu)
986{
987 unsigned long flags;
988 u32 sts;
989 cycles_t start_time = get_cycles();
990
991 if (!ecap_qis(iommu->ecap))
992 return;
993
994 spin_lock_irqsave(&iommu->register_lock, flags);
995
996 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
997 if (!(sts & DMA_GSTS_QIES))
998 goto end;
999
1000 /*
1001 * Give a chance to HW to complete the pending invalidation requests.
1002 */
1003 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1004 readl(iommu->reg + DMAR_IQH_REG)) &&
1005 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1006 cpu_relax();
1007
1008 iommu->gcmd &= ~DMA_GCMD_QIE;
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001009 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1010
1011 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1012 !(sts & DMA_GSTS_QIES), sts);
1013end:
1014 spin_unlock_irqrestore(&iommu->register_lock, flags);
1015}
1016
1017/*
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001018 * Enable queued invalidation.
1019 */
1020static void __dmar_enable_qi(struct intel_iommu *iommu)
1021{
David Woodhousec416daa2009-05-10 20:30:58 +01001022 u32 sts;
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001023 unsigned long flags;
1024 struct q_inval *qi = iommu->qi;
1025
1026 qi->free_head = qi->free_tail = 0;
1027 qi->free_cnt = QI_LENGTH;
1028
1029 spin_lock_irqsave(&iommu->register_lock, flags);
1030
1031 /* write zero to the tail reg */
1032 writel(0, iommu->reg + DMAR_IQT_REG);
1033
1034 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1035
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001036 iommu->gcmd |= DMA_GCMD_QIE;
David Woodhousec416daa2009-05-10 20:30:58 +01001037 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001038
1039 /* Make sure hardware complete it */
1040 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1041
1042 spin_unlock_irqrestore(&iommu->register_lock, flags);
1043}
1044
1045/*
Suresh Siddhafe962e92008-07-10 11:16:42 -07001046 * Enable Queued Invalidation interface. This is a must to support
1047 * interrupt-remapping. Also used by DMA-remapping, which replaces
1048 * register based IOTLB invalidation.
1049 */
1050int dmar_enable_qi(struct intel_iommu *iommu)
1051{
Suresh Siddhafe962e92008-07-10 11:16:42 -07001052 struct q_inval *qi;
1053
1054 if (!ecap_qis(iommu->ecap))
1055 return -ENOENT;
1056
1057 /*
1058 * queued invalidation is already setup and enabled.
1059 */
1060 if (iommu->qi)
1061 return 0;
1062
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001063 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001064 if (!iommu->qi)
1065 return -ENOMEM;
1066
1067 qi = iommu->qi;
1068
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001069 qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
Suresh Siddhafe962e92008-07-10 11:16:42 -07001070 if (!qi->desc) {
1071 kfree(qi);
1072 iommu->qi = 0;
1073 return -ENOMEM;
1074 }
1075
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001076 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001077 if (!qi->desc_status) {
1078 free_page((unsigned long) qi->desc);
1079 kfree(qi);
1080 iommu->qi = 0;
1081 return -ENOMEM;
1082 }
1083
1084 qi->free_head = qi->free_tail = 0;
1085 qi->free_cnt = QI_LENGTH;
1086
1087 spin_lock_init(&qi->q_lock);
1088
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001089 __dmar_enable_qi(iommu);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001090
1091 return 0;
1092}
Suresh Siddha0ac24912009-03-16 17:04:54 -07001093
1094/* iommu interrupt handling. Most stuff are MSI-like. */
1095
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001096enum faulttype {
1097 DMA_REMAP,
1098 INTR_REMAP,
1099 UNKNOWN,
1100};
1101
1102static const char *dma_remap_fault_reasons[] =
Suresh Siddha0ac24912009-03-16 17:04:54 -07001103{
1104 "Software",
1105 "Present bit in root entry is clear",
1106 "Present bit in context entry is clear",
1107 "Invalid context entry",
1108 "Access beyond MGAW",
1109 "PTE Write access is not set",
1110 "PTE Read access is not set",
1111 "Next page table ptr is invalid",
1112 "Root table address invalid",
1113 "Context table ptr is invalid",
1114 "non-zero reserved fields in RTP",
1115 "non-zero reserved fields in CTP",
1116 "non-zero reserved fields in PTE",
1117};
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001118
1119static const char *intr_remap_fault_reasons[] =
1120{
1121 "Detected reserved fields in the decoded interrupt-remapped request",
1122 "Interrupt index exceeded the interrupt-remapping table size",
1123 "Present field in the IRTE entry is clear",
1124 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1125 "Detected reserved fields in the IRTE entry",
1126 "Blocked a compatibility format interrupt request",
1127 "Blocked an interrupt request due to source-id verification failure",
1128};
1129
Suresh Siddha0ac24912009-03-16 17:04:54 -07001130#define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1131
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001132const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001133{
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001134 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1135 ARRAY_SIZE(intr_remap_fault_reasons))) {
1136 *fault_type = INTR_REMAP;
1137 return intr_remap_fault_reasons[fault_reason - 0x20];
1138 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1139 *fault_type = DMA_REMAP;
1140 return dma_remap_fault_reasons[fault_reason];
1141 } else {
1142 *fault_type = UNKNOWN;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001143 return "Unknown";
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001144 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001145}
1146
1147void dmar_msi_unmask(unsigned int irq)
1148{
1149 struct intel_iommu *iommu = get_irq_data(irq);
1150 unsigned long flag;
1151
1152 /* unmask it */
1153 spin_lock_irqsave(&iommu->register_lock, flag);
1154 writel(0, iommu->reg + DMAR_FECTL_REG);
1155 /* Read a reg to force flush the post write */
1156 readl(iommu->reg + DMAR_FECTL_REG);
1157 spin_unlock_irqrestore(&iommu->register_lock, flag);
1158}
1159
1160void dmar_msi_mask(unsigned int irq)
1161{
1162 unsigned long flag;
1163 struct intel_iommu *iommu = get_irq_data(irq);
1164
1165 /* mask it */
1166 spin_lock_irqsave(&iommu->register_lock, flag);
1167 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1168 /* Read a reg to force flush the post write */
1169 readl(iommu->reg + DMAR_FECTL_REG);
1170 spin_unlock_irqrestore(&iommu->register_lock, flag);
1171}
1172
1173void dmar_msi_write(int irq, struct msi_msg *msg)
1174{
1175 struct intel_iommu *iommu = get_irq_data(irq);
1176 unsigned long flag;
1177
1178 spin_lock_irqsave(&iommu->register_lock, flag);
1179 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1180 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1181 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1182 spin_unlock_irqrestore(&iommu->register_lock, flag);
1183}
1184
1185void dmar_msi_read(int irq, struct msi_msg *msg)
1186{
1187 struct intel_iommu *iommu = get_irq_data(irq);
1188 unsigned long flag;
1189
1190 spin_lock_irqsave(&iommu->register_lock, flag);
1191 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1192 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1193 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1194 spin_unlock_irqrestore(&iommu->register_lock, flag);
1195}
1196
1197static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1198 u8 fault_reason, u16 source_id, unsigned long long addr)
1199{
1200 const char *reason;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001201 int fault_type;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001202
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001203 reason = dmar_get_fault_reason(fault_reason, &fault_type);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001204
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001205 if (fault_type == INTR_REMAP)
1206 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1207 "fault index %llx\n"
1208 "INTR-REMAP:[fault reason %02d] %s\n",
1209 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1210 PCI_FUNC(source_id & 0xFF), addr >> 48,
1211 fault_reason, reason);
1212 else
1213 printk(KERN_ERR
1214 "DMAR:[%s] Request device [%02x:%02x.%d] "
1215 "fault addr %llx \n"
1216 "DMAR:[fault reason %02d] %s\n",
1217 (type ? "DMA Read" : "DMA Write"),
1218 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1219 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001220 return 0;
1221}
1222
1223#define PRIMARY_FAULT_REG_LEN (16)
Suresh Siddha1531a6a2009-03-16 17:04:57 -07001224irqreturn_t dmar_fault(int irq, void *dev_id)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001225{
1226 struct intel_iommu *iommu = dev_id;
1227 int reg, fault_index;
1228 u32 fault_status;
1229 unsigned long flag;
1230
1231 spin_lock_irqsave(&iommu->register_lock, flag);
1232 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001233 if (fault_status)
1234 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1235 fault_status);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001236
1237 /* TBD: ignore advanced fault log currently */
1238 if (!(fault_status & DMA_FSTS_PPF))
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001239 goto clear_rest;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001240
1241 fault_index = dma_fsts_fault_record_index(fault_status);
1242 reg = cap_fault_reg_offset(iommu->cap);
1243 while (1) {
1244 u8 fault_reason;
1245 u16 source_id;
1246 u64 guest_addr;
1247 int type;
1248 u32 data;
1249
1250 /* highest 32 bits */
1251 data = readl(iommu->reg + reg +
1252 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1253 if (!(data & DMA_FRCD_F))
1254 break;
1255
1256 fault_reason = dma_frcd_fault_reason(data);
1257 type = dma_frcd_type(data);
1258
1259 data = readl(iommu->reg + reg +
1260 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1261 source_id = dma_frcd_source_id(data);
1262
1263 guest_addr = dmar_readq(iommu->reg + reg +
1264 fault_index * PRIMARY_FAULT_REG_LEN);
1265 guest_addr = dma_frcd_page_addr(guest_addr);
1266 /* clear the fault */
1267 writel(DMA_FRCD_F, iommu->reg + reg +
1268 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1269
1270 spin_unlock_irqrestore(&iommu->register_lock, flag);
1271
1272 dmar_fault_do_one(iommu, type, fault_reason,
1273 source_id, guest_addr);
1274
1275 fault_index++;
Troy Heber8211a7b2009-08-19 15:26:11 -06001276 if (fault_index >= cap_num_fault_regs(iommu->cap))
Suresh Siddha0ac24912009-03-16 17:04:54 -07001277 fault_index = 0;
1278 spin_lock_irqsave(&iommu->register_lock, flag);
1279 }
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001280clear_rest:
1281 /* clear all the other faults */
Suresh Siddha0ac24912009-03-16 17:04:54 -07001282 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001283 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001284
1285 spin_unlock_irqrestore(&iommu->register_lock, flag);
1286 return IRQ_HANDLED;
1287}
1288
1289int dmar_set_interrupt(struct intel_iommu *iommu)
1290{
1291 int irq, ret;
1292
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001293 /*
1294 * Check if the fault interrupt is already initialized.
1295 */
1296 if (iommu->irq)
1297 return 0;
1298
Suresh Siddha0ac24912009-03-16 17:04:54 -07001299 irq = create_irq();
1300 if (!irq) {
1301 printk(KERN_ERR "IOMMU: no free vectors\n");
1302 return -EINVAL;
1303 }
1304
1305 set_irq_data(irq, iommu);
1306 iommu->irq = irq;
1307
1308 ret = arch_setup_dmar_msi(irq);
1309 if (ret) {
1310 set_irq_data(irq, NULL);
1311 iommu->irq = 0;
1312 destroy_irq(irq);
Chris Wrightdd726432009-05-13 15:55:52 -07001313 return ret;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001314 }
1315
Suresh Siddha0ac24912009-03-16 17:04:54 -07001316 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1317 if (ret)
1318 printk(KERN_ERR "IOMMU: can't request irq\n");
1319 return ret;
1320}
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001321
1322int __init enable_drhd_fault_handling(void)
1323{
1324 struct dmar_drhd_unit *drhd;
1325
1326 /*
1327 * Enable fault control interrupt.
1328 */
1329 for_each_drhd_unit(drhd) {
1330 int ret;
1331 struct intel_iommu *iommu = drhd->iommu;
1332 ret = dmar_set_interrupt(iommu);
1333
1334 if (ret) {
1335 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1336 " interrupt, ret %d\n",
1337 (unsigned long long)drhd->reg_base_addr, ret);
1338 return -1;
1339 }
1340 }
1341
1342 return 0;
1343}
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001344
1345/*
1346 * Re-enable Queued Invalidation interface.
1347 */
1348int dmar_reenable_qi(struct intel_iommu *iommu)
1349{
1350 if (!ecap_qis(iommu->ecap))
1351 return -ENOENT;
1352
1353 if (!iommu->qi)
1354 return -ENOENT;
1355
1356 /*
1357 * First disable queued invalidation.
1358 */
1359 dmar_disable_qi(iommu);
1360 /*
1361 * Then enable queued invalidation again. Since there is no pending
1362 * invalidation requests now, it's safe to re-enable queued
1363 * invalidation.
1364 */
1365 __dmar_enable_qi(iommu);
1366
1367 return 0;
1368}
Youquan Song074835f2009-09-09 12:05:39 -04001369
1370/*
1371 * Check interrupt remapping support in DMAR table description.
1372 */
1373int dmar_ir_support(void)
1374{
1375 struct acpi_table_dmar *dmar;
1376 dmar = (struct acpi_table_dmar *)dmar_tbl;
1377 return dmar->flags & 0x1;
1378}