blob: 60ab474bfff38e58ba1e246d7da6ca3a201677cb [file] [log] [blame]
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -07001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
mark gross98bcef52008-02-23 15:23:35 -080017 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070021 *
Suresh Siddhae61d98d2008-07-10 11:16:35 -070022 * This file implements early detection/parsing of Remapping Devices
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070023 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
Suresh Siddhae61d98d2008-07-10 11:16:35 -070025 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070027 */
28
Donald Dutilee9071b02012-06-08 17:13:11 -040029#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
30
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070031#include <linux/pci.h>
32#include <linux/dmar.h>
Kay, Allen M38717942008-09-09 18:37:29 +030033#include <linux/iova.h>
34#include <linux/intel-iommu.h>
Suresh Siddhafe962e92008-07-10 11:16:42 -070035#include <linux/timer.h>
Suresh Siddha0ac24912009-03-16 17:04:54 -070036#include <linux/irq.h>
37#include <linux/interrupt.h>
Shane Wang69575d32009-09-01 18:25:07 -070038#include <linux/tboot.h>
Len Browneb27cae2009-07-06 23:40:19 -040039#include <linux/dmi.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090040#include <linux/slab.h>
Alex Williamsona5459cf2014-06-12 16:12:31 -060041#include <linux/iommu.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070042#include <asm/irq_remapping.h>
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -040043#include <asm/iommu_table.h>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070044
Joerg Roedel078e1ee2012-09-26 12:44:43 +020045#include "irq_remapping.h"
46
Jiang Liu3a5670e2014-02-19 14:07:33 +080047/*
48 * Assumptions:
49 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
50 * before IO devices managed by that unit.
51 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
52 * after IO devices managed by that unit.
53 * 3) Hotplug events are rare.
54 *
55 * Locking rules for DMA and interrupt remapping related global data structures:
56 * 1) Use dmar_global_lock in process context
57 * 2) Use RCU in interrupt context
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070058 */
Jiang Liu3a5670e2014-02-19 14:07:33 +080059DECLARE_RWSEM(dmar_global_lock);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070060LIST_HEAD(dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070061
Suresh Siddha41750d32011-08-23 17:05:18 -070062struct acpi_table_header * __initdata dmar_tbl;
Yinghai Lu8e1568f2009-02-11 01:06:59 -080063static acpi_size dmar_tbl_size;
Jiang Liu2e455282014-02-19 14:07:36 +080064static int dmar_dev_scope_status = 1;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070065
Jiang Liu694835d2014-01-06 14:18:16 +080066static int alloc_iommu(struct dmar_drhd_unit *drhd);
Jiang Liua868e6b2014-01-06 14:18:20 +080067static void free_iommu(struct intel_iommu *iommu);
Jiang Liu694835d2014-01-06 14:18:16 +080068
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070069static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
70{
71 /*
72 * add INCLUDE_ALL at the tail, so scan the list will find it at
73 * the very end.
74 */
75 if (drhd->include_all)
Jiang Liu0e242612014-02-19 14:07:34 +080076 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070077 else
Jiang Liu0e242612014-02-19 14:07:34 +080078 list_add_rcu(&drhd->list, &dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070079}
80
Jiang Liubb3a6b72014-02-19 14:07:24 +080081void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070082{
83 struct acpi_dmar_device_scope *scope;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070084
85 *cnt = 0;
86 while (start < end) {
87 scope = start;
Bob Moore83118b02014-07-30 12:21:00 +080088 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
David Woodhouse07cb52f2014-03-07 14:39:27 +000089 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070090 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
91 (*cnt)++;
Linn Crosettoae3e7f32013-04-23 12:26:45 -060092 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
93 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
Donald Dutilee9071b02012-06-08 17:13:11 -040094 pr_warn("Unsupported device scope\n");
Yinghai Lu5715f0f2010-04-08 19:58:22 +010095 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070096 start += scope->length;
97 }
98 if (*cnt == 0)
Jiang Liubb3a6b72014-02-19 14:07:24 +080099 return NULL;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700100
David Woodhouse832bd852014-03-07 15:08:36 +0000101 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
Jiang Liubb3a6b72014-02-19 14:07:24 +0800102}
103
David Woodhouse832bd852014-03-07 15:08:36 +0000104void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
Jiang Liuada4d4b2014-01-06 14:18:09 +0800105{
Jiang Liub683b232014-02-19 14:07:32 +0800106 int i;
David Woodhouse832bd852014-03-07 15:08:36 +0000107 struct device *tmp_dev;
Jiang Liub683b232014-02-19 14:07:32 +0800108
Jiang Liuada4d4b2014-01-06 14:18:09 +0800109 if (*devices && *cnt) {
Jiang Liub683b232014-02-19 14:07:32 +0800110 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
David Woodhouse832bd852014-03-07 15:08:36 +0000111 put_device(tmp_dev);
Jiang Liuada4d4b2014-01-06 14:18:09 +0800112 kfree(*devices);
Jiang Liuada4d4b2014-01-06 14:18:09 +0800113 }
Jiang Liu0e242612014-02-19 14:07:34 +0800114
115 *devices = NULL;
116 *cnt = 0;
Jiang Liuada4d4b2014-01-06 14:18:09 +0800117}
118
Jiang Liu59ce0512014-02-19 14:07:35 +0800119/* Optimize out kzalloc()/kfree() for normal cases */
120static char dmar_pci_notify_info_buf[64];
121
122static struct dmar_pci_notify_info *
123dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
124{
125 int level = 0;
126 size_t size;
127 struct pci_dev *tmp;
128 struct dmar_pci_notify_info *info;
129
130 BUG_ON(dev->is_virtfn);
131
132 /* Only generate path[] for device addition event */
133 if (event == BUS_NOTIFY_ADD_DEVICE)
134 for (tmp = dev; tmp; tmp = tmp->bus->self)
135 level++;
136
137 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
138 if (size <= sizeof(dmar_pci_notify_info_buf)) {
139 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
140 } else {
141 info = kzalloc(size, GFP_KERNEL);
142 if (!info) {
143 pr_warn("Out of memory when allocating notify_info "
144 "for %s.\n", pci_name(dev));
Jiang Liu2e455282014-02-19 14:07:36 +0800145 if (dmar_dev_scope_status == 0)
146 dmar_dev_scope_status = -ENOMEM;
Jiang Liu59ce0512014-02-19 14:07:35 +0800147 return NULL;
148 }
149 }
150
151 info->event = event;
152 info->dev = dev;
153 info->seg = pci_domain_nr(dev->bus);
154 info->level = level;
155 if (event == BUS_NOTIFY_ADD_DEVICE) {
Jiang Liu5ae05662014-04-15 10:35:35 +0800156 for (tmp = dev; tmp; tmp = tmp->bus->self) {
157 level--;
Jiang Liu59ce0512014-02-19 14:07:35 +0800158 info->path[level].device = PCI_SLOT(tmp->devfn);
159 info->path[level].function = PCI_FUNC(tmp->devfn);
160 if (pci_is_root_bus(tmp->bus))
161 info->bus = tmp->bus->number;
162 }
163 }
164
165 return info;
166}
167
168static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
169{
170 if ((void *)info != dmar_pci_notify_info_buf)
171 kfree(info);
172}
173
174static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
175 struct acpi_dmar_pci_path *path, int count)
176{
177 int i;
178
179 if (info->bus != bus)
180 return false;
181 if (info->level != count)
182 return false;
183
184 for (i = 0; i < count; i++) {
185 if (path[i].device != info->path[i].device ||
186 path[i].function != info->path[i].function)
187 return false;
188 }
189
190 return true;
191}
192
193/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
194int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
195 void *start, void*end, u16 segment,
David Woodhouse832bd852014-03-07 15:08:36 +0000196 struct dmar_dev_scope *devices,
197 int devices_cnt)
Jiang Liu59ce0512014-02-19 14:07:35 +0800198{
199 int i, level;
David Woodhouse832bd852014-03-07 15:08:36 +0000200 struct device *tmp, *dev = &info->dev->dev;
Jiang Liu59ce0512014-02-19 14:07:35 +0800201 struct acpi_dmar_device_scope *scope;
202 struct acpi_dmar_pci_path *path;
203
204 if (segment != info->seg)
205 return 0;
206
207 for (; start < end; start += scope->length) {
208 scope = start;
209 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
210 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
211 continue;
212
213 path = (struct acpi_dmar_pci_path *)(scope + 1);
214 level = (scope->length - sizeof(*scope)) / sizeof(*path);
215 if (!dmar_match_pci_path(info, scope->bus, path, level))
216 continue;
217
218 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
David Woodhouse832bd852014-03-07 15:08:36 +0000219 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
Jiang Liu59ce0512014-02-19 14:07:35 +0800220 pr_warn("Device scope type does not match for %s\n",
David Woodhouse832bd852014-03-07 15:08:36 +0000221 pci_name(info->dev));
Jiang Liu59ce0512014-02-19 14:07:35 +0800222 return -EINVAL;
223 }
224
225 for_each_dev_scope(devices, devices_cnt, i, tmp)
226 if (tmp == NULL) {
David Woodhouse832bd852014-03-07 15:08:36 +0000227 devices[i].bus = info->dev->bus->number;
228 devices[i].devfn = info->dev->devfn;
229 rcu_assign_pointer(devices[i].dev,
230 get_device(dev));
Jiang Liu59ce0512014-02-19 14:07:35 +0800231 return 1;
232 }
233 BUG_ON(i >= devices_cnt);
234 }
235
236 return 0;
237}
238
239int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
David Woodhouse832bd852014-03-07 15:08:36 +0000240 struct dmar_dev_scope *devices, int count)
Jiang Liu59ce0512014-02-19 14:07:35 +0800241{
242 int index;
David Woodhouse832bd852014-03-07 15:08:36 +0000243 struct device *tmp;
Jiang Liu59ce0512014-02-19 14:07:35 +0800244
245 if (info->seg != segment)
246 return 0;
247
248 for_each_active_dev_scope(devices, count, index, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +0000249 if (tmp == &info->dev->dev) {
250 rcu_assign_pointer(devices[index].dev, NULL);
Jiang Liu59ce0512014-02-19 14:07:35 +0800251 synchronize_rcu();
David Woodhouse832bd852014-03-07 15:08:36 +0000252 put_device(tmp);
Jiang Liu59ce0512014-02-19 14:07:35 +0800253 return 1;
254 }
255
256 return 0;
257}
258
259static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
260{
261 int ret = 0;
262 struct dmar_drhd_unit *dmaru;
263 struct acpi_dmar_hardware_unit *drhd;
264
265 for_each_drhd_unit(dmaru) {
266 if (dmaru->include_all)
267 continue;
268
269 drhd = container_of(dmaru->hdr,
270 struct acpi_dmar_hardware_unit, header);
271 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
272 ((void *)drhd) + drhd->header.length,
273 dmaru->segment,
274 dmaru->devices, dmaru->devices_cnt);
275 if (ret != 0)
276 break;
277 }
278 if (ret >= 0)
279 ret = dmar_iommu_notify_scope_dev(info);
Jiang Liu2e455282014-02-19 14:07:36 +0800280 if (ret < 0 && dmar_dev_scope_status == 0)
281 dmar_dev_scope_status = ret;
Jiang Liu59ce0512014-02-19 14:07:35 +0800282
283 return ret;
284}
285
286static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
287{
288 struct dmar_drhd_unit *dmaru;
289
290 for_each_drhd_unit(dmaru)
291 if (dmar_remove_dev_scope(info, dmaru->segment,
292 dmaru->devices, dmaru->devices_cnt))
293 break;
294 dmar_iommu_notify_scope_dev(info);
295}
296
297static int dmar_pci_bus_notifier(struct notifier_block *nb,
298 unsigned long action, void *data)
299{
300 struct pci_dev *pdev = to_pci_dev(data);
301 struct dmar_pci_notify_info *info;
302
303 /* Only care about add/remove events for physical functions */
304 if (pdev->is_virtfn)
305 return NOTIFY_DONE;
306 if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
307 return NOTIFY_DONE;
308
309 info = dmar_alloc_pci_notify_info(pdev, action);
310 if (!info)
311 return NOTIFY_DONE;
312
313 down_write(&dmar_global_lock);
314 if (action == BUS_NOTIFY_ADD_DEVICE)
315 dmar_pci_bus_add_dev(info);
316 else if (action == BUS_NOTIFY_DEL_DEVICE)
317 dmar_pci_bus_del_dev(info);
318 up_write(&dmar_global_lock);
319
320 dmar_free_pci_notify_info(info);
321
322 return NOTIFY_OK;
323}
324
325static struct notifier_block dmar_pci_bus_nb = {
326 .notifier_call = dmar_pci_bus_notifier,
327 .priority = INT_MIN,
328};
329
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700330/**
331 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
332 * structure which uniquely represent one DMA remapping hardware unit
333 * present in the platform
334 */
335static int __init
336dmar_parse_one_drhd(struct acpi_dmar_header *header)
337{
338 struct acpi_dmar_hardware_unit *drhd;
339 struct dmar_drhd_unit *dmaru;
340 int ret = 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700341
David Woodhousee523b382009-04-10 22:27:48 -0700342 drhd = (struct acpi_dmar_hardware_unit *)header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700343 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
344 if (!dmaru)
345 return -ENOMEM;
346
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700347 dmaru->hdr = header;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700348 dmaru->reg_base_addr = drhd->address;
David Woodhouse276dbf992009-04-04 01:45:37 +0100349 dmaru->segment = drhd->segment;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700350 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
David Woodhouse07cb52f2014-03-07 14:39:27 +0000351 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
352 ((void *)drhd) + drhd->header.length,
353 &dmaru->devices_cnt);
354 if (dmaru->devices_cnt && dmaru->devices == NULL) {
355 kfree(dmaru);
356 return -ENOMEM;
Jiang Liu2e455282014-02-19 14:07:36 +0800357 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700358
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700359 ret = alloc_iommu(dmaru);
360 if (ret) {
David Woodhouse07cb52f2014-03-07 14:39:27 +0000361 dmar_free_dev_scope(&dmaru->devices,
362 &dmaru->devices_cnt);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700363 kfree(dmaru);
364 return ret;
365 }
366 dmar_register_drhd_unit(dmaru);
367 return 0;
368}
369
Jiang Liua868e6b2014-01-06 14:18:20 +0800370static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
371{
372 if (dmaru->devices && dmaru->devices_cnt)
373 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
374 if (dmaru->iommu)
375 free_iommu(dmaru->iommu);
376 kfree(dmaru);
377}
378
David Woodhousee625b4a2014-03-07 14:34:38 +0000379static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
380{
381 struct acpi_dmar_andd *andd = (void *)header;
382
383 /* Check for NUL termination within the designated length */
Bob Moore83118b02014-07-30 12:21:00 +0800384 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
David Woodhousee625b4a2014-03-07 14:34:38 +0000385 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
386 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
387 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
388 dmi_get_system_info(DMI_BIOS_VENDOR),
389 dmi_get_system_info(DMI_BIOS_VERSION),
390 dmi_get_system_info(DMI_PRODUCT_VERSION));
391 return -EINVAL;
392 }
393 pr_info("ANDD device: %x name: %s\n", andd->device_number,
Bob Moore83118b02014-07-30 12:21:00 +0800394 andd->device_name);
David Woodhousee625b4a2014-03-07 14:34:38 +0000395
396 return 0;
397}
398
David Woodhouseaa697072009-10-07 12:18:00 +0100399#ifdef CONFIG_ACPI_NUMA
Suresh Siddhaee34b322009-10-02 11:01:21 -0700400static int __init
401dmar_parse_one_rhsa(struct acpi_dmar_header *header)
402{
403 struct acpi_dmar_rhsa *rhsa;
404 struct dmar_drhd_unit *drhd;
405
406 rhsa = (struct acpi_dmar_rhsa *)header;
David Woodhouseaa697072009-10-07 12:18:00 +0100407 for_each_drhd_unit(drhd) {
Suresh Siddhaee34b322009-10-02 11:01:21 -0700408 if (drhd->reg_base_addr == rhsa->base_address) {
409 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
410
411 if (!node_online(node))
412 node = -1;
413 drhd->iommu->node = node;
David Woodhouseaa697072009-10-07 12:18:00 +0100414 return 0;
415 }
Suresh Siddhaee34b322009-10-02 11:01:21 -0700416 }
Ben Hutchingsfd0c8892010-04-03 19:38:43 +0100417 WARN_TAINT(
418 1, TAINT_FIRMWARE_WORKAROUND,
419 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
420 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
421 drhd->reg_base_addr,
422 dmi_get_system_info(DMI_BIOS_VENDOR),
423 dmi_get_system_info(DMI_BIOS_VERSION),
424 dmi_get_system_info(DMI_PRODUCT_VERSION));
Suresh Siddhaee34b322009-10-02 11:01:21 -0700425
David Woodhouseaa697072009-10-07 12:18:00 +0100426 return 0;
Suresh Siddhaee34b322009-10-02 11:01:21 -0700427}
David Woodhouseaa697072009-10-07 12:18:00 +0100428#endif
Suresh Siddhaee34b322009-10-02 11:01:21 -0700429
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700430static void __init
431dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
432{
433 struct acpi_dmar_hardware_unit *drhd;
434 struct acpi_dmar_reserved_memory *rmrr;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800435 struct acpi_dmar_atsr *atsr;
Roland Dreier17b60972009-09-24 12:14:00 -0700436 struct acpi_dmar_rhsa *rhsa;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700437
438 switch (header->type) {
439 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800440 drhd = container_of(header, struct acpi_dmar_hardware_unit,
441 header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400442 pr_info("DRHD base: %#016Lx flags: %#x\n",
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800443 (unsigned long long)drhd->address, drhd->flags);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700444 break;
445 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800446 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
447 header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400448 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700449 (unsigned long long)rmrr->base_address,
450 (unsigned long long)rmrr->end_address);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700451 break;
Bob Moore83118b02014-07-30 12:21:00 +0800452 case ACPI_DMAR_TYPE_ROOT_ATS:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800453 atsr = container_of(header, struct acpi_dmar_atsr, header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400454 pr_info("ATSR flags: %#x\n", atsr->flags);
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800455 break;
Bob Moore83118b02014-07-30 12:21:00 +0800456 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
Roland Dreier17b60972009-09-24 12:14:00 -0700457 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400458 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
Roland Dreier17b60972009-09-24 12:14:00 -0700459 (unsigned long long)rhsa->base_address,
460 rhsa->proximity_domain);
461 break;
Bob Moore83118b02014-07-30 12:21:00 +0800462 case ACPI_DMAR_TYPE_NAMESPACE:
David Woodhousee625b4a2014-03-07 14:34:38 +0000463 /* We don't print this here because we need to sanity-check
464 it first. So print it in dmar_parse_one_andd() instead. */
465 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700466 }
467}
468
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700469/**
470 * dmar_table_detect - checks to see if the platform supports DMAR devices
471 */
472static int __init dmar_table_detect(void)
473{
474 acpi_status status = AE_OK;
475
476 /* if we could find DMAR table, then there are DMAR devices */
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800477 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
478 (struct acpi_table_header **)&dmar_tbl,
479 &dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700480
481 if (ACPI_SUCCESS(status) && !dmar_tbl) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400482 pr_warn("Unable to map DMAR\n");
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700483 status = AE_NOT_FOUND;
484 }
485
486 return (ACPI_SUCCESS(status) ? 1 : 0);
487}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700488
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700489/**
490 * parse_dmar_table - parses the DMA reporting table
491 */
492static int __init
493parse_dmar_table(void)
494{
495 struct acpi_table_dmar *dmar;
496 struct acpi_dmar_header *entry_header;
497 int ret = 0;
Li, Zhen-Hua7cef3342013-05-20 15:57:32 +0800498 int drhd_count = 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700499
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700500 /*
501 * Do it again, earlier dmar_tbl mapping could be mapped with
502 * fixed map.
503 */
504 dmar_table_detect();
505
Joseph Cihulaa59b50e2009-06-30 19:31:10 -0700506 /*
507 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
508 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
509 */
510 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
511
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700512 dmar = (struct acpi_table_dmar *)dmar_tbl;
513 if (!dmar)
514 return -ENODEV;
515
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700516 if (dmar->width < PAGE_SHIFT - 1) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400517 pr_warn("Invalid DMAR haw\n");
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700518 return -EINVAL;
519 }
520
Donald Dutilee9071b02012-06-08 17:13:11 -0400521 pr_info("Host address width %d\n", dmar->width + 1);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700522
523 entry_header = (struct acpi_dmar_header *)(dmar + 1);
524 while (((unsigned long)entry_header) <
525 (((unsigned long)dmar) + dmar_tbl->length)) {
Tony Battersby084eb962009-02-11 13:24:19 -0800526 /* Avoid looping forever on bad ACPI tables */
527 if (entry_header->length == 0) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400528 pr_warn("Invalid 0-length structure\n");
Tony Battersby084eb962009-02-11 13:24:19 -0800529 ret = -EINVAL;
530 break;
531 }
532
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700533 dmar_table_print_dmar_entry(entry_header);
534
535 switch (entry_header->type) {
536 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
Li, Zhen-Hua7cef3342013-05-20 15:57:32 +0800537 drhd_count++;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700538 ret = dmar_parse_one_drhd(entry_header);
539 break;
540 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
541 ret = dmar_parse_one_rmrr(entry_header);
542 break;
Bob Moore83118b02014-07-30 12:21:00 +0800543 case ACPI_DMAR_TYPE_ROOT_ATS:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800544 ret = dmar_parse_one_atsr(entry_header);
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800545 break;
Bob Moore83118b02014-07-30 12:21:00 +0800546 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
David Woodhouseaa697072009-10-07 12:18:00 +0100547#ifdef CONFIG_ACPI_NUMA
Suresh Siddhaee34b322009-10-02 11:01:21 -0700548 ret = dmar_parse_one_rhsa(entry_header);
David Woodhouseaa697072009-10-07 12:18:00 +0100549#endif
Roland Dreier17b60972009-09-24 12:14:00 -0700550 break;
Bob Moore83118b02014-07-30 12:21:00 +0800551 case ACPI_DMAR_TYPE_NAMESPACE:
David Woodhousee625b4a2014-03-07 14:34:38 +0000552 ret = dmar_parse_one_andd(entry_header);
553 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700554 default:
Donald Dutilee9071b02012-06-08 17:13:11 -0400555 pr_warn("Unknown DMAR structure type %d\n",
Roland Dreier4de75cf2009-09-24 01:01:29 +0100556 entry_header->type);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700557 ret = 0; /* for forward compatibility */
558 break;
559 }
560 if (ret)
561 break;
562
563 entry_header = ((void *)entry_header + entry_header->length);
564 }
Li, Zhen-Hua7cef3342013-05-20 15:57:32 +0800565 if (drhd_count == 0)
566 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700567 return ret;
568}
569
David Woodhouse832bd852014-03-07 15:08:36 +0000570static int dmar_pci_device_match(struct dmar_dev_scope devices[],
571 int cnt, struct pci_dev *dev)
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700572{
573 int index;
David Woodhouse832bd852014-03-07 15:08:36 +0000574 struct device *tmp;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700575
576 while (dev) {
Jiang Liub683b232014-02-19 14:07:32 +0800577 for_each_active_dev_scope(devices, cnt, index, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +0000578 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700579 return 1;
580
581 /* Check our parent */
582 dev = dev->bus->self;
583 }
584
585 return 0;
586}
587
588struct dmar_drhd_unit *
589dmar_find_matched_drhd_unit(struct pci_dev *dev)
590{
Jiang Liu0e242612014-02-19 14:07:34 +0800591 struct dmar_drhd_unit *dmaru;
Yu Zhao2e824f72008-12-22 16:54:58 +0800592 struct acpi_dmar_hardware_unit *drhd;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700593
Yinghaidda56542010-04-09 01:07:55 +0100594 dev = pci_physfn(dev);
595
Jiang Liu0e242612014-02-19 14:07:34 +0800596 rcu_read_lock();
Yijing Wang8b161f02013-10-31 17:25:16 +0800597 for_each_drhd_unit(dmaru) {
Yu Zhao2e824f72008-12-22 16:54:58 +0800598 drhd = container_of(dmaru->hdr,
599 struct acpi_dmar_hardware_unit,
600 header);
601
602 if (dmaru->include_all &&
603 drhd->segment == pci_domain_nr(dev->bus))
Jiang Liu0e242612014-02-19 14:07:34 +0800604 goto out;
Yu Zhao2e824f72008-12-22 16:54:58 +0800605
606 if (dmar_pci_device_match(dmaru->devices,
607 dmaru->devices_cnt, dev))
Jiang Liu0e242612014-02-19 14:07:34 +0800608 goto out;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700609 }
Jiang Liu0e242612014-02-19 14:07:34 +0800610 dmaru = NULL;
611out:
612 rcu_read_unlock();
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700613
Jiang Liu0e242612014-02-19 14:07:34 +0800614 return dmaru;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700615}
616
David Woodhouseed403562014-03-07 23:15:42 +0000617static void __init dmar_acpi_insert_dev_scope(u8 device_number,
618 struct acpi_device *adev)
619{
620 struct dmar_drhd_unit *dmaru;
621 struct acpi_dmar_hardware_unit *drhd;
622 struct acpi_dmar_device_scope *scope;
623 struct device *tmp;
624 int i;
625 struct acpi_dmar_pci_path *path;
626
627 for_each_drhd_unit(dmaru) {
628 drhd = container_of(dmaru->hdr,
629 struct acpi_dmar_hardware_unit,
630 header);
631
632 for (scope = (void *)(drhd + 1);
633 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
634 scope = ((void *)scope) + scope->length) {
Bob Moore83118b02014-07-30 12:21:00 +0800635 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
David Woodhouseed403562014-03-07 23:15:42 +0000636 continue;
637 if (scope->enumeration_id != device_number)
638 continue;
639
640 path = (void *)(scope + 1);
641 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
642 dev_name(&adev->dev), dmaru->reg_base_addr,
643 scope->bus, path->device, path->function);
644 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
645 if (tmp == NULL) {
646 dmaru->devices[i].bus = scope->bus;
647 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
648 path->function);
649 rcu_assign_pointer(dmaru->devices[i].dev,
650 get_device(&adev->dev));
651 return;
652 }
653 BUG_ON(i >= dmaru->devices_cnt);
654 }
655 }
656 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
657 device_number, dev_name(&adev->dev));
658}
659
660static int __init dmar_acpi_dev_scope_init(void)
661{
Joerg Roedel11f1a772014-03-25 20:16:40 +0100662 struct acpi_dmar_andd *andd;
663
664 if (dmar_tbl == NULL)
665 return -ENODEV;
666
David Woodhouse7713ec02014-04-01 14:58:36 +0100667 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
668 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
669 andd = ((void *)andd) + andd->header.length) {
Bob Moore83118b02014-07-30 12:21:00 +0800670 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
David Woodhouseed403562014-03-07 23:15:42 +0000671 acpi_handle h;
672 struct acpi_device *adev;
673
674 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
Bob Moore83118b02014-07-30 12:21:00 +0800675 andd->device_name,
David Woodhouseed403562014-03-07 23:15:42 +0000676 &h))) {
677 pr_err("Failed to find handle for ACPI object %s\n",
Bob Moore83118b02014-07-30 12:21:00 +0800678 andd->device_name);
David Woodhouseed403562014-03-07 23:15:42 +0000679 continue;
680 }
681 acpi_bus_get_device(h, &adev);
682 if (!adev) {
683 pr_err("Failed to get device for ACPI object %s\n",
Bob Moore83118b02014-07-30 12:21:00 +0800684 andd->device_name);
David Woodhouseed403562014-03-07 23:15:42 +0000685 continue;
686 }
687 dmar_acpi_insert_dev_scope(andd->device_number, adev);
688 }
David Woodhouseed403562014-03-07 23:15:42 +0000689 }
690 return 0;
691}
692
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700693int __init dmar_dev_scope_init(void)
694{
Jiang Liu2e455282014-02-19 14:07:36 +0800695 struct pci_dev *dev = NULL;
696 struct dmar_pci_notify_info *info;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700697
Jiang Liu2e455282014-02-19 14:07:36 +0800698 if (dmar_dev_scope_status != 1)
699 return dmar_dev_scope_status;
Suresh Siddhac2c72862011-08-23 17:05:19 -0700700
Jiang Liu2e455282014-02-19 14:07:36 +0800701 if (list_empty(&dmar_drhd_units)) {
702 dmar_dev_scope_status = -ENODEV;
703 } else {
704 dmar_dev_scope_status = 0;
Suresh Siddha318fe7d2011-08-23 17:05:20 -0700705
David Woodhouse63b42622014-03-28 11:28:40 +0000706 dmar_acpi_dev_scope_init();
707
Jiang Liu2e455282014-02-19 14:07:36 +0800708 for_each_pci_dev(dev) {
709 if (dev->is_virtfn)
710 continue;
711
712 info = dmar_alloc_pci_notify_info(dev,
713 BUS_NOTIFY_ADD_DEVICE);
714 if (!info) {
715 return dmar_dev_scope_status;
716 } else {
717 dmar_pci_bus_add_dev(info);
718 dmar_free_pci_notify_info(info);
719 }
720 }
721
722 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700723 }
724
Jiang Liu2e455282014-02-19 14:07:36 +0800725 return dmar_dev_scope_status;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700726}
727
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700728
729int __init dmar_table_init(void)
730{
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700731 static int dmar_table_initialized;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800732 int ret;
733
Jiang Liucc053012014-01-06 14:18:24 +0800734 if (dmar_table_initialized == 0) {
735 ret = parse_dmar_table();
736 if (ret < 0) {
737 if (ret != -ENODEV)
738 pr_info("parse DMAR table failure.\n");
739 } else if (list_empty(&dmar_drhd_units)) {
740 pr_info("No DMAR devices found\n");
741 ret = -ENODEV;
742 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700743
Jiang Liucc053012014-01-06 14:18:24 +0800744 if (ret < 0)
745 dmar_table_initialized = ret;
746 else
747 dmar_table_initialized = 1;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800748 }
749
Jiang Liucc053012014-01-06 14:18:24 +0800750 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700751}
752
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100753static void warn_invalid_dmar(u64 addr, const char *message)
754{
Ben Hutchingsfd0c8892010-04-03 19:38:43 +0100755 WARN_TAINT_ONCE(
756 1, TAINT_FIRMWARE_WORKAROUND,
757 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
758 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
759 addr, message,
760 dmi_get_system_info(DMI_BIOS_VENDOR),
761 dmi_get_system_info(DMI_BIOS_VERSION),
762 dmi_get_system_info(DMI_PRODUCT_VERSION));
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100763}
David Woodhouse6ecbf012009-12-02 09:20:27 +0000764
Rashika Kheria21004dc2013-12-18 12:01:46 +0530765static int __init check_zero_address(void)
David Woodhouse86cf8982009-11-09 22:15:15 +0000766{
767 struct acpi_table_dmar *dmar;
768 struct acpi_dmar_header *entry_header;
769 struct acpi_dmar_hardware_unit *drhd;
770
771 dmar = (struct acpi_table_dmar *)dmar_tbl;
772 entry_header = (struct acpi_dmar_header *)(dmar + 1);
773
774 while (((unsigned long)entry_header) <
775 (((unsigned long)dmar) + dmar_tbl->length)) {
776 /* Avoid looping forever on bad ACPI tables */
777 if (entry_header->length == 0) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400778 pr_warn("Invalid 0-length structure\n");
David Woodhouse86cf8982009-11-09 22:15:15 +0000779 return 0;
780 }
781
782 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
Chris Wright2c992202009-12-02 09:17:13 +0000783 void __iomem *addr;
784 u64 cap, ecap;
785
David Woodhouse86cf8982009-11-09 22:15:15 +0000786 drhd = (void *)entry_header;
787 if (!drhd->address) {
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100788 warn_invalid_dmar(0, "");
Chris Wright2c992202009-12-02 09:17:13 +0000789 goto failed;
David Woodhouse86cf8982009-11-09 22:15:15 +0000790 }
Chris Wright2c992202009-12-02 09:17:13 +0000791
792 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
793 if (!addr ) {
794 printk("IOMMU: can't validate: %llx\n", drhd->address);
795 goto failed;
796 }
797 cap = dmar_readq(addr + DMAR_CAP_REG);
798 ecap = dmar_readq(addr + DMAR_ECAP_REG);
799 early_iounmap(addr, VTD_PAGE_SIZE);
800 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100801 warn_invalid_dmar(drhd->address,
802 " returns all ones");
Chris Wright2c992202009-12-02 09:17:13 +0000803 goto failed;
804 }
David Woodhouse86cf8982009-11-09 22:15:15 +0000805 }
806
807 entry_header = ((void *)entry_header + entry_header->length);
808 }
809 return 1;
Chris Wright2c992202009-12-02 09:17:13 +0000810
811failed:
Chris Wright2c992202009-12-02 09:17:13 +0000812 return 0;
David Woodhouse86cf8982009-11-09 22:15:15 +0000813}
814
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -0400815int __init detect_intel_iommu(void)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700816{
817 int ret;
818
Jiang Liu3a5670e2014-02-19 14:07:33 +0800819 down_write(&dmar_global_lock);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700820 ret = dmar_table_detect();
David Woodhouse86cf8982009-11-09 22:15:15 +0000821 if (ret)
822 ret = check_zero_address();
Suresh Siddha2ae21012008-07-10 11:16:43 -0700823 {
Linus Torvalds11bd04f2009-12-11 12:18:16 -0800824 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
Suresh Siddha2ae21012008-07-10 11:16:43 -0700825 iommu_detected = 1;
Chris Wright5d990b62009-12-04 12:15:21 -0800826 /* Make sure ACS will be enabled */
827 pci_request_acs();
828 }
Suresh Siddhaf5d1b972011-08-23 17:05:22 -0700829
FUJITA Tomonori9d5ce732009-11-10 19:46:16 +0900830#ifdef CONFIG_X86
831 if (ret)
832 x86_init.iommu.iommu_init = intel_iommu_init;
833#endif
Youquan Songcacd4212008-10-16 16:31:57 -0700834 }
Jiang Liub707cb02014-01-06 14:18:26 +0800835 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700836 dmar_tbl = NULL;
Jiang Liu3a5670e2014-02-19 14:07:33 +0800837 up_write(&dmar_global_lock);
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -0400838
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -0400839 return ret ? 1 : -ENODEV;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700840}
841
842
Donald Dutile6f5cf522012-06-04 17:29:02 -0400843static void unmap_iommu(struct intel_iommu *iommu)
844{
845 iounmap(iommu->reg);
846 release_mem_region(iommu->reg_phys, iommu->reg_size);
847}
848
849/**
850 * map_iommu: map the iommu's registers
851 * @iommu: the iommu to map
852 * @phys_addr: the physical address of the base resgister
Donald Dutilee9071b02012-06-08 17:13:11 -0400853 *
Donald Dutile6f5cf522012-06-04 17:29:02 -0400854 * Memory map the iommu's registers. Start w/ a single page, and
Donald Dutilee9071b02012-06-08 17:13:11 -0400855 * possibly expand if that turns out to be insufficent.
Donald Dutile6f5cf522012-06-04 17:29:02 -0400856 */
857static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
858{
859 int map_size, err=0;
860
861 iommu->reg_phys = phys_addr;
862 iommu->reg_size = VTD_PAGE_SIZE;
863
864 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
865 pr_err("IOMMU: can't reserve memory\n");
866 err = -EBUSY;
867 goto out;
868 }
869
870 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
871 if (!iommu->reg) {
872 pr_err("IOMMU: can't map the region\n");
873 err = -ENOMEM;
874 goto release;
875 }
876
877 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
878 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
879
880 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
881 err = -EINVAL;
882 warn_invalid_dmar(phys_addr, " returns all ones");
883 goto unmap;
884 }
885
886 /* the registers might be more than one page */
887 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
888 cap_max_fault_reg_offset(iommu->cap));
889 map_size = VTD_PAGE_ALIGN(map_size);
890 if (map_size > iommu->reg_size) {
891 iounmap(iommu->reg);
892 release_mem_region(iommu->reg_phys, iommu->reg_size);
893 iommu->reg_size = map_size;
894 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
895 iommu->name)) {
896 pr_err("IOMMU: can't reserve memory\n");
897 err = -EBUSY;
898 goto out;
899 }
900 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
901 if (!iommu->reg) {
902 pr_err("IOMMU: can't map the region\n");
903 err = -ENOMEM;
904 goto release;
905 }
906 }
907 err = 0;
908 goto out;
909
910unmap:
911 iounmap(iommu->reg);
912release:
913 release_mem_region(iommu->reg_phys, iommu->reg_size);
914out:
915 return err;
916}
917
Jiang Liu694835d2014-01-06 14:18:16 +0800918static int alloc_iommu(struct dmar_drhd_unit *drhd)
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700919{
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700920 struct intel_iommu *iommu;
Takao Indoh3a93c842013-04-23 17:35:03 +0900921 u32 ver, sts;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700922 static int iommu_allocated = 0;
Joerg Roedel43f73922009-01-03 23:56:27 +0100923 int agaw = 0;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700924 int msagaw = 0;
Donald Dutile6f5cf522012-06-04 17:29:02 -0400925 int err;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700926
David Woodhouse6ecbf012009-12-02 09:20:27 +0000927 if (!drhd->reg_base_addr) {
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100928 warn_invalid_dmar(0, "");
David Woodhouse6ecbf012009-12-02 09:20:27 +0000929 return -EINVAL;
930 }
931
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700932 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
933 if (!iommu)
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700934 return -ENOMEM;
Suresh Siddhac42d9f32008-07-10 11:16:36 -0700935
936 iommu->seq_id = iommu_allocated++;
Suresh Siddha9d783ba2009-03-16 17:04:55 -0700937 sprintf (iommu->name, "dmar%d", iommu->seq_id);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700938
Donald Dutile6f5cf522012-06-04 17:29:02 -0400939 err = map_iommu(iommu, drhd->reg_base_addr);
940 if (err) {
941 pr_err("IOMMU: failed to map %s\n", iommu->name);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700942 goto error;
943 }
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700944
Donald Dutile6f5cf522012-06-04 17:29:02 -0400945 err = -EINVAL;
Weidong Han1b573682008-12-08 15:34:06 +0800946 agaw = iommu_calculate_agaw(iommu);
947 if (agaw < 0) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -0400948 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
949 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100950 goto err_unmap;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700951 }
952 msagaw = iommu_calculate_max_sagaw(iommu);
953 if (msagaw < 0) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -0400954 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
Weidong Han1b573682008-12-08 15:34:06 +0800955 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +0100956 goto err_unmap;
Weidong Han1b573682008-12-08 15:34:06 +0800957 }
958 iommu->agaw = agaw;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700959 iommu->msagaw = msagaw;
David Woodhouse67ccac42014-03-09 13:49:45 -0700960 iommu->segment = drhd->segment;
Weidong Han1b573682008-12-08 15:34:06 +0800961
Suresh Siddhaee34b322009-10-02 11:01:21 -0700962 iommu->node = -1;
963
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700964 ver = readl(iommu->reg + DMAR_VER_REG);
Yinghai Lu680a7522010-04-08 19:58:23 +0100965 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
966 iommu->seq_id,
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700967 (unsigned long long)drhd->reg_base_addr,
968 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
969 (unsigned long long)iommu->cap,
970 (unsigned long long)iommu->ecap);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700971
Takao Indoh3a93c842013-04-23 17:35:03 +0900972 /* Reflect status in gcmd */
973 sts = readl(iommu->reg + DMAR_GSTS_REG);
974 if (sts & DMA_GSTS_IRES)
975 iommu->gcmd |= DMA_GCMD_IRE;
976 if (sts & DMA_GSTS_TES)
977 iommu->gcmd |= DMA_GCMD_TE;
978 if (sts & DMA_GSTS_QIES)
979 iommu->gcmd |= DMA_GCMD_QIE;
980
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200981 raw_spin_lock_init(&iommu->register_lock);
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700982
983 drhd->iommu = iommu;
Alex Williamsona5459cf2014-06-12 16:12:31 -0600984
985 if (intel_iommu_enabled)
986 iommu->iommu_dev = iommu_device_create(NULL, iommu,
987 intel_iommu_groups,
988 iommu->name);
989
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700990 return 0;
David Woodhouse08155652009-08-04 09:17:20 +0100991
992 err_unmap:
Donald Dutile6f5cf522012-06-04 17:29:02 -0400993 unmap_iommu(iommu);
David Woodhouse08155652009-08-04 09:17:20 +0100994 error:
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700995 kfree(iommu);
Donald Dutile6f5cf522012-06-04 17:29:02 -0400996 return err;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700997}
998
Jiang Liua868e6b2014-01-06 14:18:20 +0800999static void free_iommu(struct intel_iommu *iommu)
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001000{
Alex Williamsona5459cf2014-06-12 16:12:31 -06001001 iommu_device_destroy(iommu->iommu_dev);
1002
Jiang Liua868e6b2014-01-06 14:18:20 +08001003 if (iommu->irq) {
1004 free_irq(iommu->irq, iommu);
1005 irq_set_handler_data(iommu->irq, NULL);
Thomas Gleixnera553b142014-05-07 15:44:11 +00001006 dmar_free_hwirq(iommu->irq);
Jiang Liua868e6b2014-01-06 14:18:20 +08001007 }
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001008
Jiang Liua84da702014-01-06 14:18:23 +08001009 if (iommu->qi) {
1010 free_page((unsigned long)iommu->qi->desc);
1011 kfree(iommu->qi->desc_status);
1012 kfree(iommu->qi);
1013 }
1014
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001015 if (iommu->reg)
Donald Dutile6f5cf522012-06-04 17:29:02 -04001016 unmap_iommu(iommu);
1017
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001018 kfree(iommu);
1019}
Suresh Siddhafe962e92008-07-10 11:16:42 -07001020
1021/*
1022 * Reclaim all the submitted descriptors which have completed its work.
1023 */
1024static inline void reclaim_free_desc(struct q_inval *qi)
1025{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001026 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1027 qi->desc_status[qi->free_tail] == QI_ABORT) {
Suresh Siddhafe962e92008-07-10 11:16:42 -07001028 qi->desc_status[qi->free_tail] = QI_FREE;
1029 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1030 qi->free_cnt++;
1031 }
1032}
1033
Yu Zhao704126a2009-01-04 16:28:52 +08001034static int qi_check_fault(struct intel_iommu *iommu, int index)
1035{
1036 u32 fault;
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001037 int head, tail;
Yu Zhao704126a2009-01-04 16:28:52 +08001038 struct q_inval *qi = iommu->qi;
1039 int wait_index = (index + 1) % QI_LENGTH;
1040
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001041 if (qi->desc_status[wait_index] == QI_ABORT)
1042 return -EAGAIN;
1043
Yu Zhao704126a2009-01-04 16:28:52 +08001044 fault = readl(iommu->reg + DMAR_FSTS_REG);
1045
1046 /*
1047 * If IQE happens, the head points to the descriptor associated
1048 * with the error. No new descriptors are fetched until the IQE
1049 * is cleared.
1050 */
1051 if (fault & DMA_FSTS_IQE) {
1052 head = readl(iommu->reg + DMAR_IQH_REG);
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001053 if ((head >> DMAR_IQ_SHIFT) == index) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001054 pr_err("VT-d detected invalid descriptor: "
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001055 "low=%llx, high=%llx\n",
1056 (unsigned long long)qi->desc[index].low,
1057 (unsigned long long)qi->desc[index].high);
Yu Zhao704126a2009-01-04 16:28:52 +08001058 memcpy(&qi->desc[index], &qi->desc[wait_index],
1059 sizeof(struct qi_desc));
1060 __iommu_flush_cache(iommu, &qi->desc[index],
1061 sizeof(struct qi_desc));
1062 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1063 return -EINVAL;
1064 }
1065 }
1066
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001067 /*
1068 * If ITE happens, all pending wait_desc commands are aborted.
1069 * No new descriptors are fetched until the ITE is cleared.
1070 */
1071 if (fault & DMA_FSTS_ITE) {
1072 head = readl(iommu->reg + DMAR_IQH_REG);
1073 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1074 head |= 1;
1075 tail = readl(iommu->reg + DMAR_IQT_REG);
1076 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1077
1078 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1079
1080 do {
1081 if (qi->desc_status[head] == QI_IN_USE)
1082 qi->desc_status[head] = QI_ABORT;
1083 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1084 } while (head != tail);
1085
1086 if (qi->desc_status[wait_index] == QI_ABORT)
1087 return -EAGAIN;
1088 }
1089
1090 if (fault & DMA_FSTS_ICE)
1091 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1092
Yu Zhao704126a2009-01-04 16:28:52 +08001093 return 0;
1094}
1095
Suresh Siddhafe962e92008-07-10 11:16:42 -07001096/*
1097 * Submit the queued invalidation descriptor to the remapping
1098 * hardware unit and wait for its completion.
1099 */
Yu Zhao704126a2009-01-04 16:28:52 +08001100int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
Suresh Siddhafe962e92008-07-10 11:16:42 -07001101{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001102 int rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001103 struct q_inval *qi = iommu->qi;
1104 struct qi_desc *hw, wait_desc;
1105 int wait_index, index;
1106 unsigned long flags;
1107
1108 if (!qi)
Yu Zhao704126a2009-01-04 16:28:52 +08001109 return 0;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001110
1111 hw = qi->desc;
1112
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001113restart:
1114 rc = 0;
1115
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001116 raw_spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001117 while (qi->free_cnt < 3) {
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001118 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001119 cpu_relax();
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001120 raw_spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001121 }
1122
1123 index = qi->free_head;
1124 wait_index = (index + 1) % QI_LENGTH;
1125
1126 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1127
1128 hw[index] = *desc;
1129
Yu Zhao704126a2009-01-04 16:28:52 +08001130 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1131 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001132 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1133
1134 hw[wait_index] = wait_desc;
1135
1136 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1137 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1138
1139 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1140 qi->free_cnt -= 2;
1141
Suresh Siddhafe962e92008-07-10 11:16:42 -07001142 /*
1143 * update the HW tail register indicating the presence of
1144 * new descriptors.
1145 */
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001146 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001147
1148 while (qi->desc_status[wait_index] != QI_DONE) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -07001149 /*
1150 * We will leave the interrupts disabled, to prevent interrupt
1151 * context to queue another cmd while a cmd is already submitted
1152 * and waiting for completion on this cpu. This is to avoid
1153 * a deadlock where the interrupt context can wait indefinitely
1154 * for free slots in the queue.
1155 */
Yu Zhao704126a2009-01-04 16:28:52 +08001156 rc = qi_check_fault(iommu, index);
1157 if (rc)
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001158 break;
Yu Zhao704126a2009-01-04 16:28:52 +08001159
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001160 raw_spin_unlock(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001161 cpu_relax();
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001162 raw_spin_lock(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001163 }
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001164
1165 qi->desc_status[index] = QI_DONE;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001166
1167 reclaim_free_desc(qi);
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001168 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +08001169
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001170 if (rc == -EAGAIN)
1171 goto restart;
1172
Yu Zhao704126a2009-01-04 16:28:52 +08001173 return rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001174}
1175
1176/*
1177 * Flush the global interrupt entry cache.
1178 */
1179void qi_global_iec(struct intel_iommu *iommu)
1180{
1181 struct qi_desc desc;
1182
1183 desc.low = QI_IEC_TYPE;
1184 desc.high = 0;
1185
Yu Zhao704126a2009-01-04 16:28:52 +08001186 /* should never fail */
Suresh Siddhafe962e92008-07-10 11:16:42 -07001187 qi_submit_sync(&desc, iommu);
1188}
1189
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001190void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1191 u64 type)
Youquan Song3481f212008-10-16 16:31:55 -07001192{
Youquan Song3481f212008-10-16 16:31:55 -07001193 struct qi_desc desc;
1194
Youquan Song3481f212008-10-16 16:31:55 -07001195 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1196 | QI_CC_GRAN(type) | QI_CC_TYPE;
1197 desc.high = 0;
1198
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001199 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -07001200}
1201
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001202void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1203 unsigned int size_order, u64 type)
Youquan Song3481f212008-10-16 16:31:55 -07001204{
1205 u8 dw = 0, dr = 0;
1206
1207 struct qi_desc desc;
1208 int ih = 0;
1209
Youquan Song3481f212008-10-16 16:31:55 -07001210 if (cap_write_drain(iommu->cap))
1211 dw = 1;
1212
1213 if (cap_read_drain(iommu->cap))
1214 dr = 1;
1215
1216 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1217 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1218 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1219 | QI_IOTLB_AM(size_order);
1220
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001221 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -07001222}
1223
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001224void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1225 u64 addr, unsigned mask)
1226{
1227 struct qi_desc desc;
1228
1229 if (mask) {
1230 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1231 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1232 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1233 } else
1234 desc.high = QI_DEV_IOTLB_ADDR(addr);
1235
1236 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1237 qdep = 0;
1238
1239 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1240 QI_DIOTLB_TYPE;
1241
1242 qi_submit_sync(&desc, iommu);
1243}
1244
Suresh Siddhafe962e92008-07-10 11:16:42 -07001245/*
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001246 * Disable Queued Invalidation interface.
1247 */
1248void dmar_disable_qi(struct intel_iommu *iommu)
1249{
1250 unsigned long flags;
1251 u32 sts;
1252 cycles_t start_time = get_cycles();
1253
1254 if (!ecap_qis(iommu->ecap))
1255 return;
1256
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001257 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001258
1259 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1260 if (!(sts & DMA_GSTS_QIES))
1261 goto end;
1262
1263 /*
1264 * Give a chance to HW to complete the pending invalidation requests.
1265 */
1266 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1267 readl(iommu->reg + DMAR_IQH_REG)) &&
1268 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1269 cpu_relax();
1270
1271 iommu->gcmd &= ~DMA_GCMD_QIE;
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001272 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1273
1274 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1275 !(sts & DMA_GSTS_QIES), sts);
1276end:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001277 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001278}
1279
1280/*
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001281 * Enable queued invalidation.
1282 */
1283static void __dmar_enable_qi(struct intel_iommu *iommu)
1284{
David Woodhousec416daa2009-05-10 20:30:58 +01001285 u32 sts;
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001286 unsigned long flags;
1287 struct q_inval *qi = iommu->qi;
1288
1289 qi->free_head = qi->free_tail = 0;
1290 qi->free_cnt = QI_LENGTH;
1291
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001292 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001293
1294 /* write zero to the tail reg */
1295 writel(0, iommu->reg + DMAR_IQT_REG);
1296
1297 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1298
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001299 iommu->gcmd |= DMA_GCMD_QIE;
David Woodhousec416daa2009-05-10 20:30:58 +01001300 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001301
1302 /* Make sure hardware complete it */
1303 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1304
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001305 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001306}
1307
1308/*
Suresh Siddhafe962e92008-07-10 11:16:42 -07001309 * Enable Queued Invalidation interface. This is a must to support
1310 * interrupt-remapping. Also used by DMA-remapping, which replaces
1311 * register based IOTLB invalidation.
1312 */
1313int dmar_enable_qi(struct intel_iommu *iommu)
1314{
Suresh Siddhafe962e92008-07-10 11:16:42 -07001315 struct q_inval *qi;
Suresh Siddha751cafe2009-10-02 11:01:22 -07001316 struct page *desc_page;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001317
1318 if (!ecap_qis(iommu->ecap))
1319 return -ENOENT;
1320
1321 /*
1322 * queued invalidation is already setup and enabled.
1323 */
1324 if (iommu->qi)
1325 return 0;
1326
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001327 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001328 if (!iommu->qi)
1329 return -ENOMEM;
1330
1331 qi = iommu->qi;
1332
Suresh Siddha751cafe2009-10-02 11:01:22 -07001333
1334 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1335 if (!desc_page) {
Suresh Siddhafe962e92008-07-10 11:16:42 -07001336 kfree(qi);
Jiang Liub707cb02014-01-06 14:18:26 +08001337 iommu->qi = NULL;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001338 return -ENOMEM;
1339 }
1340
Suresh Siddha751cafe2009-10-02 11:01:22 -07001341 qi->desc = page_address(desc_page);
1342
Hannes Reinecke37a40712013-02-06 09:50:10 +01001343 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001344 if (!qi->desc_status) {
1345 free_page((unsigned long) qi->desc);
1346 kfree(qi);
Jiang Liub707cb02014-01-06 14:18:26 +08001347 iommu->qi = NULL;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001348 return -ENOMEM;
1349 }
1350
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001351 raw_spin_lock_init(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001352
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001353 __dmar_enable_qi(iommu);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001354
1355 return 0;
1356}
Suresh Siddha0ac24912009-03-16 17:04:54 -07001357
1358/* iommu interrupt handling. Most stuff are MSI-like. */
1359
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001360enum faulttype {
1361 DMA_REMAP,
1362 INTR_REMAP,
1363 UNKNOWN,
1364};
1365
1366static const char *dma_remap_fault_reasons[] =
Suresh Siddha0ac24912009-03-16 17:04:54 -07001367{
1368 "Software",
1369 "Present bit in root entry is clear",
1370 "Present bit in context entry is clear",
1371 "Invalid context entry",
1372 "Access beyond MGAW",
1373 "PTE Write access is not set",
1374 "PTE Read access is not set",
1375 "Next page table ptr is invalid",
1376 "Root table address invalid",
1377 "Context table ptr is invalid",
1378 "non-zero reserved fields in RTP",
1379 "non-zero reserved fields in CTP",
1380 "non-zero reserved fields in PTE",
Li, Zhen-Hua4ecccd92013-03-06 10:43:17 +08001381 "PCE for translation request specifies blocking",
Suresh Siddha0ac24912009-03-16 17:04:54 -07001382};
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001383
Suresh Siddha95a02e92012-03-30 11:47:07 -07001384static const char *irq_remap_fault_reasons[] =
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001385{
1386 "Detected reserved fields in the decoded interrupt-remapped request",
1387 "Interrupt index exceeded the interrupt-remapping table size",
1388 "Present field in the IRTE entry is clear",
1389 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1390 "Detected reserved fields in the IRTE entry",
1391 "Blocked a compatibility format interrupt request",
1392 "Blocked an interrupt request due to source-id verification failure",
1393};
1394
Rashika Kheria21004dc2013-12-18 12:01:46 +05301395static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001396{
Dan Carpenterfefe1ed2012-05-13 20:09:38 +03001397 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1398 ARRAY_SIZE(irq_remap_fault_reasons))) {
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001399 *fault_type = INTR_REMAP;
Suresh Siddha95a02e92012-03-30 11:47:07 -07001400 return irq_remap_fault_reasons[fault_reason - 0x20];
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001401 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1402 *fault_type = DMA_REMAP;
1403 return dma_remap_fault_reasons[fault_reason];
1404 } else {
1405 *fault_type = UNKNOWN;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001406 return "Unknown";
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001407 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001408}
1409
Thomas Gleixner5c2837f2010-09-28 17:15:11 +02001410void dmar_msi_unmask(struct irq_data *data)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001411{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001412 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001413 unsigned long flag;
1414
1415 /* unmask it */
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001416 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001417 writel(0, iommu->reg + DMAR_FECTL_REG);
1418 /* Read a reg to force flush the post write */
1419 readl(iommu->reg + DMAR_FECTL_REG);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001420 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001421}
1422
Thomas Gleixner5c2837f2010-09-28 17:15:11 +02001423void dmar_msi_mask(struct irq_data *data)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001424{
1425 unsigned long flag;
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001426 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001427
1428 /* mask it */
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001429 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001430 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1431 /* Read a reg to force flush the post write */
1432 readl(iommu->reg + DMAR_FECTL_REG);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001433 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001434}
1435
1436void dmar_msi_write(int irq, struct msi_msg *msg)
1437{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001438 struct intel_iommu *iommu = irq_get_handler_data(irq);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001439 unsigned long flag;
1440
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001441 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001442 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1443 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1444 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001445 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001446}
1447
1448void dmar_msi_read(int irq, struct msi_msg *msg)
1449{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001450 struct intel_iommu *iommu = irq_get_handler_data(irq);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001451 unsigned long flag;
1452
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001453 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001454 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1455 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1456 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001457 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001458}
1459
1460static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1461 u8 fault_reason, u16 source_id, unsigned long long addr)
1462{
1463 const char *reason;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001464 int fault_type;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001465
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001466 reason = dmar_get_fault_reason(fault_reason, &fault_type);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001467
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001468 if (fault_type == INTR_REMAP)
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001469 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001470 "fault index %llx\n"
1471 "INTR-REMAP:[fault reason %02d] %s\n",
1472 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1473 PCI_FUNC(source_id & 0xFF), addr >> 48,
1474 fault_reason, reason);
1475 else
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001476 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001477 "fault addr %llx \n"
1478 "DMAR:[fault reason %02d] %s\n",
1479 (type ? "DMA Read" : "DMA Write"),
1480 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1481 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001482 return 0;
1483}
1484
1485#define PRIMARY_FAULT_REG_LEN (16)
Suresh Siddha1531a6a2009-03-16 17:04:57 -07001486irqreturn_t dmar_fault(int irq, void *dev_id)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001487{
1488 struct intel_iommu *iommu = dev_id;
1489 int reg, fault_index;
1490 u32 fault_status;
1491 unsigned long flag;
1492
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001493 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001494 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001495 if (fault_status)
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001496 pr_err("DRHD: handling fault status reg %x\n", fault_status);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001497
1498 /* TBD: ignore advanced fault log currently */
1499 if (!(fault_status & DMA_FSTS_PPF))
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001500 goto unlock_exit;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001501
1502 fault_index = dma_fsts_fault_record_index(fault_status);
1503 reg = cap_fault_reg_offset(iommu->cap);
1504 while (1) {
1505 u8 fault_reason;
1506 u16 source_id;
1507 u64 guest_addr;
1508 int type;
1509 u32 data;
1510
1511 /* highest 32 bits */
1512 data = readl(iommu->reg + reg +
1513 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1514 if (!(data & DMA_FRCD_F))
1515 break;
1516
1517 fault_reason = dma_frcd_fault_reason(data);
1518 type = dma_frcd_type(data);
1519
1520 data = readl(iommu->reg + reg +
1521 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1522 source_id = dma_frcd_source_id(data);
1523
1524 guest_addr = dmar_readq(iommu->reg + reg +
1525 fault_index * PRIMARY_FAULT_REG_LEN);
1526 guest_addr = dma_frcd_page_addr(guest_addr);
1527 /* clear the fault */
1528 writel(DMA_FRCD_F, iommu->reg + reg +
1529 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1530
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001531 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001532
1533 dmar_fault_do_one(iommu, type, fault_reason,
1534 source_id, guest_addr);
1535
1536 fault_index++;
Troy Heber8211a7b2009-08-19 15:26:11 -06001537 if (fault_index >= cap_num_fault_regs(iommu->cap))
Suresh Siddha0ac24912009-03-16 17:04:54 -07001538 fault_index = 0;
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001539 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001540 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001541
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001542 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1543
1544unlock_exit:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001545 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001546 return IRQ_HANDLED;
1547}
1548
1549int dmar_set_interrupt(struct intel_iommu *iommu)
1550{
1551 int irq, ret;
1552
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001553 /*
1554 * Check if the fault interrupt is already initialized.
1555 */
1556 if (iommu->irq)
1557 return 0;
1558
Thomas Gleixnera553b142014-05-07 15:44:11 +00001559 irq = dmar_alloc_hwirq();
Thomas Gleixneraa5125a2014-05-07 15:44:10 +00001560 if (irq <= 0) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001561 pr_err("IOMMU: no free vectors\n");
Suresh Siddha0ac24912009-03-16 17:04:54 -07001562 return -EINVAL;
1563 }
1564
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001565 irq_set_handler_data(irq, iommu);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001566 iommu->irq = irq;
1567
1568 ret = arch_setup_dmar_msi(irq);
1569 if (ret) {
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001570 irq_set_handler_data(irq, NULL);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001571 iommu->irq = 0;
Thomas Gleixnera553b142014-05-07 15:44:11 +00001572 dmar_free_hwirq(irq);
Chris Wrightdd726432009-05-13 15:55:52 -07001573 return ret;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001574 }
1575
Thomas Gleixner477694e2011-07-19 16:25:42 +02001576 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001577 if (ret)
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001578 pr_err("IOMMU: can't request irq\n");
Suresh Siddha0ac24912009-03-16 17:04:54 -07001579 return ret;
1580}
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001581
1582int __init enable_drhd_fault_handling(void)
1583{
1584 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +08001585 struct intel_iommu *iommu;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001586
1587 /*
1588 * Enable fault control interrupt.
1589 */
Jiang Liu7c919772014-01-06 14:18:18 +08001590 for_each_iommu(iommu, drhd) {
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001591 u32 fault_status;
Jiang Liu7c919772014-01-06 14:18:18 +08001592 int ret = dmar_set_interrupt(iommu);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001593
1594 if (ret) {
Donald Dutilee9071b02012-06-08 17:13:11 -04001595 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001596 (unsigned long long)drhd->reg_base_addr, ret);
1597 return -1;
1598 }
Suresh Siddha7f99d942010-11-30 22:22:29 -08001599
1600 /*
1601 * Clear any previous faults.
1602 */
1603 dmar_fault(iommu->irq, iommu);
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001604 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1605 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001606 }
1607
1608 return 0;
1609}
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001610
1611/*
1612 * Re-enable Queued Invalidation interface.
1613 */
1614int dmar_reenable_qi(struct intel_iommu *iommu)
1615{
1616 if (!ecap_qis(iommu->ecap))
1617 return -ENOENT;
1618
1619 if (!iommu->qi)
1620 return -ENOENT;
1621
1622 /*
1623 * First disable queued invalidation.
1624 */
1625 dmar_disable_qi(iommu);
1626 /*
1627 * Then enable queued invalidation again. Since there is no pending
1628 * invalidation requests now, it's safe to re-enable queued
1629 * invalidation.
1630 */
1631 __dmar_enable_qi(iommu);
1632
1633 return 0;
1634}
Youquan Song074835f2009-09-09 12:05:39 -04001635
1636/*
1637 * Check interrupt remapping support in DMAR table description.
1638 */
Luck, Tony0b8973a2009-12-16 22:59:29 +00001639int __init dmar_ir_support(void)
Youquan Song074835f2009-09-09 12:05:39 -04001640{
1641 struct acpi_table_dmar *dmar;
1642 dmar = (struct acpi_table_dmar *)dmar_tbl;
Arnaud Patard4f506e02010-03-25 18:02:58 +00001643 if (!dmar)
1644 return 0;
Youquan Song074835f2009-09-09 12:05:39 -04001645 return dmar->flags & 0x1;
1646}
Jiang Liu694835d2014-01-06 14:18:16 +08001647
Jiang Liua868e6b2014-01-06 14:18:20 +08001648static int __init dmar_free_unused_resources(void)
1649{
1650 struct dmar_drhd_unit *dmaru, *dmaru_n;
1651
1652 /* DMAR units are in use */
1653 if (irq_remapping_enabled || intel_iommu_enabled)
1654 return 0;
1655
Jiang Liu2e455282014-02-19 14:07:36 +08001656 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1657 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
Jiang Liu59ce0512014-02-19 14:07:35 +08001658
Jiang Liu3a5670e2014-02-19 14:07:33 +08001659 down_write(&dmar_global_lock);
Jiang Liua868e6b2014-01-06 14:18:20 +08001660 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1661 list_del(&dmaru->list);
1662 dmar_free_drhd(dmaru);
1663 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08001664 up_write(&dmar_global_lock);
Jiang Liua868e6b2014-01-06 14:18:20 +08001665
1666 return 0;
1667}
1668
1669late_initcall(dmar_free_unused_resources);
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -04001670IOMMU_INIT_POST(detect_intel_iommu);