blob: 6a86b5d1defa0513f6bc5cc8e88e125aecfaf629 [file] [log] [blame]
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -07001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
mark gross98bcef52008-02-23 15:23:35 -080017 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070021 *
Suresh Siddhae61d98d2008-07-10 11:16:35 -070022 * This file implements early detection/parsing of Remapping Devices
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070023 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
Suresh Siddhae61d98d2008-07-10 11:16:35 -070025 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070027 */
28
Joerg Roedel9f10e5b2015-06-12 09:57:06 +020029#define pr_fmt(fmt) "DMAR: " fmt
Donald Dutilee9071b02012-06-08 17:13:11 -040030
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070031#include <linux/pci.h>
32#include <linux/dmar.h>
Kay, Allen M38717942008-09-09 18:37:29 +030033#include <linux/iova.h>
34#include <linux/intel-iommu.h>
Suresh Siddhafe962e92008-07-10 11:16:42 -070035#include <linux/timer.h>
Suresh Siddha0ac24912009-03-16 17:04:54 -070036#include <linux/irq.h>
37#include <linux/interrupt.h>
Shane Wang69575d32009-09-01 18:25:07 -070038#include <linux/tboot.h>
Len Browneb27cae2009-07-06 23:40:19 -040039#include <linux/dmi.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090040#include <linux/slab.h>
Alex Williamsona5459cf2014-06-12 16:12:31 -060041#include <linux/iommu.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070042#include <asm/irq_remapping.h>
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -040043#include <asm/iommu_table.h>
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070044
Joerg Roedel078e1ee2012-09-26 12:44:43 +020045#include "irq_remapping.h"
46
Jiang Liuc2a0b532014-11-09 22:47:56 +080047typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
52 bool print_entry;
53};
54
Jiang Liu3a5670e2014-02-19 14:07:33 +080055/*
56 * Assumptions:
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
62 *
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070066 */
Jiang Liu3a5670e2014-02-19 14:07:33 +080067DECLARE_RWSEM(dmar_global_lock);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070068LIST_HEAD(dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070069
Suresh Siddha41750d32011-08-23 17:05:18 -070070struct acpi_table_header * __initdata dmar_tbl;
Yinghai Lu8e1568f2009-02-11 01:06:59 -080071static acpi_size dmar_tbl_size;
Jiang Liu2e455282014-02-19 14:07:36 +080072static int dmar_dev_scope_status = 1;
Jiang Liu78d8e702014-11-09 22:47:57 +080073static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070074
Jiang Liu694835d2014-01-06 14:18:16 +080075static int alloc_iommu(struct dmar_drhd_unit *drhd);
Jiang Liua868e6b2014-01-06 14:18:20 +080076static void free_iommu(struct intel_iommu *iommu);
Jiang Liu694835d2014-01-06 14:18:16 +080077
Jiang Liu6b197242014-11-09 22:47:58 +080078static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070079{
80 /*
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
82 * the very end.
83 */
84 if (drhd->include_all)
Jiang Liu0e242612014-02-19 14:07:34 +080085 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070086 else
Jiang Liu0e242612014-02-19 14:07:34 +080087 list_add_rcu(&drhd->list, &dmar_drhd_units);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070088}
89
Jiang Liubb3a6b72014-02-19 14:07:24 +080090void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070091{
92 struct acpi_dmar_device_scope *scope;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070093
94 *cnt = 0;
95 while (start < end) {
96 scope = start;
Bob Moore83118b02014-07-30 12:21:00 +080097 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
David Woodhouse07cb52f2014-03-07 14:39:27 +000098 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -070099 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
100 (*cnt)++;
Linn Crosettoae3e7f32013-04-23 12:26:45 -0600101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400103 pr_warn("Unsupported device scope\n");
Yinghai Lu5715f0f2010-04-08 19:58:22 +0100104 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700105 start += scope->length;
106 }
107 if (*cnt == 0)
Jiang Liubb3a6b72014-02-19 14:07:24 +0800108 return NULL;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700109
David Woodhouse832bd852014-03-07 15:08:36 +0000110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
Jiang Liubb3a6b72014-02-19 14:07:24 +0800111}
112
David Woodhouse832bd852014-03-07 15:08:36 +0000113void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
Jiang Liuada4d4b2014-01-06 14:18:09 +0800114{
Jiang Liub683b232014-02-19 14:07:32 +0800115 int i;
David Woodhouse832bd852014-03-07 15:08:36 +0000116 struct device *tmp_dev;
Jiang Liub683b232014-02-19 14:07:32 +0800117
Jiang Liuada4d4b2014-01-06 14:18:09 +0800118 if (*devices && *cnt) {
Jiang Liub683b232014-02-19 14:07:32 +0800119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
David Woodhouse832bd852014-03-07 15:08:36 +0000120 put_device(tmp_dev);
Jiang Liuada4d4b2014-01-06 14:18:09 +0800121 kfree(*devices);
Jiang Liuada4d4b2014-01-06 14:18:09 +0800122 }
Jiang Liu0e242612014-02-19 14:07:34 +0800123
124 *devices = NULL;
125 *cnt = 0;
Jiang Liuada4d4b2014-01-06 14:18:09 +0800126}
127
Jiang Liu59ce0512014-02-19 14:07:35 +0800128/* Optimize out kzalloc()/kfree() for normal cases */
129static char dmar_pci_notify_info_buf[64];
130
131static struct dmar_pci_notify_info *
132dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
133{
134 int level = 0;
135 size_t size;
136 struct pci_dev *tmp;
137 struct dmar_pci_notify_info *info;
138
139 BUG_ON(dev->is_virtfn);
140
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
144 level++;
145
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
149 } else {
150 info = kzalloc(size, GFP_KERNEL);
151 if (!info) {
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
Jiang Liu2e455282014-02-19 14:07:36 +0800154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
Jiang Liu59ce0512014-02-19 14:07:35 +0800156 return NULL;
157 }
158 }
159
160 info->event = event;
161 info->dev = dev;
162 info->seg = pci_domain_nr(dev->bus);
163 info->level = level;
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
Jiang Liu5ae05662014-04-15 10:35:35 +0800165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
166 level--;
Joerg Roedel57384592014-10-02 11:50:25 +0200167 info->path[level].bus = tmp->bus->number;
Jiang Liu59ce0512014-02-19 14:07:35 +0800168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
172 }
173 }
174
175 return info;
176}
177
178static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
179{
180 if ((void *)info != dmar_pci_notify_info_buf)
181 kfree(info);
182}
183
184static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
186{
187 int i;
188
189 if (info->bus != bus)
Joerg Roedel80f7b3d2014-09-22 16:30:22 +0200190 goto fallback;
Jiang Liu59ce0512014-02-19 14:07:35 +0800191 if (info->level != count)
Joerg Roedel80f7b3d2014-09-22 16:30:22 +0200192 goto fallback;
Jiang Liu59ce0512014-02-19 14:07:35 +0800193
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
Joerg Roedel80f7b3d2014-09-22 16:30:22 +0200197 goto fallback;
Jiang Liu59ce0512014-02-19 14:07:35 +0800198 }
199
200 return true;
Joerg Roedel80f7b3d2014-09-22 16:30:22 +0200201
202fallback:
203
204 if (count != 1)
205 return false;
206
207 i = info->level - 1;
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
213 return true;
214 }
215
216 return false;
Jiang Liu59ce0512014-02-19 14:07:35 +0800217}
218
219/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
David Woodhouse832bd852014-03-07 15:08:36 +0000222 struct dmar_dev_scope *devices,
223 int devices_cnt)
Jiang Liu59ce0512014-02-19 14:07:35 +0800224{
225 int i, level;
David Woodhouse832bd852014-03-07 15:08:36 +0000226 struct device *tmp, *dev = &info->dev->dev;
Jiang Liu59ce0512014-02-19 14:07:35 +0800227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
229
230 if (segment != info->seg)
231 return 0;
232
233 for (; start < end; start += scope->length) {
234 scope = start;
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
237 continue;
238
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue;
243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
David Woodhouse832bd852014-03-07 15:08:36 +0000245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
Jiang Liu59ce0512014-02-19 14:07:35 +0800246 pr_warn("Device scope type does not match for %s\n",
David Woodhouse832bd852014-03-07 15:08:36 +0000247 pci_name(info->dev));
Jiang Liu59ce0512014-02-19 14:07:35 +0800248 return -EINVAL;
249 }
250
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
252 if (tmp == NULL) {
David Woodhouse832bd852014-03-07 15:08:36 +0000253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
256 get_device(dev));
Jiang Liu59ce0512014-02-19 14:07:35 +0800257 return 1;
258 }
259 BUG_ON(i >= devices_cnt);
260 }
261
262 return 0;
263}
264
265int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
David Woodhouse832bd852014-03-07 15:08:36 +0000266 struct dmar_dev_scope *devices, int count)
Jiang Liu59ce0512014-02-19 14:07:35 +0800267{
268 int index;
David Woodhouse832bd852014-03-07 15:08:36 +0000269 struct device *tmp;
Jiang Liu59ce0512014-02-19 14:07:35 +0800270
271 if (info->seg != segment)
272 return 0;
273
274 for_each_active_dev_scope(devices, count, index, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +0000275 if (tmp == &info->dev->dev) {
Andreea-Cristina Bernateecbad72014-08-18 15:20:56 +0300276 RCU_INIT_POINTER(devices[index].dev, NULL);
Jiang Liu59ce0512014-02-19 14:07:35 +0800277 synchronize_rcu();
David Woodhouse832bd852014-03-07 15:08:36 +0000278 put_device(tmp);
Jiang Liu59ce0512014-02-19 14:07:35 +0800279 return 1;
280 }
281
282 return 0;
283}
284
285static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
286{
287 int ret = 0;
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
290
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
293 continue;
294
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
299 dmaru->segment,
300 dmaru->devices, dmaru->devices_cnt);
301 if (ret != 0)
302 break;
303 }
304 if (ret >= 0)
305 ret = dmar_iommu_notify_scope_dev(info);
Jiang Liu2e455282014-02-19 14:07:36 +0800306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
Jiang Liu59ce0512014-02-19 14:07:35 +0800308
309 return ret;
310}
311
312static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
313{
314 struct dmar_drhd_unit *dmaru;
315
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
319 break;
320 dmar_iommu_notify_scope_dev(info);
321}
322
323static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
325{
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
328
329 /* Only care about add/remove events for physical functions */
330 if (pdev->is_virtfn)
331 return NOTIFY_DONE;
Joerg Roedele6a8c9b2016-02-29 23:49:47 +0100332 if (action != BUS_NOTIFY_ADD_DEVICE &&
333 action != BUS_NOTIFY_REMOVED_DEVICE)
Jiang Liu59ce0512014-02-19 14:07:35 +0800334 return NOTIFY_DONE;
335
336 info = dmar_alloc_pci_notify_info(pdev, action);
337 if (!info)
338 return NOTIFY_DONE;
339
340 down_write(&dmar_global_lock);
341 if (action == BUS_NOTIFY_ADD_DEVICE)
342 dmar_pci_bus_add_dev(info);
Joerg Roedele6a8c9b2016-02-29 23:49:47 +0100343 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
Jiang Liu59ce0512014-02-19 14:07:35 +0800344 dmar_pci_bus_del_dev(info);
345 up_write(&dmar_global_lock);
346
347 dmar_free_pci_notify_info(info);
348
349 return NOTIFY_OK;
350}
351
352static struct notifier_block dmar_pci_bus_nb = {
353 .notifier_call = dmar_pci_bus_notifier,
354 .priority = INT_MIN,
355};
356
Jiang Liu6b197242014-11-09 22:47:58 +0800357static struct dmar_drhd_unit *
358dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
359{
360 struct dmar_drhd_unit *dmaru;
361
362 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
363 if (dmaru->segment == drhd->segment &&
364 dmaru->reg_base_addr == drhd->address)
365 return dmaru;
366
367 return NULL;
368}
369
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700370/**
371 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
372 * structure which uniquely represent one DMA remapping hardware unit
373 * present in the platform
374 */
Jiang Liu6b197242014-11-09 22:47:58 +0800375static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700376{
377 struct acpi_dmar_hardware_unit *drhd;
378 struct dmar_drhd_unit *dmaru;
379 int ret = 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700380
David Woodhousee523b382009-04-10 22:27:48 -0700381 drhd = (struct acpi_dmar_hardware_unit *)header;
Jiang Liu6b197242014-11-09 22:47:58 +0800382 dmaru = dmar_find_dmaru(drhd);
383 if (dmaru)
384 goto out;
385
386 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700387 if (!dmaru)
388 return -ENOMEM;
389
Jiang Liu6b197242014-11-09 22:47:58 +0800390 /*
391 * If header is allocated from slab by ACPI _DSM method, we need to
392 * copy the content because the memory buffer will be freed on return.
393 */
394 dmaru->hdr = (void *)(dmaru + 1);
395 memcpy(dmaru->hdr, header, header->length);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700396 dmaru->reg_base_addr = drhd->address;
David Woodhouse276dbf992009-04-04 01:45:37 +0100397 dmaru->segment = drhd->segment;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700398 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
David Woodhouse07cb52f2014-03-07 14:39:27 +0000399 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
400 ((void *)drhd) + drhd->header.length,
401 &dmaru->devices_cnt);
402 if (dmaru->devices_cnt && dmaru->devices == NULL) {
403 kfree(dmaru);
404 return -ENOMEM;
Jiang Liu2e455282014-02-19 14:07:36 +0800405 }
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700406
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700407 ret = alloc_iommu(dmaru);
408 if (ret) {
David Woodhouse07cb52f2014-03-07 14:39:27 +0000409 dmar_free_dev_scope(&dmaru->devices,
410 &dmaru->devices_cnt);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700411 kfree(dmaru);
412 return ret;
413 }
414 dmar_register_drhd_unit(dmaru);
Jiang Liuc2a0b532014-11-09 22:47:56 +0800415
Jiang Liu6b197242014-11-09 22:47:58 +0800416out:
Jiang Liuc2a0b532014-11-09 22:47:56 +0800417 if (arg)
418 (*(int *)arg)++;
419
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700420 return 0;
421}
422
Jiang Liua868e6b2014-01-06 14:18:20 +0800423static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
424{
425 if (dmaru->devices && dmaru->devices_cnt)
426 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
427 if (dmaru->iommu)
428 free_iommu(dmaru->iommu);
429 kfree(dmaru);
430}
431
Jiang Liuc2a0b532014-11-09 22:47:56 +0800432static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
433 void *arg)
David Woodhousee625b4a2014-03-07 14:34:38 +0000434{
435 struct acpi_dmar_andd *andd = (void *)header;
436
437 /* Check for NUL termination within the designated length */
Bob Moore83118b02014-07-30 12:21:00 +0800438 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
David Woodhousee625b4a2014-03-07 14:34:38 +0000439 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
440 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
441 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
442 dmi_get_system_info(DMI_BIOS_VENDOR),
443 dmi_get_system_info(DMI_BIOS_VERSION),
444 dmi_get_system_info(DMI_PRODUCT_VERSION));
445 return -EINVAL;
446 }
447 pr_info("ANDD device: %x name: %s\n", andd->device_number,
Bob Moore83118b02014-07-30 12:21:00 +0800448 andd->device_name);
David Woodhousee625b4a2014-03-07 14:34:38 +0000449
450 return 0;
451}
452
David Woodhouseaa697072009-10-07 12:18:00 +0100453#ifdef CONFIG_ACPI_NUMA
Jiang Liu6b197242014-11-09 22:47:58 +0800454static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
Suresh Siddhaee34b322009-10-02 11:01:21 -0700455{
456 struct acpi_dmar_rhsa *rhsa;
457 struct dmar_drhd_unit *drhd;
458
459 rhsa = (struct acpi_dmar_rhsa *)header;
David Woodhouseaa697072009-10-07 12:18:00 +0100460 for_each_drhd_unit(drhd) {
Suresh Siddhaee34b322009-10-02 11:01:21 -0700461 if (drhd->reg_base_addr == rhsa->base_address) {
462 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
463
464 if (!node_online(node))
465 node = -1;
466 drhd->iommu->node = node;
David Woodhouseaa697072009-10-07 12:18:00 +0100467 return 0;
468 }
Suresh Siddhaee34b322009-10-02 11:01:21 -0700469 }
Ben Hutchingsfd0c8892010-04-03 19:38:43 +0100470 WARN_TAINT(
471 1, TAINT_FIRMWARE_WORKAROUND,
472 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
473 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
474 drhd->reg_base_addr,
475 dmi_get_system_info(DMI_BIOS_VENDOR),
476 dmi_get_system_info(DMI_BIOS_VERSION),
477 dmi_get_system_info(DMI_PRODUCT_VERSION));
Suresh Siddhaee34b322009-10-02 11:01:21 -0700478
David Woodhouseaa697072009-10-07 12:18:00 +0100479 return 0;
Suresh Siddhaee34b322009-10-02 11:01:21 -0700480}
Jiang Liuc2a0b532014-11-09 22:47:56 +0800481#else
482#define dmar_parse_one_rhsa dmar_res_noop
David Woodhouseaa697072009-10-07 12:18:00 +0100483#endif
Suresh Siddhaee34b322009-10-02 11:01:21 -0700484
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700485static void __init
486dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
487{
488 struct acpi_dmar_hardware_unit *drhd;
489 struct acpi_dmar_reserved_memory *rmrr;
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800490 struct acpi_dmar_atsr *atsr;
Roland Dreier17b60972009-09-24 12:14:00 -0700491 struct acpi_dmar_rhsa *rhsa;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700492
493 switch (header->type) {
494 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800495 drhd = container_of(header, struct acpi_dmar_hardware_unit,
496 header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400497 pr_info("DRHD base: %#016Lx flags: %#x\n",
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800498 (unsigned long long)drhd->address, drhd->flags);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700499 break;
500 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800501 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
502 header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400503 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700504 (unsigned long long)rmrr->base_address,
505 (unsigned long long)rmrr->end_address);
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700506 break;
Bob Moore83118b02014-07-30 12:21:00 +0800507 case ACPI_DMAR_TYPE_ROOT_ATS:
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800508 atsr = container_of(header, struct acpi_dmar_atsr, header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400509 pr_info("ATSR flags: %#x\n", atsr->flags);
Yu Zhaoaa5d2b52009-05-18 13:51:34 +0800510 break;
Bob Moore83118b02014-07-30 12:21:00 +0800511 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
Roland Dreier17b60972009-09-24 12:14:00 -0700512 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
Donald Dutilee9071b02012-06-08 17:13:11 -0400513 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
Roland Dreier17b60972009-09-24 12:14:00 -0700514 (unsigned long long)rhsa->base_address,
515 rhsa->proximity_domain);
516 break;
Bob Moore83118b02014-07-30 12:21:00 +0800517 case ACPI_DMAR_TYPE_NAMESPACE:
David Woodhousee625b4a2014-03-07 14:34:38 +0000518 /* We don't print this here because we need to sanity-check
519 it first. So print it in dmar_parse_one_andd() instead. */
520 break;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700521 }
522}
523
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700524/**
525 * dmar_table_detect - checks to see if the platform supports DMAR devices
526 */
527static int __init dmar_table_detect(void)
528{
529 acpi_status status = AE_OK;
530
531 /* if we could find DMAR table, then there are DMAR devices */
Yinghai Lu8e1568f2009-02-11 01:06:59 -0800532 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
533 (struct acpi_table_header **)&dmar_tbl,
534 &dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700535
536 if (ACPI_SUCCESS(status) && !dmar_tbl) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400537 pr_warn("Unable to map DMAR\n");
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700538 status = AE_NOT_FOUND;
539 }
540
541 return (ACPI_SUCCESS(status) ? 1 : 0);
542}
Suresh Siddhaaaa9d1d2008-07-10 11:16:38 -0700543
Jiang Liuc2a0b532014-11-09 22:47:56 +0800544static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
545 size_t len, struct dmar_res_callback *cb)
546{
547 int ret = 0;
548 struct acpi_dmar_header *iter, *next;
549 struct acpi_dmar_header *end = ((void *)start) + len;
550
551 for (iter = start; iter < end && ret == 0; iter = next) {
552 next = (void *)iter + iter->length;
553 if (iter->length == 0) {
554 /* Avoid looping forever on bad ACPI tables */
555 pr_debug(FW_BUG "Invalid 0-length structure\n");
556 break;
557 } else if (next > end) {
558 /* Avoid passing table end */
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200559 pr_warn(FW_BUG "Record passes table end\n");
Jiang Liuc2a0b532014-11-09 22:47:56 +0800560 ret = -EINVAL;
561 break;
562 }
563
564 if (cb->print_entry)
565 dmar_table_print_dmar_entry(iter);
566
567 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
568 /* continue for forward compatibility */
569 pr_debug("Unknown DMAR structure type %d\n",
570 iter->type);
571 } else if (cb->cb[iter->type]) {
572 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
573 } else if (!cb->ignore_unhandled) {
574 pr_warn("No handler for DMAR structure type %d\n",
575 iter->type);
576 ret = -EINVAL;
577 }
578 }
579
580 return ret;
581}
582
583static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
584 struct dmar_res_callback *cb)
585{
586 return dmar_walk_remapping_entries((void *)(dmar + 1),
587 dmar->header.length - sizeof(*dmar), cb);
588}
589
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700590/**
591 * parse_dmar_table - parses the DMA reporting table
592 */
593static int __init
594parse_dmar_table(void)
595{
596 struct acpi_table_dmar *dmar;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700597 int ret = 0;
Li, Zhen-Hua7cef3342013-05-20 15:57:32 +0800598 int drhd_count = 0;
Jiang Liuc2a0b532014-11-09 22:47:56 +0800599 struct dmar_res_callback cb = {
600 .print_entry = true,
601 .ignore_unhandled = true,
602 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
603 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
604 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
605 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
606 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
607 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
608 };
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700609
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700610 /*
611 * Do it again, earlier dmar_tbl mapping could be mapped with
612 * fixed map.
613 */
614 dmar_table_detect();
615
Joseph Cihulaa59b50e2009-06-30 19:31:10 -0700616 /*
617 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
618 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
619 */
620 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
621
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700622 dmar = (struct acpi_table_dmar *)dmar_tbl;
623 if (!dmar)
624 return -ENODEV;
625
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700626 if (dmar->width < PAGE_SHIFT - 1) {
Donald Dutilee9071b02012-06-08 17:13:11 -0400627 pr_warn("Invalid DMAR haw\n");
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700628 return -EINVAL;
629 }
630
Donald Dutilee9071b02012-06-08 17:13:11 -0400631 pr_info("Host address width %d\n", dmar->width + 1);
Jiang Liuc2a0b532014-11-09 22:47:56 +0800632 ret = dmar_walk_dmar_table(dmar, &cb);
633 if (ret == 0 && drhd_count == 0)
Li, Zhen-Hua7cef3342013-05-20 15:57:32 +0800634 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
Jiang Liuc2a0b532014-11-09 22:47:56 +0800635
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700636 return ret;
637}
638
David Woodhouse832bd852014-03-07 15:08:36 +0000639static int dmar_pci_device_match(struct dmar_dev_scope devices[],
640 int cnt, struct pci_dev *dev)
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700641{
642 int index;
David Woodhouse832bd852014-03-07 15:08:36 +0000643 struct device *tmp;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700644
645 while (dev) {
Jiang Liub683b232014-02-19 14:07:32 +0800646 for_each_active_dev_scope(devices, cnt, index, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +0000647 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700648 return 1;
649
650 /* Check our parent */
651 dev = dev->bus->self;
652 }
653
654 return 0;
655}
656
657struct dmar_drhd_unit *
658dmar_find_matched_drhd_unit(struct pci_dev *dev)
659{
Jiang Liu0e242612014-02-19 14:07:34 +0800660 struct dmar_drhd_unit *dmaru;
Yu Zhao2e824f72008-12-22 16:54:58 +0800661 struct acpi_dmar_hardware_unit *drhd;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700662
Yinghaidda56542010-04-09 01:07:55 +0100663 dev = pci_physfn(dev);
664
Jiang Liu0e242612014-02-19 14:07:34 +0800665 rcu_read_lock();
Yijing Wang8b161f02013-10-31 17:25:16 +0800666 for_each_drhd_unit(dmaru) {
Yu Zhao2e824f72008-12-22 16:54:58 +0800667 drhd = container_of(dmaru->hdr,
668 struct acpi_dmar_hardware_unit,
669 header);
670
671 if (dmaru->include_all &&
672 drhd->segment == pci_domain_nr(dev->bus))
Jiang Liu0e242612014-02-19 14:07:34 +0800673 goto out;
Yu Zhao2e824f72008-12-22 16:54:58 +0800674
675 if (dmar_pci_device_match(dmaru->devices,
676 dmaru->devices_cnt, dev))
Jiang Liu0e242612014-02-19 14:07:34 +0800677 goto out;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700678 }
Jiang Liu0e242612014-02-19 14:07:34 +0800679 dmaru = NULL;
680out:
681 rcu_read_unlock();
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700682
Jiang Liu0e242612014-02-19 14:07:34 +0800683 return dmaru;
Suresh Siddhae61d98d2008-07-10 11:16:35 -0700684}
685
David Woodhouseed403562014-03-07 23:15:42 +0000686static void __init dmar_acpi_insert_dev_scope(u8 device_number,
687 struct acpi_device *adev)
688{
689 struct dmar_drhd_unit *dmaru;
690 struct acpi_dmar_hardware_unit *drhd;
691 struct acpi_dmar_device_scope *scope;
692 struct device *tmp;
693 int i;
694 struct acpi_dmar_pci_path *path;
695
696 for_each_drhd_unit(dmaru) {
697 drhd = container_of(dmaru->hdr,
698 struct acpi_dmar_hardware_unit,
699 header);
700
701 for (scope = (void *)(drhd + 1);
702 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
703 scope = ((void *)scope) + scope->length) {
Bob Moore83118b02014-07-30 12:21:00 +0800704 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
David Woodhouseed403562014-03-07 23:15:42 +0000705 continue;
706 if (scope->enumeration_id != device_number)
707 continue;
708
709 path = (void *)(scope + 1);
710 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
711 dev_name(&adev->dev), dmaru->reg_base_addr,
712 scope->bus, path->device, path->function);
713 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
714 if (tmp == NULL) {
715 dmaru->devices[i].bus = scope->bus;
716 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
717 path->function);
718 rcu_assign_pointer(dmaru->devices[i].dev,
719 get_device(&adev->dev));
720 return;
721 }
722 BUG_ON(i >= dmaru->devices_cnt);
723 }
724 }
725 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
726 device_number, dev_name(&adev->dev));
727}
728
729static int __init dmar_acpi_dev_scope_init(void)
730{
Joerg Roedel11f1a772014-03-25 20:16:40 +0100731 struct acpi_dmar_andd *andd;
732
733 if (dmar_tbl == NULL)
734 return -ENODEV;
735
David Woodhouse7713ec02014-04-01 14:58:36 +0100736 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
737 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
738 andd = ((void *)andd) + andd->header.length) {
Bob Moore83118b02014-07-30 12:21:00 +0800739 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
David Woodhouseed403562014-03-07 23:15:42 +0000740 acpi_handle h;
741 struct acpi_device *adev;
742
743 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
Bob Moore83118b02014-07-30 12:21:00 +0800744 andd->device_name,
David Woodhouseed403562014-03-07 23:15:42 +0000745 &h))) {
746 pr_err("Failed to find handle for ACPI object %s\n",
Bob Moore83118b02014-07-30 12:21:00 +0800747 andd->device_name);
David Woodhouseed403562014-03-07 23:15:42 +0000748 continue;
749 }
Joerg Roedelc0df9752014-08-21 23:06:48 +0200750 if (acpi_bus_get_device(h, &adev)) {
David Woodhouseed403562014-03-07 23:15:42 +0000751 pr_err("Failed to get device for ACPI object %s\n",
Bob Moore83118b02014-07-30 12:21:00 +0800752 andd->device_name);
David Woodhouseed403562014-03-07 23:15:42 +0000753 continue;
754 }
755 dmar_acpi_insert_dev_scope(andd->device_number, adev);
756 }
David Woodhouseed403562014-03-07 23:15:42 +0000757 }
758 return 0;
759}
760
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700761int __init dmar_dev_scope_init(void)
762{
Jiang Liu2e455282014-02-19 14:07:36 +0800763 struct pci_dev *dev = NULL;
764 struct dmar_pci_notify_info *info;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700765
Jiang Liu2e455282014-02-19 14:07:36 +0800766 if (dmar_dev_scope_status != 1)
767 return dmar_dev_scope_status;
Suresh Siddhac2c72862011-08-23 17:05:19 -0700768
Jiang Liu2e455282014-02-19 14:07:36 +0800769 if (list_empty(&dmar_drhd_units)) {
770 dmar_dev_scope_status = -ENODEV;
771 } else {
772 dmar_dev_scope_status = 0;
Suresh Siddha318fe7d2011-08-23 17:05:20 -0700773
David Woodhouse63b42622014-03-28 11:28:40 +0000774 dmar_acpi_dev_scope_init();
775
Jiang Liu2e455282014-02-19 14:07:36 +0800776 for_each_pci_dev(dev) {
777 if (dev->is_virtfn)
778 continue;
779
780 info = dmar_alloc_pci_notify_info(dev,
781 BUS_NOTIFY_ADD_DEVICE);
782 if (!info) {
783 return dmar_dev_scope_status;
784 } else {
785 dmar_pci_bus_add_dev(info);
786 dmar_free_pci_notify_info(info);
787 }
788 }
789
790 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700791 }
792
Jiang Liu2e455282014-02-19 14:07:36 +0800793 return dmar_dev_scope_status;
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700794}
795
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700796
797int __init dmar_table_init(void)
798{
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700799 static int dmar_table_initialized;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800800 int ret;
801
Jiang Liucc053012014-01-06 14:18:24 +0800802 if (dmar_table_initialized == 0) {
803 ret = parse_dmar_table();
804 if (ret < 0) {
805 if (ret != -ENODEV)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200806 pr_info("Parse DMAR table failure.\n");
Jiang Liucc053012014-01-06 14:18:24 +0800807 } else if (list_empty(&dmar_drhd_units)) {
808 pr_info("No DMAR devices found\n");
809 ret = -ENODEV;
810 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -0700811
Jiang Liucc053012014-01-06 14:18:24 +0800812 if (ret < 0)
813 dmar_table_initialized = ret;
814 else
815 dmar_table_initialized = 1;
Fenghua Yu093f87d2007-11-21 15:07:14 -0800816 }
817
Jiang Liucc053012014-01-06 14:18:24 +0800818 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
Keshavamurthy, Anil S10e52472007-10-21 16:41:41 -0700819}
820
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100821static void warn_invalid_dmar(u64 addr, const char *message)
822{
Ben Hutchingsfd0c8892010-04-03 19:38:43 +0100823 WARN_TAINT_ONCE(
824 1, TAINT_FIRMWARE_WORKAROUND,
825 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
826 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
827 addr, message,
828 dmi_get_system_info(DMI_BIOS_VENDOR),
829 dmi_get_system_info(DMI_BIOS_VERSION),
830 dmi_get_system_info(DMI_PRODUCT_VERSION));
Ben Hutchings3a8663e2010-04-03 19:37:23 +0100831}
David Woodhouse6ecbf012009-12-02 09:20:27 +0000832
Jiang Liuc2a0b532014-11-09 22:47:56 +0800833static int __ref
834dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
David Woodhouse86cf8982009-11-09 22:15:15 +0000835{
David Woodhouse86cf8982009-11-09 22:15:15 +0000836 struct acpi_dmar_hardware_unit *drhd;
Jiang Liuc2a0b532014-11-09 22:47:56 +0800837 void __iomem *addr;
838 u64 cap, ecap;
David Woodhouse86cf8982009-11-09 22:15:15 +0000839
Jiang Liuc2a0b532014-11-09 22:47:56 +0800840 drhd = (void *)entry;
841 if (!drhd->address) {
842 warn_invalid_dmar(0, "");
843 return -EINVAL;
David Woodhouse86cf8982009-11-09 22:15:15 +0000844 }
Chris Wright2c992202009-12-02 09:17:13 +0000845
Jiang Liu6b197242014-11-09 22:47:58 +0800846 if (arg)
847 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
848 else
849 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
Jiang Liuc2a0b532014-11-09 22:47:56 +0800850 if (!addr) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200851 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
Jiang Liuc2a0b532014-11-09 22:47:56 +0800852 return -EINVAL;
853 }
Jiang Liu6b197242014-11-09 22:47:58 +0800854
Jiang Liuc2a0b532014-11-09 22:47:56 +0800855 cap = dmar_readq(addr + DMAR_CAP_REG);
856 ecap = dmar_readq(addr + DMAR_ECAP_REG);
Jiang Liu6b197242014-11-09 22:47:58 +0800857
858 if (arg)
859 iounmap(addr);
860 else
861 early_iounmap(addr, VTD_PAGE_SIZE);
Jiang Liuc2a0b532014-11-09 22:47:56 +0800862
863 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
864 warn_invalid_dmar(drhd->address, " returns all ones");
865 return -EINVAL;
866 }
867
Chris Wright2c992202009-12-02 09:17:13 +0000868 return 0;
David Woodhouse86cf8982009-11-09 22:15:15 +0000869}
870
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -0400871int __init detect_intel_iommu(void)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700872{
873 int ret;
Jiang Liuc2a0b532014-11-09 22:47:56 +0800874 struct dmar_res_callback validate_drhd_cb = {
875 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
876 .ignore_unhandled = true,
877 };
Suresh Siddha2ae21012008-07-10 11:16:43 -0700878
Jiang Liu3a5670e2014-02-19 14:07:33 +0800879 down_write(&dmar_global_lock);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700880 ret = dmar_table_detect();
David Woodhouse86cf8982009-11-09 22:15:15 +0000881 if (ret)
Jiang Liuc2a0b532014-11-09 22:47:56 +0800882 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
883 &validate_drhd_cb);
884 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
885 iommu_detected = 1;
886 /* Make sure ACS will be enabled */
887 pci_request_acs();
888 }
Suresh Siddhaf5d1b972011-08-23 17:05:22 -0700889
FUJITA Tomonori9d5ce732009-11-10 19:46:16 +0900890#ifdef CONFIG_X86
Jiang Liuc2a0b532014-11-09 22:47:56 +0800891 if (ret)
892 x86_init.iommu.iommu_init = intel_iommu_init;
FUJITA Tomonori9d5ce732009-11-10 19:46:16 +0900893#endif
Jiang Liuc2a0b532014-11-09 22:47:56 +0800894
Jiang Liub707cb02014-01-06 14:18:26 +0800895 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
Yinghai Luf6dd5c32008-09-03 16:58:32 -0700896 dmar_tbl = NULL;
Jiang Liu3a5670e2014-02-19 14:07:33 +0800897 up_write(&dmar_global_lock);
Konrad Rzeszutek Wilk480125b2010-08-26 13:57:57 -0400898
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -0400899 return ret ? 1 : -ENODEV;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700900}
901
902
Donald Dutile6f5cf522012-06-04 17:29:02 -0400903static void unmap_iommu(struct intel_iommu *iommu)
904{
905 iounmap(iommu->reg);
906 release_mem_region(iommu->reg_phys, iommu->reg_size);
907}
908
909/**
910 * map_iommu: map the iommu's registers
911 * @iommu: the iommu to map
912 * @phys_addr: the physical address of the base resgister
Donald Dutilee9071b02012-06-08 17:13:11 -0400913 *
Donald Dutile6f5cf522012-06-04 17:29:02 -0400914 * Memory map the iommu's registers. Start w/ a single page, and
Donald Dutilee9071b02012-06-08 17:13:11 -0400915 * possibly expand if that turns out to be insufficent.
Donald Dutile6f5cf522012-06-04 17:29:02 -0400916 */
917static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
918{
919 int map_size, err=0;
920
921 iommu->reg_phys = phys_addr;
922 iommu->reg_size = VTD_PAGE_SIZE;
923
924 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200925 pr_err("Can't reserve memory\n");
Donald Dutile6f5cf522012-06-04 17:29:02 -0400926 err = -EBUSY;
927 goto out;
928 }
929
930 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
931 if (!iommu->reg) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200932 pr_err("Can't map the region\n");
Donald Dutile6f5cf522012-06-04 17:29:02 -0400933 err = -ENOMEM;
934 goto release;
935 }
936
937 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
938 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
939
940 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
941 err = -EINVAL;
942 warn_invalid_dmar(phys_addr, " returns all ones");
943 goto unmap;
944 }
945
946 /* the registers might be more than one page */
947 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
948 cap_max_fault_reg_offset(iommu->cap));
949 map_size = VTD_PAGE_ALIGN(map_size);
950 if (map_size > iommu->reg_size) {
951 iounmap(iommu->reg);
952 release_mem_region(iommu->reg_phys, iommu->reg_size);
953 iommu->reg_size = map_size;
954 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
955 iommu->name)) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200956 pr_err("Can't reserve memory\n");
Donald Dutile6f5cf522012-06-04 17:29:02 -0400957 err = -EBUSY;
958 goto out;
959 }
960 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
961 if (!iommu->reg) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +0200962 pr_err("Can't map the region\n");
Donald Dutile6f5cf522012-06-04 17:29:02 -0400963 err = -ENOMEM;
964 goto release;
965 }
966 }
967 err = 0;
968 goto out;
969
970unmap:
971 iounmap(iommu->reg);
972release:
973 release_mem_region(iommu->reg_phys, iommu->reg_size);
974out:
975 return err;
976}
977
Jiang Liu78d8e702014-11-09 22:47:57 +0800978static int dmar_alloc_seq_id(struct intel_iommu *iommu)
979{
980 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
981 DMAR_UNITS_SUPPORTED);
982 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
983 iommu->seq_id = -1;
984 } else {
985 set_bit(iommu->seq_id, dmar_seq_ids);
986 sprintf(iommu->name, "dmar%d", iommu->seq_id);
987 }
988
989 return iommu->seq_id;
990}
991
992static void dmar_free_seq_id(struct intel_iommu *iommu)
993{
994 if (iommu->seq_id >= 0) {
995 clear_bit(iommu->seq_id, dmar_seq_ids);
996 iommu->seq_id = -1;
997 }
998}
999
Jiang Liu694835d2014-01-06 14:18:16 +08001000static int alloc_iommu(struct dmar_drhd_unit *drhd)
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001001{
Suresh Siddhac42d9f32008-07-10 11:16:36 -07001002 struct intel_iommu *iommu;
Takao Indoh3a93c842013-04-23 17:35:03 +09001003 u32 ver, sts;
Joerg Roedel43f73922009-01-03 23:56:27 +01001004 int agaw = 0;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001005 int msagaw = 0;
Donald Dutile6f5cf522012-06-04 17:29:02 -04001006 int err;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07001007
David Woodhouse6ecbf012009-12-02 09:20:27 +00001008 if (!drhd->reg_base_addr) {
Ben Hutchings3a8663e2010-04-03 19:37:23 +01001009 warn_invalid_dmar(0, "");
David Woodhouse6ecbf012009-12-02 09:20:27 +00001010 return -EINVAL;
1011 }
1012
Suresh Siddhac42d9f32008-07-10 11:16:36 -07001013 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1014 if (!iommu)
Suresh Siddha1886e8a2008-07-10 11:16:37 -07001015 return -ENOMEM;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07001016
Jiang Liu78d8e702014-11-09 22:47:57 +08001017 if (dmar_alloc_seq_id(iommu) < 0) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001018 pr_err("Failed to allocate seq_id\n");
Jiang Liu78d8e702014-11-09 22:47:57 +08001019 err = -ENOSPC;
1020 goto error;
1021 }
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001022
Donald Dutile6f5cf522012-06-04 17:29:02 -04001023 err = map_iommu(iommu, drhd->reg_base_addr);
1024 if (err) {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001025 pr_err("Failed to map %s\n", iommu->name);
Jiang Liu78d8e702014-11-09 22:47:57 +08001026 goto error_free_seq_id;
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001027 }
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001028
Donald Dutile6f5cf522012-06-04 17:29:02 -04001029 err = -EINVAL;
Weidong Han1b573682008-12-08 15:34:06 +08001030 agaw = iommu_calculate_agaw(iommu);
1031 if (agaw < 0) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001032 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1033 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +01001034 goto err_unmap;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001035 }
1036 msagaw = iommu_calculate_max_sagaw(iommu);
1037 if (msagaw < 0) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001038 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
Weidong Han1b573682008-12-08 15:34:06 +08001039 iommu->seq_id);
David Woodhouse08155652009-08-04 09:17:20 +01001040 goto err_unmap;
Weidong Han1b573682008-12-08 15:34:06 +08001041 }
1042 iommu->agaw = agaw;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001043 iommu->msagaw = msagaw;
David Woodhouse67ccac42014-03-09 13:49:45 -07001044 iommu->segment = drhd->segment;
Weidong Han1b573682008-12-08 15:34:06 +08001045
Suresh Siddhaee34b322009-10-02 11:01:21 -07001046 iommu->node = -1;
1047
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001048 ver = readl(iommu->reg + DMAR_VER_REG);
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001049 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1050 iommu->name,
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001051 (unsigned long long)drhd->reg_base_addr,
1052 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1053 (unsigned long long)iommu->cap,
1054 (unsigned long long)iommu->ecap);
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001055
Takao Indoh3a93c842013-04-23 17:35:03 +09001056 /* Reflect status in gcmd */
1057 sts = readl(iommu->reg + DMAR_GSTS_REG);
1058 if (sts & DMA_GSTS_IRES)
1059 iommu->gcmd |= DMA_GCMD_IRE;
1060 if (sts & DMA_GSTS_TES)
1061 iommu->gcmd |= DMA_GCMD_TE;
1062 if (sts & DMA_GSTS_QIES)
1063 iommu->gcmd |= DMA_GCMD_QIE;
1064
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001065 raw_spin_lock_init(&iommu->register_lock);
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001066
Joerg Roedelbc847452016-01-07 12:16:51 +01001067 if (intel_iommu_enabled) {
Alex Williamsona5459cf2014-06-12 16:12:31 -06001068 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1069 intel_iommu_groups,
Kees Cook2439d4a2015-07-24 16:27:57 -07001070 "%s", iommu->name);
Alex Williamsona5459cf2014-06-12 16:12:31 -06001071
Joerg Roedelbc847452016-01-07 12:16:51 +01001072 if (IS_ERR(iommu->iommu_dev)) {
1073 err = PTR_ERR(iommu->iommu_dev);
1074 goto err_unmap;
1075 }
Nicholas Krause59203372016-01-04 18:27:57 -05001076 }
1077
Joerg Roedelbc847452016-01-07 12:16:51 +01001078 drhd->iommu = iommu;
1079
Suresh Siddha1886e8a2008-07-10 11:16:37 -07001080 return 0;
David Woodhouse08155652009-08-04 09:17:20 +01001081
Jiang Liu78d8e702014-11-09 22:47:57 +08001082err_unmap:
Donald Dutile6f5cf522012-06-04 17:29:02 -04001083 unmap_iommu(iommu);
Jiang Liu78d8e702014-11-09 22:47:57 +08001084error_free_seq_id:
1085 dmar_free_seq_id(iommu);
1086error:
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001087 kfree(iommu);
Donald Dutile6f5cf522012-06-04 17:29:02 -04001088 return err;
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001089}
1090
Jiang Liua868e6b2014-01-06 14:18:20 +08001091static void free_iommu(struct intel_iommu *iommu)
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001092{
Alex Williamsona5459cf2014-06-12 16:12:31 -06001093 iommu_device_destroy(iommu->iommu_dev);
1094
Jiang Liua868e6b2014-01-06 14:18:20 +08001095 if (iommu->irq) {
David Woodhouse12082252015-10-07 15:37:03 +01001096 if (iommu->pr_irq) {
1097 free_irq(iommu->pr_irq, iommu);
1098 dmar_free_hwirq(iommu->pr_irq);
1099 iommu->pr_irq = 0;
1100 }
Jiang Liua868e6b2014-01-06 14:18:20 +08001101 free_irq(iommu->irq, iommu);
Thomas Gleixnera553b142014-05-07 15:44:11 +00001102 dmar_free_hwirq(iommu->irq);
Jiang Liu34742db2015-04-13 14:11:41 +08001103 iommu->irq = 0;
Jiang Liua868e6b2014-01-06 14:18:20 +08001104 }
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001105
Jiang Liua84da702014-01-06 14:18:23 +08001106 if (iommu->qi) {
1107 free_page((unsigned long)iommu->qi->desc);
1108 kfree(iommu->qi->desc_status);
1109 kfree(iommu->qi);
1110 }
1111
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001112 if (iommu->reg)
Donald Dutile6f5cf522012-06-04 17:29:02 -04001113 unmap_iommu(iommu);
1114
Jiang Liu78d8e702014-11-09 22:47:57 +08001115 dmar_free_seq_id(iommu);
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001116 kfree(iommu);
1117}
Suresh Siddhafe962e92008-07-10 11:16:42 -07001118
1119/*
1120 * Reclaim all the submitted descriptors which have completed its work.
1121 */
1122static inline void reclaim_free_desc(struct q_inval *qi)
1123{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001124 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1125 qi->desc_status[qi->free_tail] == QI_ABORT) {
Suresh Siddhafe962e92008-07-10 11:16:42 -07001126 qi->desc_status[qi->free_tail] = QI_FREE;
1127 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1128 qi->free_cnt++;
1129 }
1130}
1131
Yu Zhao704126a2009-01-04 16:28:52 +08001132static int qi_check_fault(struct intel_iommu *iommu, int index)
1133{
1134 u32 fault;
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001135 int head, tail;
Yu Zhao704126a2009-01-04 16:28:52 +08001136 struct q_inval *qi = iommu->qi;
1137 int wait_index = (index + 1) % QI_LENGTH;
1138
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001139 if (qi->desc_status[wait_index] == QI_ABORT)
1140 return -EAGAIN;
1141
Yu Zhao704126a2009-01-04 16:28:52 +08001142 fault = readl(iommu->reg + DMAR_FSTS_REG);
1143
1144 /*
1145 * If IQE happens, the head points to the descriptor associated
1146 * with the error. No new descriptors are fetched until the IQE
1147 * is cleared.
1148 */
1149 if (fault & DMA_FSTS_IQE) {
1150 head = readl(iommu->reg + DMAR_IQH_REG);
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001151 if ((head >> DMAR_IQ_SHIFT) == index) {
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001152 pr_err("VT-d detected invalid descriptor: "
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001153 "low=%llx, high=%llx\n",
1154 (unsigned long long)qi->desc[index].low,
1155 (unsigned long long)qi->desc[index].high);
Yu Zhao704126a2009-01-04 16:28:52 +08001156 memcpy(&qi->desc[index], &qi->desc[wait_index],
1157 sizeof(struct qi_desc));
1158 __iommu_flush_cache(iommu, &qi->desc[index],
1159 sizeof(struct qi_desc));
1160 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1161 return -EINVAL;
1162 }
1163 }
1164
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001165 /*
1166 * If ITE happens, all pending wait_desc commands are aborted.
1167 * No new descriptors are fetched until the ITE is cleared.
1168 */
1169 if (fault & DMA_FSTS_ITE) {
1170 head = readl(iommu->reg + DMAR_IQH_REG);
1171 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1172 head |= 1;
1173 tail = readl(iommu->reg + DMAR_IQT_REG);
1174 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1175
1176 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1177
1178 do {
1179 if (qi->desc_status[head] == QI_IN_USE)
1180 qi->desc_status[head] = QI_ABORT;
1181 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1182 } while (head != tail);
1183
1184 if (qi->desc_status[wait_index] == QI_ABORT)
1185 return -EAGAIN;
1186 }
1187
1188 if (fault & DMA_FSTS_ICE)
1189 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1190
Yu Zhao704126a2009-01-04 16:28:52 +08001191 return 0;
1192}
1193
Suresh Siddhafe962e92008-07-10 11:16:42 -07001194/*
1195 * Submit the queued invalidation descriptor to the remapping
1196 * hardware unit and wait for its completion.
1197 */
Yu Zhao704126a2009-01-04 16:28:52 +08001198int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
Suresh Siddhafe962e92008-07-10 11:16:42 -07001199{
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001200 int rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001201 struct q_inval *qi = iommu->qi;
1202 struct qi_desc *hw, wait_desc;
1203 int wait_index, index;
1204 unsigned long flags;
1205
1206 if (!qi)
Yu Zhao704126a2009-01-04 16:28:52 +08001207 return 0;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001208
1209 hw = qi->desc;
1210
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001211restart:
1212 rc = 0;
1213
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001214 raw_spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001215 while (qi->free_cnt < 3) {
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001216 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001217 cpu_relax();
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001218 raw_spin_lock_irqsave(&qi->q_lock, flags);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001219 }
1220
1221 index = qi->free_head;
1222 wait_index = (index + 1) % QI_LENGTH;
1223
1224 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1225
1226 hw[index] = *desc;
1227
Yu Zhao704126a2009-01-04 16:28:52 +08001228 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1229 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001230 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1231
1232 hw[wait_index] = wait_desc;
1233
1234 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1235 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1236
1237 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1238 qi->free_cnt -= 2;
1239
Suresh Siddhafe962e92008-07-10 11:16:42 -07001240 /*
1241 * update the HW tail register indicating the presence of
1242 * new descriptors.
1243 */
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001244 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001245
1246 while (qi->desc_status[wait_index] != QI_DONE) {
Suresh Siddhaf05810c2008-10-16 16:31:54 -07001247 /*
1248 * We will leave the interrupts disabled, to prevent interrupt
1249 * context to queue another cmd while a cmd is already submitted
1250 * and waiting for completion on this cpu. This is to avoid
1251 * a deadlock where the interrupt context can wait indefinitely
1252 * for free slots in the queue.
1253 */
Yu Zhao704126a2009-01-04 16:28:52 +08001254 rc = qi_check_fault(iommu, index);
1255 if (rc)
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001256 break;
Yu Zhao704126a2009-01-04 16:28:52 +08001257
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001258 raw_spin_unlock(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001259 cpu_relax();
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001260 raw_spin_lock(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001261 }
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001262
1263 qi->desc_status[index] = QI_DONE;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001264
1265 reclaim_free_desc(qi);
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001266 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +08001267
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001268 if (rc == -EAGAIN)
1269 goto restart;
1270
Yu Zhao704126a2009-01-04 16:28:52 +08001271 return rc;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001272}
1273
1274/*
1275 * Flush the global interrupt entry cache.
1276 */
1277void qi_global_iec(struct intel_iommu *iommu)
1278{
1279 struct qi_desc desc;
1280
1281 desc.low = QI_IEC_TYPE;
1282 desc.high = 0;
1283
Yu Zhao704126a2009-01-04 16:28:52 +08001284 /* should never fail */
Suresh Siddhafe962e92008-07-10 11:16:42 -07001285 qi_submit_sync(&desc, iommu);
1286}
1287
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001288void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1289 u64 type)
Youquan Song3481f212008-10-16 16:31:55 -07001290{
Youquan Song3481f212008-10-16 16:31:55 -07001291 struct qi_desc desc;
1292
Youquan Song3481f212008-10-16 16:31:55 -07001293 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1294 | QI_CC_GRAN(type) | QI_CC_TYPE;
1295 desc.high = 0;
1296
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001297 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -07001298}
1299
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001300void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1301 unsigned int size_order, u64 type)
Youquan Song3481f212008-10-16 16:31:55 -07001302{
1303 u8 dw = 0, dr = 0;
1304
1305 struct qi_desc desc;
1306 int ih = 0;
1307
Youquan Song3481f212008-10-16 16:31:55 -07001308 if (cap_write_drain(iommu->cap))
1309 dw = 1;
1310
1311 if (cap_read_drain(iommu->cap))
1312 dr = 1;
1313
1314 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1315 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1316 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1317 | QI_IOTLB_AM(size_order);
1318
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001319 qi_submit_sync(&desc, iommu);
Youquan Song3481f212008-10-16 16:31:55 -07001320}
1321
Yu Zhao6ba6c3a2009-05-18 13:51:35 +08001322void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1323 u64 addr, unsigned mask)
1324{
1325 struct qi_desc desc;
1326
1327 if (mask) {
1328 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1329 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1330 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1331 } else
1332 desc.high = QI_DEV_IOTLB_ADDR(addr);
1333
1334 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1335 qdep = 0;
1336
1337 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1338 QI_DIOTLB_TYPE;
1339
1340 qi_submit_sync(&desc, iommu);
1341}
1342
Suresh Siddhafe962e92008-07-10 11:16:42 -07001343/*
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001344 * Disable Queued Invalidation interface.
1345 */
1346void dmar_disable_qi(struct intel_iommu *iommu)
1347{
1348 unsigned long flags;
1349 u32 sts;
1350 cycles_t start_time = get_cycles();
1351
1352 if (!ecap_qis(iommu->ecap))
1353 return;
1354
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001355 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001356
CQ Tangfda3bec2016-01-13 21:15:03 +00001357 sts = readl(iommu->reg + DMAR_GSTS_REG);
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001358 if (!(sts & DMA_GSTS_QIES))
1359 goto end;
1360
1361 /*
1362 * Give a chance to HW to complete the pending invalidation requests.
1363 */
1364 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1365 readl(iommu->reg + DMAR_IQH_REG)) &&
1366 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1367 cpu_relax();
1368
1369 iommu->gcmd &= ~DMA_GCMD_QIE;
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001370 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1371
1372 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1373 !(sts & DMA_GSTS_QIES), sts);
1374end:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001375 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -07001376}
1377
1378/*
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001379 * Enable queued invalidation.
1380 */
1381static void __dmar_enable_qi(struct intel_iommu *iommu)
1382{
David Woodhousec416daa2009-05-10 20:30:58 +01001383 u32 sts;
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001384 unsigned long flags;
1385 struct q_inval *qi = iommu->qi;
1386
1387 qi->free_head = qi->free_tail = 0;
1388 qi->free_cnt = QI_LENGTH;
1389
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001390 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001391
1392 /* write zero to the tail reg */
1393 writel(0, iommu->reg + DMAR_IQT_REG);
1394
1395 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1396
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001397 iommu->gcmd |= DMA_GCMD_QIE;
David Woodhousec416daa2009-05-10 20:30:58 +01001398 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001399
1400 /* Make sure hardware complete it */
1401 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1402
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001403 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001404}
1405
1406/*
Suresh Siddhafe962e92008-07-10 11:16:42 -07001407 * Enable Queued Invalidation interface. This is a must to support
1408 * interrupt-remapping. Also used by DMA-remapping, which replaces
1409 * register based IOTLB invalidation.
1410 */
1411int dmar_enable_qi(struct intel_iommu *iommu)
1412{
Suresh Siddhafe962e92008-07-10 11:16:42 -07001413 struct q_inval *qi;
Suresh Siddha751cafe2009-10-02 11:01:22 -07001414 struct page *desc_page;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001415
1416 if (!ecap_qis(iommu->ecap))
1417 return -ENOENT;
1418
1419 /*
1420 * queued invalidation is already setup and enabled.
1421 */
1422 if (iommu->qi)
1423 return 0;
1424
Suresh Siddhafa4b57c2009-03-16 17:05:05 -07001425 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001426 if (!iommu->qi)
1427 return -ENOMEM;
1428
1429 qi = iommu->qi;
1430
Suresh Siddha751cafe2009-10-02 11:01:22 -07001431
1432 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1433 if (!desc_page) {
Suresh Siddhafe962e92008-07-10 11:16:42 -07001434 kfree(qi);
Jiang Liub707cb02014-01-06 14:18:26 +08001435 iommu->qi = NULL;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001436 return -ENOMEM;
1437 }
1438
Suresh Siddha751cafe2009-10-02 11:01:22 -07001439 qi->desc = page_address(desc_page);
1440
Hannes Reinecke37a40712013-02-06 09:50:10 +01001441 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001442 if (!qi->desc_status) {
1443 free_page((unsigned long) qi->desc);
1444 kfree(qi);
Jiang Liub707cb02014-01-06 14:18:26 +08001445 iommu->qi = NULL;
Suresh Siddhafe962e92008-07-10 11:16:42 -07001446 return -ENOMEM;
1447 }
1448
Thomas Gleixner3b8f4042011-07-19 17:02:07 +02001449 raw_spin_lock_init(&qi->q_lock);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001450
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001451 __dmar_enable_qi(iommu);
Suresh Siddhafe962e92008-07-10 11:16:42 -07001452
1453 return 0;
1454}
Suresh Siddha0ac24912009-03-16 17:04:54 -07001455
1456/* iommu interrupt handling. Most stuff are MSI-like. */
1457
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001458enum faulttype {
1459 DMA_REMAP,
1460 INTR_REMAP,
1461 UNKNOWN,
1462};
1463
1464static const char *dma_remap_fault_reasons[] =
Suresh Siddha0ac24912009-03-16 17:04:54 -07001465{
1466 "Software",
1467 "Present bit in root entry is clear",
1468 "Present bit in context entry is clear",
1469 "Invalid context entry",
1470 "Access beyond MGAW",
1471 "PTE Write access is not set",
1472 "PTE Read access is not set",
1473 "Next page table ptr is invalid",
1474 "Root table address invalid",
1475 "Context table ptr is invalid",
1476 "non-zero reserved fields in RTP",
1477 "non-zero reserved fields in CTP",
1478 "non-zero reserved fields in PTE",
Li, Zhen-Hua4ecccd92013-03-06 10:43:17 +08001479 "PCE for translation request specifies blocking",
Suresh Siddha0ac24912009-03-16 17:04:54 -07001480};
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001481
Suresh Siddha95a02e92012-03-30 11:47:07 -07001482static const char *irq_remap_fault_reasons[] =
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001483{
1484 "Detected reserved fields in the decoded interrupt-remapped request",
1485 "Interrupt index exceeded the interrupt-remapping table size",
1486 "Present field in the IRTE entry is clear",
1487 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1488 "Detected reserved fields in the IRTE entry",
1489 "Blocked a compatibility format interrupt request",
1490 "Blocked an interrupt request due to source-id verification failure",
1491};
1492
Rashika Kheria21004dc2013-12-18 12:01:46 +05301493static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001494{
Dan Carpenterfefe1ed2012-05-13 20:09:38 +03001495 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1496 ARRAY_SIZE(irq_remap_fault_reasons))) {
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001497 *fault_type = INTR_REMAP;
Suresh Siddha95a02e92012-03-30 11:47:07 -07001498 return irq_remap_fault_reasons[fault_reason - 0x20];
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001499 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1500 *fault_type = DMA_REMAP;
1501 return dma_remap_fault_reasons[fault_reason];
1502 } else {
1503 *fault_type = UNKNOWN;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001504 return "Unknown";
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001505 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001506}
1507
David Woodhouse12082252015-10-07 15:37:03 +01001508
1509static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1510{
1511 if (iommu->irq == irq)
1512 return DMAR_FECTL_REG;
1513 else if (iommu->pr_irq == irq)
1514 return DMAR_PECTL_REG;
1515 else
1516 BUG();
1517}
1518
Thomas Gleixner5c2837f2010-09-28 17:15:11 +02001519void dmar_msi_unmask(struct irq_data *data)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001520{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001521 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
David Woodhouse12082252015-10-07 15:37:03 +01001522 int reg = dmar_msi_reg(iommu, data->irq);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001523 unsigned long flag;
1524
1525 /* unmask it */
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001526 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse12082252015-10-07 15:37:03 +01001527 writel(0, iommu->reg + reg);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001528 /* Read a reg to force flush the post write */
David Woodhouse12082252015-10-07 15:37:03 +01001529 readl(iommu->reg + reg);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001530 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001531}
1532
Thomas Gleixner5c2837f2010-09-28 17:15:11 +02001533void dmar_msi_mask(struct irq_data *data)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001534{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001535 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
David Woodhouse12082252015-10-07 15:37:03 +01001536 int reg = dmar_msi_reg(iommu, data->irq);
1537 unsigned long flag;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001538
1539 /* mask it */
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001540 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse12082252015-10-07 15:37:03 +01001541 writel(DMA_FECTL_IM, iommu->reg + reg);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001542 /* Read a reg to force flush the post write */
David Woodhouse12082252015-10-07 15:37:03 +01001543 readl(iommu->reg + reg);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001544 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001545}
1546
1547void dmar_msi_write(int irq, struct msi_msg *msg)
1548{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001549 struct intel_iommu *iommu = irq_get_handler_data(irq);
David Woodhouse12082252015-10-07 15:37:03 +01001550 int reg = dmar_msi_reg(iommu, irq);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001551 unsigned long flag;
1552
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001553 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse12082252015-10-07 15:37:03 +01001554 writel(msg->data, iommu->reg + reg + 4);
1555 writel(msg->address_lo, iommu->reg + reg + 8);
1556 writel(msg->address_hi, iommu->reg + reg + 12);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001557 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001558}
1559
1560void dmar_msi_read(int irq, struct msi_msg *msg)
1561{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +02001562 struct intel_iommu *iommu = irq_get_handler_data(irq);
David Woodhouse12082252015-10-07 15:37:03 +01001563 int reg = dmar_msi_reg(iommu, irq);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001564 unsigned long flag;
1565
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001566 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse12082252015-10-07 15:37:03 +01001567 msg->data = readl(iommu->reg + reg + 4);
1568 msg->address_lo = readl(iommu->reg + reg + 8);
1569 msg->address_hi = readl(iommu->reg + reg + 12);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001570 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001571}
1572
1573static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1574 u8 fault_reason, u16 source_id, unsigned long long addr)
1575{
1576 const char *reason;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001577 int fault_type;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001578
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001579 reason = dmar_get_fault_reason(fault_reason, &fault_type);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001580
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001581 if (fault_type == INTR_REMAP)
Alex Williamsona0fe14d2016-03-17 14:12:31 -06001582 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1583 source_id >> 8, PCI_SLOT(source_id & 0xFF),
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001584 PCI_FUNC(source_id & 0xFF), addr >> 48,
1585 fault_reason, reason);
1586 else
Alex Williamsona0fe14d2016-03-17 14:12:31 -06001587 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1588 type ? "DMA Read" : "DMA Write",
1589 source_id >> 8, PCI_SLOT(source_id & 0xFF),
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001590 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001591 return 0;
1592}
1593
1594#define PRIMARY_FAULT_REG_LEN (16)
Suresh Siddha1531a6a2009-03-16 17:04:57 -07001595irqreturn_t dmar_fault(int irq, void *dev_id)
Suresh Siddha0ac24912009-03-16 17:04:54 -07001596{
1597 struct intel_iommu *iommu = dev_id;
1598 int reg, fault_index;
1599 u32 fault_status;
1600 unsigned long flag;
Alex Williamsonc43fce42016-03-17 14:12:25 -06001601 bool ratelimited;
1602 static DEFINE_RATELIMIT_STATE(rs,
1603 DEFAULT_RATELIMIT_INTERVAL,
1604 DEFAULT_RATELIMIT_BURST);
1605
1606 /* Disable printing, simply clear the fault when ratelimited */
1607 ratelimited = !__ratelimit(&rs);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001608
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001609 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001610 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
Alex Williamsonc43fce42016-03-17 14:12:25 -06001611 if (fault_status && !ratelimited)
Donald Dutilebf947fcb2012-06-04 17:29:01 -04001612 pr_err("DRHD: handling fault status reg %x\n", fault_status);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001613
1614 /* TBD: ignore advanced fault log currently */
1615 if (!(fault_status & DMA_FSTS_PPF))
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001616 goto unlock_exit;
Suresh Siddha0ac24912009-03-16 17:04:54 -07001617
1618 fault_index = dma_fsts_fault_record_index(fault_status);
1619 reg = cap_fault_reg_offset(iommu->cap);
1620 while (1) {
1621 u8 fault_reason;
1622 u16 source_id;
1623 u64 guest_addr;
1624 int type;
1625 u32 data;
1626
1627 /* highest 32 bits */
1628 data = readl(iommu->reg + reg +
1629 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1630 if (!(data & DMA_FRCD_F))
1631 break;
1632
Alex Williamsonc43fce42016-03-17 14:12:25 -06001633 if (!ratelimited) {
1634 fault_reason = dma_frcd_fault_reason(data);
1635 type = dma_frcd_type(data);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001636
Alex Williamsonc43fce42016-03-17 14:12:25 -06001637 data = readl(iommu->reg + reg +
1638 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1639 source_id = dma_frcd_source_id(data);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001640
Alex Williamsonc43fce42016-03-17 14:12:25 -06001641 guest_addr = dmar_readq(iommu->reg + reg +
1642 fault_index * PRIMARY_FAULT_REG_LEN);
1643 guest_addr = dma_frcd_page_addr(guest_addr);
1644 }
1645
Suresh Siddha0ac24912009-03-16 17:04:54 -07001646 /* clear the fault */
1647 writel(DMA_FRCD_F, iommu->reg + reg +
1648 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1649
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001650 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001651
Alex Williamsonc43fce42016-03-17 14:12:25 -06001652 if (!ratelimited)
1653 dmar_fault_do_one(iommu, type, fault_reason,
1654 source_id, guest_addr);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001655
1656 fault_index++;
Troy Heber8211a7b2009-08-19 15:26:11 -06001657 if (fault_index >= cap_num_fault_regs(iommu->cap))
Suresh Siddha0ac24912009-03-16 17:04:54 -07001658 fault_index = 0;
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001659 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001660 }
Suresh Siddha0ac24912009-03-16 17:04:54 -07001661
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001662 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1663
1664unlock_exit:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001665 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001666 return IRQ_HANDLED;
1667}
1668
1669int dmar_set_interrupt(struct intel_iommu *iommu)
1670{
1671 int irq, ret;
1672
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001673 /*
1674 * Check if the fault interrupt is already initialized.
1675 */
1676 if (iommu->irq)
1677 return 0;
1678
Jiang Liu34742db2015-04-13 14:11:41 +08001679 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1680 if (irq > 0) {
1681 iommu->irq = irq;
1682 } else {
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001683 pr_err("No free IRQ vectors\n");
Suresh Siddha0ac24912009-03-16 17:04:54 -07001684 return -EINVAL;
1685 }
1686
Thomas Gleixner477694e2011-07-19 16:25:42 +02001687 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
Suresh Siddha0ac24912009-03-16 17:04:54 -07001688 if (ret)
Joerg Roedel9f10e5b2015-06-12 09:57:06 +02001689 pr_err("Can't request irq\n");
Suresh Siddha0ac24912009-03-16 17:04:54 -07001690 return ret;
1691}
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001692
1693int __init enable_drhd_fault_handling(void)
1694{
1695 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +08001696 struct intel_iommu *iommu;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001697
1698 /*
1699 * Enable fault control interrupt.
1700 */
Jiang Liu7c919772014-01-06 14:18:18 +08001701 for_each_iommu(iommu, drhd) {
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001702 u32 fault_status;
Jiang Liu7c919772014-01-06 14:18:18 +08001703 int ret = dmar_set_interrupt(iommu);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001704
1705 if (ret) {
Donald Dutilee9071b02012-06-08 17:13:11 -04001706 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001707 (unsigned long long)drhd->reg_base_addr, ret);
1708 return -1;
1709 }
Suresh Siddha7f99d942010-11-30 22:22:29 -08001710
1711 /*
1712 * Clear any previous faults.
1713 */
1714 dmar_fault(iommu->irq, iommu);
Li, Zhen-Huabd5cdad2013-03-25 16:20:52 +08001715 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1716 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
Suresh Siddha9d783ba2009-03-16 17:04:55 -07001717 }
1718
1719 return 0;
1720}
Fenghua Yueb4a52b2009-03-27 14:22:43 -07001721
1722/*
1723 * Re-enable Queued Invalidation interface.
1724 */
1725int dmar_reenable_qi(struct intel_iommu *iommu)
1726{
1727 if (!ecap_qis(iommu->ecap))
1728 return -ENOENT;
1729
1730 if (!iommu->qi)
1731 return -ENOENT;
1732
1733 /*
1734 * First disable queued invalidation.
1735 */
1736 dmar_disable_qi(iommu);
1737 /*
1738 * Then enable queued invalidation again. Since there is no pending
1739 * invalidation requests now, it's safe to re-enable queued
1740 * invalidation.
1741 */
1742 __dmar_enable_qi(iommu);
1743
1744 return 0;
1745}
Youquan Song074835f2009-09-09 12:05:39 -04001746
1747/*
1748 * Check interrupt remapping support in DMAR table description.
1749 */
Luck, Tony0b8973a2009-12-16 22:59:29 +00001750int __init dmar_ir_support(void)
Youquan Song074835f2009-09-09 12:05:39 -04001751{
1752 struct acpi_table_dmar *dmar;
1753 dmar = (struct acpi_table_dmar *)dmar_tbl;
Arnaud Patard4f506e02010-03-25 18:02:58 +00001754 if (!dmar)
1755 return 0;
Youquan Song074835f2009-09-09 12:05:39 -04001756 return dmar->flags & 0x1;
1757}
Jiang Liu694835d2014-01-06 14:18:16 +08001758
Jiang Liu6b197242014-11-09 22:47:58 +08001759/* Check whether DMAR units are in use */
1760static inline bool dmar_in_use(void)
1761{
1762 return irq_remapping_enabled || intel_iommu_enabled;
1763}
1764
Jiang Liua868e6b2014-01-06 14:18:20 +08001765static int __init dmar_free_unused_resources(void)
1766{
1767 struct dmar_drhd_unit *dmaru, *dmaru_n;
1768
Jiang Liu6b197242014-11-09 22:47:58 +08001769 if (dmar_in_use())
Jiang Liua868e6b2014-01-06 14:18:20 +08001770 return 0;
1771
Jiang Liu2e455282014-02-19 14:07:36 +08001772 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1773 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
Jiang Liu59ce0512014-02-19 14:07:35 +08001774
Jiang Liu3a5670e2014-02-19 14:07:33 +08001775 down_write(&dmar_global_lock);
Jiang Liua868e6b2014-01-06 14:18:20 +08001776 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1777 list_del(&dmaru->list);
1778 dmar_free_drhd(dmaru);
1779 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08001780 up_write(&dmar_global_lock);
Jiang Liua868e6b2014-01-06 14:18:20 +08001781
1782 return 0;
1783}
1784
1785late_initcall(dmar_free_unused_resources);
Konrad Rzeszutek Wilk4db77ff2010-08-26 13:58:04 -04001786IOMMU_INIT_POST(detect_intel_iommu);
Jiang Liu6b197242014-11-09 22:47:58 +08001787
1788/*
1789 * DMAR Hotplug Support
1790 * For more details, please refer to Intel(R) Virtualization Technology
1791 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1792 * "Remapping Hardware Unit Hot Plug".
1793 */
1794static u8 dmar_hp_uuid[] = {
1795 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1796 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1797};
1798
1799/*
1800 * Currently there's only one revision and BIOS will not check the revision id,
1801 * so use 0 for safety.
1802 */
1803#define DMAR_DSM_REV_ID 0
1804#define DMAR_DSM_FUNC_DRHD 1
1805#define DMAR_DSM_FUNC_ATSR 2
1806#define DMAR_DSM_FUNC_RHSA 3
1807
1808static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1809{
1810 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1811}
1812
1813static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1814 dmar_res_handler_t handler, void *arg)
1815{
1816 int ret = -ENODEV;
1817 union acpi_object *obj;
1818 struct acpi_dmar_header *start;
1819 struct dmar_res_callback callback;
1820 static int res_type[] = {
1821 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1822 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1823 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1824 };
1825
1826 if (!dmar_detect_dsm(handle, func))
1827 return 0;
1828
1829 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1830 func, NULL, ACPI_TYPE_BUFFER);
1831 if (!obj)
1832 return -ENODEV;
1833
1834 memset(&callback, 0, sizeof(callback));
1835 callback.cb[res_type[func]] = handler;
1836 callback.arg[res_type[func]] = arg;
1837 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1838 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1839
1840 ACPI_FREE(obj);
1841
1842 return ret;
1843}
1844
1845static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1846{
1847 int ret;
1848 struct dmar_drhd_unit *dmaru;
1849
1850 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1851 if (!dmaru)
1852 return -ENODEV;
1853
1854 ret = dmar_ir_hotplug(dmaru, true);
1855 if (ret == 0)
1856 ret = dmar_iommu_hotplug(dmaru, true);
1857
1858 return ret;
1859}
1860
1861static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1862{
1863 int i, ret;
1864 struct device *dev;
1865 struct dmar_drhd_unit *dmaru;
1866
1867 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1868 if (!dmaru)
1869 return 0;
1870
1871 /*
1872 * All PCI devices managed by this unit should have been destroyed.
1873 */
1874 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1875 for_each_active_dev_scope(dmaru->devices,
1876 dmaru->devices_cnt, i, dev)
1877 return -EBUSY;
1878
1879 ret = dmar_ir_hotplug(dmaru, false);
1880 if (ret == 0)
1881 ret = dmar_iommu_hotplug(dmaru, false);
1882
1883 return ret;
1884}
1885
1886static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1887{
1888 struct dmar_drhd_unit *dmaru;
1889
1890 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1891 if (dmaru) {
1892 list_del_rcu(&dmaru->list);
1893 synchronize_rcu();
1894 dmar_free_drhd(dmaru);
1895 }
1896
1897 return 0;
1898}
1899
1900static int dmar_hotplug_insert(acpi_handle handle)
1901{
1902 int ret;
1903 int drhd_count = 0;
1904
1905 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1906 &dmar_validate_one_drhd, (void *)1);
1907 if (ret)
1908 goto out;
1909
1910 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1911 &dmar_parse_one_drhd, (void *)&drhd_count);
1912 if (ret == 0 && drhd_count == 0) {
1913 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1914 goto out;
1915 } else if (ret) {
1916 goto release_drhd;
1917 }
1918
1919 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1920 &dmar_parse_one_rhsa, NULL);
1921 if (ret)
1922 goto release_drhd;
1923
1924 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1925 &dmar_parse_one_atsr, NULL);
1926 if (ret)
1927 goto release_atsr;
1928
1929 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1930 &dmar_hp_add_drhd, NULL);
1931 if (!ret)
1932 return 0;
1933
1934 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1935 &dmar_hp_remove_drhd, NULL);
1936release_atsr:
1937 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1938 &dmar_release_one_atsr, NULL);
1939release_drhd:
1940 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1941 &dmar_hp_release_drhd, NULL);
1942out:
1943 return ret;
1944}
1945
1946static int dmar_hotplug_remove(acpi_handle handle)
1947{
1948 int ret;
1949
1950 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1951 &dmar_check_one_atsr, NULL);
1952 if (ret)
1953 return ret;
1954
1955 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1956 &dmar_hp_remove_drhd, NULL);
1957 if (ret == 0) {
1958 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1959 &dmar_release_one_atsr, NULL));
1960 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1961 &dmar_hp_release_drhd, NULL));
1962 } else {
1963 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1964 &dmar_hp_add_drhd, NULL);
1965 }
1966
1967 return ret;
1968}
1969
Jiang Liud35165a2014-11-09 22:47:59 +08001970static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1971 void *context, void **retval)
1972{
1973 acpi_handle *phdl = retval;
1974
1975 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1976 *phdl = handle;
1977 return AE_CTRL_TERMINATE;
1978 }
1979
1980 return AE_OK;
1981}
1982
Jiang Liu6b197242014-11-09 22:47:58 +08001983static int dmar_device_hotplug(acpi_handle handle, bool insert)
1984{
1985 int ret;
Jiang Liud35165a2014-11-09 22:47:59 +08001986 acpi_handle tmp = NULL;
1987 acpi_status status;
Jiang Liu6b197242014-11-09 22:47:58 +08001988
1989 if (!dmar_in_use())
1990 return 0;
1991
Jiang Liud35165a2014-11-09 22:47:59 +08001992 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1993 tmp = handle;
1994 } else {
1995 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1996 ACPI_UINT32_MAX,
1997 dmar_get_dsm_handle,
1998 NULL, NULL, &tmp);
1999 if (ACPI_FAILURE(status)) {
2000 pr_warn("Failed to locate _DSM method.\n");
2001 return -ENXIO;
2002 }
2003 }
2004 if (tmp == NULL)
Jiang Liu6b197242014-11-09 22:47:58 +08002005 return 0;
2006
2007 down_write(&dmar_global_lock);
2008 if (insert)
Jiang Liud35165a2014-11-09 22:47:59 +08002009 ret = dmar_hotplug_insert(tmp);
Jiang Liu6b197242014-11-09 22:47:58 +08002010 else
Jiang Liud35165a2014-11-09 22:47:59 +08002011 ret = dmar_hotplug_remove(tmp);
Jiang Liu6b197242014-11-09 22:47:58 +08002012 up_write(&dmar_global_lock);
2013
2014 return ret;
2015}
2016
2017int dmar_device_add(acpi_handle handle)
2018{
2019 return dmar_device_hotplug(handle, true);
2020}
2021
2022int dmar_device_remove(acpi_handle handle)
2023{
2024 return dmar_device_hotplug(handle, false);
2025}