blob: cd1ba24c766ad74368707e9ace7f0afd968a6ca3 [file] [log] [blame]
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001/*
David Woodhouseea8ea462014-03-05 17:09:32 +00002 * Copyright © 2006-2014 Intel Corporation.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
David Woodhouseea8ea462014-03-05 17:09:32 +000013 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070018 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
mark gross5e0d2a62008-03-04 15:22:08 -080022#include <linux/debugfs.h>
Paul Gortmaker54485c32011-10-29 10:26:25 -040023#include <linux/export.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070024#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070027#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
Jiang Liu75f05562014-02-19 14:07:37 +080032#include <linux/memory.h>
mark gross5e0d2a62008-03-04 15:22:08 -080033#include <linux/timer.h>
Kay, Allen M38717942008-09-09 18:37:29 +030034#include <linux/iova.h>
Joerg Roedel5d450802008-12-03 14:52:32 +010035#include <linux/iommu.h>
Kay, Allen M38717942008-09-09 18:37:29 +030036#include <linux/intel-iommu.h>
Rafael J. Wysocki134fac32011-03-23 22:16:14 +010037#include <linux/syscore_ops.h>
Shane Wang69575d32009-09-01 18:25:07 -070038#include <linux/tboot.h>
Stephen Rothwelladb2fe02009-08-31 15:24:23 +100039#include <linux/dmi.h>
Joerg Roedel5cdede22011-04-04 15:55:18 +020040#include <linux/pci-ats.h>
Tejun Heo0ee332c2011-12-08 10:22:09 -080041#include <linux/memblock.h>
Akinobu Mita36746432014-06-04 16:06:51 -070042#include <linux/dma-contiguous.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070043#include <asm/irq_remapping.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070044#include <asm/cacheflush.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090045#include <asm/iommu.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070046
Joerg Roedel078e1ee2012-09-26 12:44:43 +020047#include "irq_remapping.h"
48
Fenghua Yu5b6985c2008-10-16 18:02:32 -070049#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070052#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
David Woodhousee0fc7e02009-09-30 09:12:17 -070054#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070055
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070062#define MAX_AGAW_WIDTH 64
Jiang Liu5c645b32014-01-06 14:18:12 +080063#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070064
David Woodhouse2ebe3152009-09-19 07:34:04 -070065#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070073
Mark McLoughlinf27be032008-11-20 15:49:43 +000074#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
Yang Hongyang284901a2009-04-06 19:01:15 -070075#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
Yang Hongyang6a355282009-04-06 19:01:13 -070076#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
mark gross5e0d2a62008-03-04 15:22:08 -080077
Andrew Mortondf08cdc2010-09-22 13:05:11 -070078/* page table handling */
79#define LEVEL_STRIDE (9)
80#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
Ohad Ben-Cohen6d1c56a2011-11-10 11:32:30 +020082/*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700100static inline int agaw_to_level(int agaw)
101{
102 return agaw + 2;
103}
104
105static inline int agaw_to_width(int agaw)
106{
Jiang Liu5c645b32014-01-06 14:18:12 +0800107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700108}
109
110static inline int width_to_agaw(int width)
111{
Jiang Liu5c645b32014-01-06 14:18:12 +0800112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
Andrew Mortondf08cdc2010-09-22 13:05:11 -0700113}
114
115static inline unsigned int level_to_offset_bits(int level)
116{
117 return (level - 1) * LEVEL_STRIDE;
118}
119
120static inline int pfn_level_offset(unsigned long pfn, int level)
121{
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123}
124
125static inline unsigned long level_mask(int level)
126{
127 return -1UL << level_to_offset_bits(level);
128}
129
130static inline unsigned long level_size(int level)
131{
132 return 1UL << level_to_offset_bits(level);
133}
134
135static inline unsigned long align_to_level(unsigned long pfn, int level)
136{
137 return (pfn + level_size(level) - 1) & level_mask(level);
138}
David Woodhousefd18de52009-05-10 23:57:41 +0100139
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100140static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141{
Jiang Liu5c645b32014-01-06 14:18:12 +0800142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100143}
144
David Woodhousedd4e8312009-06-27 16:21:20 +0100145/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148{
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150}
151
152static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153{
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155}
156static inline unsigned long page_to_dma_pfn(struct page *pg)
157{
158 return mm_to_dma_pfn(page_to_pfn(pg));
159}
160static inline unsigned long virt_to_dma_pfn(void *p)
161{
162 return page_to_dma_pfn(virt_to_page(p));
163}
164
Weidong Hand9630fe2008-12-08 11:06:32 +0800165/* global iommu list, set NULL for ignored DMAR units */
166static struct intel_iommu **g_iommus;
167
David Woodhousee0fc7e02009-09-30 09:12:17 -0700168static void __init check_tylersburg_isoch(void);
David Woodhouse9af88142009-02-13 23:18:03 +0000169static int rwbf_quirk;
170
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000171/*
Joseph Cihulab7792602011-05-03 00:08:37 -0700172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175static int force_on = 0;
176
177/*
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183struct root_entry {
184 u64 val;
185 u64 rsvd1;
186};
187#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188static inline bool root_present(struct root_entry *root)
189{
190 return (root->val & 1);
191}
192static inline void set_root_present(struct root_entry *root)
193{
194 root->val |= 1;
195}
196static inline void set_root_value(struct root_entry *root, unsigned long value)
197{
198 root->val |= value & VTD_PAGE_MASK;
199}
200
201static inline struct context_entry *
202get_context_addr_from_root(struct root_entry *root)
203{
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
207 NULL);
208}
209
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000210/*
211 * low 64 bits:
212 * 0: present
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
216 * high 64 bits:
217 * 0-2: address width
218 * 3-6: aval
219 * 8-23: domain id
220 */
221struct context_entry {
222 u64 lo;
223 u64 hi;
224};
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000225
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000226static inline bool context_present(struct context_entry *context)
227{
228 return (context->lo & 1);
229}
230static inline void context_set_present(struct context_entry *context)
231{
232 context->lo |= 1;
233}
234
235static inline void context_set_fault_enable(struct context_entry *context)
236{
237 context->lo &= (((u64)-1) << 2) | 1;
238}
239
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000240static inline void context_set_translation_type(struct context_entry *context,
241 unsigned long value)
242{
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
245}
246
247static inline void context_set_address_root(struct context_entry *context,
248 unsigned long value)
249{
250 context->lo |= value & VTD_PAGE_MASK;
251}
252
253static inline void context_set_address_width(struct context_entry *context,
254 unsigned long value)
255{
256 context->hi |= value & 7;
257}
258
259static inline void context_set_domain_id(struct context_entry *context,
260 unsigned long value)
261{
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
263}
264
265static inline void context_clear_entry(struct context_entry *context)
266{
267 context->lo = 0;
268 context->hi = 0;
269}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000270
Mark McLoughlin622ba122008-11-20 15:49:46 +0000271/*
272 * 0: readable
273 * 1: writable
274 * 2-6: reserved
275 * 7: super page
Sheng Yang9cf066972009-03-18 15:33:07 +0800276 * 8-10: available
277 * 11: snoop behavior
Mark McLoughlin622ba122008-11-20 15:49:46 +0000278 * 12-63: Host physcial address
279 */
280struct dma_pte {
281 u64 val;
282};
Mark McLoughlin622ba122008-11-20 15:49:46 +0000283
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000284static inline void dma_clear_pte(struct dma_pte *pte)
285{
286 pte->val = 0;
287}
288
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
David Woodhousec85994e2009-07-01 19:21:24 +0100291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
David Woodhouse1a8bd482010-08-10 01:38:53 +0100295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
David Woodhousec85994e2009-07-01 19:21:24 +0100296#endif
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000297}
298
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000299static inline bool dma_pte_present(struct dma_pte *pte)
300{
301 return (pte->val & 3) != 0;
302}
Mark McLoughlin622ba122008-11-20 15:49:46 +0000303
Allen Kay4399c8b2011-10-14 12:32:46 -0700304static inline bool dma_pte_superpage(struct dma_pte *pte)
305{
Joerg Roedelc3c75eb2014-07-04 11:19:10 +0200306 return (pte->val & DMA_PTE_LARGE_PAGE);
Allen Kay4399c8b2011-10-14 12:32:46 -0700307}
308
David Woodhouse75e6bf92009-07-02 11:21:16 +0100309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
David Woodhouse19943b02009-08-04 16:19:20 +0100320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700322
Weidong Han1ce28fe2008-12-08 16:35:39 +0800323/* domain represents a virtual machine, more than one devices
324 * across iommus may be owned in one domain, e.g. kvm guest.
325 */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800326#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
Weidong Han1ce28fe2008-12-08 16:35:39 +0800327
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700328/* si_domain contains mulitple devices */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800329#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700330
Mike Travis1b198bb2012-03-05 15:05:16 -0800331/* define the limit of IOMMUs supported in each domain */
332#ifdef CONFIG_X86
333# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
334#else
335# define IOMMU_UNITS_SUPPORTED 64
336#endif
337
Mark McLoughlin99126f72008-11-20 15:49:47 +0000338struct dmar_domain {
339 int id; /* domain id */
Suresh Siddha4c923d42009-10-02 11:01:24 -0700340 int nid; /* node id */
Mike Travis1b198bb2012-03-05 15:05:16 -0800341 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
342 /* bitmap of iommus this domain uses*/
Mark McLoughlin99126f72008-11-20 15:49:47 +0000343
344 struct list_head devices; /* all devices' list */
345 struct iova_domain iovad; /* iova's that belong to this domain */
346
347 struct dma_pte *pgd; /* virtual address */
Mark McLoughlin99126f72008-11-20 15:49:47 +0000348 int gaw; /* max guest address width */
349
350 /* adjusted guest address width, 0 is level 2 30-bit */
351 int agaw;
352
Weidong Han3b5410e2008-12-08 09:17:15 +0800353 int flags; /* flags to find out type of domain */
Weidong Han8e6040972008-12-08 15:49:06 +0800354
355 int iommu_coherency;/* indicate coherency of iommu access */
Sheng Yang58c610b2009-03-18 15:33:05 +0800356 int iommu_snooping; /* indicate snooping control feature*/
Weidong Hanc7151a82008-12-08 22:51:37 +0800357 int iommu_count; /* reference count of iommu */
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100358 int iommu_superpage;/* Level of superpages supported:
359 0 == 4KiB (no superpages), 1 == 2MiB,
360 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
Weidong Hanc7151a82008-12-08 22:51:37 +0800361 spinlock_t iommu_lock; /* protect iommu set in domain */
Weidong Hanfe40f1e2008-12-08 23:10:23 +0800362 u64 max_addr; /* maximum mapped address */
Mark McLoughlin99126f72008-11-20 15:49:47 +0000363};
364
Mark McLoughlina647dac2008-11-20 15:49:48 +0000365/* PCI domain-device relationship */
366struct device_domain_info {
367 struct list_head link; /* link to domain siblings */
368 struct list_head global; /* link to global list */
David Woodhouse276dbf992009-04-04 01:45:37 +0100369 u8 bus; /* PCI bus number */
Mark McLoughlina647dac2008-11-20 15:49:48 +0000370 u8 devfn; /* PCI devfn number */
David Woodhouse0bcb3e22014-03-06 17:12:03 +0000371 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
Yu Zhao93a23a72009-05-18 13:51:37 +0800372 struct intel_iommu *iommu; /* IOMMU used by this device */
Mark McLoughlina647dac2008-11-20 15:49:48 +0000373 struct dmar_domain *domain; /* pointer to domain */
374};
375
Jiang Liub94e4112014-02-19 14:07:25 +0800376struct dmar_rmrr_unit {
377 struct list_head list; /* list of rmrr units */
378 struct acpi_dmar_header *hdr; /* ACPI header */
379 u64 base_address; /* reserved base address*/
380 u64 end_address; /* reserved end address */
David Woodhouse832bd852014-03-07 15:08:36 +0000381 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800382 int devices_cnt; /* target device count */
383};
384
385struct dmar_atsr_unit {
386 struct list_head list; /* list of ATSR units */
387 struct acpi_dmar_header *hdr; /* ACPI header */
David Woodhouse832bd852014-03-07 15:08:36 +0000388 struct dmar_dev_scope *devices; /* target devices */
Jiang Liub94e4112014-02-19 14:07:25 +0800389 int devices_cnt; /* target device count */
390 u8 include_all:1; /* include all ports */
391};
392
393static LIST_HEAD(dmar_atsr_units);
394static LIST_HEAD(dmar_rmrr_units);
395
396#define for_each_rmrr_units(rmrr) \
397 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
398
mark gross5e0d2a62008-03-04 15:22:08 -0800399static void flush_unmaps_timeout(unsigned long data);
400
Jiang Liub707cb02014-01-06 14:18:26 +0800401static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
mark gross5e0d2a62008-03-04 15:22:08 -0800402
mark gross80b20dd2008-04-18 13:53:58 -0700403#define HIGH_WATER_MARK 250
404struct deferred_flush_tables {
405 int next;
406 struct iova *iova[HIGH_WATER_MARK];
407 struct dmar_domain *domain[HIGH_WATER_MARK];
David Woodhouseea8ea462014-03-05 17:09:32 +0000408 struct page *freelist[HIGH_WATER_MARK];
mark gross80b20dd2008-04-18 13:53:58 -0700409};
410
411static struct deferred_flush_tables *deferred_flush;
412
mark gross5e0d2a62008-03-04 15:22:08 -0800413/* bitmap for indexing intel_iommus */
mark gross5e0d2a62008-03-04 15:22:08 -0800414static int g_num_of_iommus;
415
416static DEFINE_SPINLOCK(async_umap_flush_lock);
417static LIST_HEAD(unmaps_to_do);
418
419static int timer_on;
420static long list_size;
mark gross5e0d2a62008-03-04 15:22:08 -0800421
Jiang Liu92d03cc2014-02-19 14:07:28 +0800422static void domain_exit(struct dmar_domain *domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700423static void domain_remove_dev_info(struct dmar_domain *domain);
Jiang Liub94e4112014-02-19 14:07:25 +0800424static void domain_remove_one_dev_info(struct dmar_domain *domain,
David Woodhousebf9c9ed2014-03-09 16:19:13 -0700425 struct device *dev);
Jiang Liu92d03cc2014-02-19 14:07:28 +0800426static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
David Woodhouse0bcb3e22014-03-06 17:12:03 +0000427 struct device *dev);
Jiang Liu2a46ddf2014-07-11 14:19:30 +0800428static int domain_detach_iommu(struct dmar_domain *domain,
429 struct intel_iommu *iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700430
Suresh Siddhad3f13812011-08-23 17:05:25 -0700431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800432int dmar_disabled = 0;
433#else
434int dmar_disabled = 1;
Suresh Siddhad3f13812011-08-23 17:05:25 -0700435#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800436
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -0200437int intel_iommu_enabled = 0;
438EXPORT_SYMBOL_GPL(intel_iommu_enabled);
439
David Woodhouse2d9e6672010-06-15 10:57:57 +0100440static int dmar_map_gfx = 1;
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700441static int dmar_forcedac;
mark gross5e0d2a62008-03-04 15:22:08 -0800442static int intel_iommu_strict;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100443static int intel_iommu_superpage = 1;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700444
David Woodhousec0771df2011-10-14 20:59:46 +0100445int intel_iommu_gfx_mapped;
446EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
447
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700448#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
449static DEFINE_SPINLOCK(device_domain_lock);
450static LIST_HEAD(device_domain_list);
451
Thierry Redingb22f6432014-06-27 09:03:12 +0200452static const struct iommu_ops intel_iommu_ops;
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +0100453
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700454static int __init intel_iommu_setup(char *str)
455{
456 if (!str)
457 return -EINVAL;
458 while (*str) {
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800459 if (!strncmp(str, "on", 2)) {
460 dmar_disabled = 0;
461 printk(KERN_INFO "Intel-IOMMU: enabled\n");
462 } else if (!strncmp(str, "off", 3)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700463 dmar_disabled = 1;
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800464 printk(KERN_INFO "Intel-IOMMU: disabled\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700465 } else if (!strncmp(str, "igfx_off", 8)) {
466 dmar_map_gfx = 0;
467 printk(KERN_INFO
468 "Intel-IOMMU: disable GFX device mapping\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700469 } else if (!strncmp(str, "forcedac", 8)) {
mark gross5e0d2a62008-03-04 15:22:08 -0800470 printk(KERN_INFO
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700471 "Intel-IOMMU: Forcing DAC for PCI devices\n");
472 dmar_forcedac = 1;
mark gross5e0d2a62008-03-04 15:22:08 -0800473 } else if (!strncmp(str, "strict", 6)) {
474 printk(KERN_INFO
475 "Intel-IOMMU: disable batched IOTLB flush\n");
476 intel_iommu_strict = 1;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100477 } else if (!strncmp(str, "sp_off", 6)) {
478 printk(KERN_INFO
479 "Intel-IOMMU: disable supported super page\n");
480 intel_iommu_superpage = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700481 }
482
483 str += strcspn(str, ",");
484 while (*str == ',')
485 str++;
486 }
487 return 0;
488}
489__setup("intel_iommu=", intel_iommu_setup);
490
491static struct kmem_cache *iommu_domain_cache;
492static struct kmem_cache *iommu_devinfo_cache;
493static struct kmem_cache *iommu_iova_cache;
494
Suresh Siddha4c923d42009-10-02 11:01:24 -0700495static inline void *alloc_pgtable_page(int node)
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700496{
Suresh Siddha4c923d42009-10-02 11:01:24 -0700497 struct page *page;
498 void *vaddr = NULL;
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700499
Suresh Siddha4c923d42009-10-02 11:01:24 -0700500 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
501 if (page)
502 vaddr = page_address(page);
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700503 return vaddr;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700504}
505
506static inline void free_pgtable_page(void *vaddr)
507{
508 free_page((unsigned long)vaddr);
509}
510
511static inline void *alloc_domain_mem(void)
512{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900513 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700514}
515
Kay, Allen M38717942008-09-09 18:37:29 +0300516static void free_domain_mem(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700517{
518 kmem_cache_free(iommu_domain_cache, vaddr);
519}
520
521static inline void * alloc_devinfo_mem(void)
522{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900523 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700524}
525
526static inline void free_devinfo_mem(void *vaddr)
527{
528 kmem_cache_free(iommu_devinfo_cache, vaddr);
529}
530
531struct iova *alloc_iova_mem(void)
532{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900533 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700534}
535
536void free_iova_mem(struct iova *iova)
537{
538 kmem_cache_free(iommu_iova_cache, iova);
539}
540
Jiang Liuab8dfe22014-07-11 14:19:27 +0800541static inline int domain_type_is_vm(struct dmar_domain *domain)
542{
543 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
544}
545
546static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
547{
548 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
549 DOMAIN_FLAG_STATIC_IDENTITY);
550}
Weidong Han1b573682008-12-08 15:34:06 +0800551
Jiang Liu162d1b12014-07-11 14:19:35 +0800552static inline int domain_pfn_supported(struct dmar_domain *domain,
553 unsigned long pfn)
554{
555 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
556
557 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
558}
559
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700560static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
Weidong Han1b573682008-12-08 15:34:06 +0800561{
562 unsigned long sagaw;
563 int agaw = -1;
564
565 sagaw = cap_sagaw(iommu->cap);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700566 for (agaw = width_to_agaw(max_gaw);
Weidong Han1b573682008-12-08 15:34:06 +0800567 agaw >= 0; agaw--) {
568 if (test_bit(agaw, &sagaw))
569 break;
570 }
571
572 return agaw;
573}
574
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700575/*
576 * Calculate max SAGAW for each iommu.
577 */
578int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
579{
580 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
581}
582
583/*
584 * calculate agaw for each iommu.
585 * "SAGAW" may be different across iommus, use a default agaw, and
586 * get a supported less agaw for iommus that don't support the default agaw.
587 */
588int iommu_calculate_agaw(struct intel_iommu *iommu)
589{
590 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
591}
592
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700593/* This functionin only returns single iommu in a domain */
Weidong Han8c11e792008-12-08 15:29:22 +0800594static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
595{
596 int iommu_id;
597
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700598 /* si_domain and vm domain should not get here. */
Jiang Liuab8dfe22014-07-11 14:19:27 +0800599 BUG_ON(domain_type_is_vm_or_si(domain));
Mike Travis1b198bb2012-03-05 15:05:16 -0800600 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
Weidong Han8c11e792008-12-08 15:29:22 +0800601 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
602 return NULL;
603
604 return g_iommus[iommu_id];
605}
606
Weidong Han8e6040972008-12-08 15:49:06 +0800607static void domain_update_iommu_coherency(struct dmar_domain *domain)
608{
David Woodhoused0501962014-03-11 17:10:29 -0700609 struct dmar_drhd_unit *drhd;
610 struct intel_iommu *iommu;
611 int i, found = 0;
Weidong Han8e6040972008-12-08 15:49:06 +0800612
David Woodhoused0501962014-03-11 17:10:29 -0700613 domain->iommu_coherency = 1;
Weidong Han8e6040972008-12-08 15:49:06 +0800614
Mike Travis1b198bb2012-03-05 15:05:16 -0800615 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
David Woodhoused0501962014-03-11 17:10:29 -0700616 found = 1;
Weidong Han8e6040972008-12-08 15:49:06 +0800617 if (!ecap_coherent(g_iommus[i]->ecap)) {
618 domain->iommu_coherency = 0;
619 break;
620 }
Weidong Han8e6040972008-12-08 15:49:06 +0800621 }
David Woodhoused0501962014-03-11 17:10:29 -0700622 if (found)
623 return;
624
625 /* No hardware attached; use lowest common denominator */
626 rcu_read_lock();
627 for_each_active_iommu(iommu, drhd) {
628 if (!ecap_coherent(iommu->ecap)) {
629 domain->iommu_coherency = 0;
630 break;
631 }
632 }
633 rcu_read_unlock();
Weidong Han8e6040972008-12-08 15:49:06 +0800634}
635
Sheng Yang58c610b2009-03-18 15:33:05 +0800636static void domain_update_iommu_snooping(struct dmar_domain *domain)
637{
638 int i;
639
640 domain->iommu_snooping = 1;
641
Mike Travis1b198bb2012-03-05 15:05:16 -0800642 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
Sheng Yang58c610b2009-03-18 15:33:05 +0800643 if (!ecap_sc_support(g_iommus[i]->ecap)) {
644 domain->iommu_snooping = 0;
645 break;
646 }
Sheng Yang58c610b2009-03-18 15:33:05 +0800647 }
648}
649
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100650static void domain_update_iommu_superpage(struct dmar_domain *domain)
651{
Allen Kay8140a952011-10-14 12:32:17 -0700652 struct dmar_drhd_unit *drhd;
653 struct intel_iommu *iommu = NULL;
654 int mask = 0xf;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100655
656 if (!intel_iommu_superpage) {
657 domain->iommu_superpage = 0;
658 return;
659 }
660
Allen Kay8140a952011-10-14 12:32:17 -0700661 /* set iommu_superpage to the smallest common denominator */
Jiang Liu0e242612014-02-19 14:07:34 +0800662 rcu_read_lock();
Allen Kay8140a952011-10-14 12:32:17 -0700663 for_each_active_iommu(iommu, drhd) {
664 mask &= cap_super_page_val(iommu->cap);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100665 if (!mask) {
666 break;
667 }
668 }
Jiang Liu0e242612014-02-19 14:07:34 +0800669 rcu_read_unlock();
670
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100671 domain->iommu_superpage = fls(mask);
672}
673
Sheng Yang58c610b2009-03-18 15:33:05 +0800674/* Some capabilities may be different across iommus */
675static void domain_update_iommu_cap(struct dmar_domain *domain)
676{
677 domain_update_iommu_coherency(domain);
678 domain_update_iommu_snooping(domain);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100679 domain_update_iommu_superpage(domain);
Sheng Yang58c610b2009-03-18 15:33:05 +0800680}
681
David Woodhouse156baca2014-03-09 14:00:57 -0700682static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
Weidong Hanc7151a82008-12-08 22:51:37 +0800683{
684 struct dmar_drhd_unit *drhd = NULL;
Jiang Liub683b232014-02-19 14:07:32 +0800685 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -0700686 struct device *tmp;
687 struct pci_dev *ptmp, *pdev = NULL;
Yijing Wangaa4d0662014-05-26 20:14:06 +0800688 u16 segment = 0;
Weidong Hanc7151a82008-12-08 22:51:37 +0800689 int i;
690
David Woodhouse156baca2014-03-09 14:00:57 -0700691 if (dev_is_pci(dev)) {
692 pdev = to_pci_dev(dev);
693 segment = pci_domain_nr(pdev->bus);
694 } else if (ACPI_COMPANION(dev))
695 dev = &ACPI_COMPANION(dev)->dev;
696
Jiang Liu0e242612014-02-19 14:07:34 +0800697 rcu_read_lock();
Jiang Liub683b232014-02-19 14:07:32 +0800698 for_each_active_iommu(iommu, drhd) {
David Woodhouse156baca2014-03-09 14:00:57 -0700699 if (pdev && segment != drhd->segment)
David Woodhouse276dbf992009-04-04 01:45:37 +0100700 continue;
Weidong Hanc7151a82008-12-08 22:51:37 +0800701
Jiang Liub683b232014-02-19 14:07:32 +0800702 for_each_active_dev_scope(drhd->devices,
David Woodhouse156baca2014-03-09 14:00:57 -0700703 drhd->devices_cnt, i, tmp) {
704 if (tmp == dev) {
705 *bus = drhd->devices[i].bus;
706 *devfn = drhd->devices[i].devfn;
707 goto out;
708 }
709
710 if (!pdev || !dev_is_pci(tmp))
David Woodhouse832bd852014-03-07 15:08:36 +0000711 continue;
David Woodhouse156baca2014-03-09 14:00:57 -0700712
713 ptmp = to_pci_dev(tmp);
714 if (ptmp->subordinate &&
715 ptmp->subordinate->number <= pdev->bus->number &&
716 ptmp->subordinate->busn_res.end >= pdev->bus->number)
717 goto got_pdev;
David Woodhouse924b6232009-04-04 00:39:25 +0100718 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800719
David Woodhouse156baca2014-03-09 14:00:57 -0700720 if (pdev && drhd->include_all) {
721 got_pdev:
722 *bus = pdev->bus->number;
723 *devfn = pdev->devfn;
Jiang Liub683b232014-02-19 14:07:32 +0800724 goto out;
David Woodhouse156baca2014-03-09 14:00:57 -0700725 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800726 }
Jiang Liub683b232014-02-19 14:07:32 +0800727 iommu = NULL;
David Woodhouse156baca2014-03-09 14:00:57 -0700728 out:
Jiang Liu0e242612014-02-19 14:07:34 +0800729 rcu_read_unlock();
Weidong Hanc7151a82008-12-08 22:51:37 +0800730
Jiang Liub683b232014-02-19 14:07:32 +0800731 return iommu;
Weidong Hanc7151a82008-12-08 22:51:37 +0800732}
733
Weidong Han5331fe62008-12-08 23:00:00 +0800734static void domain_flush_cache(struct dmar_domain *domain,
735 void *addr, int size)
736{
737 if (!domain->iommu_coherency)
738 clflush_cache_range(addr, size);
739}
740
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700741/* Gets context entry for a given bus and devfn */
742static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
743 u8 bus, u8 devfn)
744{
745 struct root_entry *root;
746 struct context_entry *context;
747 unsigned long phy_addr;
748 unsigned long flags;
749
750 spin_lock_irqsave(&iommu->lock, flags);
751 root = &iommu->root_entry[bus];
752 context = get_context_addr_from_root(root);
753 if (!context) {
Suresh Siddha4c923d42009-10-02 11:01:24 -0700754 context = (struct context_entry *)
755 alloc_pgtable_page(iommu->node);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700756 if (!context) {
757 spin_unlock_irqrestore(&iommu->lock, flags);
758 return NULL;
759 }
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700760 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700761 phy_addr = virt_to_phys((void *)context);
762 set_root_value(root, phy_addr);
763 set_root_present(root);
764 __iommu_flush_cache(iommu, root, sizeof(*root));
765 }
766 spin_unlock_irqrestore(&iommu->lock, flags);
767 return &context[devfn];
768}
769
770static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
771{
772 struct root_entry *root;
773 struct context_entry *context;
774 int ret;
775 unsigned long flags;
776
777 spin_lock_irqsave(&iommu->lock, flags);
778 root = &iommu->root_entry[bus];
779 context = get_context_addr_from_root(root);
780 if (!context) {
781 ret = 0;
782 goto out;
783 }
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000784 ret = context_present(&context[devfn]);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700785out:
786 spin_unlock_irqrestore(&iommu->lock, flags);
787 return ret;
788}
789
790static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
791{
792 struct root_entry *root;
793 struct context_entry *context;
794 unsigned long flags;
795
796 spin_lock_irqsave(&iommu->lock, flags);
797 root = &iommu->root_entry[bus];
798 context = get_context_addr_from_root(root);
799 if (context) {
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000800 context_clear_entry(&context[devfn]);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700801 __iommu_flush_cache(iommu, &context[devfn], \
802 sizeof(*context));
803 }
804 spin_unlock_irqrestore(&iommu->lock, flags);
805}
806
807static void free_context_table(struct intel_iommu *iommu)
808{
809 struct root_entry *root;
810 int i;
811 unsigned long flags;
812 struct context_entry *context;
813
814 spin_lock_irqsave(&iommu->lock, flags);
815 if (!iommu->root_entry) {
816 goto out;
817 }
818 for (i = 0; i < ROOT_ENTRY_NR; i++) {
819 root = &iommu->root_entry[i];
820 context = get_context_addr_from_root(root);
821 if (context)
822 free_pgtable_page(context);
823 }
824 free_pgtable_page(iommu->root_entry);
825 iommu->root_entry = NULL;
826out:
827 spin_unlock_irqrestore(&iommu->lock, flags);
828}
829
David Woodhouseb026fd22009-06-28 10:37:25 +0100830static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
David Woodhouse5cf0a762014-03-19 16:07:49 +0000831 unsigned long pfn, int *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700832{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700833 struct dma_pte *parent, *pte = NULL;
834 int level = agaw_to_level(domain->agaw);
Allen Kay4399c8b2011-10-14 12:32:46 -0700835 int offset;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700836
837 BUG_ON(!domain->pgd);
Julian Stecklinaf9423602013-10-09 10:03:52 +0200838
Jiang Liu162d1b12014-07-11 14:19:35 +0800839 if (!domain_pfn_supported(domain, pfn))
Julian Stecklinaf9423602013-10-09 10:03:52 +0200840 /* Address beyond IOMMU's addressing capabilities. */
841 return NULL;
842
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700843 parent = domain->pgd;
844
David Woodhouse5cf0a762014-03-19 16:07:49 +0000845 while (1) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700846 void *tmp_page;
847
David Woodhouseb026fd22009-06-28 10:37:25 +0100848 offset = pfn_level_offset(pfn, level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700849 pte = &parent[offset];
David Woodhouse5cf0a762014-03-19 16:07:49 +0000850 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100851 break;
David Woodhouse5cf0a762014-03-19 16:07:49 +0000852 if (level == *target_level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700853 break;
854
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000855 if (!dma_pte_present(pte)) {
David Woodhousec85994e2009-07-01 19:21:24 +0100856 uint64_t pteval;
857
Suresh Siddha4c923d42009-10-02 11:01:24 -0700858 tmp_page = alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700859
David Woodhouse206a73c12009-07-01 19:30:28 +0100860 if (!tmp_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700861 return NULL;
David Woodhouse206a73c12009-07-01 19:30:28 +0100862
David Woodhousec85994e2009-07-01 19:21:24 +0100863 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
Benjamin LaHaise64de5af2009-09-16 21:05:55 -0400864 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
Yijing Wangeffad4b2014-05-26 20:13:47 +0800865 if (cmpxchg64(&pte->val, 0ULL, pteval))
David Woodhousec85994e2009-07-01 19:21:24 +0100866 /* Someone else set it while we were thinking; use theirs. */
867 free_pgtable_page(tmp_page);
Yijing Wangeffad4b2014-05-26 20:13:47 +0800868 else
David Woodhousec85994e2009-07-01 19:21:24 +0100869 domain_flush_cache(domain, pte, sizeof(*pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700870 }
David Woodhouse5cf0a762014-03-19 16:07:49 +0000871 if (level == 1)
872 break;
873
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000874 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700875 level--;
876 }
877
David Woodhouse5cf0a762014-03-19 16:07:49 +0000878 if (!*target_level)
879 *target_level = level;
880
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700881 return pte;
882}
883
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100884
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700885/* return address's pte at specific level */
David Woodhouse90dcfb52009-06-27 17:14:59 +0100886static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
887 unsigned long pfn,
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100888 int level, int *large_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700889{
890 struct dma_pte *parent, *pte = NULL;
891 int total = agaw_to_level(domain->agaw);
892 int offset;
893
894 parent = domain->pgd;
895 while (level <= total) {
David Woodhouse90dcfb52009-06-27 17:14:59 +0100896 offset = pfn_level_offset(pfn, total);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700897 pte = &parent[offset];
898 if (level == total)
899 return pte;
900
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100901 if (!dma_pte_present(pte)) {
902 *large_page = total;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700903 break;
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100904 }
905
Yijing Wange16922a2014-05-20 20:37:51 +0800906 if (dma_pte_superpage(pte)) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100907 *large_page = total;
908 return pte;
909 }
910
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000911 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700912 total--;
913 }
914 return NULL;
915}
916
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700917/* clear last level pte, a tlb flush should be followed */
David Woodhouse5cf0a762014-03-19 16:07:49 +0000918static void dma_pte_clear_range(struct dmar_domain *domain,
David Woodhouse595badf2009-06-27 22:09:11 +0100919 unsigned long start_pfn,
920 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700921{
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100922 unsigned int large_page = 1;
David Woodhouse310a5ab2009-06-28 18:52:20 +0100923 struct dma_pte *first_pte, *pte;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700924
Jiang Liu162d1b12014-07-11 14:19:35 +0800925 BUG_ON(!domain_pfn_supported(domain, start_pfn));
926 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -0700927 BUG_ON(start_pfn > last_pfn);
David Woodhouse66eae842009-06-27 19:00:32 +0100928
David Woodhouse04b18e62009-06-27 19:15:01 +0100929 /* we don't need lock here; nobody else touches the iova range */
David Woodhouse59c36282009-09-19 07:36:28 -0700930 do {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100931 large_page = 1;
932 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100933 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100934 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100935 continue;
936 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100937 do {
David Woodhouse310a5ab2009-06-28 18:52:20 +0100938 dma_clear_pte(pte);
Youquan Song6dd9a7c2011-05-25 19:13:49 +0100939 start_pfn += lvl_to_nr_pages(large_page);
David Woodhouse310a5ab2009-06-28 18:52:20 +0100940 pte++;
David Woodhouse75e6bf92009-07-02 11:21:16 +0100941 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
942
David Woodhouse310a5ab2009-06-28 18:52:20 +0100943 domain_flush_cache(domain, first_pte,
944 (void *)pte - (void *)first_pte);
David Woodhouse59c36282009-09-19 07:36:28 -0700945
946 } while (start_pfn && start_pfn <= last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700947}
948
Alex Williamson3269ee02013-06-15 10:27:19 -0600949static void dma_pte_free_level(struct dmar_domain *domain, int level,
950 struct dma_pte *pte, unsigned long pfn,
951 unsigned long start_pfn, unsigned long last_pfn)
952{
953 pfn = max(start_pfn, pfn);
954 pte = &pte[pfn_level_offset(pfn, level)];
955
956 do {
957 unsigned long level_pfn;
958 struct dma_pte *level_pte;
959
960 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
961 goto next;
962
963 level_pfn = pfn & level_mask(level - 1);
964 level_pte = phys_to_virt(dma_pte_addr(pte));
965
966 if (level > 2)
967 dma_pte_free_level(domain, level - 1, level_pte,
968 level_pfn, start_pfn, last_pfn);
969
970 /* If range covers entire pagetable, free it */
971 if (!(start_pfn > level_pfn ||
Alex Williamson08336fd2014-01-21 15:48:18 -0800972 last_pfn < level_pfn + level_size(level) - 1)) {
Alex Williamson3269ee02013-06-15 10:27:19 -0600973 dma_clear_pte(pte);
974 domain_flush_cache(domain, pte, sizeof(*pte));
975 free_pgtable_page(level_pte);
976 }
977next:
978 pfn += level_size(level);
979 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
980}
981
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700982/* free page table pages. last level pte should already be cleared */
983static void dma_pte_free_pagetable(struct dmar_domain *domain,
David Woodhoused794dc92009-06-28 00:27:49 +0100984 unsigned long start_pfn,
985 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700986{
Jiang Liu162d1b12014-07-11 14:19:35 +0800987 BUG_ON(!domain_pfn_supported(domain, start_pfn));
988 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouse59c36282009-09-19 07:36:28 -0700989 BUG_ON(start_pfn > last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700990
Jiang Liud41a4ad2014-07-11 14:19:34 +0800991 dma_pte_clear_range(domain, start_pfn, last_pfn);
992
David Woodhousef3a0a522009-06-30 03:40:07 +0100993 /* We don't need lock here; nobody else touches the iova range */
Alex Williamson3269ee02013-06-15 10:27:19 -0600994 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
995 domain->pgd, 0, start_pfn, last_pfn);
David Woodhouse6660c632009-06-27 22:41:00 +0100996
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700997 /* free pgd */
David Woodhoused794dc92009-06-28 00:27:49 +0100998 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700999 free_pgtable_page(domain->pgd);
1000 domain->pgd = NULL;
1001 }
1002}
1003
David Woodhouseea8ea462014-03-05 17:09:32 +00001004/* When a page at a given level is being unlinked from its parent, we don't
1005 need to *modify* it at all. All we need to do is make a list of all the
1006 pages which can be freed just as soon as we've flushed the IOTLB and we
1007 know the hardware page-walk will no longer touch them.
1008 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1009 be freed. */
1010static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1011 int level, struct dma_pte *pte,
1012 struct page *freelist)
1013{
1014 struct page *pg;
1015
1016 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1017 pg->freelist = freelist;
1018 freelist = pg;
1019
1020 if (level == 1)
1021 return freelist;
1022
Jiang Liuadeb2592014-04-09 10:20:39 +08001023 pte = page_address(pg);
1024 do {
David Woodhouseea8ea462014-03-05 17:09:32 +00001025 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1026 freelist = dma_pte_list_pagetables(domain, level - 1,
1027 pte, freelist);
Jiang Liuadeb2592014-04-09 10:20:39 +08001028 pte++;
1029 } while (!first_pte_in_page(pte));
David Woodhouseea8ea462014-03-05 17:09:32 +00001030
1031 return freelist;
1032}
1033
1034static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1035 struct dma_pte *pte, unsigned long pfn,
1036 unsigned long start_pfn,
1037 unsigned long last_pfn,
1038 struct page *freelist)
1039{
1040 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1041
1042 pfn = max(start_pfn, pfn);
1043 pte = &pte[pfn_level_offset(pfn, level)];
1044
1045 do {
1046 unsigned long level_pfn;
1047
1048 if (!dma_pte_present(pte))
1049 goto next;
1050
1051 level_pfn = pfn & level_mask(level);
1052
1053 /* If range covers entire pagetable, free it */
1054 if (start_pfn <= level_pfn &&
1055 last_pfn >= level_pfn + level_size(level) - 1) {
1056 /* These suborbinate page tables are going away entirely. Don't
1057 bother to clear them; we're just going to *free* them. */
1058 if (level > 1 && !dma_pte_superpage(pte))
1059 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1060
1061 dma_clear_pte(pte);
1062 if (!first_pte)
1063 first_pte = pte;
1064 last_pte = pte;
1065 } else if (level > 1) {
1066 /* Recurse down into a level that isn't *entirely* obsolete */
1067 freelist = dma_pte_clear_level(domain, level - 1,
1068 phys_to_virt(dma_pte_addr(pte)),
1069 level_pfn, start_pfn, last_pfn,
1070 freelist);
1071 }
1072next:
1073 pfn += level_size(level);
1074 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1075
1076 if (first_pte)
1077 domain_flush_cache(domain, first_pte,
1078 (void *)++last_pte - (void *)first_pte);
1079
1080 return freelist;
1081}
1082
1083/* We can't just free the pages because the IOMMU may still be walking
1084 the page tables, and may have cached the intermediate levels. The
1085 pages can only be freed after the IOTLB flush has been done. */
1086struct page *domain_unmap(struct dmar_domain *domain,
1087 unsigned long start_pfn,
1088 unsigned long last_pfn)
1089{
David Woodhouseea8ea462014-03-05 17:09:32 +00001090 struct page *freelist = NULL;
1091
Jiang Liu162d1b12014-07-11 14:19:35 +08001092 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1093 BUG_ON(!domain_pfn_supported(domain, last_pfn));
David Woodhouseea8ea462014-03-05 17:09:32 +00001094 BUG_ON(start_pfn > last_pfn);
1095
1096 /* we don't need lock here; nobody else touches the iova range */
1097 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1098 domain->pgd, 0, start_pfn, last_pfn, NULL);
1099
1100 /* free pgd */
1101 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1102 struct page *pgd_page = virt_to_page(domain->pgd);
1103 pgd_page->freelist = freelist;
1104 freelist = pgd_page;
1105
1106 domain->pgd = NULL;
1107 }
1108
1109 return freelist;
1110}
1111
1112void dma_free_pagelist(struct page *freelist)
1113{
1114 struct page *pg;
1115
1116 while ((pg = freelist)) {
1117 freelist = pg->freelist;
1118 free_pgtable_page(page_address(pg));
1119 }
1120}
1121
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001122/* iommu handling */
1123static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1124{
1125 struct root_entry *root;
1126 unsigned long flags;
1127
Suresh Siddha4c923d42009-10-02 11:01:24 -07001128 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001129 if (!root)
1130 return -ENOMEM;
1131
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001132 __iommu_flush_cache(iommu, root, ROOT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001133
1134 spin_lock_irqsave(&iommu->lock, flags);
1135 iommu->root_entry = root;
1136 spin_unlock_irqrestore(&iommu->lock, flags);
1137
1138 return 0;
1139}
1140
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001141static void iommu_set_root_entry(struct intel_iommu *iommu)
1142{
1143 void *addr;
David Woodhousec416daa2009-05-10 20:30:58 +01001144 u32 sts;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001145 unsigned long flag;
1146
1147 addr = iommu->root_entry;
1148
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001149 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001150 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1151
David Woodhousec416daa2009-05-10 20:30:58 +01001152 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001153
1154 /* Make sure hardware complete it */
1155 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001156 readl, (sts & DMA_GSTS_RTPS), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001157
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001158 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001159}
1160
1161static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1162{
1163 u32 val;
1164 unsigned long flag;
1165
David Woodhouse9af88142009-02-13 23:18:03 +00001166 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001167 return;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001168
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001169 raw_spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse462b60f2009-05-10 20:18:18 +01001170 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001171
1172 /* Make sure hardware complete it */
1173 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001174 readl, (!(val & DMA_GSTS_WBFS)), val);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001175
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001176 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001177}
1178
1179/* return value determine if we need a write buffer flush */
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001180static void __iommu_flush_context(struct intel_iommu *iommu,
1181 u16 did, u16 source_id, u8 function_mask,
1182 u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001183{
1184 u64 val = 0;
1185 unsigned long flag;
1186
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001187 switch (type) {
1188 case DMA_CCMD_GLOBAL_INVL:
1189 val = DMA_CCMD_GLOBAL_INVL;
1190 break;
1191 case DMA_CCMD_DOMAIN_INVL:
1192 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1193 break;
1194 case DMA_CCMD_DEVICE_INVL:
1195 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1196 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1197 break;
1198 default:
1199 BUG();
1200 }
1201 val |= DMA_CCMD_ICC;
1202
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001203 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001204 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1205
1206 /* Make sure hardware complete it */
1207 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1208 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1209
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001210 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001211}
1212
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001213/* return value determine if we need a write buffer flush */
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001214static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1215 u64 addr, unsigned int size_order, u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001216{
1217 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1218 u64 val = 0, val_iva = 0;
1219 unsigned long flag;
1220
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001221 switch (type) {
1222 case DMA_TLB_GLOBAL_FLUSH:
1223 /* global flush doesn't need set IVA_REG */
1224 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1225 break;
1226 case DMA_TLB_DSI_FLUSH:
1227 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1228 break;
1229 case DMA_TLB_PSI_FLUSH:
1230 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
David Woodhouseea8ea462014-03-05 17:09:32 +00001231 /* IH bit is passed in as part of address */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001232 val_iva = size_order | addr;
1233 break;
1234 default:
1235 BUG();
1236 }
1237 /* Note: set drain read/write */
1238#if 0
1239 /*
1240 * This is probably to be super secure.. Looks like we can
1241 * ignore it without any impact.
1242 */
1243 if (cap_read_drain(iommu->cap))
1244 val |= DMA_TLB_READ_DRAIN;
1245#endif
1246 if (cap_write_drain(iommu->cap))
1247 val |= DMA_TLB_WRITE_DRAIN;
1248
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001249 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001250 /* Note: Only uses first TLB reg currently */
1251 if (val_iva)
1252 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1253 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1254
1255 /* Make sure hardware complete it */
1256 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1257 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1258
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001259 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001260
1261 /* check IOTLB invalidation granularity */
1262 if (DMA_TLB_IAIG(val) == 0)
1263 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1264 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1265 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001266 (unsigned long long)DMA_TLB_IIRG(type),
1267 (unsigned long long)DMA_TLB_IAIG(val));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001268}
1269
David Woodhouse64ae8922014-03-09 12:52:30 -07001270static struct device_domain_info *
1271iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1272 u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001273{
Yu Zhao93a23a72009-05-18 13:51:37 +08001274 int found = 0;
1275 unsigned long flags;
1276 struct device_domain_info *info;
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001277 struct pci_dev *pdev;
Yu Zhao93a23a72009-05-18 13:51:37 +08001278
1279 if (!ecap_dev_iotlb_support(iommu->ecap))
1280 return NULL;
1281
1282 if (!iommu->qi)
1283 return NULL;
1284
1285 spin_lock_irqsave(&device_domain_lock, flags);
1286 list_for_each_entry(info, &domain->devices, link)
Jiang Liuc3b497c2014-07-11 14:19:25 +08001287 if (info->iommu == iommu && info->bus == bus &&
1288 info->devfn == devfn) {
Yu Zhao93a23a72009-05-18 13:51:37 +08001289 found = 1;
1290 break;
1291 }
1292 spin_unlock_irqrestore(&device_domain_lock, flags);
1293
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001294 if (!found || !info->dev || !dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001295 return NULL;
1296
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001297 pdev = to_pci_dev(info->dev);
1298
1299 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
Yu Zhao93a23a72009-05-18 13:51:37 +08001300 return NULL;
1301
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001302 if (!dmar_find_matched_atsr_unit(pdev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001303 return NULL;
1304
Yu Zhao93a23a72009-05-18 13:51:37 +08001305 return info;
1306}
1307
1308static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1309{
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001310 if (!info || !dev_is_pci(info->dev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001311 return;
1312
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001313 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
Yu Zhao93a23a72009-05-18 13:51:37 +08001314}
1315
1316static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1317{
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001318 if (!info->dev || !dev_is_pci(info->dev) ||
1319 !pci_ats_enabled(to_pci_dev(info->dev)))
Yu Zhao93a23a72009-05-18 13:51:37 +08001320 return;
1321
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001322 pci_disable_ats(to_pci_dev(info->dev));
Yu Zhao93a23a72009-05-18 13:51:37 +08001323}
1324
1325static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1326 u64 addr, unsigned mask)
1327{
1328 u16 sid, qdep;
1329 unsigned long flags;
1330 struct device_domain_info *info;
1331
1332 spin_lock_irqsave(&device_domain_lock, flags);
1333 list_for_each_entry(info, &domain->devices, link) {
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001334 struct pci_dev *pdev;
1335 if (!info->dev || !dev_is_pci(info->dev))
1336 continue;
1337
1338 pdev = to_pci_dev(info->dev);
1339 if (!pci_ats_enabled(pdev))
Yu Zhao93a23a72009-05-18 13:51:37 +08001340 continue;
1341
1342 sid = info->bus << 8 | info->devfn;
David Woodhouse0bcb3e22014-03-06 17:12:03 +00001343 qdep = pci_ats_queue_depth(pdev);
Yu Zhao93a23a72009-05-18 13:51:37 +08001344 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1345 }
1346 spin_unlock_irqrestore(&device_domain_lock, flags);
1347}
1348
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001349static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
David Woodhouseea8ea462014-03-05 17:09:32 +00001350 unsigned long pfn, unsigned int pages, int ih, int map)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001351{
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001352 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
David Woodhouse03d6a242009-06-28 15:33:46 +01001353 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001354
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001355 BUG_ON(pages == 0);
1356
David Woodhouseea8ea462014-03-05 17:09:32 +00001357 if (ih)
1358 ih = 1 << 6;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001359 /*
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001360 * Fallback to domain selective flush if no PSI support or the size is
1361 * too big.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001362 * PSI requires page size to be 2 ^ x, and the base address is naturally
1363 * aligned to the size
1364 */
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001365 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1366 iommu->flush.flush_iotlb(iommu, did, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001367 DMA_TLB_DSI_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001368 else
David Woodhouseea8ea462014-03-05 17:09:32 +00001369 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001370 DMA_TLB_PSI_FLUSH);
Yu Zhaobf92df32009-06-29 11:31:45 +08001371
1372 /*
Nadav Amit82653632010-04-01 13:24:40 +03001373 * In caching mode, changes of pages from non-present to present require
1374 * flush. However, device IOTLB doesn't need to be flushed in this case.
Yu Zhaobf92df32009-06-29 11:31:45 +08001375 */
Nadav Amit82653632010-04-01 13:24:40 +03001376 if (!cap_caching_mode(iommu->cap) || !map)
Yu Zhao93a23a72009-05-18 13:51:37 +08001377 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001378}
1379
mark grossf8bab732008-02-08 04:18:38 -08001380static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1381{
1382 u32 pmen;
1383 unsigned long flags;
1384
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001385 raw_spin_lock_irqsave(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001386 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1387 pmen &= ~DMA_PMEN_EPM;
1388 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1389
1390 /* wait for the protected region status bit to clear */
1391 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1392 readl, !(pmen & DMA_PMEN_PRS), pmen);
1393
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001394 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
mark grossf8bab732008-02-08 04:18:38 -08001395}
1396
Jiang Liu2a41cce2014-07-11 14:19:33 +08001397static void iommu_enable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001398{
1399 u32 sts;
1400 unsigned long flags;
1401
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001402 raw_spin_lock_irqsave(&iommu->register_lock, flags);
David Woodhousec416daa2009-05-10 20:30:58 +01001403 iommu->gcmd |= DMA_GCMD_TE;
1404 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001405
1406 /* Make sure hardware complete it */
1407 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001408 readl, (sts & DMA_GSTS_TES), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001409
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001410 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001411}
1412
Jiang Liu2a41cce2014-07-11 14:19:33 +08001413static void iommu_disable_translation(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001414{
1415 u32 sts;
1416 unsigned long flag;
1417
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001418 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001419 iommu->gcmd &= ~DMA_GCMD_TE;
1420 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1421
1422 /* Make sure hardware complete it */
1423 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001424 readl, (!(sts & DMA_GSTS_TES)), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001425
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02001426 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001427}
1428
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07001429
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001430static int iommu_init_domains(struct intel_iommu *iommu)
1431{
1432 unsigned long ndomains;
1433 unsigned long nlongs;
1434
1435 ndomains = cap_ndoms(iommu->cap);
Jiang Liu852bdb02014-01-06 14:18:11 +08001436 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1437 iommu->seq_id, ndomains);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001438 nlongs = BITS_TO_LONGS(ndomains);
1439
Donald Dutile94a91b52009-08-20 16:51:34 -04001440 spin_lock_init(&iommu->lock);
1441
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001442 /* TBD: there might be 64K domains,
1443 * consider other allocation for future chip
1444 */
1445 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1446 if (!iommu->domain_ids) {
Jiang Liu852bdb02014-01-06 14:18:11 +08001447 pr_err("IOMMU%d: allocating domain id array failed\n",
1448 iommu->seq_id);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001449 return -ENOMEM;
1450 }
1451 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1452 GFP_KERNEL);
1453 if (!iommu->domains) {
Jiang Liu852bdb02014-01-06 14:18:11 +08001454 pr_err("IOMMU%d: allocating domain array failed\n",
1455 iommu->seq_id);
1456 kfree(iommu->domain_ids);
1457 iommu->domain_ids = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001458 return -ENOMEM;
1459 }
1460
1461 /*
1462 * if Caching mode is set, then invalid translations are tagged
1463 * with domainid 0. Hence we need to pre-allocate it.
1464 */
1465 if (cap_caching_mode(iommu->cap))
1466 set_bit(0, iommu->domain_ids);
1467 return 0;
1468}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001469
Jiang Liua868e6b2014-01-06 14:18:20 +08001470static void free_dmar_iommu(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001471{
1472 struct dmar_domain *domain;
Jiang Liu2a46ddf2014-07-11 14:19:30 +08001473 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001474
Donald Dutile94a91b52009-08-20 16:51:34 -04001475 if ((iommu->domains) && (iommu->domain_ids)) {
Akinobu Mitaa45946a2010-03-11 14:04:08 -08001476 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
Jiang Liua4eaa862014-02-19 14:07:30 +08001477 /*
1478 * Domain id 0 is reserved for invalid translation
1479 * if hardware supports caching mode.
1480 */
1481 if (cap_caching_mode(iommu->cap) && i == 0)
1482 continue;
1483
Donald Dutile94a91b52009-08-20 16:51:34 -04001484 domain = iommu->domains[i];
1485 clear_bit(i, iommu->domain_ids);
Jiang Liu129ad282014-07-11 14:19:31 +08001486 if (domain_detach_iommu(domain, iommu) == 0 &&
1487 !domain_type_is_vm(domain))
Jiang Liu92d03cc2014-02-19 14:07:28 +08001488 domain_exit(domain);
Weidong Han5e98c4b2008-12-08 23:03:27 +08001489 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001490 }
1491
1492 if (iommu->gcmd & DMA_GCMD_TE)
1493 iommu_disable_translation(iommu);
1494
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001495 kfree(iommu->domains);
1496 kfree(iommu->domain_ids);
Jiang Liua868e6b2014-01-06 14:18:20 +08001497 iommu->domains = NULL;
1498 iommu->domain_ids = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001499
Weidong Hand9630fe2008-12-08 11:06:32 +08001500 g_iommus[iommu->seq_id] = NULL;
1501
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001502 /* free context mapping */
1503 free_context_table(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001504}
1505
Jiang Liuab8dfe22014-07-11 14:19:27 +08001506static struct dmar_domain *alloc_domain(int flags)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001507{
Jiang Liu92d03cc2014-02-19 14:07:28 +08001508 /* domain id for virtual machine, it won't be set in context */
1509 static atomic_t vm_domid = ATOMIC_INIT(0);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001510 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001511
1512 domain = alloc_domain_mem();
1513 if (!domain)
1514 return NULL;
1515
Jiang Liuab8dfe22014-07-11 14:19:27 +08001516 memset(domain, 0, sizeof(*domain));
Suresh Siddha4c923d42009-10-02 11:01:24 -07001517 domain->nid = -1;
Jiang Liuab8dfe22014-07-11 14:19:27 +08001518 domain->flags = flags;
Jiang Liu92d03cc2014-02-19 14:07:28 +08001519 spin_lock_init(&domain->iommu_lock);
1520 INIT_LIST_HEAD(&domain->devices);
Jiang Liuab8dfe22014-07-11 14:19:27 +08001521 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
Jiang Liu92d03cc2014-02-19 14:07:28 +08001522 domain->id = atomic_inc_return(&vm_domid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001523
1524 return domain;
1525}
1526
Jiang Liufb170fb2014-07-11 14:19:28 +08001527static int __iommu_attach_domain(struct dmar_domain *domain,
1528 struct intel_iommu *iommu)
1529{
1530 int num;
1531 unsigned long ndomains;
1532
1533 ndomains = cap_ndoms(iommu->cap);
1534 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1535 if (num < ndomains) {
1536 set_bit(num, iommu->domain_ids);
1537 iommu->domains[num] = domain;
1538 } else {
1539 num = -ENOSPC;
1540 }
1541
1542 return num;
1543}
1544
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001545static int iommu_attach_domain(struct dmar_domain *domain,
1546 struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001547{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001548 int num;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001549 unsigned long flags;
1550
Weidong Han8c11e792008-12-08 15:29:22 +08001551 spin_lock_irqsave(&iommu->lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08001552 num = __iommu_attach_domain(domain, iommu);
Jiang Liu44bde612014-07-11 14:19:29 +08001553 spin_unlock_irqrestore(&iommu->lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08001554 if (num < 0)
1555 pr_err("IOMMU: no free domain ids\n");
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001556
Jiang Liufb170fb2014-07-11 14:19:28 +08001557 return num;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001558}
1559
Jiang Liu44bde612014-07-11 14:19:29 +08001560static int iommu_attach_vm_domain(struct dmar_domain *domain,
1561 struct intel_iommu *iommu)
1562{
1563 int num;
1564 unsigned long ndomains;
1565
1566 ndomains = cap_ndoms(iommu->cap);
1567 for_each_set_bit(num, iommu->domain_ids, ndomains)
1568 if (iommu->domains[num] == domain)
1569 return num;
1570
1571 return __iommu_attach_domain(domain, iommu);
1572}
1573
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001574static void iommu_detach_domain(struct dmar_domain *domain,
1575 struct intel_iommu *iommu)
1576{
1577 unsigned long flags;
1578 int num, ndomains;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001579
1580 spin_lock_irqsave(&iommu->lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08001581 if (domain_type_is_vm_or_si(domain)) {
1582 ndomains = cap_ndoms(iommu->cap);
1583 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1584 if (iommu->domains[num] == domain) {
1585 clear_bit(num, iommu->domain_ids);
1586 iommu->domains[num] = NULL;
1587 break;
1588 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001589 }
Jiang Liufb170fb2014-07-11 14:19:28 +08001590 } else {
1591 clear_bit(domain->id, iommu->domain_ids);
1592 iommu->domains[domain->id] = NULL;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001593 }
Weidong Han8c11e792008-12-08 15:29:22 +08001594 spin_unlock_irqrestore(&iommu->lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001595}
1596
Jiang Liufb170fb2014-07-11 14:19:28 +08001597static void domain_attach_iommu(struct dmar_domain *domain,
1598 struct intel_iommu *iommu)
1599{
1600 unsigned long flags;
1601
1602 spin_lock_irqsave(&domain->iommu_lock, flags);
1603 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1604 domain->iommu_count++;
1605 if (domain->iommu_count == 1)
1606 domain->nid = iommu->node;
1607 domain_update_iommu_cap(domain);
1608 }
1609 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1610}
1611
1612static int domain_detach_iommu(struct dmar_domain *domain,
1613 struct intel_iommu *iommu)
1614{
1615 unsigned long flags;
1616 int count = INT_MAX;
1617
1618 spin_lock_irqsave(&domain->iommu_lock, flags);
1619 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1620 count = --domain->iommu_count;
1621 domain_update_iommu_cap(domain);
1622 }
1623 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1624
1625 return count;
1626}
1627
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001628static struct iova_domain reserved_iova_list;
Mark Gross8a443df2008-03-04 14:59:31 -08001629static struct lock_class_key reserved_rbtree_key;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001630
Joseph Cihula51a63e62011-03-21 11:04:24 -07001631static int dmar_init_reserved_ranges(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001632{
1633 struct pci_dev *pdev = NULL;
1634 struct iova *iova;
1635 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001636
David Millerf6611972008-02-06 01:36:23 -08001637 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001638
Mark Gross8a443df2008-03-04 14:59:31 -08001639 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1640 &reserved_rbtree_key);
1641
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001642 /* IOAPIC ranges shouldn't be accessed by DMA */
1643 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1644 IOVA_PFN(IOAPIC_RANGE_END));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001645 if (!iova) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001646 printk(KERN_ERR "Reserve IOAPIC range failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001647 return -ENODEV;
1648 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001649
1650 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1651 for_each_pci_dev(pdev) {
1652 struct resource *r;
1653
1654 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1655 r = &pdev->resource[i];
1656 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1657 continue;
David Woodhouse1a4a4552009-06-28 16:00:42 +01001658 iova = reserve_iova(&reserved_iova_list,
1659 IOVA_PFN(r->start),
1660 IOVA_PFN(r->end));
Joseph Cihula51a63e62011-03-21 11:04:24 -07001661 if (!iova) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001662 printk(KERN_ERR "Reserve iova failed\n");
Joseph Cihula51a63e62011-03-21 11:04:24 -07001663 return -ENODEV;
1664 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001665 }
1666 }
Joseph Cihula51a63e62011-03-21 11:04:24 -07001667 return 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001668}
1669
1670static void domain_reserve_special_ranges(struct dmar_domain *domain)
1671{
1672 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1673}
1674
1675static inline int guestwidth_to_adjustwidth(int gaw)
1676{
1677 int agaw;
1678 int r = (gaw - 12) % 9;
1679
1680 if (r == 0)
1681 agaw = gaw;
1682 else
1683 agaw = gaw + 9 - r;
1684 if (agaw > 64)
1685 agaw = 64;
1686 return agaw;
1687}
1688
1689static int domain_init(struct dmar_domain *domain, int guest_width)
1690{
1691 struct intel_iommu *iommu;
1692 int adjust_width, agaw;
1693 unsigned long sagaw;
1694
David Millerf6611972008-02-06 01:36:23 -08001695 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001696 domain_reserve_special_ranges(domain);
1697
1698 /* calculate AGAW */
Weidong Han8c11e792008-12-08 15:29:22 +08001699 iommu = domain_get_iommu(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001700 if (guest_width > cap_mgaw(iommu->cap))
1701 guest_width = cap_mgaw(iommu->cap);
1702 domain->gaw = guest_width;
1703 adjust_width = guestwidth_to_adjustwidth(guest_width);
1704 agaw = width_to_agaw(adjust_width);
1705 sagaw = cap_sagaw(iommu->cap);
1706 if (!test_bit(agaw, &sagaw)) {
1707 /* hardware doesn't support it, choose a bigger one */
1708 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1709 agaw = find_next_bit(&sagaw, 5, agaw);
1710 if (agaw >= 5)
1711 return -ENODEV;
1712 }
1713 domain->agaw = agaw;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001714
Weidong Han8e6040972008-12-08 15:49:06 +08001715 if (ecap_coherent(iommu->ecap))
1716 domain->iommu_coherency = 1;
1717 else
1718 domain->iommu_coherency = 0;
1719
Sheng Yang58c610b2009-03-18 15:33:05 +08001720 if (ecap_sc_support(iommu->ecap))
1721 domain->iommu_snooping = 1;
1722 else
1723 domain->iommu_snooping = 0;
1724
David Woodhouse214e39a2014-03-19 10:38:49 +00001725 if (intel_iommu_superpage)
1726 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1727 else
1728 domain->iommu_superpage = 0;
1729
Suresh Siddha4c923d42009-10-02 11:01:24 -07001730 domain->nid = iommu->node;
Weidong Hanc7151a82008-12-08 22:51:37 +08001731
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001732 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07001733 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001734 if (!domain->pgd)
1735 return -ENOMEM;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001736 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001737 return 0;
1738}
1739
1740static void domain_exit(struct dmar_domain *domain)
1741{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001742 struct dmar_drhd_unit *drhd;
1743 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00001744 struct page *freelist = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001745
1746 /* Domain 0 is reserved, so dont process it */
1747 if (!domain)
1748 return;
1749
Alex Williamson7b668352011-05-24 12:02:41 +01001750 /* Flush any lazy unmaps that may reference this domain */
1751 if (!intel_iommu_strict)
1752 flush_unmaps_timeout(0);
1753
Jiang Liu92d03cc2014-02-19 14:07:28 +08001754 /* remove associated devices */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001755 domain_remove_dev_info(domain);
Jiang Liu92d03cc2014-02-19 14:07:28 +08001756
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001757 /* destroy iovas */
1758 put_iova_domain(&domain->iovad);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001759
David Woodhouseea8ea462014-03-05 17:09:32 +00001760 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001761
Jiang Liu92d03cc2014-02-19 14:07:28 +08001762 /* clear attached or cached domains */
Jiang Liu0e242612014-02-19 14:07:34 +08001763 rcu_read_lock();
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001764 for_each_active_iommu(iommu, drhd)
Jiang Liufb170fb2014-07-11 14:19:28 +08001765 iommu_detach_domain(domain, iommu);
Jiang Liu0e242612014-02-19 14:07:34 +08001766 rcu_read_unlock();
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001767
David Woodhouseea8ea462014-03-05 17:09:32 +00001768 dma_free_pagelist(freelist);
1769
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001770 free_domain_mem(domain);
1771}
1772
David Woodhouse64ae8922014-03-09 12:52:30 -07001773static int domain_context_mapping_one(struct dmar_domain *domain,
1774 struct intel_iommu *iommu,
1775 u8 bus, u8 devfn, int translation)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001776{
1777 struct context_entry *context;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001778 unsigned long flags;
Weidong Hanea6606b2008-12-08 23:08:15 +08001779 struct dma_pte *pgd;
Weidong Hanea6606b2008-12-08 23:08:15 +08001780 int id;
1781 int agaw;
Yu Zhao93a23a72009-05-18 13:51:37 +08001782 struct device_domain_info *info = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001783
1784 pr_debug("Set context mapping for %02x:%02x.%d\n",
1785 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001786
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001787 BUG_ON(!domain->pgd);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001788 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1789 translation != CONTEXT_TT_MULTI_LEVEL);
Weidong Han5331fe62008-12-08 23:00:00 +08001790
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001791 context = device_to_context_entry(iommu, bus, devfn);
1792 if (!context)
1793 return -ENOMEM;
1794 spin_lock_irqsave(&iommu->lock, flags);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00001795 if (context_present(context)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001796 spin_unlock_irqrestore(&iommu->lock, flags);
1797 return 0;
1798 }
1799
Weidong Hanea6606b2008-12-08 23:08:15 +08001800 id = domain->id;
1801 pgd = domain->pgd;
1802
Jiang Liuab8dfe22014-07-11 14:19:27 +08001803 if (domain_type_is_vm_or_si(domain)) {
Jiang Liu44bde612014-07-11 14:19:29 +08001804 if (domain_type_is_vm(domain)) {
1805 id = iommu_attach_vm_domain(domain, iommu);
Jiang Liufb170fb2014-07-11 14:19:28 +08001806 if (id < 0) {
Weidong Hanea6606b2008-12-08 23:08:15 +08001807 spin_unlock_irqrestore(&iommu->lock, flags);
Jiang Liufb170fb2014-07-11 14:19:28 +08001808 pr_err("IOMMU: no free domain ids\n");
Weidong Hanea6606b2008-12-08 23:08:15 +08001809 return -EFAULT;
1810 }
Weidong Hanea6606b2008-12-08 23:08:15 +08001811 }
1812
1813 /* Skip top levels of page tables for
1814 * iommu which has less agaw than default.
Chris Wright1672af12009-12-02 12:06:34 -08001815 * Unnecessary for PT mode.
Weidong Hanea6606b2008-12-08 23:08:15 +08001816 */
Chris Wright1672af12009-12-02 12:06:34 -08001817 if (translation != CONTEXT_TT_PASS_THROUGH) {
1818 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1819 pgd = phys_to_virt(dma_pte_addr(pgd));
1820 if (!dma_pte_present(pgd)) {
1821 spin_unlock_irqrestore(&iommu->lock, flags);
1822 return -ENOMEM;
1823 }
Weidong Hanea6606b2008-12-08 23:08:15 +08001824 }
1825 }
1826 }
1827
1828 context_set_domain_id(context, id);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001829
Yu Zhao93a23a72009-05-18 13:51:37 +08001830 if (translation != CONTEXT_TT_PASS_THROUGH) {
David Woodhouse64ae8922014-03-09 12:52:30 -07001831 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
Yu Zhao93a23a72009-05-18 13:51:37 +08001832 translation = info ? CONTEXT_TT_DEV_IOTLB :
1833 CONTEXT_TT_MULTI_LEVEL;
1834 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001835 /*
1836 * In pass through mode, AW must be programmed to indicate the largest
1837 * AGAW value supported by hardware. And ASR is ignored by hardware.
1838 */
Yu Zhao93a23a72009-05-18 13:51:37 +08001839 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001840 context_set_address_width(context, iommu->msagaw);
Yu Zhao93a23a72009-05-18 13:51:37 +08001841 else {
1842 context_set_address_root(context, virt_to_phys(pgd));
1843 context_set_address_width(context, iommu->agaw);
1844 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001845
1846 context_set_translation_type(context, translation);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00001847 context_set_fault_enable(context);
1848 context_set_present(context);
Weidong Han5331fe62008-12-08 23:00:00 +08001849 domain_flush_cache(domain, context, sizeof(*context));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001850
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001851 /*
1852 * It's a non-present to present mapping. If hardware doesn't cache
1853 * non-present entry we only need to flush the write-buffer. If the
1854 * _does_ cache non-present entries, then it does so in the special
1855 * domain #0, which we have to flush:
1856 */
1857 if (cap_caching_mode(iommu->cap)) {
1858 iommu->flush.flush_context(iommu, 0,
1859 (((u16)bus) << 8) | devfn,
1860 DMA_CCMD_MASK_NOBIT,
1861 DMA_CCMD_DEVICE_INVL);
Jiang Liu18fd7792014-07-11 14:19:26 +08001862 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001863 } else {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001864 iommu_flush_write_buffer(iommu);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001865 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001866 iommu_enable_dev_iotlb(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001867 spin_unlock_irqrestore(&iommu->lock, flags);
Weidong Hanc7151a82008-12-08 22:51:37 +08001868
Jiang Liufb170fb2014-07-11 14:19:28 +08001869 domain_attach_iommu(domain, iommu);
1870
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001871 return 0;
1872}
1873
Alex Williamson579305f2014-07-03 09:51:43 -06001874struct domain_context_mapping_data {
1875 struct dmar_domain *domain;
1876 struct intel_iommu *iommu;
1877 int translation;
1878};
1879
1880static int domain_context_mapping_cb(struct pci_dev *pdev,
1881 u16 alias, void *opaque)
1882{
1883 struct domain_context_mapping_data *data = opaque;
1884
1885 return domain_context_mapping_one(data->domain, data->iommu,
1886 PCI_BUS_NUM(alias), alias & 0xff,
1887 data->translation);
1888}
1889
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001890static int
David Woodhousee1f167f2014-03-09 15:24:46 -07001891domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1892 int translation)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001893{
David Woodhouse64ae8922014-03-09 12:52:30 -07001894 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07001895 u8 bus, devfn;
Alex Williamson579305f2014-07-03 09:51:43 -06001896 struct domain_context_mapping_data data;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001897
David Woodhousee1f167f2014-03-09 15:24:46 -07001898 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse64ae8922014-03-09 12:52:30 -07001899 if (!iommu)
1900 return -ENODEV;
1901
Alex Williamson579305f2014-07-03 09:51:43 -06001902 if (!dev_is_pci(dev))
1903 return domain_context_mapping_one(domain, iommu, bus, devfn,
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001904 translation);
Alex Williamson579305f2014-07-03 09:51:43 -06001905
1906 data.domain = domain;
1907 data.iommu = iommu;
1908 data.translation = translation;
1909
1910 return pci_for_each_dma_alias(to_pci_dev(dev),
1911 &domain_context_mapping_cb, &data);
1912}
1913
1914static int domain_context_mapped_cb(struct pci_dev *pdev,
1915 u16 alias, void *opaque)
1916{
1917 struct intel_iommu *iommu = opaque;
1918
1919 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001920}
1921
David Woodhousee1f167f2014-03-09 15:24:46 -07001922static int domain_context_mapped(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001923{
Weidong Han5331fe62008-12-08 23:00:00 +08001924 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07001925 u8 bus, devfn;
Weidong Han5331fe62008-12-08 23:00:00 +08001926
David Woodhousee1f167f2014-03-09 15:24:46 -07001927 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Han5331fe62008-12-08 23:00:00 +08001928 if (!iommu)
1929 return -ENODEV;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001930
Alex Williamson579305f2014-07-03 09:51:43 -06001931 if (!dev_is_pci(dev))
1932 return device_context_mapped(iommu, bus, devfn);
David Woodhousee1f167f2014-03-09 15:24:46 -07001933
Alex Williamson579305f2014-07-03 09:51:43 -06001934 return !pci_for_each_dma_alias(to_pci_dev(dev),
1935 domain_context_mapped_cb, iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001936}
1937
Fenghua Yuf5329592009-08-04 15:09:37 -07001938/* Returns a number of VTD pages, but aligned to MM page size */
1939static inline unsigned long aligned_nrpages(unsigned long host_addr,
1940 size_t size)
1941{
1942 host_addr &= ~PAGE_MASK;
1943 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1944}
1945
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001946/* Return largest possible superpage level for a given mapping */
1947static inline int hardware_largepage_caps(struct dmar_domain *domain,
1948 unsigned long iov_pfn,
1949 unsigned long phy_pfn,
1950 unsigned long pages)
1951{
1952 int support, level = 1;
1953 unsigned long pfnmerge;
1954
1955 support = domain->iommu_superpage;
1956
1957 /* To use a large page, the virtual *and* physical addresses
1958 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1959 of them will mean we have to use smaller pages. So just
1960 merge them and check both at once. */
1961 pfnmerge = iov_pfn | phy_pfn;
1962
1963 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1964 pages >>= VTD_STRIDE_SHIFT;
1965 if (!pages)
1966 break;
1967 pfnmerge >>= VTD_STRIDE_SHIFT;
1968 level++;
1969 support--;
1970 }
1971 return level;
1972}
1973
David Woodhouse9051aa02009-06-29 12:30:54 +01001974static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1975 struct scatterlist *sg, unsigned long phys_pfn,
1976 unsigned long nr_pages, int prot)
David Woodhousee1605492009-06-29 11:17:38 +01001977{
1978 struct dma_pte *first_pte = NULL, *pte = NULL;
David Woodhouse9051aa02009-06-29 12:30:54 +01001979 phys_addr_t uninitialized_var(pteval);
David Woodhouse9051aa02009-06-29 12:30:54 +01001980 unsigned long sg_res;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001981 unsigned int largepage_lvl = 0;
1982 unsigned long lvl_pages = 0;
David Woodhousee1605492009-06-29 11:17:38 +01001983
Jiang Liu162d1b12014-07-11 14:19:35 +08001984 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
David Woodhousee1605492009-06-29 11:17:38 +01001985
1986 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1987 return -EINVAL;
1988
1989 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1990
David Woodhouse9051aa02009-06-29 12:30:54 +01001991 if (sg)
1992 sg_res = 0;
1993 else {
1994 sg_res = nr_pages + 1;
1995 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1996 }
1997
Youquan Song6dd9a7c2011-05-25 19:13:49 +01001998 while (nr_pages > 0) {
David Woodhousec85994e2009-07-01 19:21:24 +01001999 uint64_t tmp;
2000
David Woodhousee1605492009-06-29 11:17:38 +01002001 if (!sg_res) {
Fenghua Yuf5329592009-08-04 15:09:37 -07002002 sg_res = aligned_nrpages(sg->offset, sg->length);
David Woodhousee1605492009-06-29 11:17:38 +01002003 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2004 sg->dma_length = sg->length;
2005 pteval = page_to_phys(sg_page(sg)) | prot;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002006 phys_pfn = pteval >> VTD_PAGE_SHIFT;
David Woodhousee1605492009-06-29 11:17:38 +01002007 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002008
David Woodhousee1605492009-06-29 11:17:38 +01002009 if (!pte) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002010 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2011
David Woodhouse5cf0a762014-03-19 16:07:49 +00002012 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
David Woodhousee1605492009-06-29 11:17:38 +01002013 if (!pte)
2014 return -ENOMEM;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002015 /* It is large page*/
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002016 if (largepage_lvl > 1) {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002017 pteval |= DMA_PTE_LARGE_PAGE;
Jiang Liud41a4ad2014-07-11 14:19:34 +08002018 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2019 /*
2020 * Ensure that old small page tables are
2021 * removed to make room for superpage,
2022 * if they exist.
2023 */
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002024 dma_pte_free_pagetable(domain, iov_pfn,
Jiang Liud41a4ad2014-07-11 14:19:34 +08002025 iov_pfn + lvl_pages - 1);
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002026 } else {
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002027 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
Woodhouse, David6491d4d2012-12-19 13:25:35 +00002028 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002029
David Woodhousee1605492009-06-29 11:17:38 +01002030 }
2031 /* We don't need lock here, nobody else
2032 * touches the iova range
2033 */
David Woodhouse7766a3f2009-07-01 20:27:03 +01002034 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
David Woodhousec85994e2009-07-01 19:21:24 +01002035 if (tmp) {
David Woodhouse1bf20f02009-06-29 22:06:43 +01002036 static int dumps = 5;
David Woodhousec85994e2009-07-01 19:21:24 +01002037 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2038 iov_pfn, tmp, (unsigned long long)pteval);
David Woodhouse1bf20f02009-06-29 22:06:43 +01002039 if (dumps) {
2040 dumps--;
2041 debug_dma_dump_mappings(NULL);
2042 }
2043 WARN_ON(1);
2044 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002045
2046 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2047
2048 BUG_ON(nr_pages < lvl_pages);
2049 BUG_ON(sg_res < lvl_pages);
2050
2051 nr_pages -= lvl_pages;
2052 iov_pfn += lvl_pages;
2053 phys_pfn += lvl_pages;
2054 pteval += lvl_pages * VTD_PAGE_SIZE;
2055 sg_res -= lvl_pages;
2056
2057 /* If the next PTE would be the first in a new page, then we
2058 need to flush the cache on the entries we've just written.
2059 And then we'll need to recalculate 'pte', so clear it and
2060 let it get set again in the if (!pte) block above.
2061
2062 If we're done (!nr_pages) we need to flush the cache too.
2063
2064 Also if we've been setting superpages, we may need to
2065 recalculate 'pte' and switch back to smaller pages for the
2066 end of the mapping, if the trailing size is not enough to
2067 use another superpage (i.e. sg_res < lvl_pages). */
David Woodhousee1605492009-06-29 11:17:38 +01002068 pte++;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002069 if (!nr_pages || first_pte_in_page(pte) ||
2070 (largepage_lvl > 1 && sg_res < lvl_pages)) {
David Woodhousee1605492009-06-29 11:17:38 +01002071 domain_flush_cache(domain, first_pte,
2072 (void *)pte - (void *)first_pte);
2073 pte = NULL;
2074 }
Youquan Song6dd9a7c2011-05-25 19:13:49 +01002075
2076 if (!sg_res && nr_pages)
David Woodhousee1605492009-06-29 11:17:38 +01002077 sg = sg_next(sg);
2078 }
2079 return 0;
2080}
2081
David Woodhouse9051aa02009-06-29 12:30:54 +01002082static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2083 struct scatterlist *sg, unsigned long nr_pages,
2084 int prot)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002085{
David Woodhouse9051aa02009-06-29 12:30:54 +01002086 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2087}
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002088
David Woodhouse9051aa02009-06-29 12:30:54 +01002089static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2090 unsigned long phys_pfn, unsigned long nr_pages,
2091 int prot)
2092{
2093 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002094}
2095
Weidong Hanc7151a82008-12-08 22:51:37 +08002096static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002097{
Weidong Hanc7151a82008-12-08 22:51:37 +08002098 if (!iommu)
2099 return;
Weidong Han8c11e792008-12-08 15:29:22 +08002100
2101 clear_context_table(iommu, bus, devfn);
2102 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002103 DMA_CCMD_GLOBAL_INVL);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002104 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002105}
2106
David Woodhouse109b9b02012-05-25 17:43:02 +01002107static inline void unlink_domain_info(struct device_domain_info *info)
2108{
2109 assert_spin_locked(&device_domain_lock);
2110 list_del(&info->link);
2111 list_del(&info->global);
2112 if (info->dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002113 info->dev->archdata.iommu = NULL;
David Woodhouse109b9b02012-05-25 17:43:02 +01002114}
2115
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002116static void domain_remove_dev_info(struct dmar_domain *domain)
2117{
Yijing Wang3a74ca02014-05-20 20:37:47 +08002118 struct device_domain_info *info, *tmp;
Jiang Liufb170fb2014-07-11 14:19:28 +08002119 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002120
2121 spin_lock_irqsave(&device_domain_lock, flags);
Yijing Wang3a74ca02014-05-20 20:37:47 +08002122 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
David Woodhouse109b9b02012-05-25 17:43:02 +01002123 unlink_domain_info(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002124 spin_unlock_irqrestore(&device_domain_lock, flags);
2125
Yu Zhao93a23a72009-05-18 13:51:37 +08002126 iommu_disable_dev_iotlb(info);
David Woodhouse7c7faa12014-03-09 13:33:06 -07002127 iommu_detach_dev(info->iommu, info->bus, info->devfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002128
Jiang Liuab8dfe22014-07-11 14:19:27 +08002129 if (domain_type_is_vm(domain)) {
David Woodhouse7c7faa12014-03-09 13:33:06 -07002130 iommu_detach_dependent_devices(info->iommu, info->dev);
Jiang Liufb170fb2014-07-11 14:19:28 +08002131 domain_detach_iommu(domain, info->iommu);
Jiang Liu92d03cc2014-02-19 14:07:28 +08002132 }
2133
2134 free_devinfo_mem(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002135 spin_lock_irqsave(&device_domain_lock, flags);
2136 }
2137 spin_unlock_irqrestore(&device_domain_lock, flags);
2138}
2139
2140/*
2141 * find_domain
David Woodhouse1525a292014-03-06 16:19:30 +00002142 * Note: we use struct device->archdata.iommu stores the info
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002143 */
David Woodhouse1525a292014-03-06 16:19:30 +00002144static struct dmar_domain *find_domain(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002145{
2146 struct device_domain_info *info;
2147
2148 /* No lock here, assumes no domain exit in normal case */
David Woodhouse1525a292014-03-06 16:19:30 +00002149 info = dev->archdata.iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002150 if (info)
2151 return info->domain;
2152 return NULL;
2153}
2154
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002155static inline struct device_domain_info *
Jiang Liu745f2582014-02-19 14:07:26 +08002156dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2157{
2158 struct device_domain_info *info;
2159
2160 list_for_each_entry(info, &device_domain_list, global)
David Woodhouse41e80dca2014-03-09 13:55:54 -07002161 if (info->iommu->segment == segment && info->bus == bus &&
Jiang Liu745f2582014-02-19 14:07:26 +08002162 info->devfn == devfn)
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002163 return info;
Jiang Liu745f2582014-02-19 14:07:26 +08002164
2165 return NULL;
2166}
2167
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002168static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
David Woodhouse41e80dca2014-03-09 13:55:54 -07002169 int bus, int devfn,
David Woodhouseb718cd32014-03-09 13:11:33 -07002170 struct device *dev,
2171 struct dmar_domain *domain)
Jiang Liu745f2582014-02-19 14:07:26 +08002172{
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002173 struct dmar_domain *found = NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002174 struct device_domain_info *info;
2175 unsigned long flags;
2176
2177 info = alloc_devinfo_mem();
2178 if (!info)
David Woodhouseb718cd32014-03-09 13:11:33 -07002179 return NULL;
Jiang Liu745f2582014-02-19 14:07:26 +08002180
Jiang Liu745f2582014-02-19 14:07:26 +08002181 info->bus = bus;
2182 info->devfn = devfn;
2183 info->dev = dev;
2184 info->domain = domain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002185 info->iommu = iommu;
Jiang Liu745f2582014-02-19 14:07:26 +08002186
2187 spin_lock_irqsave(&device_domain_lock, flags);
2188 if (dev)
David Woodhouse0bcb3e22014-03-06 17:12:03 +00002189 found = find_domain(dev);
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002190 else {
2191 struct device_domain_info *info2;
David Woodhouse41e80dca2014-03-09 13:55:54 -07002192 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002193 if (info2)
2194 found = info2->domain;
2195 }
Jiang Liu745f2582014-02-19 14:07:26 +08002196 if (found) {
2197 spin_unlock_irqrestore(&device_domain_lock, flags);
2198 free_devinfo_mem(info);
David Woodhouseb718cd32014-03-09 13:11:33 -07002199 /* Caller must free the original domain */
2200 return found;
Jiang Liu745f2582014-02-19 14:07:26 +08002201 }
2202
David Woodhouseb718cd32014-03-09 13:11:33 -07002203 list_add(&info->link, &domain->devices);
2204 list_add(&info->global, &device_domain_list);
2205 if (dev)
2206 dev->archdata.iommu = info;
2207 spin_unlock_irqrestore(&device_domain_lock, flags);
2208
2209 return domain;
Jiang Liu745f2582014-02-19 14:07:26 +08002210}
2211
Alex Williamson579305f2014-07-03 09:51:43 -06002212static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2213{
2214 *(u16 *)opaque = alias;
2215 return 0;
2216}
2217
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002218/* domain is initialized */
David Woodhouse146922e2014-03-09 15:44:17 -07002219static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002220{
Alex Williamson579305f2014-07-03 09:51:43 -06002221 struct dmar_domain *domain, *tmp;
2222 struct intel_iommu *iommu;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002223 struct device_domain_info *info;
Alex Williamson579305f2014-07-03 09:51:43 -06002224 u16 dma_alias;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002225 unsigned long flags;
Yijing Wangaa4d0662014-05-26 20:14:06 +08002226 u8 bus, devfn;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002227
David Woodhouse146922e2014-03-09 15:44:17 -07002228 domain = find_domain(dev);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002229 if (domain)
2230 return domain;
2231
David Woodhouse146922e2014-03-09 15:44:17 -07002232 iommu = device_to_iommu(dev, &bus, &devfn);
2233 if (!iommu)
Alex Williamson579305f2014-07-03 09:51:43 -06002234 return NULL;
2235
2236 if (dev_is_pci(dev)) {
2237 struct pci_dev *pdev = to_pci_dev(dev);
2238
2239 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2240
2241 spin_lock_irqsave(&device_domain_lock, flags);
2242 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2243 PCI_BUS_NUM(dma_alias),
2244 dma_alias & 0xff);
2245 if (info) {
2246 iommu = info->iommu;
2247 domain = info->domain;
2248 }
2249 spin_unlock_irqrestore(&device_domain_lock, flags);
2250
2251 /* DMA alias already has a domain, uses it */
2252 if (info)
2253 goto found_domain;
2254 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002255
David Woodhouse146922e2014-03-09 15:44:17 -07002256 /* Allocate and initialize new domain for the device */
Jiang Liuab8dfe22014-07-11 14:19:27 +08002257 domain = alloc_domain(0);
Jiang Liu745f2582014-02-19 14:07:26 +08002258 if (!domain)
Alex Williamson579305f2014-07-03 09:51:43 -06002259 return NULL;
Jiang Liu44bde612014-07-11 14:19:29 +08002260 domain->id = iommu_attach_domain(domain, iommu);
2261 if (domain->id < 0) {
Alex Williamson2fe9723d2011-03-04 14:52:30 -07002262 free_domain_mem(domain);
Alex Williamson579305f2014-07-03 09:51:43 -06002263 return NULL;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002264 }
Jiang Liufb170fb2014-07-11 14:19:28 +08002265 domain_attach_iommu(domain, iommu);
Alex Williamson579305f2014-07-03 09:51:43 -06002266 if (domain_init(domain, gaw)) {
2267 domain_exit(domain);
2268 return NULL;
2269 }
2270
2271 /* register PCI DMA alias device */
2272 if (dev_is_pci(dev)) {
2273 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2274 dma_alias & 0xff, NULL, domain);
2275
2276 if (!tmp || tmp != domain) {
2277 domain_exit(domain);
2278 domain = tmp;
2279 }
2280
David Woodhouseb718cd32014-03-09 13:11:33 -07002281 if (!domain)
Alex Williamson579305f2014-07-03 09:51:43 -06002282 return NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002283 }
2284
2285found_domain:
Alex Williamson579305f2014-07-03 09:51:43 -06002286 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2287
2288 if (!tmp || tmp != domain) {
2289 domain_exit(domain);
2290 domain = tmp;
2291 }
David Woodhouseb718cd32014-03-09 13:11:33 -07002292
2293 return domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002294}
2295
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002296static int iommu_identity_mapping;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002297#define IDENTMAP_ALL 1
2298#define IDENTMAP_GFX 2
2299#define IDENTMAP_AZALIA 4
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002300
David Woodhouseb2132032009-06-26 18:50:28 +01002301static int iommu_domain_identity_map(struct dmar_domain *domain,
2302 unsigned long long start,
2303 unsigned long long end)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002304{
David Woodhousec5395d52009-06-28 16:35:56 +01002305 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2306 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002307
David Woodhousec5395d52009-06-28 16:35:56 +01002308 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2309 dma_to_mm_pfn(last_vpfn))) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002310 printk(KERN_ERR "IOMMU: reserve iova failed\n");
David Woodhouseb2132032009-06-26 18:50:28 +01002311 return -ENOMEM;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002312 }
2313
David Woodhousec5395d52009-06-28 16:35:56 +01002314 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2315 start, end, domain->id);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002316 /*
2317 * RMRR range might have overlap with physical memory range,
2318 * clear it first
2319 */
David Woodhousec5395d52009-06-28 16:35:56 +01002320 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002321
David Woodhousec5395d52009-06-28 16:35:56 +01002322 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2323 last_vpfn - first_vpfn + 1,
David Woodhouse61df7442009-06-28 11:55:58 +01002324 DMA_PTE_READ|DMA_PTE_WRITE);
David Woodhouseb2132032009-06-26 18:50:28 +01002325}
2326
David Woodhouse0b9d9752014-03-09 15:48:15 -07002327static int iommu_prepare_identity_map(struct device *dev,
David Woodhouseb2132032009-06-26 18:50:28 +01002328 unsigned long long start,
2329 unsigned long long end)
2330{
2331 struct dmar_domain *domain;
2332 int ret;
2333
David Woodhouse0b9d9752014-03-09 15:48:15 -07002334 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
David Woodhouseb2132032009-06-26 18:50:28 +01002335 if (!domain)
2336 return -ENOMEM;
2337
David Woodhouse19943b02009-08-04 16:19:20 +01002338 /* For _hardware_ passthrough, don't bother. But for software
2339 passthrough, we do it anyway -- it may indicate a memory
2340 range which is reserved in E820, so which didn't get set
2341 up to start with in si_domain */
2342 if (domain == si_domain && hw_pass_through) {
2343 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
David Woodhouse0b9d9752014-03-09 15:48:15 -07002344 dev_name(dev), start, end);
David Woodhouse19943b02009-08-04 16:19:20 +01002345 return 0;
2346 }
2347
2348 printk(KERN_INFO
2349 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
David Woodhouse0b9d9752014-03-09 15:48:15 -07002350 dev_name(dev), start, end);
David Woodhouse2ff729f2009-08-26 14:25:41 +01002351
David Woodhouse5595b522009-12-02 09:21:55 +00002352 if (end < start) {
2353 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2354 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2355 dmi_get_system_info(DMI_BIOS_VENDOR),
2356 dmi_get_system_info(DMI_BIOS_VERSION),
2357 dmi_get_system_info(DMI_PRODUCT_VERSION));
2358 ret = -EIO;
2359 goto error;
2360 }
2361
David Woodhouse2ff729f2009-08-26 14:25:41 +01002362 if (end >> agaw_to_width(domain->agaw)) {
2363 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2364 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2365 agaw_to_width(domain->agaw),
2366 dmi_get_system_info(DMI_BIOS_VENDOR),
2367 dmi_get_system_info(DMI_BIOS_VERSION),
2368 dmi_get_system_info(DMI_PRODUCT_VERSION));
2369 ret = -EIO;
2370 goto error;
2371 }
David Woodhouse19943b02009-08-04 16:19:20 +01002372
David Woodhouseb2132032009-06-26 18:50:28 +01002373 ret = iommu_domain_identity_map(domain, start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002374 if (ret)
2375 goto error;
2376
2377 /* context entry init */
David Woodhouse0b9d9752014-03-09 15:48:15 -07002378 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
David Woodhouseb2132032009-06-26 18:50:28 +01002379 if (ret)
2380 goto error;
2381
2382 return 0;
2383
2384 error:
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002385 domain_exit(domain);
2386 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002387}
2388
2389static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
David Woodhouse0b9d9752014-03-09 15:48:15 -07002390 struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002391{
David Woodhouse0b9d9752014-03-09 15:48:15 -07002392 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002393 return 0;
David Woodhouse0b9d9752014-03-09 15:48:15 -07002394 return iommu_prepare_identity_map(dev, rmrr->base_address,
2395 rmrr->end_address);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002396}
2397
Suresh Siddhad3f13812011-08-23 17:05:25 -07002398#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002399static inline void iommu_prepare_isa(void)
2400{
2401 struct pci_dev *pdev;
2402 int ret;
2403
2404 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2405 if (!pdev)
2406 return;
2407
David Woodhousec7ab48d2009-06-26 19:10:36 +01002408 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
David Woodhouse0b9d9752014-03-09 15:48:15 -07002409 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002410
2411 if (ret)
David Woodhousec7ab48d2009-06-26 19:10:36 +01002412 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2413 "floppy might not work\n");
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002414
Yijing Wang9b27e822014-05-20 20:37:52 +08002415 pci_dev_put(pdev);
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002416}
2417#else
2418static inline void iommu_prepare_isa(void)
2419{
2420 return;
2421}
Suresh Siddhad3f13812011-08-23 17:05:25 -07002422#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002423
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002424static int md_domain_init(struct dmar_domain *domain, int guest_width);
David Woodhousec7ab48d2009-06-26 19:10:36 +01002425
Matt Kraai071e1372009-08-23 22:30:22 -07002426static int __init si_domain_init(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002427{
2428 struct dmar_drhd_unit *drhd;
2429 struct intel_iommu *iommu;
David Woodhousec7ab48d2009-06-26 19:10:36 +01002430 int nid, ret = 0;
Jiang Liu44bde612014-07-11 14:19:29 +08002431 bool first = true;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002432
Jiang Liuab8dfe22014-07-11 14:19:27 +08002433 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002434 if (!si_domain)
2435 return -EFAULT;
2436
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002437 for_each_active_iommu(iommu, drhd) {
2438 ret = iommu_attach_domain(si_domain, iommu);
Jiang Liufb170fb2014-07-11 14:19:28 +08002439 if (ret < 0) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002440 domain_exit(si_domain);
2441 return -EFAULT;
Jiang Liu44bde612014-07-11 14:19:29 +08002442 } else if (first) {
2443 si_domain->id = ret;
2444 first = false;
2445 } else if (si_domain->id != ret) {
2446 domain_exit(si_domain);
2447 return -EFAULT;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002448 }
Jiang Liufb170fb2014-07-11 14:19:28 +08002449 domain_attach_iommu(si_domain, iommu);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002450 }
2451
2452 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2453 domain_exit(si_domain);
2454 return -EFAULT;
2455 }
2456
Jiang Liu9544c002014-01-06 14:18:13 +08002457 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2458 si_domain->id);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002459
David Woodhouse19943b02009-08-04 16:19:20 +01002460 if (hw)
2461 return 0;
2462
David Woodhousec7ab48d2009-06-26 19:10:36 +01002463 for_each_online_node(nid) {
Tejun Heod4bbf7e2011-11-28 09:46:22 -08002464 unsigned long start_pfn, end_pfn;
2465 int i;
2466
2467 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2468 ret = iommu_domain_identity_map(si_domain,
2469 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2470 if (ret)
2471 return ret;
2472 }
David Woodhousec7ab48d2009-06-26 19:10:36 +01002473 }
2474
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002475 return 0;
2476}
2477
David Woodhouse9b226622014-03-09 14:03:28 -07002478static int identity_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002479{
2480 struct device_domain_info *info;
2481
2482 if (likely(!iommu_identity_mapping))
2483 return 0;
2484
David Woodhouse9b226622014-03-09 14:03:28 -07002485 info = dev->archdata.iommu;
Mike Traviscb452a42011-05-28 13:15:03 -05002486 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2487 return (info->domain == si_domain);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002488
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002489 return 0;
2490}
2491
2492static int domain_add_dev_info(struct dmar_domain *domain,
David Woodhouse5913c9b2014-03-09 16:27:31 -07002493 struct device *dev, int translation)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002494{
David Woodhouse0ac72662014-03-09 13:19:22 -07002495 struct dmar_domain *ndomain;
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002496 struct intel_iommu *iommu;
David Woodhouse156baca2014-03-09 14:00:57 -07002497 u8 bus, devfn;
David Woodhouse5fe60f42009-08-09 10:53:41 +01002498 int ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002499
David Woodhouse5913c9b2014-03-09 16:27:31 -07002500 iommu = device_to_iommu(dev, &bus, &devfn);
David Woodhouse5a8f40e2014-03-09 13:31:18 -07002501 if (!iommu)
2502 return -ENODEV;
2503
David Woodhouse5913c9b2014-03-09 16:27:31 -07002504 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
David Woodhouse0ac72662014-03-09 13:19:22 -07002505 if (ndomain != domain)
2506 return -EBUSY;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002507
David Woodhouse5913c9b2014-03-09 16:27:31 -07002508 ret = domain_context_mapping(domain, dev, translation);
David Woodhousee2ad23d2012-05-25 17:42:54 +01002509 if (ret) {
David Woodhouse5913c9b2014-03-09 16:27:31 -07002510 domain_remove_one_dev_info(domain, dev);
David Woodhousee2ad23d2012-05-25 17:42:54 +01002511 return ret;
2512 }
2513
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002514 return 0;
2515}
2516
David Woodhouse0b9d9752014-03-09 15:48:15 -07002517static bool device_has_rmrr(struct device *dev)
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002518{
2519 struct dmar_rmrr_unit *rmrr;
David Woodhouse832bd852014-03-07 15:08:36 +00002520 struct device *tmp;
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002521 int i;
2522
Jiang Liu0e242612014-02-19 14:07:34 +08002523 rcu_read_lock();
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002524 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08002525 /*
2526 * Return TRUE if this RMRR contains the device that
2527 * is passed in.
2528 */
2529 for_each_active_dev_scope(rmrr->devices,
2530 rmrr->devices_cnt, i, tmp)
David Woodhouse0b9d9752014-03-09 15:48:15 -07002531 if (tmp == dev) {
Jiang Liu0e242612014-02-19 14:07:34 +08002532 rcu_read_unlock();
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002533 return true;
Jiang Liub683b232014-02-19 14:07:32 +08002534 }
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002535 }
Jiang Liu0e242612014-02-19 14:07:34 +08002536 rcu_read_unlock();
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002537 return false;
2538}
2539
David Woodhouse3bdb2592014-03-09 16:03:08 -07002540static int iommu_should_identity_map(struct device *dev, int startup)
David Woodhouse6941af22009-07-04 18:24:27 +01002541{
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002542
David Woodhouse3bdb2592014-03-09 16:03:08 -07002543 if (dev_is_pci(dev)) {
2544 struct pci_dev *pdev = to_pci_dev(dev);
Tom Mingarelliea2447f2012-11-20 19:43:17 +00002545
David Woodhouse3bdb2592014-03-09 16:03:08 -07002546 /*
2547 * We want to prevent any device associated with an RMRR from
2548 * getting placed into the SI Domain. This is done because
2549 * problems exist when devices are moved in and out of domains
2550 * and their respective RMRR info is lost. We exempt USB devices
2551 * from this process due to their usage of RMRRs that are known
2552 * to not be needed after BIOS hand-off to OS.
2553 */
2554 if (device_has_rmrr(dev) &&
2555 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2556 return 0;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002557
David Woodhouse3bdb2592014-03-09 16:03:08 -07002558 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2559 return 1;
David Woodhousee0fc7e02009-09-30 09:12:17 -07002560
David Woodhouse3bdb2592014-03-09 16:03:08 -07002561 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2562 return 1;
2563
2564 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2565 return 0;
2566
2567 /*
2568 * We want to start off with all devices in the 1:1 domain, and
2569 * take them out later if we find they can't access all of memory.
2570 *
2571 * However, we can't do this for PCI devices behind bridges,
2572 * because all PCI devices behind the same bridge will end up
2573 * with the same source-id on their transactions.
2574 *
2575 * Practically speaking, we can't change things around for these
2576 * devices at run-time, because we can't be sure there'll be no
2577 * DMA transactions in flight for any of their siblings.
2578 *
2579 * So PCI devices (unless they're on the root bus) as well as
2580 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2581 * the 1:1 domain, just in _case_ one of their siblings turns out
2582 * not to be able to map all of memory.
2583 */
2584 if (!pci_is_pcie(pdev)) {
2585 if (!pci_is_root_bus(pdev->bus))
2586 return 0;
2587 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2588 return 0;
2589 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2590 return 0;
2591 } else {
2592 if (device_has_rmrr(dev))
2593 return 0;
2594 }
David Woodhouse6941af22009-07-04 18:24:27 +01002595
David Woodhouse3dfc8132009-07-04 19:11:08 +01002596 /*
David Woodhouse3dfc8132009-07-04 19:11:08 +01002597 * At boot time, we don't yet know if devices will be 64-bit capable.
David Woodhouse3bdb2592014-03-09 16:03:08 -07002598 * Assume that they will — if they turn out not to be, then we can
David Woodhouse3dfc8132009-07-04 19:11:08 +01002599 * take them out of the 1:1 domain later.
2600 */
Chris Wright8fcc5372011-05-28 13:15:02 -05002601 if (!startup) {
2602 /*
2603 * If the device's dma_mask is less than the system's memory
2604 * size then this is not a candidate for identity mapping.
2605 */
David Woodhouse3bdb2592014-03-09 16:03:08 -07002606 u64 dma_mask = *dev->dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002607
David Woodhouse3bdb2592014-03-09 16:03:08 -07002608 if (dev->coherent_dma_mask &&
2609 dev->coherent_dma_mask < dma_mask)
2610 dma_mask = dev->coherent_dma_mask;
Chris Wright8fcc5372011-05-28 13:15:02 -05002611
David Woodhouse3bdb2592014-03-09 16:03:08 -07002612 return dma_mask >= dma_get_required_mask(dev);
Chris Wright8fcc5372011-05-28 13:15:02 -05002613 }
David Woodhouse6941af22009-07-04 18:24:27 +01002614
2615 return 1;
2616}
2617
David Woodhousecf04eee2014-03-21 16:49:04 +00002618static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2619{
2620 int ret;
2621
2622 if (!iommu_should_identity_map(dev, 1))
2623 return 0;
2624
2625 ret = domain_add_dev_info(si_domain, dev,
2626 hw ? CONTEXT_TT_PASS_THROUGH :
2627 CONTEXT_TT_MULTI_LEVEL);
2628 if (!ret)
2629 pr_info("IOMMU: %s identity mapping for device %s\n",
2630 hw ? "hardware" : "software", dev_name(dev));
2631 else if (ret == -ENODEV)
2632 /* device not associated with an iommu */
2633 ret = 0;
2634
2635 return ret;
2636}
2637
2638
Matt Kraai071e1372009-08-23 22:30:22 -07002639static int __init iommu_prepare_static_identity_mapping(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002640{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002641 struct pci_dev *pdev = NULL;
David Woodhousecf04eee2014-03-21 16:49:04 +00002642 struct dmar_drhd_unit *drhd;
2643 struct intel_iommu *iommu;
2644 struct device *dev;
2645 int i;
2646 int ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002647
David Woodhouse19943b02009-08-04 16:19:20 +01002648 ret = si_domain_init(hw);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002649 if (ret)
2650 return -EFAULT;
2651
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002652 for_each_pci_dev(pdev) {
David Woodhousecf04eee2014-03-21 16:49:04 +00002653 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2654 if (ret)
2655 return ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002656 }
2657
David Woodhousecf04eee2014-03-21 16:49:04 +00002658 for_each_active_iommu(iommu, drhd)
2659 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2660 struct acpi_device_physical_node *pn;
2661 struct acpi_device *adev;
2662
2663 if (dev->bus != &acpi_bus_type)
2664 continue;
2665
2666 adev= to_acpi_device(dev);
2667 mutex_lock(&adev->physical_node_lock);
2668 list_for_each_entry(pn, &adev->physical_node_list, node) {
2669 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2670 if (ret)
2671 break;
2672 }
2673 mutex_unlock(&adev->physical_node_lock);
2674 if (ret)
2675 return ret;
2676 }
2677
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002678 return 0;
2679}
2680
Joseph Cihulab7792602011-05-03 00:08:37 -07002681static int __init init_dmars(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002682{
2683 struct dmar_drhd_unit *drhd;
2684 struct dmar_rmrr_unit *rmrr;
David Woodhouse832bd852014-03-07 15:08:36 +00002685 struct device *dev;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002686 struct intel_iommu *iommu;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07002687 int i, ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002688
2689 /*
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002690 * for each drhd
2691 * allocate root
2692 * initialize and program root entry to not present
2693 * endfor
2694 */
2695 for_each_drhd_unit(drhd) {
mark gross5e0d2a62008-03-04 15:22:08 -08002696 /*
2697 * lock not needed as this is only incremented in the single
2698 * threaded kernel __init code path all other access are read
2699 * only
2700 */
Mike Travis1b198bb2012-03-05 15:05:16 -08002701 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2702 g_num_of_iommus++;
2703 continue;
2704 }
2705 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2706 IOMMU_UNITS_SUPPORTED);
mark gross5e0d2a62008-03-04 15:22:08 -08002707 }
2708
Weidong Hand9630fe2008-12-08 11:06:32 +08002709 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2710 GFP_KERNEL);
2711 if (!g_iommus) {
2712 printk(KERN_ERR "Allocating global iommu array failed\n");
2713 ret = -ENOMEM;
2714 goto error;
2715 }
2716
mark gross80b20dd2008-04-18 13:53:58 -07002717 deferred_flush = kzalloc(g_num_of_iommus *
2718 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2719 if (!deferred_flush) {
mark gross5e0d2a62008-03-04 15:22:08 -08002720 ret = -ENOMEM;
Jiang Liu989d51f2014-02-19 14:07:21 +08002721 goto free_g_iommus;
mark gross5e0d2a62008-03-04 15:22:08 -08002722 }
2723
Jiang Liu7c919772014-01-06 14:18:18 +08002724 for_each_active_iommu(iommu, drhd) {
Weidong Hand9630fe2008-12-08 11:06:32 +08002725 g_iommus[iommu->seq_id] = iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002726
Suresh Siddhae61d98d2008-07-10 11:16:35 -07002727 ret = iommu_init_domains(iommu);
2728 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08002729 goto free_iommu;
Suresh Siddhae61d98d2008-07-10 11:16:35 -07002730
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002731 /*
2732 * TBD:
2733 * we could share the same root & context tables
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002734 * among all IOMMU's. Need to Split it later.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002735 */
2736 ret = iommu_alloc_root_entry(iommu);
2737 if (ret) {
2738 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
Jiang Liu989d51f2014-02-19 14:07:21 +08002739 goto free_iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002740 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002741 if (!ecap_pass_through(iommu->ecap))
David Woodhouse19943b02009-08-04 16:19:20 +01002742 hw_pass_through = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002743 }
2744
Suresh Siddha1531a6a2009-03-16 17:04:57 -07002745 /*
2746 * Start from the sane iommu hardware state.
2747 */
Jiang Liu7c919772014-01-06 14:18:18 +08002748 for_each_active_iommu(iommu, drhd) {
Suresh Siddha1531a6a2009-03-16 17:04:57 -07002749 /*
2750 * If the queued invalidation is already initialized by us
2751 * (for example, while enabling interrupt-remapping) then
2752 * we got the things already rolling from a sane state.
2753 */
2754 if (iommu->qi)
2755 continue;
2756
2757 /*
2758 * Clear any previous faults.
2759 */
2760 dmar_fault(-1, iommu);
2761 /*
2762 * Disable queued invalidation if supported and already enabled
2763 * before OS handover.
2764 */
2765 dmar_disable_qi(iommu);
2766 }
2767
Jiang Liu7c919772014-01-06 14:18:18 +08002768 for_each_active_iommu(iommu, drhd) {
Youquan Songa77b67d2008-10-16 16:31:56 -07002769 if (dmar_enable_qi(iommu)) {
2770 /*
2771 * Queued Invalidate not enabled, use Register Based
2772 * Invalidate
2773 */
2774 iommu->flush.flush_context = __iommu_flush_context;
2775 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
Yinghai Lu680a7522010-04-08 19:58:23 +01002776 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002777 "invalidation\n",
Yinghai Lu680a7522010-04-08 19:58:23 +01002778 iommu->seq_id,
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002779 (unsigned long long)drhd->reg_base_addr);
Youquan Songa77b67d2008-10-16 16:31:56 -07002780 } else {
2781 iommu->flush.flush_context = qi_flush_context;
2782 iommu->flush.flush_iotlb = qi_flush_iotlb;
Yinghai Lu680a7522010-04-08 19:58:23 +01002783 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002784 "invalidation\n",
Yinghai Lu680a7522010-04-08 19:58:23 +01002785 iommu->seq_id,
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002786 (unsigned long long)drhd->reg_base_addr);
Youquan Songa77b67d2008-10-16 16:31:56 -07002787 }
2788 }
2789
David Woodhouse19943b02009-08-04 16:19:20 +01002790 if (iommu_pass_through)
David Woodhousee0fc7e02009-09-30 09:12:17 -07002791 iommu_identity_mapping |= IDENTMAP_ALL;
2792
Suresh Siddhad3f13812011-08-23 17:05:25 -07002793#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
David Woodhousee0fc7e02009-09-30 09:12:17 -07002794 iommu_identity_mapping |= IDENTMAP_GFX;
David Woodhouse19943b02009-08-04 16:19:20 +01002795#endif
David Woodhousee0fc7e02009-09-30 09:12:17 -07002796
2797 check_tylersburg_isoch();
2798
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002799 /*
2800 * If pass through is not set or not enabled, setup context entries for
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002801 * identity mappings for rmrr, gfx, and isa and may fall back to static
2802 * identity mapping if iommu_identity_mapping is set.
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002803 */
David Woodhouse19943b02009-08-04 16:19:20 +01002804 if (iommu_identity_mapping) {
2805 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2806 if (ret) {
2807 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
Jiang Liu989d51f2014-02-19 14:07:21 +08002808 goto free_iommu;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002809 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002810 }
David Woodhouse19943b02009-08-04 16:19:20 +01002811 /*
2812 * For each rmrr
2813 * for each dev attached to rmrr
2814 * do
2815 * locate drhd for dev, alloc domain for dev
2816 * allocate free domain
2817 * allocate page table entries for rmrr
2818 * if context not allocated for bus
2819 * allocate and init context
2820 * set present in root table for this bus
2821 * init context with domain, translation etc
2822 * endfor
2823 * endfor
2824 */
2825 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2826 for_each_rmrr_units(rmrr) {
Jiang Liub683b232014-02-19 14:07:32 +08002827 /* some BIOS lists non-exist devices in DMAR table. */
2828 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
David Woodhouse832bd852014-03-07 15:08:36 +00002829 i, dev) {
David Woodhouse0b9d9752014-03-09 15:48:15 -07002830 ret = iommu_prepare_rmrr_dev(rmrr, dev);
David Woodhouse19943b02009-08-04 16:19:20 +01002831 if (ret)
2832 printk(KERN_ERR
2833 "IOMMU: mapping reserved region failed\n");
2834 }
2835 }
2836
2837 iommu_prepare_isa();
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002838
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002839 /*
2840 * for each drhd
2841 * enable fault log
2842 * global invalidate context cache
2843 * global invalidate iotlb
2844 * enable translation
2845 */
Jiang Liu7c919772014-01-06 14:18:18 +08002846 for_each_iommu(iommu, drhd) {
Joseph Cihula51a63e62011-03-21 11:04:24 -07002847 if (drhd->ignored) {
2848 /*
2849 * we always have to disable PMRs or DMA may fail on
2850 * this device
2851 */
2852 if (force_on)
Jiang Liu7c919772014-01-06 14:18:18 +08002853 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002854 continue;
Joseph Cihula51a63e62011-03-21 11:04:24 -07002855 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002856
2857 iommu_flush_write_buffer(iommu);
2858
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07002859 ret = dmar_set_interrupt(iommu);
2860 if (ret)
Jiang Liu989d51f2014-02-19 14:07:21 +08002861 goto free_iommu;
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07002862
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002863 iommu_set_root_entry(iommu);
2864
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002865 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002866 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
Jiang Liu2a41cce2014-07-11 14:19:33 +08002867 iommu_enable_translation(iommu);
David Woodhouseb94996c2009-09-19 15:28:12 -07002868 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002869 }
2870
2871 return 0;
Jiang Liu989d51f2014-02-19 14:07:21 +08002872
2873free_iommu:
Jiang Liu7c919772014-01-06 14:18:18 +08002874 for_each_active_iommu(iommu, drhd)
Jiang Liua868e6b2014-01-06 14:18:20 +08002875 free_dmar_iommu(iommu);
Jiang Liu9bdc5312014-01-06 14:18:27 +08002876 kfree(deferred_flush);
Jiang Liu989d51f2014-02-19 14:07:21 +08002877free_g_iommus:
Weidong Hand9630fe2008-12-08 11:06:32 +08002878 kfree(g_iommus);
Jiang Liu989d51f2014-02-19 14:07:21 +08002879error:
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002880 return ret;
2881}
2882
David Woodhouse5a5e02a2009-07-04 09:35:44 +01002883/* This takes a number of _MM_ pages, not VTD pages */
David Woodhouse875764d2009-06-28 21:20:51 +01002884static struct iova *intel_alloc_iova(struct device *dev,
2885 struct dmar_domain *domain,
2886 unsigned long nrpages, uint64_t dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002887{
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002888 struct iova *iova = NULL;
2889
David Woodhouse875764d2009-06-28 21:20:51 +01002890 /* Restrict dma_mask to the width that the iommu can handle */
2891 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2892
2893 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002894 /*
2895 * First try to allocate an io virtual address in
Yang Hongyang284901a2009-04-06 19:01:15 -07002896 * DMA_BIT_MASK(32) and if that fails then try allocating
Joe Perches36098012007-12-17 11:40:11 -08002897 * from higher range
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002898 */
David Woodhouse875764d2009-06-28 21:20:51 +01002899 iova = alloc_iova(&domain->iovad, nrpages,
2900 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2901 if (iova)
2902 return iova;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002903 }
David Woodhouse875764d2009-06-28 21:20:51 +01002904 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2905 if (unlikely(!iova)) {
2906 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
David Woodhouse207e3592014-03-09 16:12:32 -07002907 nrpages, dev_name(dev));
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002908 return NULL;
2909 }
2910
2911 return iova;
2912}
2913
David Woodhoused4b709f2014-03-09 16:07:40 -07002914static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002915{
2916 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002917 int ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002918
David Woodhoused4b709f2014-03-09 16:07:40 -07002919 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002920 if (!domain) {
David Woodhoused4b709f2014-03-09 16:07:40 -07002921 printk(KERN_ERR "Allocating domain for %s failed",
2922 dev_name(dev));
Al Viro4fe05bb2007-10-29 04:51:16 +00002923 return NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002924 }
2925
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002926 /* make sure context mapping is ok */
David Woodhoused4b709f2014-03-09 16:07:40 -07002927 if (unlikely(!domain_context_mapped(dev))) {
2928 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002929 if (ret) {
David Woodhoused4b709f2014-03-09 16:07:40 -07002930 printk(KERN_ERR "Domain context map for %s failed",
2931 dev_name(dev));
Al Viro4fe05bb2007-10-29 04:51:16 +00002932 return NULL;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002933 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002934 }
2935
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002936 return domain;
2937}
2938
David Woodhoused4b709f2014-03-09 16:07:40 -07002939static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
David Woodhouse147202a2009-07-07 19:43:20 +01002940{
2941 struct device_domain_info *info;
2942
2943 /* No lock here, assumes no domain exit in normal case */
David Woodhoused4b709f2014-03-09 16:07:40 -07002944 info = dev->archdata.iommu;
David Woodhouse147202a2009-07-07 19:43:20 +01002945 if (likely(info))
2946 return info->domain;
2947
2948 return __get_valid_domain_for_dev(dev);
2949}
2950
David Woodhouse3d891942014-03-06 15:59:26 +00002951static int iommu_dummy(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002952{
David Woodhouse3d891942014-03-06 15:59:26 +00002953 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002954}
2955
David Woodhouseecb509e2014-03-09 16:29:55 -07002956/* Check if the dev needs to go through non-identity map and unmap process.*/
David Woodhouse73676832009-07-04 14:08:36 +01002957static int iommu_no_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002958{
2959 int found;
2960
David Woodhouse3d891942014-03-06 15:59:26 +00002961 if (iommu_dummy(dev))
David Woodhouse1e4c64c2009-07-04 10:40:38 +01002962 return 1;
2963
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002964 if (!iommu_identity_mapping)
David Woodhouse1e4c64c2009-07-04 10:40:38 +01002965 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002966
David Woodhouse9b226622014-03-09 14:03:28 -07002967 found = identity_mapping(dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002968 if (found) {
David Woodhouseecb509e2014-03-09 16:29:55 -07002969 if (iommu_should_identity_map(dev, 0))
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002970 return 1;
2971 else {
2972 /*
2973 * 32 bit DMA is removed from si_domain and fall back
2974 * to non-identity mapping.
2975 */
David Woodhousebf9c9ed2014-03-09 16:19:13 -07002976 domain_remove_one_dev_info(si_domain, dev);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002977 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
David Woodhouseecb509e2014-03-09 16:29:55 -07002978 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002979 return 0;
2980 }
2981 } else {
2982 /*
2983 * In case of a detached 64 bit DMA device from vm, the device
2984 * is put into si_domain for identity mapping.
2985 */
David Woodhouseecb509e2014-03-09 16:29:55 -07002986 if (iommu_should_identity_map(dev, 0)) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002987 int ret;
David Woodhouse5913c9b2014-03-09 16:27:31 -07002988 ret = domain_add_dev_info(si_domain, dev,
David Woodhouse5fe60f42009-08-09 10:53:41 +01002989 hw_pass_through ?
2990 CONTEXT_TT_PASS_THROUGH :
2991 CONTEXT_TT_MULTI_LEVEL);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002992 if (!ret) {
2993 printk(KERN_INFO "64bit %s uses identity mapping\n",
David Woodhouseecb509e2014-03-09 16:29:55 -07002994 dev_name(dev));
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002995 return 1;
2996 }
2997 }
2998 }
2999
David Woodhouse1e4c64c2009-07-04 10:40:38 +01003000 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003001}
3002
David Woodhouse5040a912014-03-09 16:14:00 -07003003static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003004 size_t size, int dir, u64 dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003005{
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003006 struct dmar_domain *domain;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07003007 phys_addr_t start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003008 struct iova *iova;
3009 int prot = 0;
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003010 int ret;
Weidong Han8c11e792008-12-08 15:29:22 +08003011 struct intel_iommu *iommu;
Fenghua Yu33041ec2009-08-04 15:10:59 -07003012 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003013
3014 BUG_ON(dir == DMA_NONE);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003015
David Woodhouse5040a912014-03-09 16:14:00 -07003016 if (iommu_no_mapping(dev))
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003017 return paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003018
David Woodhouse5040a912014-03-09 16:14:00 -07003019 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003020 if (!domain)
3021 return 0;
3022
Weidong Han8c11e792008-12-08 15:29:22 +08003023 iommu = domain_get_iommu(domain);
David Woodhouse88cb6a72009-06-28 15:03:06 +01003024 size = aligned_nrpages(paddr, size);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003025
David Woodhouse5040a912014-03-09 16:14:00 -07003026 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003027 if (!iova)
3028 goto error;
3029
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003030 /*
3031 * Check if DMAR supports zero-length reads on write only
3032 * mappings..
3033 */
3034 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003035 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003036 prot |= DMA_PTE_READ;
3037 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3038 prot |= DMA_PTE_WRITE;
3039 /*
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003040 * paddr - (paddr + size) might be partial page, we should map the whole
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003041 * page. Note: if two part of one page are separately mapped, we
Ingo Molnar6865f0d2008-04-22 11:09:04 +02003042 * might have two guest_addr mapping to the same host paddr, but this
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003043 * is not a big problem
3044 */
David Woodhouse0ab36de2009-06-28 14:01:43 +01003045 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
Fenghua Yu33041ec2009-08-04 15:10:59 -07003046 mm_to_dma_pfn(paddr_pfn), size, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003047 if (ret)
3048 goto error;
3049
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003050 /* it's a non-present to present mapping. Only flush if caching mode */
3051 if (cap_caching_mode(iommu->cap))
David Woodhouseea8ea462014-03-05 17:09:32 +00003052 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003053 else
Weidong Han8c11e792008-12-08 15:29:22 +08003054 iommu_flush_write_buffer(iommu);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003055
David Woodhouse03d6a242009-06-28 15:33:46 +01003056 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3057 start_paddr += paddr & ~PAGE_MASK;
3058 return start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003059
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003060error:
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003061 if (iova)
3062 __free_iova(&domain->iovad, iova);
David Woodhouse4cf2e752009-02-11 17:23:43 +00003063 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
David Woodhouse5040a912014-03-09 16:14:00 -07003064 dev_name(dev), size, (unsigned long long)paddr, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003065 return 0;
3066}
3067
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003068static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3069 unsigned long offset, size_t size,
3070 enum dma_data_direction dir,
3071 struct dma_attrs *attrs)
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003072{
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003073 return __intel_map_single(dev, page_to_phys(page) + offset, size,
David Woodhouse46333e32014-03-10 20:01:21 -07003074 dir, *dev->dma_mask);
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003075}
3076
mark gross5e0d2a62008-03-04 15:22:08 -08003077static void flush_unmaps(void)
3078{
mark gross80b20dd2008-04-18 13:53:58 -07003079 int i, j;
mark gross5e0d2a62008-03-04 15:22:08 -08003080
mark gross5e0d2a62008-03-04 15:22:08 -08003081 timer_on = 0;
3082
3083 /* just flush them all */
3084 for (i = 0; i < g_num_of_iommus; i++) {
Weidong Hana2bb8452008-12-08 11:24:12 +08003085 struct intel_iommu *iommu = g_iommus[i];
3086 if (!iommu)
3087 continue;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07003088
Yu Zhao9dd2fe82009-05-18 13:51:36 +08003089 if (!deferred_flush[i].next)
3090 continue;
3091
Nadav Amit78d5f0f2010-04-08 23:00:41 +03003092 /* In caching mode, global flushes turn emulation expensive */
3093 if (!cap_caching_mode(iommu->cap))
3094 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
Yu Zhao93a23a72009-05-18 13:51:37 +08003095 DMA_TLB_GLOBAL_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08003096 for (j = 0; j < deferred_flush[i].next; j++) {
Yu Zhao93a23a72009-05-18 13:51:37 +08003097 unsigned long mask;
3098 struct iova *iova = deferred_flush[i].iova[j];
Nadav Amit78d5f0f2010-04-08 23:00:41 +03003099 struct dmar_domain *domain = deferred_flush[i].domain[j];
Yu Zhao93a23a72009-05-18 13:51:37 +08003100
Nadav Amit78d5f0f2010-04-08 23:00:41 +03003101 /* On real hardware multiple invalidations are expensive */
3102 if (cap_caching_mode(iommu->cap))
3103 iommu_flush_iotlb_psi(iommu, domain->id,
Jiang Liua156ef92014-07-11 14:19:36 +08003104 iova->pfn_lo, iova_size(iova),
David Woodhouseea8ea462014-03-05 17:09:32 +00003105 !deferred_flush[i].freelist[j], 0);
Nadav Amit78d5f0f2010-04-08 23:00:41 +03003106 else {
Jiang Liua156ef92014-07-11 14:19:36 +08003107 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
Nadav Amit78d5f0f2010-04-08 23:00:41 +03003108 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3109 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3110 }
Yu Zhao93a23a72009-05-18 13:51:37 +08003111 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
David Woodhouseea8ea462014-03-05 17:09:32 +00003112 if (deferred_flush[i].freelist[j])
3113 dma_free_pagelist(deferred_flush[i].freelist[j]);
mark gross80b20dd2008-04-18 13:53:58 -07003114 }
Yu Zhao9dd2fe82009-05-18 13:51:36 +08003115 deferred_flush[i].next = 0;
mark gross5e0d2a62008-03-04 15:22:08 -08003116 }
3117
mark gross5e0d2a62008-03-04 15:22:08 -08003118 list_size = 0;
mark gross5e0d2a62008-03-04 15:22:08 -08003119}
3120
3121static void flush_unmaps_timeout(unsigned long data)
3122{
mark gross80b20dd2008-04-18 13:53:58 -07003123 unsigned long flags;
3124
3125 spin_lock_irqsave(&async_umap_flush_lock, flags);
mark gross5e0d2a62008-03-04 15:22:08 -08003126 flush_unmaps();
mark gross80b20dd2008-04-18 13:53:58 -07003127 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
mark gross5e0d2a62008-03-04 15:22:08 -08003128}
3129
David Woodhouseea8ea462014-03-05 17:09:32 +00003130static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
mark gross5e0d2a62008-03-04 15:22:08 -08003131{
3132 unsigned long flags;
mark gross80b20dd2008-04-18 13:53:58 -07003133 int next, iommu_id;
Weidong Han8c11e792008-12-08 15:29:22 +08003134 struct intel_iommu *iommu;
mark gross5e0d2a62008-03-04 15:22:08 -08003135
3136 spin_lock_irqsave(&async_umap_flush_lock, flags);
mark gross80b20dd2008-04-18 13:53:58 -07003137 if (list_size == HIGH_WATER_MARK)
3138 flush_unmaps();
3139
Weidong Han8c11e792008-12-08 15:29:22 +08003140 iommu = domain_get_iommu(dom);
3141 iommu_id = iommu->seq_id;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07003142
mark gross80b20dd2008-04-18 13:53:58 -07003143 next = deferred_flush[iommu_id].next;
3144 deferred_flush[iommu_id].domain[next] = dom;
3145 deferred_flush[iommu_id].iova[next] = iova;
David Woodhouseea8ea462014-03-05 17:09:32 +00003146 deferred_flush[iommu_id].freelist[next] = freelist;
mark gross80b20dd2008-04-18 13:53:58 -07003147 deferred_flush[iommu_id].next++;
mark gross5e0d2a62008-03-04 15:22:08 -08003148
3149 if (!timer_on) {
3150 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3151 timer_on = 1;
3152 }
3153 list_size++;
3154 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3155}
3156
Jiang Liud41a4ad2014-07-11 14:19:34 +08003157static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003158{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003159 struct dmar_domain *domain;
David Woodhoused794dc92009-06-28 00:27:49 +01003160 unsigned long start_pfn, last_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003161 struct iova *iova;
Weidong Han8c11e792008-12-08 15:29:22 +08003162 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00003163 struct page *freelist;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003164
David Woodhouse73676832009-07-04 14:08:36 +01003165 if (iommu_no_mapping(dev))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003166 return;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003167
David Woodhouse1525a292014-03-06 16:19:30 +00003168 domain = find_domain(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003169 BUG_ON(!domain);
3170
Weidong Han8c11e792008-12-08 15:29:22 +08003171 iommu = domain_get_iommu(domain);
3172
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003173 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
David Woodhouse85b98272009-07-01 19:27:53 +01003174 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3175 (unsigned long long)dev_addr))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003176 return;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003177
David Woodhoused794dc92009-06-28 00:27:49 +01003178 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3179 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003180
David Woodhoused794dc92009-06-28 00:27:49 +01003181 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
David Woodhouse207e3592014-03-09 16:12:32 -07003182 dev_name(dev), start_pfn, last_pfn);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003183
David Woodhouseea8ea462014-03-05 17:09:32 +00003184 freelist = domain_unmap(domain, start_pfn, last_pfn);
David Woodhoused794dc92009-06-28 00:27:49 +01003185
mark gross5e0d2a62008-03-04 15:22:08 -08003186 if (intel_iommu_strict) {
David Woodhouse03d6a242009-06-28 15:33:46 +01003187 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
David Woodhouseea8ea462014-03-05 17:09:32 +00003188 last_pfn - start_pfn + 1, !freelist, 0);
mark gross5e0d2a62008-03-04 15:22:08 -08003189 /* free iova */
3190 __free_iova(&domain->iovad, iova);
David Woodhouseea8ea462014-03-05 17:09:32 +00003191 dma_free_pagelist(freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003192 } else {
David Woodhouseea8ea462014-03-05 17:09:32 +00003193 add_unmap(domain, iova, freelist);
mark gross5e0d2a62008-03-04 15:22:08 -08003194 /*
3195 * queue up the release of the unmap to save the 1/6th of the
3196 * cpu used up by the iotlb flush operation...
3197 */
mark gross5e0d2a62008-03-04 15:22:08 -08003198 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003199}
3200
Jiang Liud41a4ad2014-07-11 14:19:34 +08003201static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3202 size_t size, enum dma_data_direction dir,
3203 struct dma_attrs *attrs)
3204{
3205 intel_unmap(dev, dev_addr);
3206}
3207
David Woodhouse5040a912014-03-09 16:14:00 -07003208static void *intel_alloc_coherent(struct device *dev, size_t size,
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003209 dma_addr_t *dma_handle, gfp_t flags,
3210 struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003211{
Akinobu Mita36746432014-06-04 16:06:51 -07003212 struct page *page = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003213 int order;
3214
Fenghua Yu5b6985c2008-10-16 18:02:32 -07003215 size = PAGE_ALIGN(size);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003216 order = get_order(size);
Alex Williamsone8bb9102009-11-04 15:59:34 -07003217
David Woodhouse5040a912014-03-09 16:14:00 -07003218 if (!iommu_no_mapping(dev))
Alex Williamsone8bb9102009-11-04 15:59:34 -07003219 flags &= ~(GFP_DMA | GFP_DMA32);
David Woodhouse5040a912014-03-09 16:14:00 -07003220 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3221 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
Alex Williamsone8bb9102009-11-04 15:59:34 -07003222 flags |= GFP_DMA;
3223 else
3224 flags |= GFP_DMA32;
3225 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003226
Akinobu Mita36746432014-06-04 16:06:51 -07003227 if (flags & __GFP_WAIT) {
3228 unsigned int count = size >> PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003229
Akinobu Mita36746432014-06-04 16:06:51 -07003230 page = dma_alloc_from_contiguous(dev, count, order);
3231 if (page && iommu_no_mapping(dev) &&
3232 page_to_phys(page) + size > dev->coherent_dma_mask) {
3233 dma_release_from_contiguous(dev, page, count);
3234 page = NULL;
3235 }
3236 }
3237
3238 if (!page)
3239 page = alloc_pages(flags, order);
3240 if (!page)
3241 return NULL;
3242 memset(page_address(page), 0, size);
3243
3244 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09003245 DMA_BIDIRECTIONAL,
David Woodhouse5040a912014-03-09 16:14:00 -07003246 dev->coherent_dma_mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003247 if (*dma_handle)
Akinobu Mita36746432014-06-04 16:06:51 -07003248 return page_address(page);
3249 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3250 __free_pages(page, order);
3251
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003252 return NULL;
3253}
3254
David Woodhouse5040a912014-03-09 16:14:00 -07003255static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003256 dma_addr_t dma_handle, struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003257{
3258 int order;
Akinobu Mita36746432014-06-04 16:06:51 -07003259 struct page *page = virt_to_page(vaddr);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003260
Fenghua Yu5b6985c2008-10-16 18:02:32 -07003261 size = PAGE_ALIGN(size);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003262 order = get_order(size);
3263
Jiang Liud41a4ad2014-07-11 14:19:34 +08003264 intel_unmap(dev, dma_handle);
Akinobu Mita36746432014-06-04 16:06:51 -07003265 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3266 __free_pages(page, order);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003267}
3268
David Woodhouse5040a912014-03-09 16:14:00 -07003269static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09003270 int nelems, enum dma_data_direction dir,
3271 struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003272{
Jiang Liud41a4ad2014-07-11 14:19:34 +08003273 intel_unmap(dev, sglist[0].dma_address);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003274}
3275
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003276static int intel_nontranslate_map_sg(struct device *hddev,
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003277 struct scatterlist *sglist, int nelems, int dir)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003278{
3279 int i;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003280 struct scatterlist *sg;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003281
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003282 for_each_sg(sglist, sg, nelems, i) {
FUJITA Tomonori12d4d402007-10-23 09:32:25 +02003283 BUG_ON(!sg_page(sg));
David Woodhouse4cf2e752009-02-11 17:23:43 +00003284 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003285 sg->dma_length = sg->length;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003286 }
3287 return nelems;
3288}
3289
David Woodhouse5040a912014-03-09 16:14:00 -07003290static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09003291 enum dma_data_direction dir, struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003292{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003293 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003294 struct dmar_domain *domain;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003295 size_t size = 0;
3296 int prot = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003297 struct iova *iova = NULL;
3298 int ret;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003299 struct scatterlist *sg;
David Woodhouseb536d242009-06-28 14:49:31 +01003300 unsigned long start_vpfn;
Weidong Han8c11e792008-12-08 15:29:22 +08003301 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003302
3303 BUG_ON(dir == DMA_NONE);
David Woodhouse5040a912014-03-09 16:14:00 -07003304 if (iommu_no_mapping(dev))
3305 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003306
David Woodhouse5040a912014-03-09 16:14:00 -07003307 domain = get_valid_domain_for_dev(dev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003308 if (!domain)
3309 return 0;
3310
Weidong Han8c11e792008-12-08 15:29:22 +08003311 iommu = domain_get_iommu(domain);
3312
David Woodhouseb536d242009-06-28 14:49:31 +01003313 for_each_sg(sglist, sg, nelems, i)
David Woodhouse88cb6a72009-06-28 15:03:06 +01003314 size += aligned_nrpages(sg->offset, sg->length);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003315
David Woodhouse5040a912014-03-09 16:14:00 -07003316 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3317 *dev->dma_mask);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003318 if (!iova) {
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07003319 sglist->dma_length = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003320 return 0;
3321 }
3322
3323 /*
3324 * Check if DMAR supports zero-length reads on write only
3325 * mappings..
3326 */
3327 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08003328 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003329 prot |= DMA_PTE_READ;
3330 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3331 prot |= DMA_PTE_WRITE;
3332
David Woodhouseb536d242009-06-28 14:49:31 +01003333 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
David Woodhousee1605492009-06-29 11:17:38 +01003334
Fenghua Yuf5329592009-08-04 15:09:37 -07003335 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
David Woodhousee1605492009-06-29 11:17:38 +01003336 if (unlikely(ret)) {
David Woodhousee1605492009-06-29 11:17:38 +01003337 dma_pte_free_pagetable(domain, start_vpfn,
3338 start_vpfn + size - 1);
David Woodhousee1605492009-06-29 11:17:38 +01003339 __free_iova(&domain->iovad, iova);
3340 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07003341 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003342
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003343 /* it's a non-present to present mapping. Only flush if caching mode */
3344 if (cap_caching_mode(iommu->cap))
David Woodhouseea8ea462014-03-05 17:09:32 +00003345 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003346 else
Weidong Han8c11e792008-12-08 15:29:22 +08003347 iommu_flush_write_buffer(iommu);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003348
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003349 return nelems;
3350}
3351
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09003352static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3353{
3354 return !dma_addr;
3355}
3356
FUJITA Tomonori160c1d82009-01-05 23:59:02 +09003357struct dma_map_ops intel_dma_ops = {
Andrzej Pietrasiewiczbaa676f2012-03-27 14:28:18 +02003358 .alloc = intel_alloc_coherent,
3359 .free = intel_free_coherent,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003360 .map_sg = intel_map_sg,
3361 .unmap_sg = intel_unmap_sg,
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09003362 .map_page = intel_map_page,
3363 .unmap_page = intel_unmap_page,
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09003364 .mapping_error = intel_mapping_error,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003365};
3366
3367static inline int iommu_domain_cache_init(void)
3368{
3369 int ret = 0;
3370
3371 iommu_domain_cache = kmem_cache_create("iommu_domain",
3372 sizeof(struct dmar_domain),
3373 0,
3374 SLAB_HWCACHE_ALIGN,
3375
3376 NULL);
3377 if (!iommu_domain_cache) {
3378 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3379 ret = -ENOMEM;
3380 }
3381
3382 return ret;
3383}
3384
3385static inline int iommu_devinfo_cache_init(void)
3386{
3387 int ret = 0;
3388
3389 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3390 sizeof(struct device_domain_info),
3391 0,
3392 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003393 NULL);
3394 if (!iommu_devinfo_cache) {
3395 printk(KERN_ERR "Couldn't create devinfo cache\n");
3396 ret = -ENOMEM;
3397 }
3398
3399 return ret;
3400}
3401
3402static inline int iommu_iova_cache_init(void)
3403{
3404 int ret = 0;
3405
3406 iommu_iova_cache = kmem_cache_create("iommu_iova",
3407 sizeof(struct iova),
3408 0,
3409 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003410 NULL);
3411 if (!iommu_iova_cache) {
3412 printk(KERN_ERR "Couldn't create iova cache\n");
3413 ret = -ENOMEM;
3414 }
3415
3416 return ret;
3417}
3418
3419static int __init iommu_init_mempool(void)
3420{
3421 int ret;
3422 ret = iommu_iova_cache_init();
3423 if (ret)
3424 return ret;
3425
3426 ret = iommu_domain_cache_init();
3427 if (ret)
3428 goto domain_error;
3429
3430 ret = iommu_devinfo_cache_init();
3431 if (!ret)
3432 return ret;
3433
3434 kmem_cache_destroy(iommu_domain_cache);
3435domain_error:
3436 kmem_cache_destroy(iommu_iova_cache);
3437
3438 return -ENOMEM;
3439}
3440
3441static void __init iommu_exit_mempool(void)
3442{
3443 kmem_cache_destroy(iommu_devinfo_cache);
3444 kmem_cache_destroy(iommu_domain_cache);
3445 kmem_cache_destroy(iommu_iova_cache);
3446
3447}
3448
Dan Williams556ab452010-07-23 15:47:56 -07003449static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3450{
3451 struct dmar_drhd_unit *drhd;
3452 u32 vtbar;
3453 int rc;
3454
3455 /* We know that this device on this chipset has its own IOMMU.
3456 * If we find it under a different IOMMU, then the BIOS is lying
3457 * to us. Hope that the IOMMU for this device is actually
3458 * disabled, and it needs no translation...
3459 */
3460 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3461 if (rc) {
3462 /* "can't" happen */
3463 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3464 return;
3465 }
3466 vtbar &= 0xffff0000;
3467
3468 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3469 drhd = dmar_find_matched_drhd_unit(pdev);
3470 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3471 TAINT_FIRMWARE_WORKAROUND,
3472 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3473 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3474}
3475DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3476
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003477static void __init init_no_remapping_devices(void)
3478{
3479 struct dmar_drhd_unit *drhd;
David Woodhouse832bd852014-03-07 15:08:36 +00003480 struct device *dev;
Jiang Liub683b232014-02-19 14:07:32 +08003481 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003482
3483 for_each_drhd_unit(drhd) {
3484 if (!drhd->include_all) {
Jiang Liub683b232014-02-19 14:07:32 +08003485 for_each_active_dev_scope(drhd->devices,
3486 drhd->devices_cnt, i, dev)
3487 break;
David Woodhouse832bd852014-03-07 15:08:36 +00003488 /* ignore DMAR unit if no devices exist */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003489 if (i == drhd->devices_cnt)
3490 drhd->ignored = 1;
3491 }
3492 }
3493
Jiang Liu7c919772014-01-06 14:18:18 +08003494 for_each_active_drhd_unit(drhd) {
Jiang Liu7c919772014-01-06 14:18:18 +08003495 if (drhd->include_all)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003496 continue;
3497
Jiang Liub683b232014-02-19 14:07:32 +08003498 for_each_active_dev_scope(drhd->devices,
3499 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00003500 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003501 break;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003502 if (i < drhd->devices_cnt)
3503 continue;
3504
David Woodhousec0771df2011-10-14 20:59:46 +01003505 /* This IOMMU has *only* gfx devices. Either bypass it or
3506 set the gfx_mapped flag, as appropriate */
3507 if (dmar_map_gfx) {
3508 intel_iommu_gfx_mapped = 1;
3509 } else {
3510 drhd->ignored = 1;
Jiang Liub683b232014-02-19 14:07:32 +08003511 for_each_active_dev_scope(drhd->devices,
3512 drhd->devices_cnt, i, dev)
David Woodhouse832bd852014-03-07 15:08:36 +00003513 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003514 }
3515 }
3516}
3517
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003518#ifdef CONFIG_SUSPEND
3519static int init_iommu_hw(void)
3520{
3521 struct dmar_drhd_unit *drhd;
3522 struct intel_iommu *iommu = NULL;
3523
3524 for_each_active_iommu(iommu, drhd)
3525 if (iommu->qi)
3526 dmar_reenable_qi(iommu);
3527
Joseph Cihulab7792602011-05-03 00:08:37 -07003528 for_each_iommu(iommu, drhd) {
3529 if (drhd->ignored) {
3530 /*
3531 * we always have to disable PMRs or DMA may fail on
3532 * this device
3533 */
3534 if (force_on)
3535 iommu_disable_protect_mem_regions(iommu);
3536 continue;
3537 }
3538
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003539 iommu_flush_write_buffer(iommu);
3540
3541 iommu_set_root_entry(iommu);
3542
3543 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003544 DMA_CCMD_GLOBAL_INVL);
Jiang Liu2a41cce2014-07-11 14:19:33 +08003545 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3546 iommu_enable_translation(iommu);
David Woodhouseb94996c2009-09-19 15:28:12 -07003547 iommu_disable_protect_mem_regions(iommu);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003548 }
3549
3550 return 0;
3551}
3552
3553static void iommu_flush_all(void)
3554{
3555 struct dmar_drhd_unit *drhd;
3556 struct intel_iommu *iommu;
3557
3558 for_each_active_iommu(iommu, drhd) {
3559 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003560 DMA_CCMD_GLOBAL_INVL);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003561 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003562 DMA_TLB_GLOBAL_FLUSH);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003563 }
3564}
3565
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003566static int iommu_suspend(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003567{
3568 struct dmar_drhd_unit *drhd;
3569 struct intel_iommu *iommu = NULL;
3570 unsigned long flag;
3571
3572 for_each_active_iommu(iommu, drhd) {
3573 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3574 GFP_ATOMIC);
3575 if (!iommu->iommu_state)
3576 goto nomem;
3577 }
3578
3579 iommu_flush_all();
3580
3581 for_each_active_iommu(iommu, drhd) {
3582 iommu_disable_translation(iommu);
3583
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02003584 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003585
3586 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3587 readl(iommu->reg + DMAR_FECTL_REG);
3588 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3589 readl(iommu->reg + DMAR_FEDATA_REG);
3590 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3591 readl(iommu->reg + DMAR_FEADDR_REG);
3592 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3593 readl(iommu->reg + DMAR_FEUADDR_REG);
3594
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02003595 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003596 }
3597 return 0;
3598
3599nomem:
3600 for_each_active_iommu(iommu, drhd)
3601 kfree(iommu->iommu_state);
3602
3603 return -ENOMEM;
3604}
3605
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003606static void iommu_resume(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003607{
3608 struct dmar_drhd_unit *drhd;
3609 struct intel_iommu *iommu = NULL;
3610 unsigned long flag;
3611
3612 if (init_iommu_hw()) {
Joseph Cihulab7792602011-05-03 00:08:37 -07003613 if (force_on)
3614 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3615 else
3616 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003617 return;
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003618 }
3619
3620 for_each_active_iommu(iommu, drhd) {
3621
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02003622 raw_spin_lock_irqsave(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003623
3624 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3625 iommu->reg + DMAR_FECTL_REG);
3626 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3627 iommu->reg + DMAR_FEDATA_REG);
3628 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3629 iommu->reg + DMAR_FEADDR_REG);
3630 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3631 iommu->reg + DMAR_FEUADDR_REG);
3632
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +02003633 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003634 }
3635
3636 for_each_active_iommu(iommu, drhd)
3637 kfree(iommu->iommu_state);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003638}
3639
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003640static struct syscore_ops iommu_syscore_ops = {
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003641 .resume = iommu_resume,
3642 .suspend = iommu_suspend,
3643};
3644
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003645static void __init init_iommu_pm_ops(void)
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003646{
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01003647 register_syscore_ops(&iommu_syscore_ops);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003648}
3649
3650#else
Rafael J. Wysocki99592ba2011-06-07 21:32:31 +02003651static inline void init_iommu_pm_ops(void) {}
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003652#endif /* CONFIG_PM */
3653
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003654
3655int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3656{
3657 struct acpi_dmar_reserved_memory *rmrr;
3658 struct dmar_rmrr_unit *rmrru;
3659
3660 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3661 if (!rmrru)
3662 return -ENOMEM;
3663
3664 rmrru->hdr = header;
3665 rmrr = (struct acpi_dmar_reserved_memory *)header;
3666 rmrru->base_address = rmrr->base_address;
3667 rmrru->end_address = rmrr->end_address;
Jiang Liu2e455282014-02-19 14:07:36 +08003668 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3669 ((void *)rmrr) + rmrr->header.length,
3670 &rmrru->devices_cnt);
3671 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3672 kfree(rmrru);
3673 return -ENOMEM;
3674 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003675
Jiang Liu2e455282014-02-19 14:07:36 +08003676 list_add(&rmrru->list, &dmar_rmrr_units);
3677
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003678 return 0;
3679}
3680
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003681int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3682{
3683 struct acpi_dmar_atsr *atsr;
3684 struct dmar_atsr_unit *atsru;
3685
3686 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3687 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3688 if (!atsru)
3689 return -ENOMEM;
3690
3691 atsru->hdr = hdr;
3692 atsru->include_all = atsr->flags & 0x1;
Jiang Liu2e455282014-02-19 14:07:36 +08003693 if (!atsru->include_all) {
3694 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3695 (void *)atsr + atsr->header.length,
3696 &atsru->devices_cnt);
3697 if (atsru->devices_cnt && atsru->devices == NULL) {
3698 kfree(atsru);
3699 return -ENOMEM;
3700 }
3701 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003702
Jiang Liu0e242612014-02-19 14:07:34 +08003703 list_add_rcu(&atsru->list, &dmar_atsr_units);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003704
3705 return 0;
3706}
3707
Jiang Liu9bdc5312014-01-06 14:18:27 +08003708static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3709{
3710 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3711 kfree(atsru);
3712}
3713
3714static void intel_iommu_free_dmars(void)
3715{
3716 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3717 struct dmar_atsr_unit *atsru, *atsr_n;
3718
3719 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3720 list_del(&rmrru->list);
3721 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3722 kfree(rmrru);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003723 }
3724
Jiang Liu9bdc5312014-01-06 14:18:27 +08003725 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3726 list_del(&atsru->list);
3727 intel_iommu_free_atsr(atsru);
3728 }
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003729}
3730
3731int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3732{
Jiang Liub683b232014-02-19 14:07:32 +08003733 int i, ret = 1;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003734 struct pci_bus *bus;
David Woodhouse832bd852014-03-07 15:08:36 +00003735 struct pci_dev *bridge = NULL;
3736 struct device *tmp;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003737 struct acpi_dmar_atsr *atsr;
3738 struct dmar_atsr_unit *atsru;
3739
3740 dev = pci_physfn(dev);
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003741 for (bus = dev->bus; bus; bus = bus->parent) {
Jiang Liub5f82dd2014-02-19 14:07:31 +08003742 bridge = bus->self;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003743 if (!bridge || !pci_is_pcie(bridge) ||
Yijing Wang62f87c02012-07-24 17:20:03 +08003744 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003745 return 0;
Jiang Liub5f82dd2014-02-19 14:07:31 +08003746 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003747 break;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003748 }
Jiang Liub5f82dd2014-02-19 14:07:31 +08003749 if (!bridge)
3750 return 0;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003751
Jiang Liu0e242612014-02-19 14:07:34 +08003752 rcu_read_lock();
Jiang Liub5f82dd2014-02-19 14:07:31 +08003753 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3754 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3755 if (atsr->segment != pci_domain_nr(dev->bus))
3756 continue;
3757
Jiang Liub683b232014-02-19 14:07:32 +08003758 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
David Woodhouse832bd852014-03-07 15:08:36 +00003759 if (tmp == &bridge->dev)
Jiang Liub683b232014-02-19 14:07:32 +08003760 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08003761
3762 if (atsru->include_all)
Jiang Liub683b232014-02-19 14:07:32 +08003763 goto out;
Jiang Liub5f82dd2014-02-19 14:07:31 +08003764 }
Jiang Liub683b232014-02-19 14:07:32 +08003765 ret = 0;
3766out:
Jiang Liu0e242612014-02-19 14:07:34 +08003767 rcu_read_unlock();
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003768
Jiang Liub683b232014-02-19 14:07:32 +08003769 return ret;
Suresh Siddha318fe7d2011-08-23 17:05:20 -07003770}
3771
Jiang Liu59ce0512014-02-19 14:07:35 +08003772int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3773{
3774 int ret = 0;
3775 struct dmar_rmrr_unit *rmrru;
3776 struct dmar_atsr_unit *atsru;
3777 struct acpi_dmar_atsr *atsr;
3778 struct acpi_dmar_reserved_memory *rmrr;
3779
3780 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3781 return 0;
3782
3783 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3784 rmrr = container_of(rmrru->hdr,
3785 struct acpi_dmar_reserved_memory, header);
3786 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3787 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3788 ((void *)rmrr) + rmrr->header.length,
3789 rmrr->segment, rmrru->devices,
3790 rmrru->devices_cnt);
Jiang Liu27e24952014-06-20 15:08:06 +08003791 if(ret < 0)
Jiang Liu59ce0512014-02-19 14:07:35 +08003792 return ret;
3793 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
Jiang Liu27e24952014-06-20 15:08:06 +08003794 dmar_remove_dev_scope(info, rmrr->segment,
3795 rmrru->devices, rmrru->devices_cnt);
Jiang Liu59ce0512014-02-19 14:07:35 +08003796 }
3797 }
3798
3799 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3800 if (atsru->include_all)
3801 continue;
3802
3803 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3804 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3805 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3806 (void *)atsr + atsr->header.length,
3807 atsr->segment, atsru->devices,
3808 atsru->devices_cnt);
3809 if (ret > 0)
3810 break;
3811 else if(ret < 0)
3812 return ret;
3813 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3814 if (dmar_remove_dev_scope(info, atsr->segment,
3815 atsru->devices, atsru->devices_cnt))
3816 break;
3817 }
3818 }
3819
3820 return 0;
3821}
3822
Fenghua Yu99dcade2009-11-11 07:23:06 -08003823/*
3824 * Here we only respond to action of unbound device from driver.
3825 *
3826 * Added device is not attached to its DMAR domain here yet. That will happen
3827 * when mapping the device to iova.
3828 */
3829static int device_notifier(struct notifier_block *nb,
3830 unsigned long action, void *data)
3831{
3832 struct device *dev = data;
Fenghua Yu99dcade2009-11-11 07:23:06 -08003833 struct dmar_domain *domain;
3834
David Woodhouse3d891942014-03-06 15:59:26 +00003835 if (iommu_dummy(dev))
David Woodhouse44cd6132009-12-02 10:18:30 +00003836 return 0;
3837
Jiang Liu7e7dfab2014-02-19 14:07:23 +08003838 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3839 action != BUS_NOTIFY_DEL_DEVICE)
3840 return 0;
3841
David Woodhouse1525a292014-03-06 16:19:30 +00003842 domain = find_domain(dev);
Fenghua Yu99dcade2009-11-11 07:23:06 -08003843 if (!domain)
3844 return 0;
3845
Jiang Liu3a5670e2014-02-19 14:07:33 +08003846 down_read(&dmar_global_lock);
David Woodhousebf9c9ed2014-03-09 16:19:13 -07003847 domain_remove_one_dev_info(domain, dev);
Jiang Liuab8dfe22014-07-11 14:19:27 +08003848 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
Jiang Liu7e7dfab2014-02-19 14:07:23 +08003849 domain_exit(domain);
Jiang Liu3a5670e2014-02-19 14:07:33 +08003850 up_read(&dmar_global_lock);
Alex Williamsona97590e2011-03-04 14:52:16 -07003851
Fenghua Yu99dcade2009-11-11 07:23:06 -08003852 return 0;
3853}
3854
3855static struct notifier_block device_nb = {
3856 .notifier_call = device_notifier,
3857};
3858
Jiang Liu75f05562014-02-19 14:07:37 +08003859static int intel_iommu_memory_notifier(struct notifier_block *nb,
3860 unsigned long val, void *v)
3861{
3862 struct memory_notify *mhp = v;
3863 unsigned long long start, end;
3864 unsigned long start_vpfn, last_vpfn;
3865
3866 switch (val) {
3867 case MEM_GOING_ONLINE:
3868 start = mhp->start_pfn << PAGE_SHIFT;
3869 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3870 if (iommu_domain_identity_map(si_domain, start, end)) {
3871 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3872 start, end);
3873 return NOTIFY_BAD;
3874 }
3875 break;
3876
3877 case MEM_OFFLINE:
3878 case MEM_CANCEL_ONLINE:
3879 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3880 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3881 while (start_vpfn <= last_vpfn) {
3882 struct iova *iova;
3883 struct dmar_drhd_unit *drhd;
3884 struct intel_iommu *iommu;
David Woodhouseea8ea462014-03-05 17:09:32 +00003885 struct page *freelist;
Jiang Liu75f05562014-02-19 14:07:37 +08003886
3887 iova = find_iova(&si_domain->iovad, start_vpfn);
3888 if (iova == NULL) {
3889 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3890 start_vpfn);
3891 break;
3892 }
3893
3894 iova = split_and_remove_iova(&si_domain->iovad, iova,
3895 start_vpfn, last_vpfn);
3896 if (iova == NULL) {
3897 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3898 start_vpfn, last_vpfn);
3899 return NOTIFY_BAD;
3900 }
3901
David Woodhouseea8ea462014-03-05 17:09:32 +00003902 freelist = domain_unmap(si_domain, iova->pfn_lo,
3903 iova->pfn_hi);
3904
Jiang Liu75f05562014-02-19 14:07:37 +08003905 rcu_read_lock();
3906 for_each_active_iommu(iommu, drhd)
3907 iommu_flush_iotlb_psi(iommu, si_domain->id,
Jiang Liua156ef92014-07-11 14:19:36 +08003908 iova->pfn_lo, iova_size(iova),
David Woodhouseea8ea462014-03-05 17:09:32 +00003909 !freelist, 0);
Jiang Liu75f05562014-02-19 14:07:37 +08003910 rcu_read_unlock();
David Woodhouseea8ea462014-03-05 17:09:32 +00003911 dma_free_pagelist(freelist);
Jiang Liu75f05562014-02-19 14:07:37 +08003912
3913 start_vpfn = iova->pfn_hi + 1;
3914 free_iova_mem(iova);
3915 }
3916 break;
3917 }
3918
3919 return NOTIFY_OK;
3920}
3921
3922static struct notifier_block intel_iommu_memory_nb = {
3923 .notifier_call = intel_iommu_memory_notifier,
3924 .priority = 0
3925};
3926
Alex Williamsona5459cf2014-06-12 16:12:31 -06003927
3928static ssize_t intel_iommu_show_version(struct device *dev,
3929 struct device_attribute *attr,
3930 char *buf)
3931{
3932 struct intel_iommu *iommu = dev_get_drvdata(dev);
3933 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3934 return sprintf(buf, "%d:%d\n",
3935 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3936}
3937static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
3938
3939static ssize_t intel_iommu_show_address(struct device *dev,
3940 struct device_attribute *attr,
3941 char *buf)
3942{
3943 struct intel_iommu *iommu = dev_get_drvdata(dev);
3944 return sprintf(buf, "%llx\n", iommu->reg_phys);
3945}
3946static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
3947
3948static ssize_t intel_iommu_show_cap(struct device *dev,
3949 struct device_attribute *attr,
3950 char *buf)
3951{
3952 struct intel_iommu *iommu = dev_get_drvdata(dev);
3953 return sprintf(buf, "%llx\n", iommu->cap);
3954}
3955static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
3956
3957static ssize_t intel_iommu_show_ecap(struct device *dev,
3958 struct device_attribute *attr,
3959 char *buf)
3960{
3961 struct intel_iommu *iommu = dev_get_drvdata(dev);
3962 return sprintf(buf, "%llx\n", iommu->ecap);
3963}
3964static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
3965
3966static struct attribute *intel_iommu_attrs[] = {
3967 &dev_attr_version.attr,
3968 &dev_attr_address.attr,
3969 &dev_attr_cap.attr,
3970 &dev_attr_ecap.attr,
3971 NULL,
3972};
3973
3974static struct attribute_group intel_iommu_group = {
3975 .name = "intel-iommu",
3976 .attrs = intel_iommu_attrs,
3977};
3978
3979const struct attribute_group *intel_iommu_groups[] = {
3980 &intel_iommu_group,
3981 NULL,
3982};
3983
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003984int __init intel_iommu_init(void)
3985{
Jiang Liu9bdc5312014-01-06 14:18:27 +08003986 int ret = -ENODEV;
Takao Indoh3a93c842013-04-23 17:35:03 +09003987 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +08003988 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003989
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003990 /* VT-d is required for a TXT/tboot launch, so enforce that */
3991 force_on = tboot_force_iommu();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003992
Jiang Liu3a5670e2014-02-19 14:07:33 +08003993 if (iommu_init_mempool()) {
3994 if (force_on)
3995 panic("tboot: Failed to initialize iommu memory\n");
3996 return -ENOMEM;
3997 }
3998
3999 down_write(&dmar_global_lock);
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004000 if (dmar_table_init()) {
4001 if (force_on)
4002 panic("tboot: Failed to initialize DMAR table\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004003 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004004 }
4005
Takao Indoh3a93c842013-04-23 17:35:03 +09004006 /*
4007 * Disable translation if already enabled prior to OS handover.
4008 */
Jiang Liu7c919772014-01-06 14:18:18 +08004009 for_each_active_iommu(iommu, drhd)
Takao Indoh3a93c842013-04-23 17:35:03 +09004010 if (iommu->gcmd & DMA_GCMD_TE)
4011 iommu_disable_translation(iommu);
Takao Indoh3a93c842013-04-23 17:35:03 +09004012
Suresh Siddhac2c72862011-08-23 17:05:19 -07004013 if (dmar_dev_scope_init() < 0) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004014 if (force_on)
4015 panic("tboot: Failed to initialize DMAR device scope\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004016 goto out_free_dmar;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004017 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -07004018
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09004019 if (no_iommu || dmar_disabled)
Jiang Liu9bdc5312014-01-06 14:18:27 +08004020 goto out_free_dmar;
Suresh Siddha2ae21012008-07-10 11:16:43 -07004021
Suresh Siddha318fe7d2011-08-23 17:05:20 -07004022 if (list_empty(&dmar_rmrr_units))
4023 printk(KERN_INFO "DMAR: No RMRR found\n");
4024
4025 if (list_empty(&dmar_atsr_units))
4026 printk(KERN_INFO "DMAR: No ATSR found\n");
4027
Joseph Cihula51a63e62011-03-21 11:04:24 -07004028 if (dmar_init_reserved_ranges()) {
4029 if (force_on)
4030 panic("tboot: Failed to reserve iommu ranges\n");
Jiang Liu3a5670e2014-02-19 14:07:33 +08004031 goto out_free_reserved_range;
Joseph Cihula51a63e62011-03-21 11:04:24 -07004032 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004033
4034 init_no_remapping_devices();
4035
Joseph Cihulab7792602011-05-03 00:08:37 -07004036 ret = init_dmars();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004037 if (ret) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07004038 if (force_on)
4039 panic("tboot: Failed to initialize DMARs\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004040 printk(KERN_ERR "IOMMU: dmar init failed\n");
Jiang Liu9bdc5312014-01-06 14:18:27 +08004041 goto out_free_reserved_range;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004042 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08004043 up_write(&dmar_global_lock);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004044 printk(KERN_INFO
4045 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4046
mark gross5e0d2a62008-03-04 15:22:08 -08004047 init_timer(&unmap_timer);
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09004048#ifdef CONFIG_SWIOTLB
4049 swiotlb = 0;
4050#endif
David Woodhouse19943b02009-08-04 16:19:20 +01004051 dma_ops = &intel_dma_ops;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07004052
Rafael J. Wysocki134fac32011-03-23 22:16:14 +01004053 init_iommu_pm_ops();
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004054
Alex Williamsona5459cf2014-06-12 16:12:31 -06004055 for_each_active_iommu(iommu, drhd)
4056 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4057 intel_iommu_groups,
4058 iommu->name);
4059
Joerg Roedel4236d97d2011-09-06 17:56:07 +02004060 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004061 bus_register_notifier(&pci_bus_type, &device_nb);
Jiang Liu75f05562014-02-19 14:07:37 +08004062 if (si_domain && !hw_pass_through)
4063 register_memory_notifier(&intel_iommu_memory_nb);
Fenghua Yu99dcade2009-11-11 07:23:06 -08004064
Eugeni Dodonov8bc1f852011-11-23 16:42:14 -02004065 intel_iommu_enabled = 1;
4066
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004067 return 0;
Jiang Liu9bdc5312014-01-06 14:18:27 +08004068
4069out_free_reserved_range:
4070 put_iova_domain(&reserved_iova_list);
Jiang Liu9bdc5312014-01-06 14:18:27 +08004071out_free_dmar:
4072 intel_iommu_free_dmars();
Jiang Liu3a5670e2014-02-19 14:07:33 +08004073 up_write(&dmar_global_lock);
4074 iommu_exit_mempool();
Jiang Liu9bdc5312014-01-06 14:18:27 +08004075 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07004076}
Keshavamurthy, Anil Se8204822007-10-21 16:41:55 -07004077
Alex Williamson579305f2014-07-03 09:51:43 -06004078static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4079{
4080 struct intel_iommu *iommu = opaque;
4081
4082 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4083 return 0;
4084}
4085
4086/*
4087 * NB - intel-iommu lacks any sort of reference counting for the users of
4088 * dependent devices. If multiple endpoints have intersecting dependent
4089 * devices, unbinding the driver from any one of them will possibly leave
4090 * the others unable to operate.
4091 */
Han, Weidong3199aa62009-02-26 17:31:12 +08004092static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
David Woodhouse0bcb3e22014-03-06 17:12:03 +00004093 struct device *dev)
Han, Weidong3199aa62009-02-26 17:31:12 +08004094{
David Woodhouse0bcb3e22014-03-06 17:12:03 +00004095 if (!iommu || !dev || !dev_is_pci(dev))
Han, Weidong3199aa62009-02-26 17:31:12 +08004096 return;
4097
Alex Williamson579305f2014-07-03 09:51:43 -06004098 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
Han, Weidong3199aa62009-02-26 17:31:12 +08004099}
4100
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07004101static void domain_remove_one_dev_info(struct dmar_domain *domain,
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004102 struct device *dev)
Weidong Hanc7151a82008-12-08 22:51:37 +08004103{
Yijing Wangbca2b912013-10-31 17:26:04 +08004104 struct device_domain_info *info, *tmp;
Weidong Hanc7151a82008-12-08 22:51:37 +08004105 struct intel_iommu *iommu;
4106 unsigned long flags;
4107 int found = 0;
David Woodhouse156baca2014-03-09 14:00:57 -07004108 u8 bus, devfn;
Weidong Hanc7151a82008-12-08 22:51:37 +08004109
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004110 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Hanc7151a82008-12-08 22:51:37 +08004111 if (!iommu)
4112 return;
4113
4114 spin_lock_irqsave(&device_domain_lock, flags);
Yijing Wangbca2b912013-10-31 17:26:04 +08004115 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004116 if (info->iommu == iommu && info->bus == bus &&
4117 info->devfn == devfn) {
David Woodhouse109b9b02012-05-25 17:43:02 +01004118 unlink_domain_info(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08004119 spin_unlock_irqrestore(&device_domain_lock, flags);
4120
Yu Zhao93a23a72009-05-18 13:51:37 +08004121 iommu_disable_dev_iotlb(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08004122 iommu_detach_dev(iommu, info->bus, info->devfn);
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004123 iommu_detach_dependent_devices(iommu, dev);
Weidong Hanc7151a82008-12-08 22:51:37 +08004124 free_devinfo_mem(info);
4125
4126 spin_lock_irqsave(&device_domain_lock, flags);
4127
4128 if (found)
4129 break;
4130 else
4131 continue;
4132 }
4133
4134 /* if there is no other devices under the same iommu
4135 * owned by this domain, clear this iommu in iommu_bmp
4136 * update iommu count and coherency
4137 */
David Woodhouse8bbc4412014-03-09 13:52:37 -07004138 if (info->iommu == iommu)
Weidong Hanc7151a82008-12-08 22:51:37 +08004139 found = 1;
4140 }
4141
Roland Dreier3e7abe22011-07-20 06:22:21 -07004142 spin_unlock_irqrestore(&device_domain_lock, flags);
4143
Weidong Hanc7151a82008-12-08 22:51:37 +08004144 if (found == 0) {
Jiang Liufb170fb2014-07-11 14:19:28 +08004145 domain_detach_iommu(domain, iommu);
4146 if (!domain_type_is_vm_or_si(domain))
4147 iommu_detach_domain(domain, iommu);
Weidong Hanc7151a82008-12-08 22:51:37 +08004148 }
Weidong Hanc7151a82008-12-08 22:51:37 +08004149}
4150
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07004151static int md_domain_init(struct dmar_domain *domain, int guest_width)
Weidong Han5e98c4b2008-12-08 23:03:27 +08004152{
4153 int adjust_width;
4154
4155 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004156 domain_reserve_special_ranges(domain);
4157
4158 /* calculate AGAW */
4159 domain->gaw = guest_width;
4160 adjust_width = guestwidth_to_adjustwidth(guest_width);
4161 domain->agaw = width_to_agaw(adjust_width);
4162
Weidong Han5e98c4b2008-12-08 23:03:27 +08004163 domain->iommu_coherency = 0;
Sheng Yangc5b15252009-08-06 13:31:56 +08004164 domain->iommu_snooping = 0;
Youquan Song6dd9a7c2011-05-25 19:13:49 +01004165 domain->iommu_superpage = 0;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004166 domain->max_addr = 0;
Weidong Han5e98c4b2008-12-08 23:03:27 +08004167
4168 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07004169 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Weidong Han5e98c4b2008-12-08 23:03:27 +08004170 if (!domain->pgd)
4171 return -ENOMEM;
4172 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4173 return 0;
4174}
4175
Joerg Roedel5d450802008-12-03 14:52:32 +01004176static int intel_iommu_domain_init(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03004177{
Joerg Roedel5d450802008-12-03 14:52:32 +01004178 struct dmar_domain *dmar_domain;
Kay, Allen M38717942008-09-09 18:37:29 +03004179
Jiang Liuab8dfe22014-07-11 14:19:27 +08004180 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
Joerg Roedel5d450802008-12-03 14:52:32 +01004181 if (!dmar_domain) {
Kay, Allen M38717942008-09-09 18:37:29 +03004182 printk(KERN_ERR
Joerg Roedel5d450802008-12-03 14:52:32 +01004183 "intel_iommu_domain_init: dmar_domain == NULL\n");
4184 return -ENOMEM;
Kay, Allen M38717942008-09-09 18:37:29 +03004185 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07004186 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
Kay, Allen M38717942008-09-09 18:37:29 +03004187 printk(KERN_ERR
Joerg Roedel5d450802008-12-03 14:52:32 +01004188 "intel_iommu_domain_init() failed\n");
Jiang Liu92d03cc2014-02-19 14:07:28 +08004189 domain_exit(dmar_domain);
Joerg Roedel5d450802008-12-03 14:52:32 +01004190 return -ENOMEM;
Kay, Allen M38717942008-09-09 18:37:29 +03004191 }
Allen Kay8140a952011-10-14 12:32:17 -07004192 domain_update_iommu_cap(dmar_domain);
Joerg Roedel5d450802008-12-03 14:52:32 +01004193 domain->priv = dmar_domain;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004194
Joerg Roedel8a0e7152012-01-26 19:40:54 +01004195 domain->geometry.aperture_start = 0;
4196 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4197 domain->geometry.force_aperture = true;
4198
Joerg Roedel5d450802008-12-03 14:52:32 +01004199 return 0;
Kay, Allen M38717942008-09-09 18:37:29 +03004200}
Kay, Allen M38717942008-09-09 18:37:29 +03004201
Joerg Roedel5d450802008-12-03 14:52:32 +01004202static void intel_iommu_domain_destroy(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03004203{
Joerg Roedel5d450802008-12-03 14:52:32 +01004204 struct dmar_domain *dmar_domain = domain->priv;
4205
4206 domain->priv = NULL;
Jiang Liu92d03cc2014-02-19 14:07:28 +08004207 domain_exit(dmar_domain);
Kay, Allen M38717942008-09-09 18:37:29 +03004208}
Kay, Allen M38717942008-09-09 18:37:29 +03004209
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004210static int intel_iommu_attach_device(struct iommu_domain *domain,
4211 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03004212{
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004213 struct dmar_domain *dmar_domain = domain->priv;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004214 struct intel_iommu *iommu;
4215 int addr_width;
David Woodhouse156baca2014-03-09 14:00:57 -07004216 u8 bus, devfn;
Kay, Allen M38717942008-09-09 18:37:29 +03004217
David Woodhouse7207d8f2014-03-09 16:31:06 -07004218 /* normally dev is not mapped */
4219 if (unlikely(domain_context_mapped(dev))) {
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004220 struct dmar_domain *old_domain;
4221
David Woodhouse1525a292014-03-06 16:19:30 +00004222 old_domain = find_domain(dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004223 if (old_domain) {
Jiang Liuab8dfe22014-07-11 14:19:27 +08004224 if (domain_type_is_vm_or_si(dmar_domain))
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004225 domain_remove_one_dev_info(old_domain, dev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004226 else
4227 domain_remove_dev_info(old_domain);
4228 }
4229 }
4230
David Woodhouse156baca2014-03-09 14:00:57 -07004231 iommu = device_to_iommu(dev, &bus, &devfn);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004232 if (!iommu)
4233 return -ENODEV;
4234
4235 /* check if this iommu agaw is sufficient for max mapped address */
4236 addr_width = agaw_to_width(iommu->agaw);
Tom Lyona99c47a2010-05-17 08:20:45 +01004237 if (addr_width > cap_mgaw(iommu->cap))
4238 addr_width = cap_mgaw(iommu->cap);
4239
4240 if (dmar_domain->max_addr > (1LL << addr_width)) {
4241 printk(KERN_ERR "%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004242 "sufficient for the mapped address (%llx)\n",
Tom Lyona99c47a2010-05-17 08:20:45 +01004243 __func__, addr_width, dmar_domain->max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004244 return -EFAULT;
4245 }
Tom Lyona99c47a2010-05-17 08:20:45 +01004246 dmar_domain->gaw = addr_width;
4247
4248 /*
4249 * Knock out extra levels of page tables if necessary
4250 */
4251 while (iommu->agaw < dmar_domain->agaw) {
4252 struct dma_pte *pte;
4253
4254 pte = dmar_domain->pgd;
4255 if (dma_pte_present(pte)) {
Sheng Yang25cbff12010-06-12 19:21:42 +08004256 dmar_domain->pgd = (struct dma_pte *)
4257 phys_to_virt(dma_pte_addr(pte));
Jan Kiszka7a661012010-11-02 08:05:51 +01004258 free_pgtable_page(pte);
Tom Lyona99c47a2010-05-17 08:20:45 +01004259 }
4260 dmar_domain->agaw--;
4261 }
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004262
David Woodhouse5913c9b2014-03-09 16:27:31 -07004263 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004264}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004265
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004266static void intel_iommu_detach_device(struct iommu_domain *domain,
4267 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03004268{
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004269 struct dmar_domain *dmar_domain = domain->priv;
Joerg Roedel4c5478c2008-12-03 14:58:24 +01004270
David Woodhousebf9c9ed2014-03-09 16:19:13 -07004271 domain_remove_one_dev_info(dmar_domain, dev);
Kay, Allen M38717942008-09-09 18:37:29 +03004272}
Kay, Allen M38717942008-09-09 18:37:29 +03004273
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01004274static int intel_iommu_map(struct iommu_domain *domain,
4275 unsigned long iova, phys_addr_t hpa,
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02004276 size_t size, int iommu_prot)
Kay, Allen M38717942008-09-09 18:37:29 +03004277{
Joerg Roedeldde57a22008-12-03 15:04:09 +01004278 struct dmar_domain *dmar_domain = domain->priv;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004279 u64 max_addr;
Joerg Roedeldde57a22008-12-03 15:04:09 +01004280 int prot = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004281 int ret;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004282
Joerg Roedeldde57a22008-12-03 15:04:09 +01004283 if (iommu_prot & IOMMU_READ)
4284 prot |= DMA_PTE_READ;
4285 if (iommu_prot & IOMMU_WRITE)
4286 prot |= DMA_PTE_WRITE;
Sheng Yang9cf066972009-03-18 15:33:07 +08004287 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4288 prot |= DMA_PTE_SNP;
Joerg Roedeldde57a22008-12-03 15:04:09 +01004289
David Woodhouse163cc522009-06-28 00:51:17 +01004290 max_addr = iova + size;
Joerg Roedeldde57a22008-12-03 15:04:09 +01004291 if (dmar_domain->max_addr < max_addr) {
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004292 u64 end;
4293
4294 /* check if minimum agaw is sufficient for mapped address */
Tom Lyon8954da12010-05-17 08:19:52 +01004295 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004296 if (end < max_addr) {
Tom Lyon8954da12010-05-17 08:19:52 +01004297 printk(KERN_ERR "%s: iommu width (%d) is not "
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004298 "sufficient for the mapped address (%llx)\n",
Tom Lyon8954da12010-05-17 08:19:52 +01004299 __func__, dmar_domain->gaw, max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004300 return -EFAULT;
4301 }
Joerg Roedeldde57a22008-12-03 15:04:09 +01004302 dmar_domain->max_addr = max_addr;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004303 }
David Woodhousead051222009-06-28 14:22:28 +01004304 /* Round up size to next multiple of PAGE_SIZE, if it and
4305 the low bits of hpa would take us onto the next page */
David Woodhouse88cb6a72009-06-28 15:03:06 +01004306 size = aligned_nrpages(hpa, size);
David Woodhousead051222009-06-28 14:22:28 +01004307 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4308 hpa >> VTD_PAGE_SHIFT, size, prot);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004309 return ret;
Kay, Allen M38717942008-09-09 18:37:29 +03004310}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004311
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02004312static size_t intel_iommu_unmap(struct iommu_domain *domain,
David Woodhouseea8ea462014-03-05 17:09:32 +00004313 unsigned long iova, size_t size)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004314{
Joerg Roedeldde57a22008-12-03 15:04:09 +01004315 struct dmar_domain *dmar_domain = domain->priv;
David Woodhouseea8ea462014-03-05 17:09:32 +00004316 struct page *freelist = NULL;
4317 struct intel_iommu *iommu;
4318 unsigned long start_pfn, last_pfn;
4319 unsigned int npages;
4320 int iommu_id, num, ndomains, level = 0;
Sheng Yang4b99d352009-07-08 11:52:52 +01004321
David Woodhouse5cf0a762014-03-19 16:07:49 +00004322 /* Cope with horrid API which requires us to unmap more than the
4323 size argument if it happens to be a large-page mapping. */
4324 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4325 BUG();
4326
4327 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4328 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4329
David Woodhouseea8ea462014-03-05 17:09:32 +00004330 start_pfn = iova >> VTD_PAGE_SHIFT;
4331 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4332
4333 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4334
4335 npages = last_pfn - start_pfn + 1;
4336
4337 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4338 iommu = g_iommus[iommu_id];
4339
4340 /*
4341 * find bit position of dmar_domain
4342 */
4343 ndomains = cap_ndoms(iommu->cap);
4344 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4345 if (iommu->domains[num] == dmar_domain)
4346 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4347 npages, !freelist, 0);
4348 }
4349
4350 }
4351
4352 dma_free_pagelist(freelist);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08004353
David Woodhouse163cc522009-06-28 00:51:17 +01004354 if (dmar_domain->max_addr == iova + size)
4355 dmar_domain->max_addr = iova;
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01004356
David Woodhouse5cf0a762014-03-19 16:07:49 +00004357 return size;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004358}
Kay, Allen M38717942008-09-09 18:37:29 +03004359
Joerg Roedeld14d6572008-12-03 15:06:57 +01004360static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
Varun Sethibb5547ac2013-03-29 01:23:58 +05304361 dma_addr_t iova)
Kay, Allen M38717942008-09-09 18:37:29 +03004362{
Joerg Roedeld14d6572008-12-03 15:06:57 +01004363 struct dmar_domain *dmar_domain = domain->priv;
Kay, Allen M38717942008-09-09 18:37:29 +03004364 struct dma_pte *pte;
David Woodhouse5cf0a762014-03-19 16:07:49 +00004365 int level = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004366 u64 phys = 0;
Kay, Allen M38717942008-09-09 18:37:29 +03004367
David Woodhouse5cf0a762014-03-19 16:07:49 +00004368 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
Kay, Allen M38717942008-09-09 18:37:29 +03004369 if (pte)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004370 phys = dma_pte_addr(pte);
Kay, Allen M38717942008-09-09 18:37:29 +03004371
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08004372 return phys;
Kay, Allen M38717942008-09-09 18:37:29 +03004373}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004374
Sheng Yangdbb9fd82009-03-18 15:33:06 +08004375static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4376 unsigned long cap)
4377{
4378 struct dmar_domain *dmar_domain = domain->priv;
4379
4380 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4381 return dmar_domain->iommu_snooping;
Tom Lyon323f99c2010-07-02 16:56:14 -04004382 if (cap == IOMMU_CAP_INTR_REMAP)
Suresh Siddha95a02e92012-03-30 11:47:07 -07004383 return irq_remapping_enabled;
Sheng Yangdbb9fd82009-03-18 15:33:06 +08004384
4385 return 0;
4386}
4387
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004388static int intel_iommu_add_device(struct device *dev)
Alex Williamson70ae6f02011-10-21 15:56:11 -04004389{
Alex Williamsona5459cf2014-06-12 16:12:31 -06004390 struct intel_iommu *iommu;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004391 struct iommu_group *group;
David Woodhouse156baca2014-03-09 14:00:57 -07004392 u8 bus, devfn;
Alex Williamson70ae6f02011-10-21 15:56:11 -04004393
Alex Williamsona5459cf2014-06-12 16:12:31 -06004394 iommu = device_to_iommu(dev, &bus, &devfn);
4395 if (!iommu)
Alex Williamson70ae6f02011-10-21 15:56:11 -04004396 return -ENODEV;
4397
Alex Williamsona5459cf2014-06-12 16:12:31 -06004398 iommu_device_link(iommu->iommu_dev, dev);
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004399
Alex Williamsone17f9ff2014-07-03 09:51:37 -06004400 group = iommu_group_get_for_dev(dev);
Alex Williamson783f1572012-05-30 14:19:43 -06004401
Alex Williamsone17f9ff2014-07-03 09:51:37 -06004402 if (IS_ERR(group))
4403 return PTR_ERR(group);
Alex Williamson70ae6f02011-10-21 15:56:11 -04004404
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004405 iommu_group_put(group);
Alex Williamsone17f9ff2014-07-03 09:51:37 -06004406 return 0;
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004407}
4408
4409static void intel_iommu_remove_device(struct device *dev)
4410{
Alex Williamsona5459cf2014-06-12 16:12:31 -06004411 struct intel_iommu *iommu;
4412 u8 bus, devfn;
4413
4414 iommu = device_to_iommu(dev, &bus, &devfn);
4415 if (!iommu)
4416 return;
4417
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004418 iommu_group_remove_device(dev);
Alex Williamsona5459cf2014-06-12 16:12:31 -06004419
4420 iommu_device_unlink(iommu->iommu_dev, dev);
Alex Williamson70ae6f02011-10-21 15:56:11 -04004421}
4422
Thierry Redingb22f6432014-06-27 09:03:12 +02004423static const struct iommu_ops intel_iommu_ops = {
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004424 .domain_init = intel_iommu_domain_init,
4425 .domain_destroy = intel_iommu_domain_destroy,
4426 .attach_dev = intel_iommu_attach_device,
4427 .detach_dev = intel_iommu_detach_device,
Joerg Roedelb146a1c9f2010-01-20 17:17:37 +01004428 .map = intel_iommu_map,
4429 .unmap = intel_iommu_unmap,
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004430 .iova_to_phys = intel_iommu_iova_to_phys,
Sheng Yangdbb9fd82009-03-18 15:33:06 +08004431 .domain_has_cap = intel_iommu_domain_has_cap,
Alex Williamsonabdfdde2012-05-30 14:19:19 -06004432 .add_device = intel_iommu_add_device,
4433 .remove_device = intel_iommu_remove_device,
Ohad Ben-Cohen6d1c56a2011-11-10 11:32:30 +02004434 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01004435};
David Woodhouse9af88142009-02-13 23:18:03 +00004436
Daniel Vetter94526182013-01-20 23:50:13 +01004437static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4438{
4439 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4440 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4441 dmar_map_gfx = 0;
4442}
4443
4444DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4445DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4446DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4447DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4448DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4449DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4450DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4451
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08004452static void quirk_iommu_rwbf(struct pci_dev *dev)
David Woodhouse9af88142009-02-13 23:18:03 +00004453{
4454 /*
4455 * Mobile 4 Series Chipset neglects to set RWBF capability,
Daniel Vetter210561f2013-01-21 19:48:59 +01004456 * but needs it. Same seems to hold for the desktop versions.
David Woodhouse9af88142009-02-13 23:18:03 +00004457 */
4458 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4459 rwbf_quirk = 1;
4460}
4461
4462DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Daniel Vetter210561f2013-01-21 19:48:59 +01004463DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4464DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4465DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4466DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4467DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4468DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
David Woodhousee0fc7e02009-09-30 09:12:17 -07004469
Adam Jacksoneecfd572010-08-25 21:17:34 +01004470#define GGC 0x52
4471#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4472#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4473#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4474#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4475#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4476#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4477#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4478#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4479
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08004480static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
David Woodhouse9eecabc2010-09-21 22:28:23 +01004481{
4482 unsigned short ggc;
4483
Adam Jacksoneecfd572010-08-25 21:17:34 +01004484 if (pci_read_config_word(dev, GGC, &ggc))
David Woodhouse9eecabc2010-09-21 22:28:23 +01004485 return;
4486
Adam Jacksoneecfd572010-08-25 21:17:34 +01004487 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
David Woodhouse9eecabc2010-09-21 22:28:23 +01004488 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4489 dmar_map_gfx = 0;
David Woodhouse6fbcfb32011-09-25 19:11:14 -07004490 } else if (dmar_map_gfx) {
4491 /* we have to ensure the gfx device is idle before we flush */
4492 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4493 intel_iommu_strict = 1;
4494 }
David Woodhouse9eecabc2010-09-21 22:28:23 +01004495}
4496DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4497DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4498DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4499DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4500
David Woodhousee0fc7e02009-09-30 09:12:17 -07004501/* On Tylersburg chipsets, some BIOSes have been known to enable the
4502 ISOCH DMAR unit for the Azalia sound device, but not give it any
4503 TLB entries, which causes it to deadlock. Check for that. We do
4504 this in a function called from init_dmars(), instead of in a PCI
4505 quirk, because we don't want to print the obnoxious "BIOS broken"
4506 message if VT-d is actually disabled.
4507*/
4508static void __init check_tylersburg_isoch(void)
4509{
4510 struct pci_dev *pdev;
4511 uint32_t vtisochctrl;
4512
4513 /* If there's no Azalia in the system anyway, forget it. */
4514 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4515 if (!pdev)
4516 return;
4517 pci_dev_put(pdev);
4518
4519 /* System Management Registers. Might be hidden, in which case
4520 we can't do the sanity check. But that's OK, because the
4521 known-broken BIOSes _don't_ actually hide it, so far. */
4522 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4523 if (!pdev)
4524 return;
4525
4526 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4527 pci_dev_put(pdev);
4528 return;
4529 }
4530
4531 pci_dev_put(pdev);
4532
4533 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4534 if (vtisochctrl & 1)
4535 return;
4536
4537 /* Drop all bits other than the number of TLB entries */
4538 vtisochctrl &= 0x1c;
4539
4540 /* If we have the recommended number of TLB entries (16), fine. */
4541 if (vtisochctrl == 0x10)
4542 return;
4543
4544 /* Zero TLB entries? You get to ride the short bus to school. */
4545 if (!vtisochctrl) {
4546 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4547 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4548 dmi_get_system_info(DMI_BIOS_VENDOR),
4549 dmi_get_system_info(DMI_BIOS_VERSION),
4550 dmi_get_system_info(DMI_PRODUCT_VERSION));
4551 iommu_identity_mapping |= IDENTMAP_AZALIA;
4552 return;
4553 }
4554
4555 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4556 vtisochctrl);
4557}