blob: 417312528ddff29ab4ead4fd5bc7d89e9a718121 [file] [log] [blame]
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
mark gross98bcef52008-02-23 15:23:35 -080017 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Fenghua Yu5b6985c2008-10-16 18:02:32 -070021 * Author: Fenghua Yu <fenghua.yu@intel.com>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070022 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
mark gross5e0d2a62008-03-04 15:22:08 -080026#include <linux/debugfs.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070027#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070030#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
mark gross5e0d2a62008-03-04 15:22:08 -080035#include <linux/timer.h>
Kay, Allen M38717942008-09-09 18:37:29 +030036#include <linux/iova.h>
Joerg Roedel5d450802008-12-03 14:52:32 +010037#include <linux/iommu.h>
Kay, Allen M38717942008-09-09 18:37:29 +030038#include <linux/intel-iommu.h>
Fenghua Yuf59c7b62009-03-27 14:22:42 -070039#include <linux/sysdev.h>
Shane Wang69575d32009-09-01 18:25:07 -070040#include <linux/tboot.h>
Stephen Rothwelladb2fe02009-08-31 15:24:23 +100041#include <linux/dmi.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070042#include <asm/cacheflush.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090043#include <asm/iommu.h>
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070044#include "pci.h"
45
Fenghua Yu5b6985c2008-10-16 18:02:32 -070046#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070049#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
David Woodhousee0fc7e02009-09-30 09:12:17 -070051#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070052
53#define IOAPIC_RANGE_START (0xfee00000)
54#define IOAPIC_RANGE_END (0xfeefffff)
55#define IOVA_START_ADDR (0x1000)
56
57#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -070059#define MAX_AGAW_WIDTH 64
60
David Woodhouse2ebe3152009-09-19 07:34:04 -070061#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -070069
Mark McLoughlinf27be032008-11-20 15:49:43 +000070#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
Yang Hongyang284901a2009-04-06 19:01:15 -070071#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
Yang Hongyang6a355282009-04-06 19:01:13 -070072#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
mark gross5e0d2a62008-03-04 15:22:08 -080073
David Woodhousefd18de52009-05-10 23:57:41 +010074
David Woodhousedd4e8312009-06-27 16:21:20 +010075/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
76 are never going to work. */
77static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
78{
79 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
80}
81
82static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
83{
84 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
85}
86static inline unsigned long page_to_dma_pfn(struct page *pg)
87{
88 return mm_to_dma_pfn(page_to_pfn(pg));
89}
90static inline unsigned long virt_to_dma_pfn(void *p)
91{
92 return page_to_dma_pfn(virt_to_page(p));
93}
94
Weidong Hand9630fe2008-12-08 11:06:32 +080095/* global iommu list, set NULL for ignored DMAR units */
96static struct intel_iommu **g_iommus;
97
David Woodhousee0fc7e02009-09-30 09:12:17 -070098static void __init check_tylersburg_isoch(void);
David Woodhouse9af88142009-02-13 23:18:03 +000099static int rwbf_quirk;
100
Mark McLoughlin46b08e12008-11-20 15:49:44 +0000101/*
102 * 0: Present
103 * 1-11: Reserved
104 * 12-63: Context Ptr (12 - (haw-1))
105 * 64-127: Reserved
106 */
107struct root_entry {
108 u64 val;
109 u64 rsvd1;
110};
111#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
112static inline bool root_present(struct root_entry *root)
113{
114 return (root->val & 1);
115}
116static inline void set_root_present(struct root_entry *root)
117{
118 root->val |= 1;
119}
120static inline void set_root_value(struct root_entry *root, unsigned long value)
121{
122 root->val |= value & VTD_PAGE_MASK;
123}
124
125static inline struct context_entry *
126get_context_addr_from_root(struct root_entry *root)
127{
128 return (struct context_entry *)
129 (root_present(root)?phys_to_virt(
130 root->val & VTD_PAGE_MASK) :
131 NULL);
132}
133
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000134/*
135 * low 64 bits:
136 * 0: present
137 * 1: fault processing disable
138 * 2-3: translation type
139 * 12-63: address space root
140 * high 64 bits:
141 * 0-2: address width
142 * 3-6: aval
143 * 8-23: domain id
144 */
145struct context_entry {
146 u64 lo;
147 u64 hi;
148};
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000149
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000150static inline bool context_present(struct context_entry *context)
151{
152 return (context->lo & 1);
153}
154static inline void context_set_present(struct context_entry *context)
155{
156 context->lo |= 1;
157}
158
159static inline void context_set_fault_enable(struct context_entry *context)
160{
161 context->lo &= (((u64)-1) << 2) | 1;
162}
163
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000164static inline void context_set_translation_type(struct context_entry *context,
165 unsigned long value)
166{
167 context->lo &= (((u64)-1) << 4) | 3;
168 context->lo |= (value & 3) << 2;
169}
170
171static inline void context_set_address_root(struct context_entry *context,
172 unsigned long value)
173{
174 context->lo |= value & VTD_PAGE_MASK;
175}
176
177static inline void context_set_address_width(struct context_entry *context,
178 unsigned long value)
179{
180 context->hi |= value & 7;
181}
182
183static inline void context_set_domain_id(struct context_entry *context,
184 unsigned long value)
185{
186 context->hi |= (value & ((1 << 16) - 1)) << 8;
187}
188
189static inline void context_clear_entry(struct context_entry *context)
190{
191 context->lo = 0;
192 context->hi = 0;
193}
Mark McLoughlin7a8fc252008-11-20 15:49:45 +0000194
Mark McLoughlin622ba122008-11-20 15:49:46 +0000195/*
196 * 0: readable
197 * 1: writable
198 * 2-6: reserved
199 * 7: super page
Sheng Yang9cf06692009-03-18 15:33:07 +0800200 * 8-10: available
201 * 11: snoop behavior
Mark McLoughlin622ba122008-11-20 15:49:46 +0000202 * 12-63: Host physcial address
203 */
204struct dma_pte {
205 u64 val;
206};
Mark McLoughlin622ba122008-11-20 15:49:46 +0000207
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000208static inline void dma_clear_pte(struct dma_pte *pte)
209{
210 pte->val = 0;
211}
212
213static inline void dma_set_pte_readable(struct dma_pte *pte)
214{
215 pte->val |= DMA_PTE_READ;
216}
217
218static inline void dma_set_pte_writable(struct dma_pte *pte)
219{
220 pte->val |= DMA_PTE_WRITE;
221}
222
Sheng Yang9cf06692009-03-18 15:33:07 +0800223static inline void dma_set_pte_snp(struct dma_pte *pte)
224{
225 pte->val |= DMA_PTE_SNP;
226}
227
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000228static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
229{
230 pte->val = (pte->val & ~3) | (prot & 3);
231}
232
233static inline u64 dma_pte_addr(struct dma_pte *pte)
234{
David Woodhousec85994e2009-07-01 19:21:24 +0100235#ifdef CONFIG_64BIT
236 return pte->val & VTD_PAGE_MASK;
237#else
238 /* Must have a full atomic 64-bit read */
239 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
240#endif
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000241}
242
David Woodhousedd4e8312009-06-27 16:21:20 +0100243static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000244{
David Woodhousedd4e8312009-06-27 16:21:20 +0100245 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000246}
247
248static inline bool dma_pte_present(struct dma_pte *pte)
249{
250 return (pte->val & 3) != 0;
251}
Mark McLoughlin622ba122008-11-20 15:49:46 +0000252
David Woodhouse75e6bf92009-07-02 11:21:16 +0100253static inline int first_pte_in_page(struct dma_pte *pte)
254{
255 return !((unsigned long)pte & ~VTD_PAGE_MASK);
256}
257
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700258/*
259 * This domain is a statically identity mapping domain.
260 * 1. This domain creats a static 1:1 mapping to all usable memory.
261 * 2. It maps to each iommu if successful.
262 * 3. Each iommu mapps to this domain if successful.
263 */
David Woodhouse19943b02009-08-04 16:19:20 +0100264static struct dmar_domain *si_domain;
265static int hw_pass_through = 1;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700266
Weidong Han3b5410e2008-12-08 09:17:15 +0800267/* devices under the same p2p bridge are owned in one domain */
Mike Daycdc7b832008-12-12 17:16:30 +0100268#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
Weidong Han3b5410e2008-12-08 09:17:15 +0800269
Weidong Han1ce28fe2008-12-08 16:35:39 +0800270/* domain represents a virtual machine, more than one devices
271 * across iommus may be owned in one domain, e.g. kvm guest.
272 */
273#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
274
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700275/* si_domain contains mulitple devices */
276#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
277
Mark McLoughlin99126f72008-11-20 15:49:47 +0000278struct dmar_domain {
279 int id; /* domain id */
Suresh Siddha4c923d42009-10-02 11:01:24 -0700280 int nid; /* node id */
Weidong Han8c11e792008-12-08 15:29:22 +0800281 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
Mark McLoughlin99126f72008-11-20 15:49:47 +0000282
283 struct list_head devices; /* all devices' list */
284 struct iova_domain iovad; /* iova's that belong to this domain */
285
286 struct dma_pte *pgd; /* virtual address */
Mark McLoughlin99126f72008-11-20 15:49:47 +0000287 int gaw; /* max guest address width */
288
289 /* adjusted guest address width, 0 is level 2 30-bit */
290 int agaw;
291
Weidong Han3b5410e2008-12-08 09:17:15 +0800292 int flags; /* flags to find out type of domain */
Weidong Han8e604092008-12-08 15:49:06 +0800293
294 int iommu_coherency;/* indicate coherency of iommu access */
Sheng Yang58c610b2009-03-18 15:33:05 +0800295 int iommu_snooping; /* indicate snooping control feature*/
Weidong Hanc7151a82008-12-08 22:51:37 +0800296 int iommu_count; /* reference count of iommu */
297 spinlock_t iommu_lock; /* protect iommu set in domain */
Weidong Hanfe40f1e2008-12-08 23:10:23 +0800298 u64 max_addr; /* maximum mapped address */
Mark McLoughlin99126f72008-11-20 15:49:47 +0000299};
300
Mark McLoughlina647dac2008-11-20 15:49:48 +0000301/* PCI domain-device relationship */
302struct device_domain_info {
303 struct list_head link; /* link to domain siblings */
304 struct list_head global; /* link to global list */
David Woodhouse276dbf92009-04-04 01:45:37 +0100305 int segment; /* PCI domain */
306 u8 bus; /* PCI bus number */
Mark McLoughlina647dac2008-11-20 15:49:48 +0000307 u8 devfn; /* PCI devfn number */
Stefan Assmann45e829e2009-12-03 06:49:24 -0500308 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
Yu Zhao93a23a72009-05-18 13:51:37 +0800309 struct intel_iommu *iommu; /* IOMMU used by this device */
Mark McLoughlina647dac2008-11-20 15:49:48 +0000310 struct dmar_domain *domain; /* pointer to domain */
311};
312
mark gross5e0d2a62008-03-04 15:22:08 -0800313static void flush_unmaps_timeout(unsigned long data);
314
315DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
316
mark gross80b20dd2008-04-18 13:53:58 -0700317#define HIGH_WATER_MARK 250
318struct deferred_flush_tables {
319 int next;
320 struct iova *iova[HIGH_WATER_MARK];
321 struct dmar_domain *domain[HIGH_WATER_MARK];
322};
323
324static struct deferred_flush_tables *deferred_flush;
325
mark gross5e0d2a62008-03-04 15:22:08 -0800326/* bitmap for indexing intel_iommus */
mark gross5e0d2a62008-03-04 15:22:08 -0800327static int g_num_of_iommus;
328
329static DEFINE_SPINLOCK(async_umap_flush_lock);
330static LIST_HEAD(unmaps_to_do);
331
332static int timer_on;
333static long list_size;
mark gross5e0d2a62008-03-04 15:22:08 -0800334
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700335static void domain_remove_dev_info(struct dmar_domain *domain);
336
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800337#ifdef CONFIG_DMAR_DEFAULT_ON
338int dmar_disabled = 0;
339#else
340int dmar_disabled = 1;
341#endif /*CONFIG_DMAR_DEFAULT_ON*/
342
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700343static int __initdata dmar_map_gfx = 1;
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700344static int dmar_forcedac;
mark gross5e0d2a62008-03-04 15:22:08 -0800345static int intel_iommu_strict;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700346
347#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
348static DEFINE_SPINLOCK(device_domain_lock);
349static LIST_HEAD(device_domain_list);
350
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +0100351static struct iommu_ops intel_iommu_ops;
352
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700353static int __init intel_iommu_setup(char *str)
354{
355 if (!str)
356 return -EINVAL;
357 while (*str) {
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800358 if (!strncmp(str, "on", 2)) {
359 dmar_disabled = 0;
360 printk(KERN_INFO "Intel-IOMMU: enabled\n");
361 } else if (!strncmp(str, "off", 3)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700362 dmar_disabled = 1;
Kyle McMartin0cd5c3c2009-02-04 14:29:19 -0800363 printk(KERN_INFO "Intel-IOMMU: disabled\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700364 } else if (!strncmp(str, "igfx_off", 8)) {
365 dmar_map_gfx = 0;
366 printk(KERN_INFO
367 "Intel-IOMMU: disable GFX device mapping\n");
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700368 } else if (!strncmp(str, "forcedac", 8)) {
mark gross5e0d2a62008-03-04 15:22:08 -0800369 printk(KERN_INFO
Keshavamurthy, Anil S7d3b03c2007-10-21 16:41:53 -0700370 "Intel-IOMMU: Forcing DAC for PCI devices\n");
371 dmar_forcedac = 1;
mark gross5e0d2a62008-03-04 15:22:08 -0800372 } else if (!strncmp(str, "strict", 6)) {
373 printk(KERN_INFO
374 "Intel-IOMMU: disable batched IOTLB flush\n");
375 intel_iommu_strict = 1;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700376 }
377
378 str += strcspn(str, ",");
379 while (*str == ',')
380 str++;
381 }
382 return 0;
383}
384__setup("intel_iommu=", intel_iommu_setup);
385
386static struct kmem_cache *iommu_domain_cache;
387static struct kmem_cache *iommu_devinfo_cache;
388static struct kmem_cache *iommu_iova_cache;
389
Suresh Siddha4c923d42009-10-02 11:01:24 -0700390static inline void *alloc_pgtable_page(int node)
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700391{
Suresh Siddha4c923d42009-10-02 11:01:24 -0700392 struct page *page;
393 void *vaddr = NULL;
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700394
Suresh Siddha4c923d42009-10-02 11:01:24 -0700395 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
396 if (page)
397 vaddr = page_address(page);
Keshavamurthy, Anil Seb3fa7c2007-10-21 16:41:52 -0700398 return vaddr;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700399}
400
401static inline void free_pgtable_page(void *vaddr)
402{
403 free_page((unsigned long)vaddr);
404}
405
406static inline void *alloc_domain_mem(void)
407{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900408 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700409}
410
Kay, Allen M38717942008-09-09 18:37:29 +0300411static void free_domain_mem(void *vaddr)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700412{
413 kmem_cache_free(iommu_domain_cache, vaddr);
414}
415
416static inline void * alloc_devinfo_mem(void)
417{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900418 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700419}
420
421static inline void free_devinfo_mem(void *vaddr)
422{
423 kmem_cache_free(iommu_devinfo_cache, vaddr);
424}
425
426struct iova *alloc_iova_mem(void)
427{
KOSAKI Motohiro354bb652009-11-17 16:21:09 +0900428 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700429}
430
431void free_iova_mem(struct iova *iova)
432{
433 kmem_cache_free(iommu_iova_cache, iova);
434}
435
Weidong Han1b573682008-12-08 15:34:06 +0800436
437static inline int width_to_agaw(int width);
438
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700439static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
Weidong Han1b573682008-12-08 15:34:06 +0800440{
441 unsigned long sagaw;
442 int agaw = -1;
443
444 sagaw = cap_sagaw(iommu->cap);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700445 for (agaw = width_to_agaw(max_gaw);
Weidong Han1b573682008-12-08 15:34:06 +0800446 agaw >= 0; agaw--) {
447 if (test_bit(agaw, &sagaw))
448 break;
449 }
450
451 return agaw;
452}
453
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700454/*
455 * Calculate max SAGAW for each iommu.
456 */
457int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
458{
459 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
460}
461
462/*
463 * calculate agaw for each iommu.
464 * "SAGAW" may be different across iommus, use a default agaw, and
465 * get a supported less agaw for iommus that don't support the default agaw.
466 */
467int iommu_calculate_agaw(struct intel_iommu *iommu)
468{
469 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
470}
471
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700472/* This functionin only returns single iommu in a domain */
Weidong Han8c11e792008-12-08 15:29:22 +0800473static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
474{
475 int iommu_id;
476
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700477 /* si_domain and vm domain should not get here. */
Weidong Han1ce28fe2008-12-08 16:35:39 +0800478 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -0700479 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
Weidong Han1ce28fe2008-12-08 16:35:39 +0800480
Weidong Han8c11e792008-12-08 15:29:22 +0800481 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
482 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
483 return NULL;
484
485 return g_iommus[iommu_id];
486}
487
Weidong Han8e604092008-12-08 15:49:06 +0800488static void domain_update_iommu_coherency(struct dmar_domain *domain)
489{
490 int i;
491
492 domain->iommu_coherency = 1;
493
494 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
495 for (; i < g_num_of_iommus; ) {
496 if (!ecap_coherent(g_iommus[i]->ecap)) {
497 domain->iommu_coherency = 0;
498 break;
499 }
500 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
501 }
502}
503
Sheng Yang58c610b2009-03-18 15:33:05 +0800504static void domain_update_iommu_snooping(struct dmar_domain *domain)
505{
506 int i;
507
508 domain->iommu_snooping = 1;
509
510 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
511 for (; i < g_num_of_iommus; ) {
512 if (!ecap_sc_support(g_iommus[i]->ecap)) {
513 domain->iommu_snooping = 0;
514 break;
515 }
516 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
517 }
518}
519
520/* Some capabilities may be different across iommus */
521static void domain_update_iommu_cap(struct dmar_domain *domain)
522{
523 domain_update_iommu_coherency(domain);
524 domain_update_iommu_snooping(domain);
525}
526
David Woodhouse276dbf92009-04-04 01:45:37 +0100527static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
Weidong Hanc7151a82008-12-08 22:51:37 +0800528{
529 struct dmar_drhd_unit *drhd = NULL;
530 int i;
531
532 for_each_drhd_unit(drhd) {
533 if (drhd->ignored)
534 continue;
David Woodhouse276dbf92009-04-04 01:45:37 +0100535 if (segment != drhd->segment)
536 continue;
Weidong Hanc7151a82008-12-08 22:51:37 +0800537
David Woodhouse924b6232009-04-04 00:39:25 +0100538 for (i = 0; i < drhd->devices_cnt; i++) {
Dirk Hohndel288e4872009-01-11 15:33:51 +0000539 if (drhd->devices[i] &&
540 drhd->devices[i]->bus->number == bus &&
Weidong Hanc7151a82008-12-08 22:51:37 +0800541 drhd->devices[i]->devfn == devfn)
542 return drhd->iommu;
David Woodhouse4958c5d2009-04-06 13:30:01 -0700543 if (drhd->devices[i] &&
544 drhd->devices[i]->subordinate &&
David Woodhouse924b6232009-04-04 00:39:25 +0100545 drhd->devices[i]->subordinate->number <= bus &&
546 drhd->devices[i]->subordinate->subordinate >= bus)
547 return drhd->iommu;
548 }
Weidong Hanc7151a82008-12-08 22:51:37 +0800549
550 if (drhd->include_all)
551 return drhd->iommu;
552 }
553
554 return NULL;
555}
556
Weidong Han5331fe62008-12-08 23:00:00 +0800557static void domain_flush_cache(struct dmar_domain *domain,
558 void *addr, int size)
559{
560 if (!domain->iommu_coherency)
561 clflush_cache_range(addr, size);
562}
563
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700564/* Gets context entry for a given bus and devfn */
565static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
566 u8 bus, u8 devfn)
567{
568 struct root_entry *root;
569 struct context_entry *context;
570 unsigned long phy_addr;
571 unsigned long flags;
572
573 spin_lock_irqsave(&iommu->lock, flags);
574 root = &iommu->root_entry[bus];
575 context = get_context_addr_from_root(root);
576 if (!context) {
Suresh Siddha4c923d42009-10-02 11:01:24 -0700577 context = (struct context_entry *)
578 alloc_pgtable_page(iommu->node);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700579 if (!context) {
580 spin_unlock_irqrestore(&iommu->lock, flags);
581 return NULL;
582 }
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700583 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700584 phy_addr = virt_to_phys((void *)context);
585 set_root_value(root, phy_addr);
586 set_root_present(root);
587 __iommu_flush_cache(iommu, root, sizeof(*root));
588 }
589 spin_unlock_irqrestore(&iommu->lock, flags);
590 return &context[devfn];
591}
592
593static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
594{
595 struct root_entry *root;
596 struct context_entry *context;
597 int ret;
598 unsigned long flags;
599
600 spin_lock_irqsave(&iommu->lock, flags);
601 root = &iommu->root_entry[bus];
602 context = get_context_addr_from_root(root);
603 if (!context) {
604 ret = 0;
605 goto out;
606 }
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000607 ret = context_present(&context[devfn]);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700608out:
609 spin_unlock_irqrestore(&iommu->lock, flags);
610 return ret;
611}
612
613static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
614{
615 struct root_entry *root;
616 struct context_entry *context;
617 unsigned long flags;
618
619 spin_lock_irqsave(&iommu->lock, flags);
620 root = &iommu->root_entry[bus];
621 context = get_context_addr_from_root(root);
622 if (context) {
Mark McLoughlinc07e7d22008-11-21 16:54:46 +0000623 context_clear_entry(&context[devfn]);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700624 __iommu_flush_cache(iommu, &context[devfn], \
625 sizeof(*context));
626 }
627 spin_unlock_irqrestore(&iommu->lock, flags);
628}
629
630static void free_context_table(struct intel_iommu *iommu)
631{
632 struct root_entry *root;
633 int i;
634 unsigned long flags;
635 struct context_entry *context;
636
637 spin_lock_irqsave(&iommu->lock, flags);
638 if (!iommu->root_entry) {
639 goto out;
640 }
641 for (i = 0; i < ROOT_ENTRY_NR; i++) {
642 root = &iommu->root_entry[i];
643 context = get_context_addr_from_root(root);
644 if (context)
645 free_pgtable_page(context);
646 }
647 free_pgtable_page(iommu->root_entry);
648 iommu->root_entry = NULL;
649out:
650 spin_unlock_irqrestore(&iommu->lock, flags);
651}
652
653/* page table handling */
654#define LEVEL_STRIDE (9)
655#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
656
657static inline int agaw_to_level(int agaw)
658{
659 return agaw + 2;
660}
661
662static inline int agaw_to_width(int agaw)
663{
664 return 30 + agaw * LEVEL_STRIDE;
665
666}
667
668static inline int width_to_agaw(int width)
669{
670 return (width - 30) / LEVEL_STRIDE;
671}
672
673static inline unsigned int level_to_offset_bits(int level)
674{
David Woodhouse6660c632009-06-27 22:41:00 +0100675 return (level - 1) * LEVEL_STRIDE;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700676}
677
David Woodhouse77dfa562009-06-27 16:40:08 +0100678static inline int pfn_level_offset(unsigned long pfn, int level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700679{
David Woodhouse6660c632009-06-27 22:41:00 +0100680 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700681}
682
David Woodhouse6660c632009-06-27 22:41:00 +0100683static inline unsigned long level_mask(int level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700684{
David Woodhouse6660c632009-06-27 22:41:00 +0100685 return -1UL << level_to_offset_bits(level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700686}
687
David Woodhouse6660c632009-06-27 22:41:00 +0100688static inline unsigned long level_size(int level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700689{
David Woodhouse6660c632009-06-27 22:41:00 +0100690 return 1UL << level_to_offset_bits(level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700691}
692
David Woodhouse6660c632009-06-27 22:41:00 +0100693static inline unsigned long align_to_level(unsigned long pfn, int level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700694{
David Woodhouse6660c632009-06-27 22:41:00 +0100695 return (pfn + level_size(level) - 1) & level_mask(level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700696}
697
David Woodhouseb026fd22009-06-28 10:37:25 +0100698static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
699 unsigned long pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700700{
David Woodhouseb026fd22009-06-28 10:37:25 +0100701 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700702 struct dma_pte *parent, *pte = NULL;
703 int level = agaw_to_level(domain->agaw);
704 int offset;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700705
706 BUG_ON(!domain->pgd);
David Woodhouseb026fd22009-06-28 10:37:25 +0100707 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700708 parent = domain->pgd;
709
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700710 while (level > 0) {
711 void *tmp_page;
712
David Woodhouseb026fd22009-06-28 10:37:25 +0100713 offset = pfn_level_offset(pfn, level);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700714 pte = &parent[offset];
715 if (level == 1)
716 break;
717
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000718 if (!dma_pte_present(pte)) {
David Woodhousec85994e2009-07-01 19:21:24 +0100719 uint64_t pteval;
720
Suresh Siddha4c923d42009-10-02 11:01:24 -0700721 tmp_page = alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700722
David Woodhouse206a73c2009-07-01 19:30:28 +0100723 if (!tmp_page)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700724 return NULL;
David Woodhouse206a73c2009-07-01 19:30:28 +0100725
David Woodhousec85994e2009-07-01 19:21:24 +0100726 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
Benjamin LaHaise64de5af2009-09-16 21:05:55 -0400727 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
David Woodhousec85994e2009-07-01 19:21:24 +0100728 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
729 /* Someone else set it while we were thinking; use theirs. */
730 free_pgtable_page(tmp_page);
731 } else {
732 dma_pte_addr(pte);
733 domain_flush_cache(domain, pte, sizeof(*pte));
734 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700735 }
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000736 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700737 level--;
738 }
739
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700740 return pte;
741}
742
743/* return address's pte at specific level */
David Woodhouse90dcfb52009-06-27 17:14:59 +0100744static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
745 unsigned long pfn,
746 int level)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700747{
748 struct dma_pte *parent, *pte = NULL;
749 int total = agaw_to_level(domain->agaw);
750 int offset;
751
752 parent = domain->pgd;
753 while (level <= total) {
David Woodhouse90dcfb52009-06-27 17:14:59 +0100754 offset = pfn_level_offset(pfn, total);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700755 pte = &parent[offset];
756 if (level == total)
757 return pte;
758
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000759 if (!dma_pte_present(pte))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700760 break;
Mark McLoughlin19c239c2008-11-21 16:56:53 +0000761 parent = phys_to_virt(dma_pte_addr(pte));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700762 total--;
763 }
764 return NULL;
765}
766
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700767/* clear last level pte, a tlb flush should be followed */
David Woodhouse595badf2009-06-27 22:09:11 +0100768static void dma_pte_clear_range(struct dmar_domain *domain,
769 unsigned long start_pfn,
770 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700771{
David Woodhouse04b18e62009-06-27 19:15:01 +0100772 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
David Woodhouse310a5ab2009-06-28 18:52:20 +0100773 struct dma_pte *first_pte, *pte;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700774
David Woodhouse04b18e62009-06-27 19:15:01 +0100775 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
David Woodhouse595badf2009-06-27 22:09:11 +0100776 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
David Woodhouse59c36282009-09-19 07:36:28 -0700777 BUG_ON(start_pfn > last_pfn);
David Woodhouse66eae842009-06-27 19:00:32 +0100778
David Woodhouse04b18e62009-06-27 19:15:01 +0100779 /* we don't need lock here; nobody else touches the iova range */
David Woodhouse59c36282009-09-19 07:36:28 -0700780 do {
David Woodhouse310a5ab2009-06-28 18:52:20 +0100781 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
782 if (!pte) {
783 start_pfn = align_to_level(start_pfn + 1, 2);
784 continue;
785 }
David Woodhouse75e6bf92009-07-02 11:21:16 +0100786 do {
David Woodhouse310a5ab2009-06-28 18:52:20 +0100787 dma_clear_pte(pte);
788 start_pfn++;
789 pte++;
David Woodhouse75e6bf92009-07-02 11:21:16 +0100790 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
791
David Woodhouse310a5ab2009-06-28 18:52:20 +0100792 domain_flush_cache(domain, first_pte,
793 (void *)pte - (void *)first_pte);
David Woodhouse59c36282009-09-19 07:36:28 -0700794
795 } while (start_pfn && start_pfn <= last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700796}
797
798/* free page table pages. last level pte should already be cleared */
799static void dma_pte_free_pagetable(struct dmar_domain *domain,
David Woodhoused794dc92009-06-28 00:27:49 +0100800 unsigned long start_pfn,
801 unsigned long last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700802{
David Woodhouse6660c632009-06-27 22:41:00 +0100803 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
David Woodhousef3a0a522009-06-30 03:40:07 +0100804 struct dma_pte *first_pte, *pte;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700805 int total = agaw_to_level(domain->agaw);
806 int level;
David Woodhouse6660c632009-06-27 22:41:00 +0100807 unsigned long tmp;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700808
David Woodhouse6660c632009-06-27 22:41:00 +0100809 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
810 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
David Woodhouse59c36282009-09-19 07:36:28 -0700811 BUG_ON(start_pfn > last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700812
David Woodhousef3a0a522009-06-30 03:40:07 +0100813 /* We don't need lock here; nobody else touches the iova range */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700814 level = 2;
815 while (level <= total) {
David Woodhouse6660c632009-06-27 22:41:00 +0100816 tmp = align_to_level(start_pfn, level);
817
David Woodhousef3a0a522009-06-30 03:40:07 +0100818 /* If we can't even clear one PTE at this level, we're done */
David Woodhouse6660c632009-06-27 22:41:00 +0100819 if (tmp + level_size(level) - 1 > last_pfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700820 return;
821
David Woodhouse59c36282009-09-19 07:36:28 -0700822 do {
David Woodhousef3a0a522009-06-30 03:40:07 +0100823 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
824 if (!pte) {
825 tmp = align_to_level(tmp + 1, level + 1);
826 continue;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700827 }
David Woodhouse75e6bf92009-07-02 11:21:16 +0100828 do {
David Woodhouse6a43e572009-07-02 12:02:34 +0100829 if (dma_pte_present(pte)) {
830 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
831 dma_clear_pte(pte);
832 }
David Woodhousef3a0a522009-06-30 03:40:07 +0100833 pte++;
834 tmp += level_size(level);
David Woodhouse75e6bf92009-07-02 11:21:16 +0100835 } while (!first_pte_in_page(pte) &&
836 tmp + level_size(level) - 1 <= last_pfn);
837
David Woodhousef3a0a522009-06-30 03:40:07 +0100838 domain_flush_cache(domain, first_pte,
839 (void *)pte - (void *)first_pte);
840
David Woodhouse59c36282009-09-19 07:36:28 -0700841 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700842 level++;
843 }
844 /* free pgd */
David Woodhoused794dc92009-06-28 00:27:49 +0100845 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700846 free_pgtable_page(domain->pgd);
847 domain->pgd = NULL;
848 }
849}
850
851/* iommu handling */
852static int iommu_alloc_root_entry(struct intel_iommu *iommu)
853{
854 struct root_entry *root;
855 unsigned long flags;
856
Suresh Siddha4c923d42009-10-02 11:01:24 -0700857 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700858 if (!root)
859 return -ENOMEM;
860
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700861 __iommu_flush_cache(iommu, root, ROOT_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700862
863 spin_lock_irqsave(&iommu->lock, flags);
864 iommu->root_entry = root;
865 spin_unlock_irqrestore(&iommu->lock, flags);
866
867 return 0;
868}
869
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700870static void iommu_set_root_entry(struct intel_iommu *iommu)
871{
872 void *addr;
David Woodhousec416daa2009-05-10 20:30:58 +0100873 u32 sts;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700874 unsigned long flag;
875
876 addr = iommu->root_entry;
877
878 spin_lock_irqsave(&iommu->register_lock, flag);
879 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
880
David Woodhousec416daa2009-05-10 20:30:58 +0100881 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700882
883 /* Make sure hardware complete it */
884 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +0100885 readl, (sts & DMA_GSTS_RTPS), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700886
887 spin_unlock_irqrestore(&iommu->register_lock, flag);
888}
889
890static void iommu_flush_write_buffer(struct intel_iommu *iommu)
891{
892 u32 val;
893 unsigned long flag;
894
David Woodhouse9af88142009-02-13 23:18:03 +0000895 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700896 return;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700897
898 spin_lock_irqsave(&iommu->register_lock, flag);
David Woodhouse462b60f2009-05-10 20:18:18 +0100899 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700900
901 /* Make sure hardware complete it */
902 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +0100903 readl, (!(val & DMA_GSTS_WBFS)), val);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700904
905 spin_unlock_irqrestore(&iommu->register_lock, flag);
906}
907
908/* return value determine if we need a write buffer flush */
David Woodhouse4c25a2c2009-05-10 17:16:06 +0100909static void __iommu_flush_context(struct intel_iommu *iommu,
910 u16 did, u16 source_id, u8 function_mask,
911 u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700912{
913 u64 val = 0;
914 unsigned long flag;
915
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700916 switch (type) {
917 case DMA_CCMD_GLOBAL_INVL:
918 val = DMA_CCMD_GLOBAL_INVL;
919 break;
920 case DMA_CCMD_DOMAIN_INVL:
921 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
922 break;
923 case DMA_CCMD_DEVICE_INVL:
924 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
925 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
926 break;
927 default:
928 BUG();
929 }
930 val |= DMA_CCMD_ICC;
931
932 spin_lock_irqsave(&iommu->register_lock, flag);
933 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
934
935 /* Make sure hardware complete it */
936 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
937 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
938
939 spin_unlock_irqrestore(&iommu->register_lock, flag);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700940}
941
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700942/* return value determine if we need a write buffer flush */
David Woodhouse1f0ef2a2009-05-10 19:58:49 +0100943static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
944 u64 addr, unsigned int size_order, u64 type)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700945{
946 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
947 u64 val = 0, val_iva = 0;
948 unsigned long flag;
949
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700950 switch (type) {
951 case DMA_TLB_GLOBAL_FLUSH:
952 /* global flush doesn't need set IVA_REG */
953 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
954 break;
955 case DMA_TLB_DSI_FLUSH:
956 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
957 break;
958 case DMA_TLB_PSI_FLUSH:
959 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
960 /* Note: always flush non-leaf currently */
961 val_iva = size_order | addr;
962 break;
963 default:
964 BUG();
965 }
966 /* Note: set drain read/write */
967#if 0
968 /*
969 * This is probably to be super secure.. Looks like we can
970 * ignore it without any impact.
971 */
972 if (cap_read_drain(iommu->cap))
973 val |= DMA_TLB_READ_DRAIN;
974#endif
975 if (cap_write_drain(iommu->cap))
976 val |= DMA_TLB_WRITE_DRAIN;
977
978 spin_lock_irqsave(&iommu->register_lock, flag);
979 /* Note: Only uses first TLB reg currently */
980 if (val_iva)
981 dmar_writeq(iommu->reg + tlb_offset, val_iva);
982 dmar_writeq(iommu->reg + tlb_offset + 8, val);
983
984 /* Make sure hardware complete it */
985 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
986 dmar_readq, (!(val & DMA_TLB_IVT)), val);
987
988 spin_unlock_irqrestore(&iommu->register_lock, flag);
989
990 /* check IOTLB invalidation granularity */
991 if (DMA_TLB_IAIG(val) == 0)
992 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
993 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
994 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -0700995 (unsigned long long)DMA_TLB_IIRG(type),
996 (unsigned long long)DMA_TLB_IAIG(val));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -0700997}
998
Yu Zhao93a23a72009-05-18 13:51:37 +0800999static struct device_domain_info *iommu_support_dev_iotlb(
1000 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001001{
Yu Zhao93a23a72009-05-18 13:51:37 +08001002 int found = 0;
1003 unsigned long flags;
1004 struct device_domain_info *info;
1005 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1006
1007 if (!ecap_dev_iotlb_support(iommu->ecap))
1008 return NULL;
1009
1010 if (!iommu->qi)
1011 return NULL;
1012
1013 spin_lock_irqsave(&device_domain_lock, flags);
1014 list_for_each_entry(info, &domain->devices, link)
1015 if (info->bus == bus && info->devfn == devfn) {
1016 found = 1;
1017 break;
1018 }
1019 spin_unlock_irqrestore(&device_domain_lock, flags);
1020
1021 if (!found || !info->dev)
1022 return NULL;
1023
1024 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1025 return NULL;
1026
1027 if (!dmar_find_matched_atsr_unit(info->dev))
1028 return NULL;
1029
1030 info->iommu = iommu;
1031
1032 return info;
1033}
1034
1035static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1036{
1037 if (!info)
1038 return;
1039
1040 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1041}
1042
1043static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1044{
1045 if (!info->dev || !pci_ats_enabled(info->dev))
1046 return;
1047
1048 pci_disable_ats(info->dev);
1049}
1050
1051static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1052 u64 addr, unsigned mask)
1053{
1054 u16 sid, qdep;
1055 unsigned long flags;
1056 struct device_domain_info *info;
1057
1058 spin_lock_irqsave(&device_domain_lock, flags);
1059 list_for_each_entry(info, &domain->devices, link) {
1060 if (!info->dev || !pci_ats_enabled(info->dev))
1061 continue;
1062
1063 sid = info->bus << 8 | info->devfn;
1064 qdep = pci_ats_queue_depth(info->dev);
1065 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1066 }
1067 spin_unlock_irqrestore(&device_domain_lock, flags);
1068}
1069
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001070static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
David Woodhouse03d6a242009-06-28 15:33:46 +01001071 unsigned long pfn, unsigned int pages)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001072{
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001073 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
David Woodhouse03d6a242009-06-28 15:33:46 +01001074 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001075
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001076 BUG_ON(pages == 0);
1077
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001078 /*
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001079 * Fallback to domain selective flush if no PSI support or the size is
1080 * too big.
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001081 * PSI requires page size to be 2 ^ x, and the base address is naturally
1082 * aligned to the size
1083 */
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001084 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1085 iommu->flush.flush_iotlb(iommu, did, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001086 DMA_TLB_DSI_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08001087 else
1088 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1089 DMA_TLB_PSI_FLUSH);
Yu Zhaobf92df32009-06-29 11:31:45 +08001090
1091 /*
1092 * In caching mode, domain ID 0 is reserved for non-present to present
1093 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1094 */
1095 if (!cap_caching_mode(iommu->cap) || did)
Yu Zhao93a23a72009-05-18 13:51:37 +08001096 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001097}
1098
mark grossf8bab732008-02-08 04:18:38 -08001099static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1100{
1101 u32 pmen;
1102 unsigned long flags;
1103
1104 spin_lock_irqsave(&iommu->register_lock, flags);
1105 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1106 pmen &= ~DMA_PMEN_EPM;
1107 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1108
1109 /* wait for the protected region status bit to clear */
1110 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1111 readl, !(pmen & DMA_PMEN_PRS), pmen);
1112
1113 spin_unlock_irqrestore(&iommu->register_lock, flags);
1114}
1115
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001116static int iommu_enable_translation(struct intel_iommu *iommu)
1117{
1118 u32 sts;
1119 unsigned long flags;
1120
1121 spin_lock_irqsave(&iommu->register_lock, flags);
David Woodhousec416daa2009-05-10 20:30:58 +01001122 iommu->gcmd |= DMA_GCMD_TE;
1123 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001124
1125 /* Make sure hardware complete it */
1126 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001127 readl, (sts & DMA_GSTS_TES), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001128
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001129 spin_unlock_irqrestore(&iommu->register_lock, flags);
1130 return 0;
1131}
1132
1133static int iommu_disable_translation(struct intel_iommu *iommu)
1134{
1135 u32 sts;
1136 unsigned long flag;
1137
1138 spin_lock_irqsave(&iommu->register_lock, flag);
1139 iommu->gcmd &= ~DMA_GCMD_TE;
1140 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1141
1142 /* Make sure hardware complete it */
1143 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
David Woodhousec416daa2009-05-10 20:30:58 +01001144 readl, (!(sts & DMA_GSTS_TES)), sts);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001145
1146 spin_unlock_irqrestore(&iommu->register_lock, flag);
1147 return 0;
1148}
1149
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07001150
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001151static int iommu_init_domains(struct intel_iommu *iommu)
1152{
1153 unsigned long ndomains;
1154 unsigned long nlongs;
1155
1156 ndomains = cap_ndoms(iommu->cap);
1157 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1158 nlongs = BITS_TO_LONGS(ndomains);
1159
Donald Dutile94a91b52009-08-20 16:51:34 -04001160 spin_lock_init(&iommu->lock);
1161
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001162 /* TBD: there might be 64K domains,
1163 * consider other allocation for future chip
1164 */
1165 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1166 if (!iommu->domain_ids) {
1167 printk(KERN_ERR "Allocating domain id array failed\n");
1168 return -ENOMEM;
1169 }
1170 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1171 GFP_KERNEL);
1172 if (!iommu->domains) {
1173 printk(KERN_ERR "Allocating domain array failed\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001174 return -ENOMEM;
1175 }
1176
1177 /*
1178 * if Caching mode is set, then invalid translations are tagged
1179 * with domainid 0. Hence we need to pre-allocate it.
1180 */
1181 if (cap_caching_mode(iommu->cap))
1182 set_bit(0, iommu->domain_ids);
1183 return 0;
1184}
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001185
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001186
1187static void domain_exit(struct dmar_domain *domain);
Weidong Han5e98c4b2008-12-08 23:03:27 +08001188static void vm_domain_exit(struct dmar_domain *domain);
Suresh Siddhae61d98d2008-07-10 11:16:35 -07001189
1190void free_dmar_iommu(struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001191{
1192 struct dmar_domain *domain;
1193 int i;
Weidong Hanc7151a82008-12-08 22:51:37 +08001194 unsigned long flags;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001195
Donald Dutile94a91b52009-08-20 16:51:34 -04001196 if ((iommu->domains) && (iommu->domain_ids)) {
1197 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1198 for (; i < cap_ndoms(iommu->cap); ) {
1199 domain = iommu->domains[i];
1200 clear_bit(i, iommu->domain_ids);
Weidong Hanc7151a82008-12-08 22:51:37 +08001201
Donald Dutile94a91b52009-08-20 16:51:34 -04001202 spin_lock_irqsave(&domain->iommu_lock, flags);
1203 if (--domain->iommu_count == 0) {
1204 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1205 vm_domain_exit(domain);
1206 else
1207 domain_exit(domain);
1208 }
1209 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1210
1211 i = find_next_bit(iommu->domain_ids,
1212 cap_ndoms(iommu->cap), i+1);
Weidong Han5e98c4b2008-12-08 23:03:27 +08001213 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001214 }
1215
1216 if (iommu->gcmd & DMA_GCMD_TE)
1217 iommu_disable_translation(iommu);
1218
1219 if (iommu->irq) {
1220 set_irq_data(iommu->irq, NULL);
1221 /* This will mask the irq */
1222 free_irq(iommu->irq, iommu);
1223 destroy_irq(iommu->irq);
1224 }
1225
1226 kfree(iommu->domains);
1227 kfree(iommu->domain_ids);
1228
Weidong Hand9630fe2008-12-08 11:06:32 +08001229 g_iommus[iommu->seq_id] = NULL;
1230
1231 /* if all iommus are freed, free g_iommus */
1232 for (i = 0; i < g_num_of_iommus; i++) {
1233 if (g_iommus[i])
1234 break;
1235 }
1236
1237 if (i == g_num_of_iommus)
1238 kfree(g_iommus);
1239
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001240 /* free context mapping */
1241 free_context_table(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001242}
1243
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001244static struct dmar_domain *alloc_domain(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001245{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001246 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001247
1248 domain = alloc_domain_mem();
1249 if (!domain)
1250 return NULL;
1251
Suresh Siddha4c923d42009-10-02 11:01:24 -07001252 domain->nid = -1;
Weidong Han8c11e792008-12-08 15:29:22 +08001253 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
Weidong Hand71a2f32008-12-07 21:13:41 +08001254 domain->flags = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001255
1256 return domain;
1257}
1258
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001259static int iommu_attach_domain(struct dmar_domain *domain,
1260 struct intel_iommu *iommu)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001261{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001262 int num;
1263 unsigned long ndomains;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001264 unsigned long flags;
1265
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001266 ndomains = cap_ndoms(iommu->cap);
Weidong Han8c11e792008-12-08 15:29:22 +08001267
1268 spin_lock_irqsave(&iommu->lock, flags);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001269
1270 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1271 if (num >= ndomains) {
1272 spin_unlock_irqrestore(&iommu->lock, flags);
1273 printk(KERN_ERR "IOMMU: no free domain ids\n");
1274 return -ENOMEM;
1275 }
1276
1277 domain->id = num;
1278 set_bit(num, iommu->domain_ids);
1279 set_bit(iommu->seq_id, &domain->iommu_bmp);
1280 iommu->domains[num] = domain;
1281 spin_unlock_irqrestore(&iommu->lock, flags);
1282
1283 return 0;
1284}
1285
1286static void iommu_detach_domain(struct dmar_domain *domain,
1287 struct intel_iommu *iommu)
1288{
1289 unsigned long flags;
1290 int num, ndomains;
1291 int found = 0;
1292
1293 spin_lock_irqsave(&iommu->lock, flags);
1294 ndomains = cap_ndoms(iommu->cap);
1295 num = find_first_bit(iommu->domain_ids, ndomains);
1296 for (; num < ndomains; ) {
1297 if (iommu->domains[num] == domain) {
1298 found = 1;
1299 break;
1300 }
1301 num = find_next_bit(iommu->domain_ids,
1302 cap_ndoms(iommu->cap), num+1);
1303 }
1304
1305 if (found) {
1306 clear_bit(num, iommu->domain_ids);
1307 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1308 iommu->domains[num] = NULL;
1309 }
Weidong Han8c11e792008-12-08 15:29:22 +08001310 spin_unlock_irqrestore(&iommu->lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001311}
1312
1313static struct iova_domain reserved_iova_list;
Mark Gross8a443df2008-03-04 14:59:31 -08001314static struct lock_class_key reserved_rbtree_key;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001315
1316static void dmar_init_reserved_ranges(void)
1317{
1318 struct pci_dev *pdev = NULL;
1319 struct iova *iova;
1320 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001321
David Millerf6611972008-02-06 01:36:23 -08001322 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001323
Mark Gross8a443df2008-03-04 14:59:31 -08001324 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1325 &reserved_rbtree_key);
1326
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001327 /* IOAPIC ranges shouldn't be accessed by DMA */
1328 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1329 IOVA_PFN(IOAPIC_RANGE_END));
1330 if (!iova)
1331 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1332
1333 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1334 for_each_pci_dev(pdev) {
1335 struct resource *r;
1336
1337 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1338 r = &pdev->resource[i];
1339 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1340 continue;
David Woodhouse1a4a4552009-06-28 16:00:42 +01001341 iova = reserve_iova(&reserved_iova_list,
1342 IOVA_PFN(r->start),
1343 IOVA_PFN(r->end));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001344 if (!iova)
1345 printk(KERN_ERR "Reserve iova failed\n");
1346 }
1347 }
1348
1349}
1350
1351static void domain_reserve_special_ranges(struct dmar_domain *domain)
1352{
1353 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1354}
1355
1356static inline int guestwidth_to_adjustwidth(int gaw)
1357{
1358 int agaw;
1359 int r = (gaw - 12) % 9;
1360
1361 if (r == 0)
1362 agaw = gaw;
1363 else
1364 agaw = gaw + 9 - r;
1365 if (agaw > 64)
1366 agaw = 64;
1367 return agaw;
1368}
1369
1370static int domain_init(struct dmar_domain *domain, int guest_width)
1371{
1372 struct intel_iommu *iommu;
1373 int adjust_width, agaw;
1374 unsigned long sagaw;
1375
David Millerf6611972008-02-06 01:36:23 -08001376 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
Weidong Hanc7151a82008-12-08 22:51:37 +08001377 spin_lock_init(&domain->iommu_lock);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001378
1379 domain_reserve_special_ranges(domain);
1380
1381 /* calculate AGAW */
Weidong Han8c11e792008-12-08 15:29:22 +08001382 iommu = domain_get_iommu(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001383 if (guest_width > cap_mgaw(iommu->cap))
1384 guest_width = cap_mgaw(iommu->cap);
1385 domain->gaw = guest_width;
1386 adjust_width = guestwidth_to_adjustwidth(guest_width);
1387 agaw = width_to_agaw(adjust_width);
1388 sagaw = cap_sagaw(iommu->cap);
1389 if (!test_bit(agaw, &sagaw)) {
1390 /* hardware doesn't support it, choose a bigger one */
1391 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1392 agaw = find_next_bit(&sagaw, 5, agaw);
1393 if (agaw >= 5)
1394 return -ENODEV;
1395 }
1396 domain->agaw = agaw;
1397 INIT_LIST_HEAD(&domain->devices);
1398
Weidong Han8e604092008-12-08 15:49:06 +08001399 if (ecap_coherent(iommu->ecap))
1400 domain->iommu_coherency = 1;
1401 else
1402 domain->iommu_coherency = 0;
1403
Sheng Yang58c610b2009-03-18 15:33:05 +08001404 if (ecap_sc_support(iommu->ecap))
1405 domain->iommu_snooping = 1;
1406 else
1407 domain->iommu_snooping = 0;
1408
Weidong Hanc7151a82008-12-08 22:51:37 +08001409 domain->iommu_count = 1;
Suresh Siddha4c923d42009-10-02 11:01:24 -07001410 domain->nid = iommu->node;
Weidong Hanc7151a82008-12-08 22:51:37 +08001411
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001412 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07001413 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001414 if (!domain->pgd)
1415 return -ENOMEM;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001416 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001417 return 0;
1418}
1419
1420static void domain_exit(struct dmar_domain *domain)
1421{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001422 struct dmar_drhd_unit *drhd;
1423 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001424
1425 /* Domain 0 is reserved, so dont process it */
1426 if (!domain)
1427 return;
1428
1429 domain_remove_dev_info(domain);
1430 /* destroy iovas */
1431 put_iova_domain(&domain->iovad);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001432
1433 /* clear ptes */
David Woodhouse595badf2009-06-27 22:09:11 +01001434 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001435
1436 /* free page tables */
David Woodhoused794dc92009-06-28 00:27:49 +01001437 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001438
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001439 for_each_active_iommu(iommu, drhd)
1440 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1441 iommu_detach_domain(domain, iommu);
1442
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001443 free_domain_mem(domain);
1444}
1445
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001446static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1447 u8 bus, u8 devfn, int translation)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001448{
1449 struct context_entry *context;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001450 unsigned long flags;
Weidong Han5331fe62008-12-08 23:00:00 +08001451 struct intel_iommu *iommu;
Weidong Hanea6606b2008-12-08 23:08:15 +08001452 struct dma_pte *pgd;
1453 unsigned long num;
1454 unsigned long ndomains;
1455 int id;
1456 int agaw;
Yu Zhao93a23a72009-05-18 13:51:37 +08001457 struct device_domain_info *info = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001458
1459 pr_debug("Set context mapping for %02x:%02x.%d\n",
1460 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001461
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001462 BUG_ON(!domain->pgd);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001463 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1464 translation != CONTEXT_TT_MULTI_LEVEL);
Weidong Han5331fe62008-12-08 23:00:00 +08001465
David Woodhouse276dbf92009-04-04 01:45:37 +01001466 iommu = device_to_iommu(segment, bus, devfn);
Weidong Han5331fe62008-12-08 23:00:00 +08001467 if (!iommu)
1468 return -ENODEV;
1469
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001470 context = device_to_context_entry(iommu, bus, devfn);
1471 if (!context)
1472 return -ENOMEM;
1473 spin_lock_irqsave(&iommu->lock, flags);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00001474 if (context_present(context)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001475 spin_unlock_irqrestore(&iommu->lock, flags);
1476 return 0;
1477 }
1478
Weidong Hanea6606b2008-12-08 23:08:15 +08001479 id = domain->id;
1480 pgd = domain->pgd;
1481
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001482 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1483 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
Weidong Hanea6606b2008-12-08 23:08:15 +08001484 int found = 0;
1485
1486 /* find an available domain id for this device in iommu */
1487 ndomains = cap_ndoms(iommu->cap);
1488 num = find_first_bit(iommu->domain_ids, ndomains);
1489 for (; num < ndomains; ) {
1490 if (iommu->domains[num] == domain) {
1491 id = num;
1492 found = 1;
1493 break;
1494 }
1495 num = find_next_bit(iommu->domain_ids,
1496 cap_ndoms(iommu->cap), num+1);
1497 }
1498
1499 if (found == 0) {
1500 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1501 if (num >= ndomains) {
1502 spin_unlock_irqrestore(&iommu->lock, flags);
1503 printk(KERN_ERR "IOMMU: no free domain ids\n");
1504 return -EFAULT;
1505 }
1506
1507 set_bit(num, iommu->domain_ids);
1508 iommu->domains[num] = domain;
1509 id = num;
1510 }
1511
1512 /* Skip top levels of page tables for
1513 * iommu which has less agaw than default.
Chris Wright1672af12009-12-02 12:06:34 -08001514 * Unnecessary for PT mode.
Weidong Hanea6606b2008-12-08 23:08:15 +08001515 */
Chris Wright1672af12009-12-02 12:06:34 -08001516 if (translation != CONTEXT_TT_PASS_THROUGH) {
1517 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1518 pgd = phys_to_virt(dma_pte_addr(pgd));
1519 if (!dma_pte_present(pgd)) {
1520 spin_unlock_irqrestore(&iommu->lock, flags);
1521 return -ENOMEM;
1522 }
Weidong Hanea6606b2008-12-08 23:08:15 +08001523 }
1524 }
1525 }
1526
1527 context_set_domain_id(context, id);
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001528
Yu Zhao93a23a72009-05-18 13:51:37 +08001529 if (translation != CONTEXT_TT_PASS_THROUGH) {
1530 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1531 translation = info ? CONTEXT_TT_DEV_IOTLB :
1532 CONTEXT_TT_MULTI_LEVEL;
1533 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001534 /*
1535 * In pass through mode, AW must be programmed to indicate the largest
1536 * AGAW value supported by hardware. And ASR is ignored by hardware.
1537 */
Yu Zhao93a23a72009-05-18 13:51:37 +08001538 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001539 context_set_address_width(context, iommu->msagaw);
Yu Zhao93a23a72009-05-18 13:51:37 +08001540 else {
1541 context_set_address_root(context, virt_to_phys(pgd));
1542 context_set_address_width(context, iommu->agaw);
1543 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001544
1545 context_set_translation_type(context, translation);
Mark McLoughlinc07e7d22008-11-21 16:54:46 +00001546 context_set_fault_enable(context);
1547 context_set_present(context);
Weidong Han5331fe62008-12-08 23:00:00 +08001548 domain_flush_cache(domain, context, sizeof(*context));
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001549
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001550 /*
1551 * It's a non-present to present mapping. If hardware doesn't cache
1552 * non-present entry we only need to flush the write-buffer. If the
1553 * _does_ cache non-present entries, then it does so in the special
1554 * domain #0, which we have to flush:
1555 */
1556 if (cap_caching_mode(iommu->cap)) {
1557 iommu->flush.flush_context(iommu, 0,
1558 (((u16)bus) << 8) | devfn,
1559 DMA_CCMD_MASK_NOBIT,
1560 DMA_CCMD_DEVICE_INVL);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001561 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001562 } else {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001563 iommu_flush_write_buffer(iommu);
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001564 }
Yu Zhao93a23a72009-05-18 13:51:37 +08001565 iommu_enable_dev_iotlb(info);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001566 spin_unlock_irqrestore(&iommu->lock, flags);
Weidong Hanc7151a82008-12-08 22:51:37 +08001567
1568 spin_lock_irqsave(&domain->iommu_lock, flags);
1569 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1570 domain->iommu_count++;
Suresh Siddha4c923d42009-10-02 11:01:24 -07001571 if (domain->iommu_count == 1)
1572 domain->nid = iommu->node;
Sheng Yang58c610b2009-03-18 15:33:05 +08001573 domain_update_iommu_cap(domain);
Weidong Hanc7151a82008-12-08 22:51:37 +08001574 }
1575 spin_unlock_irqrestore(&domain->iommu_lock, flags);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001576 return 0;
1577}
1578
1579static int
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001580domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1581 int translation)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001582{
1583 int ret;
1584 struct pci_dev *tmp, *parent;
1585
David Woodhouse276dbf92009-04-04 01:45:37 +01001586 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001587 pdev->bus->number, pdev->devfn,
1588 translation);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001589 if (ret)
1590 return ret;
1591
1592 /* dependent device mapping */
1593 tmp = pci_find_upstream_pcie_bridge(pdev);
1594 if (!tmp)
1595 return 0;
1596 /* Secondary interface's bus number and devfn 0 */
1597 parent = pdev->bus->self;
1598 while (parent != tmp) {
David Woodhouse276dbf92009-04-04 01:45:37 +01001599 ret = domain_context_mapping_one(domain,
1600 pci_domain_nr(parent->bus),
1601 parent->bus->number,
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001602 parent->devfn, translation);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001603 if (ret)
1604 return ret;
1605 parent = parent->bus->self;
1606 }
Stefan Assmann45e829e2009-12-03 06:49:24 -05001607 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001608 return domain_context_mapping_one(domain,
David Woodhouse276dbf92009-04-04 01:45:37 +01001609 pci_domain_nr(tmp->subordinate),
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001610 tmp->subordinate->number, 0,
1611 translation);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001612 else /* this is a legacy PCI bridge */
1613 return domain_context_mapping_one(domain,
David Woodhouse276dbf92009-04-04 01:45:37 +01001614 pci_domain_nr(tmp->bus),
1615 tmp->bus->number,
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07001616 tmp->devfn,
1617 translation);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001618}
1619
Weidong Han5331fe62008-12-08 23:00:00 +08001620static int domain_context_mapped(struct pci_dev *pdev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001621{
1622 int ret;
1623 struct pci_dev *tmp, *parent;
Weidong Han5331fe62008-12-08 23:00:00 +08001624 struct intel_iommu *iommu;
1625
David Woodhouse276dbf92009-04-04 01:45:37 +01001626 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1627 pdev->devfn);
Weidong Han5331fe62008-12-08 23:00:00 +08001628 if (!iommu)
1629 return -ENODEV;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001630
David Woodhouse276dbf92009-04-04 01:45:37 +01001631 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001632 if (!ret)
1633 return ret;
1634 /* dependent device mapping */
1635 tmp = pci_find_upstream_pcie_bridge(pdev);
1636 if (!tmp)
1637 return ret;
1638 /* Secondary interface's bus number and devfn 0 */
1639 parent = pdev->bus->self;
1640 while (parent != tmp) {
Weidong Han8c11e792008-12-08 15:29:22 +08001641 ret = device_context_mapped(iommu, parent->bus->number,
David Woodhouse276dbf92009-04-04 01:45:37 +01001642 parent->devfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001643 if (!ret)
1644 return ret;
1645 parent = parent->bus->self;
1646 }
Kenji Kaneshige5f4d91a2009-11-11 14:36:17 +09001647 if (pci_is_pcie(tmp))
David Woodhouse276dbf92009-04-04 01:45:37 +01001648 return device_context_mapped(iommu, tmp->subordinate->number,
1649 0);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001650 else
David Woodhouse276dbf92009-04-04 01:45:37 +01001651 return device_context_mapped(iommu, tmp->bus->number,
1652 tmp->devfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001653}
1654
Fenghua Yuf5329592009-08-04 15:09:37 -07001655/* Returns a number of VTD pages, but aligned to MM page size */
1656static inline unsigned long aligned_nrpages(unsigned long host_addr,
1657 size_t size)
1658{
1659 host_addr &= ~PAGE_MASK;
1660 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1661}
1662
David Woodhouse9051aa02009-06-29 12:30:54 +01001663static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1664 struct scatterlist *sg, unsigned long phys_pfn,
1665 unsigned long nr_pages, int prot)
David Woodhousee1605492009-06-29 11:17:38 +01001666{
1667 struct dma_pte *first_pte = NULL, *pte = NULL;
David Woodhouse9051aa02009-06-29 12:30:54 +01001668 phys_addr_t uninitialized_var(pteval);
David Woodhousee1605492009-06-29 11:17:38 +01001669 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
David Woodhouse9051aa02009-06-29 12:30:54 +01001670 unsigned long sg_res;
David Woodhousee1605492009-06-29 11:17:38 +01001671
1672 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1673
1674 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1675 return -EINVAL;
1676
1677 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1678
David Woodhouse9051aa02009-06-29 12:30:54 +01001679 if (sg)
1680 sg_res = 0;
1681 else {
1682 sg_res = nr_pages + 1;
1683 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1684 }
1685
David Woodhousee1605492009-06-29 11:17:38 +01001686 while (nr_pages--) {
David Woodhousec85994e2009-07-01 19:21:24 +01001687 uint64_t tmp;
1688
David Woodhousee1605492009-06-29 11:17:38 +01001689 if (!sg_res) {
Fenghua Yuf5329592009-08-04 15:09:37 -07001690 sg_res = aligned_nrpages(sg->offset, sg->length);
David Woodhousee1605492009-06-29 11:17:38 +01001691 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1692 sg->dma_length = sg->length;
1693 pteval = page_to_phys(sg_page(sg)) | prot;
1694 }
1695 if (!pte) {
1696 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1697 if (!pte)
1698 return -ENOMEM;
1699 }
1700 /* We don't need lock here, nobody else
1701 * touches the iova range
1702 */
David Woodhouse7766a3f2009-07-01 20:27:03 +01001703 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
David Woodhousec85994e2009-07-01 19:21:24 +01001704 if (tmp) {
David Woodhouse1bf20f02009-06-29 22:06:43 +01001705 static int dumps = 5;
David Woodhousec85994e2009-07-01 19:21:24 +01001706 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1707 iov_pfn, tmp, (unsigned long long)pteval);
David Woodhouse1bf20f02009-06-29 22:06:43 +01001708 if (dumps) {
1709 dumps--;
1710 debug_dma_dump_mappings(NULL);
1711 }
1712 WARN_ON(1);
1713 }
David Woodhousee1605492009-06-29 11:17:38 +01001714 pte++;
David Woodhouse75e6bf92009-07-02 11:21:16 +01001715 if (!nr_pages || first_pte_in_page(pte)) {
David Woodhousee1605492009-06-29 11:17:38 +01001716 domain_flush_cache(domain, first_pte,
1717 (void *)pte - (void *)first_pte);
1718 pte = NULL;
1719 }
1720 iov_pfn++;
1721 pteval += VTD_PAGE_SIZE;
1722 sg_res--;
1723 if (!sg_res)
1724 sg = sg_next(sg);
1725 }
1726 return 0;
1727}
1728
David Woodhouse9051aa02009-06-29 12:30:54 +01001729static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1730 struct scatterlist *sg, unsigned long nr_pages,
1731 int prot)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001732{
David Woodhouse9051aa02009-06-29 12:30:54 +01001733 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1734}
Fenghua Yu5b6985c2008-10-16 18:02:32 -07001735
David Woodhouse9051aa02009-06-29 12:30:54 +01001736static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1737 unsigned long phys_pfn, unsigned long nr_pages,
1738 int prot)
1739{
1740 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001741}
1742
Weidong Hanc7151a82008-12-08 22:51:37 +08001743static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001744{
Weidong Hanc7151a82008-12-08 22:51:37 +08001745 if (!iommu)
1746 return;
Weidong Han8c11e792008-12-08 15:29:22 +08001747
1748 clear_context_table(iommu, bus, devfn);
1749 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse4c25a2c2009-05-10 17:16:06 +01001750 DMA_CCMD_GLOBAL_INVL);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01001751 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001752}
1753
1754static void domain_remove_dev_info(struct dmar_domain *domain)
1755{
1756 struct device_domain_info *info;
1757 unsigned long flags;
Weidong Hanc7151a82008-12-08 22:51:37 +08001758 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001759
1760 spin_lock_irqsave(&device_domain_lock, flags);
1761 while (!list_empty(&domain->devices)) {
1762 info = list_entry(domain->devices.next,
1763 struct device_domain_info, link);
1764 list_del(&info->link);
1765 list_del(&info->global);
1766 if (info->dev)
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07001767 info->dev->dev.archdata.iommu = NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001768 spin_unlock_irqrestore(&device_domain_lock, flags);
1769
Yu Zhao93a23a72009-05-18 13:51:37 +08001770 iommu_disable_dev_iotlb(info);
David Woodhouse276dbf92009-04-04 01:45:37 +01001771 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
Weidong Hanc7151a82008-12-08 22:51:37 +08001772 iommu_detach_dev(iommu, info->bus, info->devfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001773 free_devinfo_mem(info);
1774
1775 spin_lock_irqsave(&device_domain_lock, flags);
1776 }
1777 spin_unlock_irqrestore(&device_domain_lock, flags);
1778}
1779
1780/*
1781 * find_domain
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07001782 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001783 */
Kay, Allen M38717942008-09-09 18:37:29 +03001784static struct dmar_domain *
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001785find_domain(struct pci_dev *pdev)
1786{
1787 struct device_domain_info *info;
1788
1789 /* No lock here, assumes no domain exit in normal case */
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07001790 info = pdev->dev.archdata.iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001791 if (info)
1792 return info->domain;
1793 return NULL;
1794}
1795
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001796/* domain is initialized */
1797static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1798{
1799 struct dmar_domain *domain, *found = NULL;
1800 struct intel_iommu *iommu;
1801 struct dmar_drhd_unit *drhd;
1802 struct device_domain_info *info, *tmp;
1803 struct pci_dev *dev_tmp;
1804 unsigned long flags;
1805 int bus = 0, devfn = 0;
David Woodhouse276dbf92009-04-04 01:45:37 +01001806 int segment;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001807 int ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001808
1809 domain = find_domain(pdev);
1810 if (domain)
1811 return domain;
1812
David Woodhouse276dbf92009-04-04 01:45:37 +01001813 segment = pci_domain_nr(pdev->bus);
1814
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001815 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1816 if (dev_tmp) {
Kenji Kaneshige5f4d91a2009-11-11 14:36:17 +09001817 if (pci_is_pcie(dev_tmp)) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001818 bus = dev_tmp->subordinate->number;
1819 devfn = 0;
1820 } else {
1821 bus = dev_tmp->bus->number;
1822 devfn = dev_tmp->devfn;
1823 }
1824 spin_lock_irqsave(&device_domain_lock, flags);
1825 list_for_each_entry(info, &device_domain_list, global) {
David Woodhouse276dbf92009-04-04 01:45:37 +01001826 if (info->segment == segment &&
1827 info->bus == bus && info->devfn == devfn) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001828 found = info->domain;
1829 break;
1830 }
1831 }
1832 spin_unlock_irqrestore(&device_domain_lock, flags);
1833 /* pcie-pci bridge already has a domain, uses it */
1834 if (found) {
1835 domain = found;
1836 goto found_domain;
1837 }
1838 }
1839
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001840 domain = alloc_domain();
1841 if (!domain)
1842 goto error;
1843
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001844 /* Allocate new domain for the device */
1845 drhd = dmar_find_matched_drhd_unit(pdev);
1846 if (!drhd) {
1847 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1848 pci_name(pdev));
1849 return NULL;
1850 }
1851 iommu = drhd->iommu;
1852
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001853 ret = iommu_attach_domain(domain, iommu);
1854 if (ret) {
1855 domain_exit(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001856 goto error;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001857 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001858
1859 if (domain_init(domain, gaw)) {
1860 domain_exit(domain);
1861 goto error;
1862 }
1863
1864 /* register pcie-to-pci device */
1865 if (dev_tmp) {
1866 info = alloc_devinfo_mem();
1867 if (!info) {
1868 domain_exit(domain);
1869 goto error;
1870 }
David Woodhouse276dbf92009-04-04 01:45:37 +01001871 info->segment = segment;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001872 info->bus = bus;
1873 info->devfn = devfn;
1874 info->dev = NULL;
1875 info->domain = domain;
1876 /* This domain is shared by devices under p2p bridge */
Weidong Han3b5410e2008-12-08 09:17:15 +08001877 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001878
1879 /* pcie-to-pci bridge already has a domain, uses it */
1880 found = NULL;
1881 spin_lock_irqsave(&device_domain_lock, flags);
1882 list_for_each_entry(tmp, &device_domain_list, global) {
David Woodhouse276dbf92009-04-04 01:45:37 +01001883 if (tmp->segment == segment &&
1884 tmp->bus == bus && tmp->devfn == devfn) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001885 found = tmp->domain;
1886 break;
1887 }
1888 }
1889 if (found) {
1890 free_devinfo_mem(info);
1891 domain_exit(domain);
1892 domain = found;
1893 } else {
1894 list_add(&info->link, &domain->devices);
1895 list_add(&info->global, &device_domain_list);
1896 }
1897 spin_unlock_irqrestore(&device_domain_lock, flags);
1898 }
1899
1900found_domain:
1901 info = alloc_devinfo_mem();
1902 if (!info)
1903 goto error;
David Woodhouse276dbf92009-04-04 01:45:37 +01001904 info->segment = segment;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001905 info->bus = pdev->bus->number;
1906 info->devfn = pdev->devfn;
1907 info->dev = pdev;
1908 info->domain = domain;
1909 spin_lock_irqsave(&device_domain_lock, flags);
1910 /* somebody is fast */
1911 found = find_domain(pdev);
1912 if (found != NULL) {
1913 spin_unlock_irqrestore(&device_domain_lock, flags);
1914 if (found != domain) {
1915 domain_exit(domain);
1916 domain = found;
1917 }
1918 free_devinfo_mem(info);
1919 return domain;
1920 }
1921 list_add(&info->link, &domain->devices);
1922 list_add(&info->global, &device_domain_list);
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07001923 pdev->dev.archdata.iommu = info;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001924 spin_unlock_irqrestore(&device_domain_lock, flags);
1925 return domain;
1926error:
1927 /* recheck it here, maybe others set it */
1928 return find_domain(pdev);
1929}
1930
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001931static int iommu_identity_mapping;
David Woodhousee0fc7e02009-09-30 09:12:17 -07001932#define IDENTMAP_ALL 1
1933#define IDENTMAP_GFX 2
1934#define IDENTMAP_AZALIA 4
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07001935
David Woodhouseb2132032009-06-26 18:50:28 +01001936static int iommu_domain_identity_map(struct dmar_domain *domain,
1937 unsigned long long start,
1938 unsigned long long end)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001939{
David Woodhousec5395d52009-06-28 16:35:56 +01001940 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1941 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001942
David Woodhousec5395d52009-06-28 16:35:56 +01001943 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1944 dma_to_mm_pfn(last_vpfn))) {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001945 printk(KERN_ERR "IOMMU: reserve iova failed\n");
David Woodhouseb2132032009-06-26 18:50:28 +01001946 return -ENOMEM;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001947 }
1948
David Woodhousec5395d52009-06-28 16:35:56 +01001949 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1950 start, end, domain->id);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001951 /*
1952 * RMRR range might have overlap with physical memory range,
1953 * clear it first
1954 */
David Woodhousec5395d52009-06-28 16:35:56 +01001955 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07001956
David Woodhousec5395d52009-06-28 16:35:56 +01001957 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1958 last_vpfn - first_vpfn + 1,
David Woodhouse61df7442009-06-28 11:55:58 +01001959 DMA_PTE_READ|DMA_PTE_WRITE);
David Woodhouseb2132032009-06-26 18:50:28 +01001960}
1961
1962static int iommu_prepare_identity_map(struct pci_dev *pdev,
1963 unsigned long long start,
1964 unsigned long long end)
1965{
1966 struct dmar_domain *domain;
1967 int ret;
1968
David Woodhousec7ab48d2009-06-26 19:10:36 +01001969 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
David Woodhouseb2132032009-06-26 18:50:28 +01001970 if (!domain)
1971 return -ENOMEM;
1972
David Woodhouse19943b02009-08-04 16:19:20 +01001973 /* For _hardware_ passthrough, don't bother. But for software
1974 passthrough, we do it anyway -- it may indicate a memory
1975 range which is reserved in E820, so which didn't get set
1976 up to start with in si_domain */
1977 if (domain == si_domain && hw_pass_through) {
1978 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1979 pci_name(pdev), start, end);
1980 return 0;
1981 }
1982
1983 printk(KERN_INFO
1984 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1985 pci_name(pdev), start, end);
David Woodhouse2ff729f2009-08-26 14:25:41 +01001986
David Woodhouse5595b522009-12-02 09:21:55 +00001987 if (end < start) {
1988 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1989 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1990 dmi_get_system_info(DMI_BIOS_VENDOR),
1991 dmi_get_system_info(DMI_BIOS_VERSION),
1992 dmi_get_system_info(DMI_PRODUCT_VERSION));
1993 ret = -EIO;
1994 goto error;
1995 }
1996
David Woodhouse2ff729f2009-08-26 14:25:41 +01001997 if (end >> agaw_to_width(domain->agaw)) {
1998 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1999 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2000 agaw_to_width(domain->agaw),
2001 dmi_get_system_info(DMI_BIOS_VENDOR),
2002 dmi_get_system_info(DMI_BIOS_VERSION),
2003 dmi_get_system_info(DMI_PRODUCT_VERSION));
2004 ret = -EIO;
2005 goto error;
2006 }
David Woodhouse19943b02009-08-04 16:19:20 +01002007
David Woodhouseb2132032009-06-26 18:50:28 +01002008 ret = iommu_domain_identity_map(domain, start, end);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002009 if (ret)
2010 goto error;
2011
2012 /* context entry init */
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002013 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
David Woodhouseb2132032009-06-26 18:50:28 +01002014 if (ret)
2015 goto error;
2016
2017 return 0;
2018
2019 error:
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002020 domain_exit(domain);
2021 return ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002022}
2023
2024static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2025 struct pci_dev *pdev)
2026{
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07002027 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002028 return 0;
2029 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2030 rmrr->end_address + 1);
2031}
2032
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002033#ifdef CONFIG_DMAR_FLOPPY_WA
2034static inline void iommu_prepare_isa(void)
2035{
2036 struct pci_dev *pdev;
2037 int ret;
2038
2039 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2040 if (!pdev)
2041 return;
2042
David Woodhousec7ab48d2009-06-26 19:10:36 +01002043 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002044 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2045
2046 if (ret)
David Woodhousec7ab48d2009-06-26 19:10:36 +01002047 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2048 "floppy might not work\n");
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002049
2050}
2051#else
2052static inline void iommu_prepare_isa(void)
2053{
2054 return;
2055}
2056#endif /* !CONFIG_DMAR_FLPY_WA */
2057
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002058static int md_domain_init(struct dmar_domain *domain, int guest_width);
David Woodhousec7ab48d2009-06-26 19:10:36 +01002059
2060static int __init si_domain_work_fn(unsigned long start_pfn,
2061 unsigned long end_pfn, void *datax)
2062{
2063 int *ret = datax;
2064
2065 *ret = iommu_domain_identity_map(si_domain,
2066 (uint64_t)start_pfn << PAGE_SHIFT,
2067 (uint64_t)end_pfn << PAGE_SHIFT);
2068 return *ret;
2069
2070}
2071
Matt Kraai071e1372009-08-23 22:30:22 -07002072static int __init si_domain_init(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002073{
2074 struct dmar_drhd_unit *drhd;
2075 struct intel_iommu *iommu;
David Woodhousec7ab48d2009-06-26 19:10:36 +01002076 int nid, ret = 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002077
2078 si_domain = alloc_domain();
2079 if (!si_domain)
2080 return -EFAULT;
2081
David Woodhousec7ab48d2009-06-26 19:10:36 +01002082 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002083
2084 for_each_active_iommu(iommu, drhd) {
2085 ret = iommu_attach_domain(si_domain, iommu);
2086 if (ret) {
2087 domain_exit(si_domain);
2088 return -EFAULT;
2089 }
2090 }
2091
2092 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2093 domain_exit(si_domain);
2094 return -EFAULT;
2095 }
2096
2097 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2098
David Woodhouse19943b02009-08-04 16:19:20 +01002099 if (hw)
2100 return 0;
2101
David Woodhousec7ab48d2009-06-26 19:10:36 +01002102 for_each_online_node(nid) {
2103 work_with_active_regions(nid, si_domain_work_fn, &ret);
2104 if (ret)
2105 return ret;
2106 }
2107
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002108 return 0;
2109}
2110
2111static void domain_remove_one_dev_info(struct dmar_domain *domain,
2112 struct pci_dev *pdev);
2113static int identity_mapping(struct pci_dev *pdev)
2114{
2115 struct device_domain_info *info;
2116
2117 if (likely(!iommu_identity_mapping))
2118 return 0;
2119
2120
2121 list_for_each_entry(info, &si_domain->devices, link)
2122 if (info->dev == pdev)
2123 return 1;
2124 return 0;
2125}
2126
2127static int domain_add_dev_info(struct dmar_domain *domain,
David Woodhouse5fe60f42009-08-09 10:53:41 +01002128 struct pci_dev *pdev,
2129 int translation)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002130{
2131 struct device_domain_info *info;
2132 unsigned long flags;
David Woodhouse5fe60f42009-08-09 10:53:41 +01002133 int ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002134
2135 info = alloc_devinfo_mem();
2136 if (!info)
2137 return -ENOMEM;
2138
David Woodhouse5fe60f42009-08-09 10:53:41 +01002139 ret = domain_context_mapping(domain, pdev, translation);
2140 if (ret) {
2141 free_devinfo_mem(info);
2142 return ret;
2143 }
2144
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002145 info->segment = pci_domain_nr(pdev->bus);
2146 info->bus = pdev->bus->number;
2147 info->devfn = pdev->devfn;
2148 info->dev = pdev;
2149 info->domain = domain;
2150
2151 spin_lock_irqsave(&device_domain_lock, flags);
2152 list_add(&info->link, &domain->devices);
2153 list_add(&info->global, &device_domain_list);
2154 pdev->dev.archdata.iommu = info;
2155 spin_unlock_irqrestore(&device_domain_lock, flags);
2156
2157 return 0;
2158}
2159
David Woodhouse6941af22009-07-04 18:24:27 +01002160static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2161{
David Woodhousee0fc7e02009-09-30 09:12:17 -07002162 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2163 return 1;
2164
2165 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2166 return 1;
2167
2168 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2169 return 0;
David Woodhouse6941af22009-07-04 18:24:27 +01002170
David Woodhouse3dfc8132009-07-04 19:11:08 +01002171 /*
2172 * We want to start off with all devices in the 1:1 domain, and
2173 * take them out later if we find they can't access all of memory.
2174 *
2175 * However, we can't do this for PCI devices behind bridges,
2176 * because all PCI devices behind the same bridge will end up
2177 * with the same source-id on their transactions.
2178 *
2179 * Practically speaking, we can't change things around for these
2180 * devices at run-time, because we can't be sure there'll be no
2181 * DMA transactions in flight for any of their siblings.
2182 *
2183 * So PCI devices (unless they're on the root bus) as well as
2184 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2185 * the 1:1 domain, just in _case_ one of their siblings turns out
2186 * not to be able to map all of memory.
2187 */
Kenji Kaneshige5f4d91a2009-11-11 14:36:17 +09002188 if (!pci_is_pcie(pdev)) {
David Woodhouse3dfc8132009-07-04 19:11:08 +01002189 if (!pci_is_root_bus(pdev->bus))
2190 return 0;
2191 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2192 return 0;
2193 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2194 return 0;
2195
2196 /*
2197 * At boot time, we don't yet know if devices will be 64-bit capable.
2198 * Assume that they will -- if they turn out not to be, then we can
2199 * take them out of the 1:1 domain later.
2200 */
David Woodhouse6941af22009-07-04 18:24:27 +01002201 if (!startup)
2202 return pdev->dma_mask > DMA_BIT_MASK(32);
2203
2204 return 1;
2205}
2206
Matt Kraai071e1372009-08-23 22:30:22 -07002207static int __init iommu_prepare_static_identity_mapping(int hw)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002208{
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002209 struct pci_dev *pdev = NULL;
2210 int ret;
2211
David Woodhouse19943b02009-08-04 16:19:20 +01002212 ret = si_domain_init(hw);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002213 if (ret)
2214 return -EFAULT;
2215
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002216 for_each_pci_dev(pdev) {
David Woodhouse6941af22009-07-04 18:24:27 +01002217 if (iommu_should_identity_map(pdev, 1)) {
David Woodhouse19943b02009-08-04 16:19:20 +01002218 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2219 hw ? "hardware" : "software", pci_name(pdev));
David Woodhousec7ab48d2009-06-26 19:10:36 +01002220
David Woodhouse5fe60f42009-08-09 10:53:41 +01002221 ret = domain_add_dev_info(si_domain, pdev,
David Woodhouse19943b02009-08-04 16:19:20 +01002222 hw ? CONTEXT_TT_PASS_THROUGH :
David Woodhouse62edf5d2009-07-04 10:59:46 +01002223 CONTEXT_TT_MULTI_LEVEL);
2224 if (ret)
2225 return ret;
David Woodhouse62edf5d2009-07-04 10:59:46 +01002226 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002227 }
2228
2229 return 0;
2230}
2231
2232int __init init_dmars(void)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002233{
2234 struct dmar_drhd_unit *drhd;
2235 struct dmar_rmrr_unit *rmrr;
2236 struct pci_dev *pdev;
2237 struct intel_iommu *iommu;
Suresh Siddha9d783ba2009-03-16 17:04:55 -07002238 int i, ret;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002239
2240 /*
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002241 * for each drhd
2242 * allocate root
2243 * initialize and program root entry to not present
2244 * endfor
2245 */
2246 for_each_drhd_unit(drhd) {
mark gross5e0d2a62008-03-04 15:22:08 -08002247 g_num_of_iommus++;
2248 /*
2249 * lock not needed as this is only incremented in the single
2250 * threaded kernel __init code path all other access are read
2251 * only
2252 */
2253 }
2254
Weidong Hand9630fe2008-12-08 11:06:32 +08002255 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2256 GFP_KERNEL);
2257 if (!g_iommus) {
2258 printk(KERN_ERR "Allocating global iommu array failed\n");
2259 ret = -ENOMEM;
2260 goto error;
2261 }
2262
mark gross80b20dd2008-04-18 13:53:58 -07002263 deferred_flush = kzalloc(g_num_of_iommus *
2264 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2265 if (!deferred_flush) {
mark gross5e0d2a62008-03-04 15:22:08 -08002266 ret = -ENOMEM;
2267 goto error;
2268 }
2269
mark gross5e0d2a62008-03-04 15:22:08 -08002270 for_each_drhd_unit(drhd) {
2271 if (drhd->ignored)
2272 continue;
Suresh Siddha1886e8a2008-07-10 11:16:37 -07002273
2274 iommu = drhd->iommu;
Weidong Hand9630fe2008-12-08 11:06:32 +08002275 g_iommus[iommu->seq_id] = iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002276
Suresh Siddhae61d98d2008-07-10 11:16:35 -07002277 ret = iommu_init_domains(iommu);
2278 if (ret)
2279 goto error;
2280
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002281 /*
2282 * TBD:
2283 * we could share the same root & context tables
2284 * amoung all IOMMU's. Need to Split it later.
2285 */
2286 ret = iommu_alloc_root_entry(iommu);
2287 if (ret) {
2288 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2289 goto error;
2290 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002291 if (!ecap_pass_through(iommu->ecap))
David Woodhouse19943b02009-08-04 16:19:20 +01002292 hw_pass_through = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002293 }
2294
Suresh Siddha1531a6a2009-03-16 17:04:57 -07002295 /*
2296 * Start from the sane iommu hardware state.
2297 */
Youquan Songa77b67d2008-10-16 16:31:56 -07002298 for_each_drhd_unit(drhd) {
2299 if (drhd->ignored)
2300 continue;
2301
2302 iommu = drhd->iommu;
Suresh Siddha1531a6a2009-03-16 17:04:57 -07002303
2304 /*
2305 * If the queued invalidation is already initialized by us
2306 * (for example, while enabling interrupt-remapping) then
2307 * we got the things already rolling from a sane state.
2308 */
2309 if (iommu->qi)
2310 continue;
2311
2312 /*
2313 * Clear any previous faults.
2314 */
2315 dmar_fault(-1, iommu);
2316 /*
2317 * Disable queued invalidation if supported and already enabled
2318 * before OS handover.
2319 */
2320 dmar_disable_qi(iommu);
2321 }
2322
2323 for_each_drhd_unit(drhd) {
2324 if (drhd->ignored)
2325 continue;
2326
2327 iommu = drhd->iommu;
2328
Youquan Songa77b67d2008-10-16 16:31:56 -07002329 if (dmar_enable_qi(iommu)) {
2330 /*
2331 * Queued Invalidate not enabled, use Register Based
2332 * Invalidate
2333 */
2334 iommu->flush.flush_context = __iommu_flush_context;
2335 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2336 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002337 "invalidation\n",
2338 (unsigned long long)drhd->reg_base_addr);
Youquan Songa77b67d2008-10-16 16:31:56 -07002339 } else {
2340 iommu->flush.flush_context = qi_flush_context;
2341 iommu->flush.flush_iotlb = qi_flush_iotlb;
2342 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
FUJITA Tomonorib4e0f9e2008-11-19 13:53:42 +09002343 "invalidation\n",
2344 (unsigned long long)drhd->reg_base_addr);
Youquan Songa77b67d2008-10-16 16:31:56 -07002345 }
2346 }
2347
David Woodhouse19943b02009-08-04 16:19:20 +01002348 if (iommu_pass_through)
David Woodhousee0fc7e02009-09-30 09:12:17 -07002349 iommu_identity_mapping |= IDENTMAP_ALL;
2350
David Woodhouse19943b02009-08-04 16:19:20 +01002351#ifdef CONFIG_DMAR_BROKEN_GFX_WA
David Woodhousee0fc7e02009-09-30 09:12:17 -07002352 iommu_identity_mapping |= IDENTMAP_GFX;
David Woodhouse19943b02009-08-04 16:19:20 +01002353#endif
David Woodhousee0fc7e02009-09-30 09:12:17 -07002354
2355 check_tylersburg_isoch();
2356
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002357 /*
2358 * If pass through is not set or not enabled, setup context entries for
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002359 * identity mappings for rmrr, gfx, and isa and may fall back to static
2360 * identity mapping if iommu_identity_mapping is set.
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002361 */
David Woodhouse19943b02009-08-04 16:19:20 +01002362 if (iommu_identity_mapping) {
2363 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2364 if (ret) {
2365 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2366 goto error;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002367 }
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002368 }
David Woodhouse19943b02009-08-04 16:19:20 +01002369 /*
2370 * For each rmrr
2371 * for each dev attached to rmrr
2372 * do
2373 * locate drhd for dev, alloc domain for dev
2374 * allocate free domain
2375 * allocate page table entries for rmrr
2376 * if context not allocated for bus
2377 * allocate and init context
2378 * set present in root table for this bus
2379 * init context with domain, translation etc
2380 * endfor
2381 * endfor
2382 */
2383 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2384 for_each_rmrr_units(rmrr) {
2385 for (i = 0; i < rmrr->devices_cnt; i++) {
2386 pdev = rmrr->devices[i];
2387 /*
2388 * some BIOS lists non-exist devices in DMAR
2389 * table.
2390 */
2391 if (!pdev)
2392 continue;
2393 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2394 if (ret)
2395 printk(KERN_ERR
2396 "IOMMU: mapping reserved region failed\n");
2397 }
2398 }
2399
2400 iommu_prepare_isa();
Keshavamurthy, Anil S49a04292007-10-21 16:41:57 -07002401
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002402 /*
2403 * for each drhd
2404 * enable fault log
2405 * global invalidate context cache
2406 * global invalidate iotlb
2407 * enable translation
2408 */
2409 for_each_drhd_unit(drhd) {
2410 if (drhd->ignored)
2411 continue;
2412 iommu = drhd->iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002413
2414 iommu_flush_write_buffer(iommu);
2415
Keshavamurthy, Anil S3460a6d2007-10-21 16:41:54 -07002416 ret = dmar_set_interrupt(iommu);
2417 if (ret)
2418 goto error;
2419
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002420 iommu_set_root_entry(iommu);
2421
David Woodhouse4c25a2c2009-05-10 17:16:06 +01002422 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002423 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
mark grossf8bab732008-02-08 04:18:38 -08002424
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002425 ret = iommu_enable_translation(iommu);
2426 if (ret)
2427 goto error;
David Woodhouseb94996c2009-09-19 15:28:12 -07002428
2429 iommu_disable_protect_mem_regions(iommu);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002430 }
2431
2432 return 0;
2433error:
2434 for_each_drhd_unit(drhd) {
2435 if (drhd->ignored)
2436 continue;
2437 iommu = drhd->iommu;
2438 free_iommu(iommu);
2439 }
Weidong Hand9630fe2008-12-08 11:06:32 +08002440 kfree(g_iommus);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002441 return ret;
2442}
2443
David Woodhouse5a5e02a2009-07-04 09:35:44 +01002444/* This takes a number of _MM_ pages, not VTD pages */
David Woodhouse875764d2009-06-28 21:20:51 +01002445static struct iova *intel_alloc_iova(struct device *dev,
2446 struct dmar_domain *domain,
2447 unsigned long nrpages, uint64_t dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002448{
2449 struct pci_dev *pdev = to_pci_dev(dev);
2450 struct iova *iova = NULL;
2451
David Woodhouse875764d2009-06-28 21:20:51 +01002452 /* Restrict dma_mask to the width that the iommu can handle */
2453 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2454
2455 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002456 /*
2457 * First try to allocate an io virtual address in
Yang Hongyang284901a2009-04-06 19:01:15 -07002458 * DMA_BIT_MASK(32) and if that fails then try allocating
Joe Perches36098012007-12-17 11:40:11 -08002459 * from higher range
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002460 */
David Woodhouse875764d2009-06-28 21:20:51 +01002461 iova = alloc_iova(&domain->iovad, nrpages,
2462 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2463 if (iova)
2464 return iova;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002465 }
David Woodhouse875764d2009-06-28 21:20:51 +01002466 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2467 if (unlikely(!iova)) {
2468 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2469 nrpages, pci_name(pdev));
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002470 return NULL;
2471 }
2472
2473 return iova;
2474}
2475
David Woodhouse147202a2009-07-07 19:43:20 +01002476static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002477{
2478 struct dmar_domain *domain;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002479 int ret;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002480
2481 domain = get_domain_for_dev(pdev,
2482 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2483 if (!domain) {
2484 printk(KERN_ERR
2485 "Allocating domain for %s failed", pci_name(pdev));
Al Viro4fe05bb2007-10-29 04:51:16 +00002486 return NULL;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002487 }
2488
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002489 /* make sure context mapping is ok */
Weidong Han5331fe62008-12-08 23:00:00 +08002490 if (unlikely(!domain_context_mapped(pdev))) {
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07002491 ret = domain_context_mapping(domain, pdev,
2492 CONTEXT_TT_MULTI_LEVEL);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002493 if (ret) {
2494 printk(KERN_ERR
2495 "Domain context map for %s failed",
2496 pci_name(pdev));
Al Viro4fe05bb2007-10-29 04:51:16 +00002497 return NULL;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002498 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002499 }
2500
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002501 return domain;
2502}
2503
David Woodhouse147202a2009-07-07 19:43:20 +01002504static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2505{
2506 struct device_domain_info *info;
2507
2508 /* No lock here, assumes no domain exit in normal case */
2509 info = dev->dev.archdata.iommu;
2510 if (likely(info))
2511 return info->domain;
2512
2513 return __get_valid_domain_for_dev(dev);
2514}
2515
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002516static int iommu_dummy(struct pci_dev *pdev)
2517{
2518 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2519}
2520
2521/* Check if the pdev needs to go through non-identity map and unmap process.*/
David Woodhouse73676832009-07-04 14:08:36 +01002522static int iommu_no_mapping(struct device *dev)
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002523{
David Woodhouse73676832009-07-04 14:08:36 +01002524 struct pci_dev *pdev;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002525 int found;
2526
David Woodhouse73676832009-07-04 14:08:36 +01002527 if (unlikely(dev->bus != &pci_bus_type))
2528 return 1;
2529
2530 pdev = to_pci_dev(dev);
David Woodhouse1e4c64c2009-07-04 10:40:38 +01002531 if (iommu_dummy(pdev))
2532 return 1;
2533
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002534 if (!iommu_identity_mapping)
David Woodhouse1e4c64c2009-07-04 10:40:38 +01002535 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002536
2537 found = identity_mapping(pdev);
2538 if (found) {
David Woodhouse6941af22009-07-04 18:24:27 +01002539 if (iommu_should_identity_map(pdev, 0))
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002540 return 1;
2541 else {
2542 /*
2543 * 32 bit DMA is removed from si_domain and fall back
2544 * to non-identity mapping.
2545 */
2546 domain_remove_one_dev_info(si_domain, pdev);
2547 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2548 pci_name(pdev));
2549 return 0;
2550 }
2551 } else {
2552 /*
2553 * In case of a detached 64 bit DMA device from vm, the device
2554 * is put into si_domain for identity mapping.
2555 */
David Woodhouse6941af22009-07-04 18:24:27 +01002556 if (iommu_should_identity_map(pdev, 0)) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002557 int ret;
David Woodhouse5fe60f42009-08-09 10:53:41 +01002558 ret = domain_add_dev_info(si_domain, pdev,
2559 hw_pass_through ?
2560 CONTEXT_TT_PASS_THROUGH :
2561 CONTEXT_TT_MULTI_LEVEL);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002562 if (!ret) {
2563 printk(KERN_INFO "64bit %s uses identity mapping\n",
2564 pci_name(pdev));
2565 return 1;
2566 }
2567 }
2568 }
2569
David Woodhouse1e4c64c2009-07-04 10:40:38 +01002570 return 0;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002571}
2572
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09002573static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2574 size_t size, int dir, u64 dma_mask)
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002575{
2576 struct pci_dev *pdev = to_pci_dev(hwdev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002577 struct dmar_domain *domain;
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002578 phys_addr_t start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002579 struct iova *iova;
2580 int prot = 0;
Ingo Molnar6865f0d2008-04-22 11:09:04 +02002581 int ret;
Weidong Han8c11e792008-12-08 15:29:22 +08002582 struct intel_iommu *iommu;
Fenghua Yu33041ec2009-08-04 15:10:59 -07002583 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002584
2585 BUG_ON(dir == DMA_NONE);
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002586
David Woodhouse73676832009-07-04 14:08:36 +01002587 if (iommu_no_mapping(hwdev))
Ingo Molnar6865f0d2008-04-22 11:09:04 +02002588 return paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002589
2590 domain = get_valid_domain_for_dev(pdev);
2591 if (!domain)
2592 return 0;
2593
Weidong Han8c11e792008-12-08 15:29:22 +08002594 iommu = domain_get_iommu(domain);
David Woodhouse88cb6a72009-06-28 15:03:06 +01002595 size = aligned_nrpages(paddr, size);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002596
David Woodhouse5a5e02a2009-07-04 09:35:44 +01002597 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2598 pdev->dma_mask);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002599 if (!iova)
2600 goto error;
2601
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002602 /*
2603 * Check if DMAR supports zero-length reads on write only
2604 * mappings..
2605 */
2606 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08002607 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002608 prot |= DMA_PTE_READ;
2609 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2610 prot |= DMA_PTE_WRITE;
2611 /*
Ingo Molnar6865f0d2008-04-22 11:09:04 +02002612 * paddr - (paddr + size) might be partial page, we should map the whole
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002613 * page. Note: if two part of one page are separately mapped, we
Ingo Molnar6865f0d2008-04-22 11:09:04 +02002614 * might have two guest_addr mapping to the same host paddr, but this
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002615 * is not a big problem
2616 */
David Woodhouse0ab36de2009-06-28 14:01:43 +01002617 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
Fenghua Yu33041ec2009-08-04 15:10:59 -07002618 mm_to_dma_pfn(paddr_pfn), size, prot);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002619 if (ret)
2620 goto error;
2621
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002622 /* it's a non-present to present mapping. Only flush if caching mode */
2623 if (cap_caching_mode(iommu->cap))
David Woodhouse03d6a242009-06-28 15:33:46 +01002624 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002625 else
Weidong Han8c11e792008-12-08 15:29:22 +08002626 iommu_flush_write_buffer(iommu);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002627
David Woodhouse03d6a242009-06-28 15:33:46 +01002628 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2629 start_paddr += paddr & ~PAGE_MASK;
2630 return start_paddr;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002631
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002632error:
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002633 if (iova)
2634 __free_iova(&domain->iovad, iova);
David Woodhouse4cf2e752009-02-11 17:23:43 +00002635 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002636 pci_name(pdev), size, (unsigned long long)paddr, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002637 return 0;
2638}
2639
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09002640static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2641 unsigned long offset, size_t size,
2642 enum dma_data_direction dir,
2643 struct dma_attrs *attrs)
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09002644{
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09002645 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2646 dir, to_pci_dev(dev)->dma_mask);
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09002647}
2648
mark gross5e0d2a62008-03-04 15:22:08 -08002649static void flush_unmaps(void)
2650{
mark gross80b20dd2008-04-18 13:53:58 -07002651 int i, j;
mark gross5e0d2a62008-03-04 15:22:08 -08002652
mark gross5e0d2a62008-03-04 15:22:08 -08002653 timer_on = 0;
2654
2655 /* just flush them all */
2656 for (i = 0; i < g_num_of_iommus; i++) {
Weidong Hana2bb8452008-12-08 11:24:12 +08002657 struct intel_iommu *iommu = g_iommus[i];
2658 if (!iommu)
2659 continue;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07002660
Yu Zhao9dd2fe82009-05-18 13:51:36 +08002661 if (!deferred_flush[i].next)
2662 continue;
2663
2664 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
Yu Zhao93a23a72009-05-18 13:51:37 +08002665 DMA_TLB_GLOBAL_FLUSH);
Yu Zhao9dd2fe82009-05-18 13:51:36 +08002666 for (j = 0; j < deferred_flush[i].next; j++) {
Yu Zhao93a23a72009-05-18 13:51:37 +08002667 unsigned long mask;
2668 struct iova *iova = deferred_flush[i].iova[j];
2669
Benjamin LaHaise64de5af2009-09-16 21:05:55 -04002670 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
Yu Zhao93a23a72009-05-18 13:51:37 +08002671 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
Benjamin LaHaise64de5af2009-09-16 21:05:55 -04002672 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
Yu Zhao93a23a72009-05-18 13:51:37 +08002673 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
mark gross80b20dd2008-04-18 13:53:58 -07002674 }
Yu Zhao9dd2fe82009-05-18 13:51:36 +08002675 deferred_flush[i].next = 0;
mark gross5e0d2a62008-03-04 15:22:08 -08002676 }
2677
mark gross5e0d2a62008-03-04 15:22:08 -08002678 list_size = 0;
mark gross5e0d2a62008-03-04 15:22:08 -08002679}
2680
2681static void flush_unmaps_timeout(unsigned long data)
2682{
mark gross80b20dd2008-04-18 13:53:58 -07002683 unsigned long flags;
2684
2685 spin_lock_irqsave(&async_umap_flush_lock, flags);
mark gross5e0d2a62008-03-04 15:22:08 -08002686 flush_unmaps();
mark gross80b20dd2008-04-18 13:53:58 -07002687 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
mark gross5e0d2a62008-03-04 15:22:08 -08002688}
2689
2690static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2691{
2692 unsigned long flags;
mark gross80b20dd2008-04-18 13:53:58 -07002693 int next, iommu_id;
Weidong Han8c11e792008-12-08 15:29:22 +08002694 struct intel_iommu *iommu;
mark gross5e0d2a62008-03-04 15:22:08 -08002695
2696 spin_lock_irqsave(&async_umap_flush_lock, flags);
mark gross80b20dd2008-04-18 13:53:58 -07002697 if (list_size == HIGH_WATER_MARK)
2698 flush_unmaps();
2699
Weidong Han8c11e792008-12-08 15:29:22 +08002700 iommu = domain_get_iommu(dom);
2701 iommu_id = iommu->seq_id;
Suresh Siddhac42d9f32008-07-10 11:16:36 -07002702
mark gross80b20dd2008-04-18 13:53:58 -07002703 next = deferred_flush[iommu_id].next;
2704 deferred_flush[iommu_id].domain[next] = dom;
2705 deferred_flush[iommu_id].iova[next] = iova;
2706 deferred_flush[iommu_id].next++;
mark gross5e0d2a62008-03-04 15:22:08 -08002707
2708 if (!timer_on) {
2709 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2710 timer_on = 1;
2711 }
2712 list_size++;
2713 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2714}
2715
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09002716static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2717 size_t size, enum dma_data_direction dir,
2718 struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002719{
2720 struct pci_dev *pdev = to_pci_dev(dev);
2721 struct dmar_domain *domain;
David Woodhoused794dc92009-06-28 00:27:49 +01002722 unsigned long start_pfn, last_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002723 struct iova *iova;
Weidong Han8c11e792008-12-08 15:29:22 +08002724 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002725
David Woodhouse73676832009-07-04 14:08:36 +01002726 if (iommu_no_mapping(dev))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002727 return;
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07002728
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002729 domain = find_domain(pdev);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002730 BUG_ON(!domain);
2731
Weidong Han8c11e792008-12-08 15:29:22 +08002732 iommu = domain_get_iommu(domain);
2733
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002734 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
David Woodhouse85b98272009-07-01 19:27:53 +01002735 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2736 (unsigned long long)dev_addr))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002737 return;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002738
David Woodhoused794dc92009-06-28 00:27:49 +01002739 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2740 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002741
David Woodhoused794dc92009-06-28 00:27:49 +01002742 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2743 pci_name(pdev), start_pfn, last_pfn);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002744
2745 /* clear the whole page */
David Woodhoused794dc92009-06-28 00:27:49 +01002746 dma_pte_clear_range(domain, start_pfn, last_pfn);
2747
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002748 /* free page tables */
David Woodhoused794dc92009-06-28 00:27:49 +01002749 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2750
mark gross5e0d2a62008-03-04 15:22:08 -08002751 if (intel_iommu_strict) {
David Woodhouse03d6a242009-06-28 15:33:46 +01002752 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
David Woodhoused794dc92009-06-28 00:27:49 +01002753 last_pfn - start_pfn + 1);
mark gross5e0d2a62008-03-04 15:22:08 -08002754 /* free iova */
2755 __free_iova(&domain->iovad, iova);
2756 } else {
2757 add_unmap(domain, iova);
2758 /*
2759 * queue up the release of the unmap to save the 1/6th of the
2760 * cpu used up by the iotlb flush operation...
2761 */
mark gross5e0d2a62008-03-04 15:22:08 -08002762 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002763}
2764
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09002765static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2766 dma_addr_t *dma_handle, gfp_t flags)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002767{
2768 void *vaddr;
2769 int order;
2770
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002771 size = PAGE_ALIGN(size);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002772 order = get_order(size);
Alex Williamsone8bb9102009-11-04 15:59:34 -07002773
2774 if (!iommu_no_mapping(hwdev))
2775 flags &= ~(GFP_DMA | GFP_DMA32);
2776 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2777 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2778 flags |= GFP_DMA;
2779 else
2780 flags |= GFP_DMA32;
2781 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002782
2783 vaddr = (void *)__get_free_pages(flags, order);
2784 if (!vaddr)
2785 return NULL;
2786 memset(vaddr, 0, size);
2787
FUJITA Tomonoribb9e6d62008-10-15 16:08:28 +09002788 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2789 DMA_BIDIRECTIONAL,
2790 hwdev->coherent_dma_mask);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002791 if (*dma_handle)
2792 return vaddr;
2793 free_pages((unsigned long)vaddr, order);
2794 return NULL;
2795}
2796
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09002797static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2798 dma_addr_t dma_handle)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002799{
2800 int order;
2801
Fenghua Yu5b6985c2008-10-16 18:02:32 -07002802 size = PAGE_ALIGN(size);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002803 order = get_order(size);
2804
David Woodhouse0db9b7a2009-07-14 02:01:57 +01002805 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002806 free_pages((unsigned long)vaddr, order);
2807}
2808
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09002809static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2810 int nelems, enum dma_data_direction dir,
2811 struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002812{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002813 struct pci_dev *pdev = to_pci_dev(hwdev);
2814 struct dmar_domain *domain;
David Woodhoused794dc92009-06-28 00:27:49 +01002815 unsigned long start_pfn, last_pfn;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002816 struct iova *iova;
Weidong Han8c11e792008-12-08 15:29:22 +08002817 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002818
David Woodhouse73676832009-07-04 14:08:36 +01002819 if (iommu_no_mapping(hwdev))
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002820 return;
2821
2822 domain = find_domain(pdev);
Weidong Han8c11e792008-12-08 15:29:22 +08002823 BUG_ON(!domain);
2824
2825 iommu = domain_get_iommu(domain);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002826
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002827 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
David Woodhouse85b98272009-07-01 19:27:53 +01002828 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2829 (unsigned long long)sglist[0].dma_address))
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002830 return;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002831
David Woodhoused794dc92009-06-28 00:27:49 +01002832 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2833 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002834
2835 /* clear the whole page */
David Woodhoused794dc92009-06-28 00:27:49 +01002836 dma_pte_clear_range(domain, start_pfn, last_pfn);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002837
David Woodhoused794dc92009-06-28 00:27:49 +01002838 /* free page tables */
2839 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2840
David Woodhouseacea0012009-07-14 01:55:11 +01002841 if (intel_iommu_strict) {
2842 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2843 last_pfn - start_pfn + 1);
2844 /* free iova */
2845 __free_iova(&domain->iovad, iova);
2846 } else {
2847 add_unmap(domain, iova);
2848 /*
2849 * queue up the release of the unmap to save the 1/6th of the
2850 * cpu used up by the iotlb flush operation...
2851 */
2852 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002853}
2854
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002855static int intel_nontranslate_map_sg(struct device *hddev,
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002856 struct scatterlist *sglist, int nelems, int dir)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002857{
2858 int i;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002859 struct scatterlist *sg;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002860
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002861 for_each_sg(sglist, sg, nelems, i) {
FUJITA Tomonori12d4d402007-10-23 09:32:25 +02002862 BUG_ON(!sg_page(sg));
David Woodhouse4cf2e752009-02-11 17:23:43 +00002863 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002864 sg->dma_length = sg->length;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002865 }
2866 return nelems;
2867}
2868
FUJITA Tomonorid7ab5c42009-01-28 21:53:18 +09002869static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2870 enum dma_data_direction dir, struct dma_attrs *attrs)
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002871{
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002872 int i;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002873 struct pci_dev *pdev = to_pci_dev(hwdev);
2874 struct dmar_domain *domain;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002875 size_t size = 0;
2876 int prot = 0;
David Woodhouseb536d242009-06-28 14:49:31 +01002877 size_t offset_pfn = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002878 struct iova *iova = NULL;
2879 int ret;
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002880 struct scatterlist *sg;
David Woodhouseb536d242009-06-28 14:49:31 +01002881 unsigned long start_vpfn;
Weidong Han8c11e792008-12-08 15:29:22 +08002882 struct intel_iommu *iommu;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002883
2884 BUG_ON(dir == DMA_NONE);
David Woodhouse73676832009-07-04 14:08:36 +01002885 if (iommu_no_mapping(hwdev))
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002886 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002887
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002888 domain = get_valid_domain_for_dev(pdev);
2889 if (!domain)
2890 return 0;
2891
Weidong Han8c11e792008-12-08 15:29:22 +08002892 iommu = domain_get_iommu(domain);
2893
David Woodhouseb536d242009-06-28 14:49:31 +01002894 for_each_sg(sglist, sg, nelems, i)
David Woodhouse88cb6a72009-06-28 15:03:06 +01002895 size += aligned_nrpages(sg->offset, sg->length);
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002896
David Woodhouse5a5e02a2009-07-04 09:35:44 +01002897 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2898 pdev->dma_mask);
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002899 if (!iova) {
FUJITA Tomonoric03ab372007-10-21 16:42:00 -07002900 sglist->dma_length = 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002901 return 0;
2902 }
2903
2904 /*
2905 * Check if DMAR supports zero-length reads on write only
2906 * mappings..
2907 */
2908 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
Weidong Han8c11e792008-12-08 15:29:22 +08002909 !cap_zlr(iommu->cap))
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002910 prot |= DMA_PTE_READ;
2911 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2912 prot |= DMA_PTE_WRITE;
2913
David Woodhouseb536d242009-06-28 14:49:31 +01002914 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
David Woodhousee1605492009-06-29 11:17:38 +01002915
Fenghua Yuf5329592009-08-04 15:09:37 -07002916 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
David Woodhousee1605492009-06-29 11:17:38 +01002917 if (unlikely(ret)) {
2918 /* clear the page */
2919 dma_pte_clear_range(domain, start_vpfn,
2920 start_vpfn + size - 1);
2921 /* free page tables */
2922 dma_pte_free_pagetable(domain, start_vpfn,
2923 start_vpfn + size - 1);
2924 /* free iova */
2925 __free_iova(&domain->iovad, iova);
2926 return 0;
Keshavamurthy, Anil Sf76aec72007-10-21 16:41:58 -07002927 }
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002928
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002929 /* it's a non-present to present mapping. Only flush if caching mode */
2930 if (cap_caching_mode(iommu->cap))
David Woodhouse03d6a242009-06-28 15:33:46 +01002931 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002932 else
Weidong Han8c11e792008-12-08 15:29:22 +08002933 iommu_flush_write_buffer(iommu);
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01002934
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002935 return nelems;
2936}
2937
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09002938static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2939{
2940 return !dma_addr;
2941}
2942
FUJITA Tomonori160c1d82009-01-05 23:59:02 +09002943struct dma_map_ops intel_dma_ops = {
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002944 .alloc_coherent = intel_alloc_coherent,
2945 .free_coherent = intel_free_coherent,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002946 .map_sg = intel_map_sg,
2947 .unmap_sg = intel_unmap_sg,
FUJITA Tomonoriffbbef52009-01-05 23:47:26 +09002948 .map_page = intel_map_page,
2949 .unmap_page = intel_unmap_page,
FUJITA Tomonoridfb805e2009-01-28 21:53:17 +09002950 .mapping_error = intel_mapping_error,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002951};
2952
2953static inline int iommu_domain_cache_init(void)
2954{
2955 int ret = 0;
2956
2957 iommu_domain_cache = kmem_cache_create("iommu_domain",
2958 sizeof(struct dmar_domain),
2959 0,
2960 SLAB_HWCACHE_ALIGN,
2961
2962 NULL);
2963 if (!iommu_domain_cache) {
2964 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2965 ret = -ENOMEM;
2966 }
2967
2968 return ret;
2969}
2970
2971static inline int iommu_devinfo_cache_init(void)
2972{
2973 int ret = 0;
2974
2975 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2976 sizeof(struct device_domain_info),
2977 0,
2978 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002979 NULL);
2980 if (!iommu_devinfo_cache) {
2981 printk(KERN_ERR "Couldn't create devinfo cache\n");
2982 ret = -ENOMEM;
2983 }
2984
2985 return ret;
2986}
2987
2988static inline int iommu_iova_cache_init(void)
2989{
2990 int ret = 0;
2991
2992 iommu_iova_cache = kmem_cache_create("iommu_iova",
2993 sizeof(struct iova),
2994 0,
2995 SLAB_HWCACHE_ALIGN,
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07002996 NULL);
2997 if (!iommu_iova_cache) {
2998 printk(KERN_ERR "Couldn't create iova cache\n");
2999 ret = -ENOMEM;
3000 }
3001
3002 return ret;
3003}
3004
3005static int __init iommu_init_mempool(void)
3006{
3007 int ret;
3008 ret = iommu_iova_cache_init();
3009 if (ret)
3010 return ret;
3011
3012 ret = iommu_domain_cache_init();
3013 if (ret)
3014 goto domain_error;
3015
3016 ret = iommu_devinfo_cache_init();
3017 if (!ret)
3018 return ret;
3019
3020 kmem_cache_destroy(iommu_domain_cache);
3021domain_error:
3022 kmem_cache_destroy(iommu_iova_cache);
3023
3024 return -ENOMEM;
3025}
3026
3027static void __init iommu_exit_mempool(void)
3028{
3029 kmem_cache_destroy(iommu_devinfo_cache);
3030 kmem_cache_destroy(iommu_domain_cache);
3031 kmem_cache_destroy(iommu_iova_cache);
3032
3033}
3034
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003035static void __init init_no_remapping_devices(void)
3036{
3037 struct dmar_drhd_unit *drhd;
3038
3039 for_each_drhd_unit(drhd) {
3040 if (!drhd->include_all) {
3041 int i;
3042 for (i = 0; i < drhd->devices_cnt; i++)
3043 if (drhd->devices[i] != NULL)
3044 break;
3045 /* ignore DMAR unit if no pci devices exist */
3046 if (i == drhd->devices_cnt)
3047 drhd->ignored = 1;
3048 }
3049 }
3050
3051 if (dmar_map_gfx)
3052 return;
3053
3054 for_each_drhd_unit(drhd) {
3055 int i;
3056 if (drhd->ignored || drhd->include_all)
3057 continue;
3058
3059 for (i = 0; i < drhd->devices_cnt; i++)
3060 if (drhd->devices[i] &&
3061 !IS_GFX_DEVICE(drhd->devices[i]))
3062 break;
3063
3064 if (i < drhd->devices_cnt)
3065 continue;
3066
3067 /* bypass IOMMU if it is just for gfx devices */
3068 drhd->ignored = 1;
3069 for (i = 0; i < drhd->devices_cnt; i++) {
3070 if (!drhd->devices[i])
3071 continue;
Keshavamurthy, Anil S358dd8a2007-10-21 16:41:59 -07003072 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003073 }
3074 }
3075}
3076
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003077#ifdef CONFIG_SUSPEND
3078static int init_iommu_hw(void)
3079{
3080 struct dmar_drhd_unit *drhd;
3081 struct intel_iommu *iommu = NULL;
3082
3083 for_each_active_iommu(iommu, drhd)
3084 if (iommu->qi)
3085 dmar_reenable_qi(iommu);
3086
3087 for_each_active_iommu(iommu, drhd) {
3088 iommu_flush_write_buffer(iommu);
3089
3090 iommu_set_root_entry(iommu);
3091
3092 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003093 DMA_CCMD_GLOBAL_INVL);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003094 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003095 DMA_TLB_GLOBAL_FLUSH);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003096 iommu_enable_translation(iommu);
David Woodhouseb94996c2009-09-19 15:28:12 -07003097 iommu_disable_protect_mem_regions(iommu);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003098 }
3099
3100 return 0;
3101}
3102
3103static void iommu_flush_all(void)
3104{
3105 struct dmar_drhd_unit *drhd;
3106 struct intel_iommu *iommu;
3107
3108 for_each_active_iommu(iommu, drhd) {
3109 iommu->flush.flush_context(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003110 DMA_CCMD_GLOBAL_INVL);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003111 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
David Woodhouse1f0ef2a2009-05-10 19:58:49 +01003112 DMA_TLB_GLOBAL_FLUSH);
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003113 }
3114}
3115
3116static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3117{
3118 struct dmar_drhd_unit *drhd;
3119 struct intel_iommu *iommu = NULL;
3120 unsigned long flag;
3121
3122 for_each_active_iommu(iommu, drhd) {
3123 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3124 GFP_ATOMIC);
3125 if (!iommu->iommu_state)
3126 goto nomem;
3127 }
3128
3129 iommu_flush_all();
3130
3131 for_each_active_iommu(iommu, drhd) {
3132 iommu_disable_translation(iommu);
3133
3134 spin_lock_irqsave(&iommu->register_lock, flag);
3135
3136 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3137 readl(iommu->reg + DMAR_FECTL_REG);
3138 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3139 readl(iommu->reg + DMAR_FEDATA_REG);
3140 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3141 readl(iommu->reg + DMAR_FEADDR_REG);
3142 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3143 readl(iommu->reg + DMAR_FEUADDR_REG);
3144
3145 spin_unlock_irqrestore(&iommu->register_lock, flag);
3146 }
3147 return 0;
3148
3149nomem:
3150 for_each_active_iommu(iommu, drhd)
3151 kfree(iommu->iommu_state);
3152
3153 return -ENOMEM;
3154}
3155
3156static int iommu_resume(struct sys_device *dev)
3157{
3158 struct dmar_drhd_unit *drhd;
3159 struct intel_iommu *iommu = NULL;
3160 unsigned long flag;
3161
3162 if (init_iommu_hw()) {
3163 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3164 return -EIO;
3165 }
3166
3167 for_each_active_iommu(iommu, drhd) {
3168
3169 spin_lock_irqsave(&iommu->register_lock, flag);
3170
3171 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3172 iommu->reg + DMAR_FECTL_REG);
3173 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3174 iommu->reg + DMAR_FEDATA_REG);
3175 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3176 iommu->reg + DMAR_FEADDR_REG);
3177 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3178 iommu->reg + DMAR_FEUADDR_REG);
3179
3180 spin_unlock_irqrestore(&iommu->register_lock, flag);
3181 }
3182
3183 for_each_active_iommu(iommu, drhd)
3184 kfree(iommu->iommu_state);
3185
3186 return 0;
3187}
3188
3189static struct sysdev_class iommu_sysclass = {
3190 .name = "iommu",
3191 .resume = iommu_resume,
3192 .suspend = iommu_suspend,
3193};
3194
3195static struct sys_device device_iommu = {
3196 .cls = &iommu_sysclass,
3197};
3198
3199static int __init init_iommu_sysfs(void)
3200{
3201 int error;
3202
3203 error = sysdev_class_register(&iommu_sysclass);
3204 if (error)
3205 return error;
3206
3207 error = sysdev_register(&device_iommu);
3208 if (error)
3209 sysdev_class_unregister(&iommu_sysclass);
3210
3211 return error;
3212}
3213
3214#else
3215static int __init init_iommu_sysfs(void)
3216{
3217 return 0;
3218}
3219#endif /* CONFIG_PM */
3220
Fenghua Yu99dcade2009-11-11 07:23:06 -08003221/*
3222 * Here we only respond to action of unbound device from driver.
3223 *
3224 * Added device is not attached to its DMAR domain here yet. That will happen
3225 * when mapping the device to iova.
3226 */
3227static int device_notifier(struct notifier_block *nb,
3228 unsigned long action, void *data)
3229{
3230 struct device *dev = data;
3231 struct pci_dev *pdev = to_pci_dev(dev);
3232 struct dmar_domain *domain;
3233
David Woodhouse44cd6132009-12-02 10:18:30 +00003234 if (iommu_no_mapping(dev))
3235 return 0;
3236
Fenghua Yu99dcade2009-11-11 07:23:06 -08003237 domain = find_domain(pdev);
3238 if (!domain)
3239 return 0;
3240
3241 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3242 domain_remove_one_dev_info(domain, pdev);
3243
3244 return 0;
3245}
3246
3247static struct notifier_block device_nb = {
3248 .notifier_call = device_notifier,
3249};
3250
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003251int __init intel_iommu_init(void)
3252{
3253 int ret = 0;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003254 int force_on = 0;
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003255
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003256 /* VT-d is required for a TXT/tboot launch, so enforce that */
3257 force_on = tboot_force_iommu();
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003258
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003259 if (dmar_table_init()) {
3260 if (force_on)
3261 panic("tboot: Failed to initialize DMAR table\n");
Suresh Siddha1886e8a2008-07-10 11:16:37 -07003262 return -ENODEV;
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003263 }
3264
3265 if (dmar_dev_scope_init()) {
3266 if (force_on)
3267 panic("tboot: Failed to initialize DMAR device scope\n");
3268 return -ENODEV;
3269 }
Suresh Siddha1886e8a2008-07-10 11:16:37 -07003270
Suresh Siddha2ae21012008-07-10 11:16:43 -07003271 /*
3272 * Check the need for DMA-remapping initialization now.
3273 * Above initialization will also be used by Interrupt-remapping.
3274 */
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09003275 if (no_iommu || dmar_disabled)
Suresh Siddha2ae21012008-07-10 11:16:43 -07003276 return -ENODEV;
3277
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003278 iommu_init_mempool();
3279 dmar_init_reserved_ranges();
3280
3281 init_no_remapping_devices();
3282
3283 ret = init_dmars();
3284 if (ret) {
Joseph Cihulaa59b50e2009-06-30 19:31:10 -07003285 if (force_on)
3286 panic("tboot: Failed to initialize DMARs\n");
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003287 printk(KERN_ERR "IOMMU: dmar init failed\n");
3288 put_iova_domain(&reserved_iova_list);
3289 iommu_exit_mempool();
3290 return ret;
3291 }
3292 printk(KERN_INFO
3293 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3294
mark gross5e0d2a62008-03-04 15:22:08 -08003295 init_timer(&unmap_timer);
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +09003296#ifdef CONFIG_SWIOTLB
3297 swiotlb = 0;
3298#endif
David Woodhouse19943b02009-08-04 16:19:20 +01003299 dma_ops = &intel_dma_ops;
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -07003300
Fenghua Yuf59c7b62009-03-27 14:22:42 -07003301 init_iommu_sysfs();
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01003302
3303 register_iommu(&intel_iommu_ops);
3304
Fenghua Yu99dcade2009-11-11 07:23:06 -08003305 bus_register_notifier(&pci_bus_type, &device_nb);
3306
Keshavamurthy, Anil Sba395922007-10-21 16:41:49 -07003307 return 0;
3308}
Keshavamurthy, Anil Se8204822007-10-21 16:41:55 -07003309
Han, Weidong3199aa62009-02-26 17:31:12 +08003310static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3311 struct pci_dev *pdev)
3312{
3313 struct pci_dev *tmp, *parent;
3314
3315 if (!iommu || !pdev)
3316 return;
3317
3318 /* dependent device detach */
3319 tmp = pci_find_upstream_pcie_bridge(pdev);
3320 /* Secondary interface's bus number and devfn 0 */
3321 if (tmp) {
3322 parent = pdev->bus->self;
3323 while (parent != tmp) {
3324 iommu_detach_dev(iommu, parent->bus->number,
David Woodhouse276dbf92009-04-04 01:45:37 +01003325 parent->devfn);
Han, Weidong3199aa62009-02-26 17:31:12 +08003326 parent = parent->bus->self;
3327 }
Stefan Assmann45e829e2009-12-03 06:49:24 -05003328 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
Han, Weidong3199aa62009-02-26 17:31:12 +08003329 iommu_detach_dev(iommu,
3330 tmp->subordinate->number, 0);
3331 else /* this is a legacy PCI bridge */
David Woodhouse276dbf92009-04-04 01:45:37 +01003332 iommu_detach_dev(iommu, tmp->bus->number,
3333 tmp->devfn);
Han, Weidong3199aa62009-02-26 17:31:12 +08003334 }
3335}
3336
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003337static void domain_remove_one_dev_info(struct dmar_domain *domain,
Weidong Hanc7151a82008-12-08 22:51:37 +08003338 struct pci_dev *pdev)
3339{
3340 struct device_domain_info *info;
3341 struct intel_iommu *iommu;
3342 unsigned long flags;
3343 int found = 0;
3344 struct list_head *entry, *tmp;
3345
David Woodhouse276dbf92009-04-04 01:45:37 +01003346 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3347 pdev->devfn);
Weidong Hanc7151a82008-12-08 22:51:37 +08003348 if (!iommu)
3349 return;
3350
3351 spin_lock_irqsave(&device_domain_lock, flags);
3352 list_for_each_safe(entry, tmp, &domain->devices) {
3353 info = list_entry(entry, struct device_domain_info, link);
David Woodhouse276dbf92009-04-04 01:45:37 +01003354 /* No need to compare PCI domain; it has to be the same */
Weidong Hanc7151a82008-12-08 22:51:37 +08003355 if (info->bus == pdev->bus->number &&
3356 info->devfn == pdev->devfn) {
3357 list_del(&info->link);
3358 list_del(&info->global);
3359 if (info->dev)
3360 info->dev->dev.archdata.iommu = NULL;
3361 spin_unlock_irqrestore(&device_domain_lock, flags);
3362
Yu Zhao93a23a72009-05-18 13:51:37 +08003363 iommu_disable_dev_iotlb(info);
Weidong Hanc7151a82008-12-08 22:51:37 +08003364 iommu_detach_dev(iommu, info->bus, info->devfn);
Han, Weidong3199aa62009-02-26 17:31:12 +08003365 iommu_detach_dependent_devices(iommu, pdev);
Weidong Hanc7151a82008-12-08 22:51:37 +08003366 free_devinfo_mem(info);
3367
3368 spin_lock_irqsave(&device_domain_lock, flags);
3369
3370 if (found)
3371 break;
3372 else
3373 continue;
3374 }
3375
3376 /* if there is no other devices under the same iommu
3377 * owned by this domain, clear this iommu in iommu_bmp
3378 * update iommu count and coherency
3379 */
David Woodhouse276dbf92009-04-04 01:45:37 +01003380 if (iommu == device_to_iommu(info->segment, info->bus,
3381 info->devfn))
Weidong Hanc7151a82008-12-08 22:51:37 +08003382 found = 1;
3383 }
3384
3385 if (found == 0) {
3386 unsigned long tmp_flags;
3387 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3388 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3389 domain->iommu_count--;
Sheng Yang58c610b2009-03-18 15:33:05 +08003390 domain_update_iommu_cap(domain);
Weidong Hanc7151a82008-12-08 22:51:37 +08003391 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3392 }
3393
3394 spin_unlock_irqrestore(&device_domain_lock, flags);
3395}
3396
3397static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3398{
3399 struct device_domain_info *info;
3400 struct intel_iommu *iommu;
3401 unsigned long flags1, flags2;
3402
3403 spin_lock_irqsave(&device_domain_lock, flags1);
3404 while (!list_empty(&domain->devices)) {
3405 info = list_entry(domain->devices.next,
3406 struct device_domain_info, link);
3407 list_del(&info->link);
3408 list_del(&info->global);
3409 if (info->dev)
3410 info->dev->dev.archdata.iommu = NULL;
3411
3412 spin_unlock_irqrestore(&device_domain_lock, flags1);
3413
Yu Zhao93a23a72009-05-18 13:51:37 +08003414 iommu_disable_dev_iotlb(info);
David Woodhouse276dbf92009-04-04 01:45:37 +01003415 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
Weidong Hanc7151a82008-12-08 22:51:37 +08003416 iommu_detach_dev(iommu, info->bus, info->devfn);
Han, Weidong3199aa62009-02-26 17:31:12 +08003417 iommu_detach_dependent_devices(iommu, info->dev);
Weidong Hanc7151a82008-12-08 22:51:37 +08003418
3419 /* clear this iommu in iommu_bmp, update iommu count
Sheng Yang58c610b2009-03-18 15:33:05 +08003420 * and capabilities
Weidong Hanc7151a82008-12-08 22:51:37 +08003421 */
3422 spin_lock_irqsave(&domain->iommu_lock, flags2);
3423 if (test_and_clear_bit(iommu->seq_id,
3424 &domain->iommu_bmp)) {
3425 domain->iommu_count--;
Sheng Yang58c610b2009-03-18 15:33:05 +08003426 domain_update_iommu_cap(domain);
Weidong Hanc7151a82008-12-08 22:51:37 +08003427 }
3428 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3429
3430 free_devinfo_mem(info);
3431 spin_lock_irqsave(&device_domain_lock, flags1);
3432 }
3433 spin_unlock_irqrestore(&device_domain_lock, flags1);
3434}
3435
Weidong Han5e98c4b2008-12-08 23:03:27 +08003436/* domain id for virtual machine, it won't be set in context */
3437static unsigned long vm_domid;
3438
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003439static int vm_domain_min_agaw(struct dmar_domain *domain)
3440{
3441 int i;
3442 int min_agaw = domain->agaw;
3443
3444 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3445 for (; i < g_num_of_iommus; ) {
3446 if (min_agaw > g_iommus[i]->agaw)
3447 min_agaw = g_iommus[i]->agaw;
3448
3449 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3450 }
3451
3452 return min_agaw;
3453}
3454
Weidong Han5e98c4b2008-12-08 23:03:27 +08003455static struct dmar_domain *iommu_alloc_vm_domain(void)
3456{
3457 struct dmar_domain *domain;
3458
3459 domain = alloc_domain_mem();
3460 if (!domain)
3461 return NULL;
3462
3463 domain->id = vm_domid++;
Suresh Siddha4c923d42009-10-02 11:01:24 -07003464 domain->nid = -1;
Weidong Han5e98c4b2008-12-08 23:03:27 +08003465 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3466 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3467
3468 return domain;
3469}
3470
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003471static int md_domain_init(struct dmar_domain *domain, int guest_width)
Weidong Han5e98c4b2008-12-08 23:03:27 +08003472{
3473 int adjust_width;
3474
3475 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
Weidong Han5e98c4b2008-12-08 23:03:27 +08003476 spin_lock_init(&domain->iommu_lock);
3477
3478 domain_reserve_special_ranges(domain);
3479
3480 /* calculate AGAW */
3481 domain->gaw = guest_width;
3482 adjust_width = guestwidth_to_adjustwidth(guest_width);
3483 domain->agaw = width_to_agaw(adjust_width);
3484
3485 INIT_LIST_HEAD(&domain->devices);
3486
3487 domain->iommu_count = 0;
3488 domain->iommu_coherency = 0;
Sheng Yangc5b15252009-08-06 13:31:56 +08003489 domain->iommu_snooping = 0;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003490 domain->max_addr = 0;
Suresh Siddha4c923d42009-10-02 11:01:24 -07003491 domain->nid = -1;
Weidong Han5e98c4b2008-12-08 23:03:27 +08003492
3493 /* always allocate the top pgd */
Suresh Siddha4c923d42009-10-02 11:01:24 -07003494 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
Weidong Han5e98c4b2008-12-08 23:03:27 +08003495 if (!domain->pgd)
3496 return -ENOMEM;
3497 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3498 return 0;
3499}
3500
3501static void iommu_free_vm_domain(struct dmar_domain *domain)
3502{
3503 unsigned long flags;
3504 struct dmar_drhd_unit *drhd;
3505 struct intel_iommu *iommu;
3506 unsigned long i;
3507 unsigned long ndomains;
3508
3509 for_each_drhd_unit(drhd) {
3510 if (drhd->ignored)
3511 continue;
3512 iommu = drhd->iommu;
3513
3514 ndomains = cap_ndoms(iommu->cap);
3515 i = find_first_bit(iommu->domain_ids, ndomains);
3516 for (; i < ndomains; ) {
3517 if (iommu->domains[i] == domain) {
3518 spin_lock_irqsave(&iommu->lock, flags);
3519 clear_bit(i, iommu->domain_ids);
3520 iommu->domains[i] = NULL;
3521 spin_unlock_irqrestore(&iommu->lock, flags);
3522 break;
3523 }
3524 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3525 }
3526 }
3527}
3528
3529static void vm_domain_exit(struct dmar_domain *domain)
3530{
Weidong Han5e98c4b2008-12-08 23:03:27 +08003531 /* Domain 0 is reserved, so dont process it */
3532 if (!domain)
3533 return;
3534
3535 vm_domain_remove_all_dev_info(domain);
3536 /* destroy iovas */
3537 put_iova_domain(&domain->iovad);
Weidong Han5e98c4b2008-12-08 23:03:27 +08003538
3539 /* clear ptes */
David Woodhouse595badf2009-06-27 22:09:11 +01003540 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Weidong Han5e98c4b2008-12-08 23:03:27 +08003541
3542 /* free page tables */
David Woodhoused794dc92009-06-28 00:27:49 +01003543 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
Weidong Han5e98c4b2008-12-08 23:03:27 +08003544
3545 iommu_free_vm_domain(domain);
3546 free_domain_mem(domain);
3547}
3548
Joerg Roedel5d450802008-12-03 14:52:32 +01003549static int intel_iommu_domain_init(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03003550{
Joerg Roedel5d450802008-12-03 14:52:32 +01003551 struct dmar_domain *dmar_domain;
Kay, Allen M38717942008-09-09 18:37:29 +03003552
Joerg Roedel5d450802008-12-03 14:52:32 +01003553 dmar_domain = iommu_alloc_vm_domain();
3554 if (!dmar_domain) {
Kay, Allen M38717942008-09-09 18:37:29 +03003555 printk(KERN_ERR
Joerg Roedel5d450802008-12-03 14:52:32 +01003556 "intel_iommu_domain_init: dmar_domain == NULL\n");
3557 return -ENOMEM;
Kay, Allen M38717942008-09-09 18:37:29 +03003558 }
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003559 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
Kay, Allen M38717942008-09-09 18:37:29 +03003560 printk(KERN_ERR
Joerg Roedel5d450802008-12-03 14:52:32 +01003561 "intel_iommu_domain_init() failed\n");
3562 vm_domain_exit(dmar_domain);
3563 return -ENOMEM;
Kay, Allen M38717942008-09-09 18:37:29 +03003564 }
Joerg Roedel5d450802008-12-03 14:52:32 +01003565 domain->priv = dmar_domain;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003566
Joerg Roedel5d450802008-12-03 14:52:32 +01003567 return 0;
Kay, Allen M38717942008-09-09 18:37:29 +03003568}
Kay, Allen M38717942008-09-09 18:37:29 +03003569
Joerg Roedel5d450802008-12-03 14:52:32 +01003570static void intel_iommu_domain_destroy(struct iommu_domain *domain)
Kay, Allen M38717942008-09-09 18:37:29 +03003571{
Joerg Roedel5d450802008-12-03 14:52:32 +01003572 struct dmar_domain *dmar_domain = domain->priv;
3573
3574 domain->priv = NULL;
3575 vm_domain_exit(dmar_domain);
Kay, Allen M38717942008-09-09 18:37:29 +03003576}
Kay, Allen M38717942008-09-09 18:37:29 +03003577
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003578static int intel_iommu_attach_device(struct iommu_domain *domain,
3579 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03003580{
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003581 struct dmar_domain *dmar_domain = domain->priv;
3582 struct pci_dev *pdev = to_pci_dev(dev);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003583 struct intel_iommu *iommu;
3584 int addr_width;
3585 u64 end;
Kay, Allen M38717942008-09-09 18:37:29 +03003586
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003587 /* normally pdev is not mapped */
3588 if (unlikely(domain_context_mapped(pdev))) {
3589 struct dmar_domain *old_domain;
3590
3591 old_domain = find_domain(pdev);
3592 if (old_domain) {
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003593 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3594 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3595 domain_remove_one_dev_info(old_domain, pdev);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003596 else
3597 domain_remove_dev_info(old_domain);
3598 }
3599 }
3600
David Woodhouse276dbf92009-04-04 01:45:37 +01003601 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3602 pdev->devfn);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003603 if (!iommu)
3604 return -ENODEV;
3605
3606 /* check if this iommu agaw is sufficient for max mapped address */
3607 addr_width = agaw_to_width(iommu->agaw);
3608 end = DOMAIN_MAX_ADDR(addr_width);
3609 end = end & VTD_PAGE_MASK;
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003610 if (end < dmar_domain->max_addr) {
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003611 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3612 "sufficient for the mapped address (%llx)\n",
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003613 __func__, iommu->agaw, dmar_domain->max_addr);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003614 return -EFAULT;
3615 }
3616
David Woodhouse5fe60f42009-08-09 10:53:41 +01003617 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003618}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003619
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003620static void intel_iommu_detach_device(struct iommu_domain *domain,
3621 struct device *dev)
Kay, Allen M38717942008-09-09 18:37:29 +03003622{
Joerg Roedel4c5478c2008-12-03 14:58:24 +01003623 struct dmar_domain *dmar_domain = domain->priv;
3624 struct pci_dev *pdev = to_pci_dev(dev);
3625
Fenghua Yu2c2e2c32009-06-19 13:47:29 -07003626 domain_remove_one_dev_info(dmar_domain, pdev);
Kay, Allen M38717942008-09-09 18:37:29 +03003627}
Kay, Allen M38717942008-09-09 18:37:29 +03003628
Joerg Roedeldde57a22008-12-03 15:04:09 +01003629static int intel_iommu_map_range(struct iommu_domain *domain,
3630 unsigned long iova, phys_addr_t hpa,
3631 size_t size, int iommu_prot)
Kay, Allen M38717942008-09-09 18:37:29 +03003632{
Joerg Roedeldde57a22008-12-03 15:04:09 +01003633 struct dmar_domain *dmar_domain = domain->priv;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003634 u64 max_addr;
3635 int addr_width;
Joerg Roedeldde57a22008-12-03 15:04:09 +01003636 int prot = 0;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003637 int ret;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003638
Joerg Roedeldde57a22008-12-03 15:04:09 +01003639 if (iommu_prot & IOMMU_READ)
3640 prot |= DMA_PTE_READ;
3641 if (iommu_prot & IOMMU_WRITE)
3642 prot |= DMA_PTE_WRITE;
Sheng Yang9cf06692009-03-18 15:33:07 +08003643 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3644 prot |= DMA_PTE_SNP;
Joerg Roedeldde57a22008-12-03 15:04:09 +01003645
David Woodhouse163cc522009-06-28 00:51:17 +01003646 max_addr = iova + size;
Joerg Roedeldde57a22008-12-03 15:04:09 +01003647 if (dmar_domain->max_addr < max_addr) {
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003648 int min_agaw;
3649 u64 end;
3650
3651 /* check if minimum agaw is sufficient for mapped address */
Joerg Roedeldde57a22008-12-03 15:04:09 +01003652 min_agaw = vm_domain_min_agaw(dmar_domain);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003653 addr_width = agaw_to_width(min_agaw);
3654 end = DOMAIN_MAX_ADDR(addr_width);
3655 end = end & VTD_PAGE_MASK;
3656 if (end < max_addr) {
3657 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3658 "sufficient for the mapped address (%llx)\n",
3659 __func__, min_agaw, max_addr);
3660 return -EFAULT;
3661 }
Joerg Roedeldde57a22008-12-03 15:04:09 +01003662 dmar_domain->max_addr = max_addr;
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003663 }
David Woodhousead051222009-06-28 14:22:28 +01003664 /* Round up size to next multiple of PAGE_SIZE, if it and
3665 the low bits of hpa would take us onto the next page */
David Woodhouse88cb6a72009-06-28 15:03:06 +01003666 size = aligned_nrpages(hpa, size);
David Woodhousead051222009-06-28 14:22:28 +01003667 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3668 hpa >> VTD_PAGE_SHIFT, size, prot);
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003669 return ret;
Kay, Allen M38717942008-09-09 18:37:29 +03003670}
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003671
Joerg Roedeldde57a22008-12-03 15:04:09 +01003672static void intel_iommu_unmap_range(struct iommu_domain *domain,
3673 unsigned long iova, size_t size)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003674{
Joerg Roedeldde57a22008-12-03 15:04:09 +01003675 struct dmar_domain *dmar_domain = domain->priv;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003676
Sheng Yang4b99d352009-07-08 11:52:52 +01003677 if (!size)
3678 return;
3679
David Woodhouse163cc522009-06-28 00:51:17 +01003680 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3681 (iova + size - 1) >> VTD_PAGE_SHIFT);
Weidong Hanfe40f1e2008-12-08 23:10:23 +08003682
David Woodhouse163cc522009-06-28 00:51:17 +01003683 if (dmar_domain->max_addr == iova + size)
3684 dmar_domain->max_addr = iova;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003685}
Kay, Allen M38717942008-09-09 18:37:29 +03003686
Joerg Roedeld14d6572008-12-03 15:06:57 +01003687static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3688 unsigned long iova)
Kay, Allen M38717942008-09-09 18:37:29 +03003689{
Joerg Roedeld14d6572008-12-03 15:06:57 +01003690 struct dmar_domain *dmar_domain = domain->priv;
Kay, Allen M38717942008-09-09 18:37:29 +03003691 struct dma_pte *pte;
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003692 u64 phys = 0;
Kay, Allen M38717942008-09-09 18:37:29 +03003693
David Woodhouseb026fd22009-06-28 10:37:25 +01003694 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
Kay, Allen M38717942008-09-09 18:37:29 +03003695 if (pte)
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003696 phys = dma_pte_addr(pte);
Kay, Allen M38717942008-09-09 18:37:29 +03003697
Weidong Hanfaa3d6f2008-12-08 23:09:29 +08003698 return phys;
Kay, Allen M38717942008-09-09 18:37:29 +03003699}
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01003700
Sheng Yangdbb9fd82009-03-18 15:33:06 +08003701static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3702 unsigned long cap)
3703{
3704 struct dmar_domain *dmar_domain = domain->priv;
3705
3706 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3707 return dmar_domain->iommu_snooping;
3708
3709 return 0;
3710}
3711
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01003712static struct iommu_ops intel_iommu_ops = {
3713 .domain_init = intel_iommu_domain_init,
3714 .domain_destroy = intel_iommu_domain_destroy,
3715 .attach_dev = intel_iommu_attach_device,
3716 .detach_dev = intel_iommu_detach_device,
3717 .map = intel_iommu_map_range,
3718 .unmap = intel_iommu_unmap_range,
3719 .iova_to_phys = intel_iommu_iova_to_phys,
Sheng Yangdbb9fd82009-03-18 15:33:06 +08003720 .domain_has_cap = intel_iommu_domain_has_cap,
Joerg Roedela8bcbb0d2008-12-03 15:14:02 +01003721};
David Woodhouse9af88142009-02-13 23:18:03 +00003722
3723static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3724{
3725 /*
3726 * Mobile 4 Series Chipset neglects to set RWBF capability,
3727 * but needs it:
3728 */
3729 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3730 rwbf_quirk = 1;
3731}
3732
3733DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
David Woodhousee0fc7e02009-09-30 09:12:17 -07003734
3735/* On Tylersburg chipsets, some BIOSes have been known to enable the
3736 ISOCH DMAR unit for the Azalia sound device, but not give it any
3737 TLB entries, which causes it to deadlock. Check for that. We do
3738 this in a function called from init_dmars(), instead of in a PCI
3739 quirk, because we don't want to print the obnoxious "BIOS broken"
3740 message if VT-d is actually disabled.
3741*/
3742static void __init check_tylersburg_isoch(void)
3743{
3744 struct pci_dev *pdev;
3745 uint32_t vtisochctrl;
3746
3747 /* If there's no Azalia in the system anyway, forget it. */
3748 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3749 if (!pdev)
3750 return;
3751 pci_dev_put(pdev);
3752
3753 /* System Management Registers. Might be hidden, in which case
3754 we can't do the sanity check. But that's OK, because the
3755 known-broken BIOSes _don't_ actually hide it, so far. */
3756 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3757 if (!pdev)
3758 return;
3759
3760 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3761 pci_dev_put(pdev);
3762 return;
3763 }
3764
3765 pci_dev_put(pdev);
3766
3767 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3768 if (vtisochctrl & 1)
3769 return;
3770
3771 /* Drop all bits other than the number of TLB entries */
3772 vtisochctrl &= 0x1c;
3773
3774 /* If we have the recommended number of TLB entries (16), fine. */
3775 if (vtisochctrl == 0x10)
3776 return;
3777
3778 /* Zero TLB entries? You get to ride the short bus to school. */
3779 if (!vtisochctrl) {
3780 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3781 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3782 dmi_get_system_info(DMI_BIOS_VENDOR),
3783 dmi_get_system_info(DMI_BIOS_VERSION),
3784 dmi_get_system_info(DMI_PRODUCT_VERSION));
3785 iommu_identity_mapping |= IDENTMAP_AZALIA;
3786 return;
3787 }
3788
3789 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3790 vtisochctrl);
3791}