blob: 1f90344d3e2c11bd47816ae7a54012b62bd3238f [file] [log] [blame]
Alex Williamson73fa0d12012-07-31 08:16:23 -06001/*
2 * VFIO: IOMMU DMA mapping support for Type1 IOMMU
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
14 *
15 * We arbitrarily define a Type1 IOMMU as one matching the below code.
16 * It could be called the x86 IOMMU as it's designed for AMD-Vi & Intel
17 * VT-d, but that makes it harder to re-use as theoretically anyone
18 * implementing a similar IOMMU could make use of this. We expect the
19 * IOMMU to support the IOMMU API and have few to no restrictions around
20 * the IOVA range that can be mapped. The Type1 IOMMU is currently
21 * optimized for relatively static mappings of a userspace process with
22 * userpsace pages pinned into memory. We also assume devices and IOMMU
23 * domains are PCI based as the IOMMU API is still centered around a
24 * device/bus interface rather than a group interface.
25 */
26
27#include <linux/compat.h>
28#include <linux/device.h>
29#include <linux/fs.h>
30#include <linux/iommu.h>
31#include <linux/module.h>
32#include <linux/mm.h>
Alex Williamsoncd9b2262013-06-21 09:37:50 -060033#include <linux/rbtree.h>
Alex Williamson73fa0d12012-07-31 08:16:23 -060034#include <linux/sched.h>
35#include <linux/slab.h>
36#include <linux/uaccess.h>
37#include <linux/vfio.h>
38#include <linux/workqueue.h>
39
40#define DRIVER_VERSION "0.2"
41#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
42#define DRIVER_DESC "Type1 IOMMU driver for VFIO"
43
44static bool allow_unsafe_interrupts;
45module_param_named(allow_unsafe_interrupts,
46 allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR);
47MODULE_PARM_DESC(allow_unsafe_interrupts,
48 "Enable VFIO IOMMU support for on platforms without interrupt remapping support.");
49
Alex Williamson5c6c2b22013-06-21 09:38:11 -060050static bool disable_hugepages;
51module_param_named(disable_hugepages,
52 disable_hugepages, bool, S_IRUGO | S_IWUSR);
53MODULE_PARM_DESC(disable_hugepages,
54 "Disable VFIO IOMMU support for IOMMU hugepages.");
55
Alex Williamson73fa0d12012-07-31 08:16:23 -060056struct vfio_iommu {
Alex Williamson1ef3e2b2014-02-26 11:38:36 -070057 struct list_head domain_list;
Alex Williamson73fa0d12012-07-31 08:16:23 -060058 struct mutex lock;
Alex Williamsoncd9b2262013-06-21 09:37:50 -060059 struct rb_root dma_list;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -070060 bool v2;
61};
62
63struct vfio_domain {
64 struct iommu_domain *domain;
65 struct list_head next;
Alex Williamson73fa0d12012-07-31 08:16:23 -060066 struct list_head group_list;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -070067 int prot; /* IOMMU_CACHE */
Alex Williamson73fa0d12012-07-31 08:16:23 -060068};
69
70struct vfio_dma {
Alex Williamsoncd9b2262013-06-21 09:37:50 -060071 struct rb_node node;
Alex Williamson73fa0d12012-07-31 08:16:23 -060072 dma_addr_t iova; /* Device address */
73 unsigned long vaddr; /* Process virtual addr */
Alex Williamson166fd7d2013-06-21 09:38:02 -060074 size_t size; /* Map size (bytes) */
Alex Williamson73fa0d12012-07-31 08:16:23 -060075 int prot; /* IOMMU_READ/WRITE */
76};
77
78struct vfio_group {
79 struct iommu_group *iommu_group;
80 struct list_head next;
81};
82
83/*
84 * This code handles mapping and unmapping of user data buffers
85 * into DMA'ble space using the IOMMU
86 */
87
Alex Williamsoncd9b2262013-06-21 09:37:50 -060088static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
89 dma_addr_t start, size_t size)
90{
91 struct rb_node *node = iommu->dma_list.rb_node;
92
93 while (node) {
94 struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
95
96 if (start + size <= dma->iova)
97 node = node->rb_left;
Alex Williamson166fd7d2013-06-21 09:38:02 -060098 else if (start >= dma->iova + dma->size)
Alex Williamsoncd9b2262013-06-21 09:37:50 -060099 node = node->rb_right;
100 else
101 return dma;
102 }
103
104 return NULL;
105}
106
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700107static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600108{
109 struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;
110 struct vfio_dma *dma;
111
112 while (*link) {
113 parent = *link;
114 dma = rb_entry(parent, struct vfio_dma, node);
115
Alex Williamson166fd7d2013-06-21 09:38:02 -0600116 if (new->iova + new->size <= dma->iova)
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600117 link = &(*link)->rb_left;
118 else
119 link = &(*link)->rb_right;
120 }
121
122 rb_link_node(&new->node, parent, link);
123 rb_insert_color(&new->node, &iommu->dma_list);
124}
125
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700126static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600127{
128 rb_erase(&old->node, &iommu->dma_list);
129}
130
Alex Williamson73fa0d12012-07-31 08:16:23 -0600131struct vwork {
132 struct mm_struct *mm;
133 long npage;
134 struct work_struct work;
135};
136
137/* delayed decrement/increment for locked_vm */
138static void vfio_lock_acct_bg(struct work_struct *work)
139{
140 struct vwork *vwork = container_of(work, struct vwork, work);
141 struct mm_struct *mm;
142
143 mm = vwork->mm;
144 down_write(&mm->mmap_sem);
145 mm->locked_vm += vwork->npage;
146 up_write(&mm->mmap_sem);
147 mmput(mm);
148 kfree(vwork);
149}
150
151static void vfio_lock_acct(long npage)
152{
153 struct vwork *vwork;
154 struct mm_struct *mm;
155
Alex Williamson166fd7d2013-06-21 09:38:02 -0600156 if (!current->mm || !npage)
157 return; /* process exited or nothing to do */
Alex Williamson73fa0d12012-07-31 08:16:23 -0600158
159 if (down_write_trylock(&current->mm->mmap_sem)) {
160 current->mm->locked_vm += npage;
161 up_write(&current->mm->mmap_sem);
162 return;
163 }
164
165 /*
166 * Couldn't get mmap_sem lock, so must setup to update
167 * mm->locked_vm later. If locked_vm were atomic, we
168 * wouldn't need this silliness
169 */
170 vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
171 if (!vwork)
172 return;
173 mm = get_task_mm(current);
174 if (!mm) {
175 kfree(vwork);
176 return;
177 }
178 INIT_WORK(&vwork->work, vfio_lock_acct_bg);
179 vwork->mm = mm;
180 vwork->npage = npage;
181 schedule_work(&vwork->work);
182}
183
184/*
185 * Some mappings aren't backed by a struct page, for example an mmap'd
186 * MMIO range for our own or another device. These use a different
187 * pfn conversion and shouldn't be tracked as locked pages.
188 */
189static bool is_invalid_reserved_pfn(unsigned long pfn)
190{
191 if (pfn_valid(pfn)) {
192 bool reserved;
193 struct page *tail = pfn_to_page(pfn);
194 struct page *head = compound_trans_head(tail);
195 reserved = !!(PageReserved(head));
196 if (head != tail) {
197 /*
198 * "head" is not a dangling pointer
199 * (compound_trans_head takes care of that)
200 * but the hugepage may have been split
201 * from under us (and we may not hold a
202 * reference count on the head page so it can
203 * be reused before we run PageReferenced), so
204 * we've to check PageTail before returning
205 * what we just read.
206 */
207 smp_rmb();
208 if (PageTail(tail))
209 return reserved;
210 }
211 return PageReserved(tail);
212 }
213
214 return true;
215}
216
217static int put_pfn(unsigned long pfn, int prot)
218{
219 if (!is_invalid_reserved_pfn(pfn)) {
220 struct page *page = pfn_to_page(pfn);
221 if (prot & IOMMU_WRITE)
222 SetPageDirty(page);
223 put_page(page);
224 return 1;
225 }
226 return 0;
227}
228
Alex Williamson73fa0d12012-07-31 08:16:23 -0600229static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
230{
231 struct page *page[1];
232 struct vm_area_struct *vma;
233 int ret = -EFAULT;
234
235 if (get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE), page) == 1) {
236 *pfn = page_to_pfn(page[0]);
237 return 0;
238 }
239
240 down_read(&current->mm->mmap_sem);
241
242 vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);
243
244 if (vma && vma->vm_flags & VM_PFNMAP) {
245 *pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
246 if (is_invalid_reserved_pfn(*pfn))
247 ret = 0;
248 }
249
250 up_read(&current->mm->mmap_sem);
251
252 return ret;
253}
254
Alex Williamson166fd7d2013-06-21 09:38:02 -0600255/*
256 * Attempt to pin pages. We really don't want to track all the pfns and
257 * the iommu can only map chunks of consecutive pfns anyway, so get the
258 * first page and all consecutive pages with the same locking.
259 */
260static long vfio_pin_pages(unsigned long vaddr, long npage,
261 int prot, unsigned long *pfn_base)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600262{
Alex Williamson166fd7d2013-06-21 09:38:02 -0600263 unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
264 bool lock_cap = capable(CAP_IPC_LOCK);
265 long ret, i;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600266
Alex Williamson166fd7d2013-06-21 09:38:02 -0600267 if (!current->mm)
268 return -ENODEV;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600269
Alex Williamson166fd7d2013-06-21 09:38:02 -0600270 ret = vaddr_get_pfn(vaddr, prot, pfn_base);
271 if (ret)
272 return ret;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600273
Alex Williamson166fd7d2013-06-21 09:38:02 -0600274 if (is_invalid_reserved_pfn(*pfn_base))
275 return 1;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600276
Alex Williamson166fd7d2013-06-21 09:38:02 -0600277 if (!lock_cap && current->mm->locked_vm + 1 > limit) {
278 put_pfn(*pfn_base, prot);
279 pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
280 limit << PAGE_SHIFT);
281 return -ENOMEM;
282 }
283
Alex Williamson5c6c2b22013-06-21 09:38:11 -0600284 if (unlikely(disable_hugepages)) {
285 vfio_lock_acct(1);
286 return 1;
287 }
288
Alex Williamson166fd7d2013-06-21 09:38:02 -0600289 /* Lock all the consecutive pages from pfn_base */
290 for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
Alex Williamson73fa0d12012-07-31 08:16:23 -0600291 unsigned long pfn = 0;
292
293 ret = vaddr_get_pfn(vaddr, prot, &pfn);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600294 if (ret)
295 break;
296
297 if (pfn != *pfn_base + i || is_invalid_reserved_pfn(pfn)) {
298 put_pfn(pfn, prot);
299 break;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600300 }
301
Alex Williamson166fd7d2013-06-21 09:38:02 -0600302 if (!lock_cap && current->mm->locked_vm + i + 1 > limit) {
Alex Williamson73fa0d12012-07-31 08:16:23 -0600303 put_pfn(pfn, prot);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600304 pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
305 __func__, limit << PAGE_SHIFT);
306 break;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600307 }
308 }
Alex Williamson166fd7d2013-06-21 09:38:02 -0600309
310 vfio_lock_acct(i);
311
312 return i;
313}
314
315static long vfio_unpin_pages(unsigned long pfn, long npage,
316 int prot, bool do_accounting)
317{
318 unsigned long unlocked = 0;
319 long i;
320
321 for (i = 0; i < npage; i++)
322 unlocked += put_pfn(pfn++, prot);
323
324 if (do_accounting)
325 vfio_lock_acct(-unlocked);
326
327 return unlocked;
328}
329
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700330static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
Alex Williamson166fd7d2013-06-21 09:38:02 -0600331{
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700332 dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
333 struct vfio_domain *domain, *d;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600334 long unlocked = 0;
335
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700336 if (!dma->size)
337 return;
338 /*
339 * We use the IOMMU to track the physical addresses, otherwise we'd
340 * need a much more complicated tracking system. Unfortunately that
341 * means we need to use one of the iommu domains to figure out the
342 * pfns to unpin. The rest need to be unmapped in advance so we have
343 * no iommu translations remaining when the pages are unpinned.
344 */
345 domain = d = list_first_entry(&iommu->domain_list,
346 struct vfio_domain, next);
347
348 list_for_each_entry_continue(d, &iommu->domain_list, next)
349 iommu_unmap(d->domain, dma->iova, dma->size);
350
Alex Williamson166fd7d2013-06-21 09:38:02 -0600351 while (iova < end) {
352 size_t unmapped;
353 phys_addr_t phys;
354
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700355 phys = iommu_iova_to_phys(domain->domain, iova);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600356 if (WARN_ON(!phys)) {
357 iova += PAGE_SIZE;
358 continue;
359 }
360
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700361 unmapped = iommu_unmap(domain->domain, iova, PAGE_SIZE);
362 if (WARN_ON(!unmapped))
Alex Williamson166fd7d2013-06-21 09:38:02 -0600363 break;
364
365 unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT,
366 unmapped >> PAGE_SHIFT,
367 dma->prot, false);
368 iova += unmapped;
369 }
370
371 vfio_lock_acct(-unlocked);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600372}
373
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700374static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600375{
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700376 vfio_unmap_unpin(iommu, dma);
377 vfio_unlink_dma(iommu, dma);
378 kfree(dma);
379}
Alex Williamson73fa0d12012-07-31 08:16:23 -0600380
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700381static unsigned long vfio_pgsize_bitmap(struct vfio_iommu *iommu)
382{
383 struct vfio_domain *domain;
384 unsigned long bitmap = PAGE_MASK;
Alex Williamsonf5bfdbf2013-06-25 16:01:44 -0600385
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700386 mutex_lock(&iommu->lock);
387 list_for_each_entry(domain, &iommu->domain_list, next)
388 bitmap &= domain->domain->ops->pgsize_bitmap;
389 mutex_unlock(&iommu->lock);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600390
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700391 return bitmap;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600392}
393
394static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
395 struct vfio_iommu_type1_dma_unmap *unmap)
396{
Alex Williamson73fa0d12012-07-31 08:16:23 -0600397 uint64_t mask;
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600398 struct vfio_dma *dma;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700399 size_t unmapped = 0;
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600400 int ret = 0;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600401
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700402 mask = ((uint64_t)1 << __ffs(vfio_pgsize_bitmap(iommu))) - 1;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600403
404 if (unmap->iova & mask)
405 return -EINVAL;
Alex Williamsonf5bfdbf2013-06-25 16:01:44 -0600406 if (!unmap->size || unmap->size & mask)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600407 return -EINVAL;
408
Alex Williamson73fa0d12012-07-31 08:16:23 -0600409 WARN_ON(mask & PAGE_MASK);
410
411 mutex_lock(&iommu->lock);
412
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700413 /*
414 * vfio-iommu-type1 (v1) - User mappings were coalesced together to
415 * avoid tracking individual mappings. This means that the granularity
416 * of the original mapping was lost and the user was allowed to attempt
417 * to unmap any range. Depending on the contiguousness of physical
418 * memory and page sizes supported by the IOMMU, arbitrary unmaps may
419 * or may not have worked. We only guaranteed unmap granularity
420 * matching the original mapping; even though it was untracked here,
421 * the original mappings are reflected in IOMMU mappings. This
422 * resulted in a couple unusual behaviors. First, if a range is not
423 * able to be unmapped, ex. a set of 4k pages that was mapped as a
424 * 2M hugepage into the IOMMU, the unmap ioctl returns success but with
425 * a zero sized unmap. Also, if an unmap request overlaps the first
426 * address of a hugepage, the IOMMU will unmap the entire hugepage.
427 * This also returns success and the returned unmap size reflects the
428 * actual size unmapped.
429 *
430 * We attempt to maintain compatibility with this "v1" interface, but
431 * we take control out of the hands of the IOMMU. Therefore, an unmap
432 * request offset from the beginning of the original mapping will
433 * return success with zero sized unmap. And an unmap request covering
434 * the first iova of mapping will unmap the entire range.
435 *
436 * The v2 version of this interface intends to be more deterministic.
437 * Unmap requests must fully cover previous mappings. Multiple
438 * mappings may still be unmaped by specifying large ranges, but there
439 * must not be any previous mappings bisected by the range. An error
440 * will be returned if these conditions are not met. The v2 interface
441 * will only return success and a size of zero if there were no
442 * mappings within the range.
443 */
444 if (iommu->v2) {
445 dma = vfio_find_dma(iommu, unmap->iova, 0);
446 if (dma && dma->iova != unmap->iova) {
447 ret = -EINVAL;
448 goto unlock;
449 }
450 dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0);
451 if (dma && dma->iova + dma->size != unmap->iova + unmap->size) {
452 ret = -EINVAL;
453 goto unlock;
454 }
Alex Williamson166fd7d2013-06-21 09:38:02 -0600455 }
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600456
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700457 while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
458 if (!iommu->v2 && unmap->iova > dma->iova)
459 break;
460 unmapped += dma->size;
461 vfio_remove_dma(iommu, dma);
462 }
463
464unlock:
Alex Williamson73fa0d12012-07-31 08:16:23 -0600465 mutex_unlock(&iommu->lock);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600466
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700467 /* Report how much was unmapped */
Alex Williamson166fd7d2013-06-21 09:38:02 -0600468 unmap->size = unmapped;
469
470 return ret;
471}
472
473/*
474 * Turns out AMD IOMMU has a page table bug where it won't map large pages
475 * to a region that previously mapped smaller pages. This should be fixed
476 * soon, so this is just a temporary workaround to break mappings down into
477 * PAGE_SIZE. Better to map smaller pages than nothing.
478 */
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700479static int map_try_harder(struct vfio_domain *domain, dma_addr_t iova,
Alex Williamson166fd7d2013-06-21 09:38:02 -0600480 unsigned long pfn, long npage, int prot)
481{
482 long i;
483 int ret;
484
485 for (i = 0; i < npage; i++, pfn++, iova += PAGE_SIZE) {
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700486 ret = iommu_map(domain->domain, iova,
Alex Williamson166fd7d2013-06-21 09:38:02 -0600487 (phys_addr_t)pfn << PAGE_SHIFT,
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700488 PAGE_SIZE, prot | domain->prot);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600489 if (ret)
490 break;
491 }
492
493 for (; i < npage && i > 0; i--, iova -= PAGE_SIZE)
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700494 iommu_unmap(domain->domain, iova, PAGE_SIZE);
495
496 return ret;
497}
498
499static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova,
500 unsigned long pfn, long npage, int prot)
501{
502 struct vfio_domain *d;
503 int ret;
504
505 list_for_each_entry(d, &iommu->domain_list, next) {
506 ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT,
507 npage << PAGE_SHIFT, prot | d->prot);
508 if (ret) {
509 if (ret != -EBUSY ||
510 map_try_harder(d, iova, pfn, npage, prot))
511 goto unwind;
512 }
513 }
514
515 return 0;
516
517unwind:
518 list_for_each_entry_continue_reverse(d, &iommu->domain_list, next)
519 iommu_unmap(d->domain, iova, npage << PAGE_SHIFT);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600520
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600521 return ret;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600522}
523
524static int vfio_dma_do_map(struct vfio_iommu *iommu,
525 struct vfio_iommu_type1_dma_map *map)
526{
Alex Williamson166fd7d2013-06-21 09:38:02 -0600527 dma_addr_t end, iova;
528 unsigned long vaddr = map->vaddr;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600529 size_t size = map->size;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600530 long npage;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600531 int ret = 0, prot = 0;
532 uint64_t mask;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700533 struct vfio_dma *dma;
Antonios Motakisd93b3ac2013-10-11 10:40:46 -0600534 unsigned long pfn;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600535
536 end = map->iova + map->size;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600537
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700538 mask = ((uint64_t)1 << __ffs(vfio_pgsize_bitmap(iommu))) - 1;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600539
540 /* READ/WRITE from device perspective */
541 if (map->flags & VFIO_DMA_MAP_FLAG_WRITE)
542 prot |= IOMMU_WRITE;
543 if (map->flags & VFIO_DMA_MAP_FLAG_READ)
544 prot |= IOMMU_READ;
545
546 if (!prot)
547 return -EINVAL; /* No READ/WRITE? */
548
549 if (vaddr & mask)
550 return -EINVAL;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600551 if (map->iova & mask)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600552 return -EINVAL;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600553 if (!map->size || map->size & mask)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600554 return -EINVAL;
555
Alex Williamson73fa0d12012-07-31 08:16:23 -0600556 WARN_ON(mask & PAGE_MASK);
557
558 /* Don't allow IOVA wrap */
Alex Williamson166fd7d2013-06-21 09:38:02 -0600559 if (end && end < map->iova)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600560 return -EINVAL;
561
562 /* Don't allow virtual address wrap */
Alex Williamson166fd7d2013-06-21 09:38:02 -0600563 if (vaddr + map->size && vaddr + map->size < vaddr)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600564 return -EINVAL;
565
Alex Williamson73fa0d12012-07-31 08:16:23 -0600566 mutex_lock(&iommu->lock);
567
Alex Williamson166fd7d2013-06-21 09:38:02 -0600568 if (vfio_find_dma(iommu, map->iova, map->size)) {
569 mutex_unlock(&iommu->lock);
570 return -EEXIST;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600571 }
572
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700573 dma = kzalloc(sizeof(*dma), GFP_KERNEL);
574 if (!dma) {
575 mutex_unlock(&iommu->lock);
576 return -ENOMEM;
577 }
Alex Williamson73fa0d12012-07-31 08:16:23 -0600578
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700579 dma->iova = map->iova;
580 dma->vaddr = map->vaddr;
581 dma->prot = prot;
582
583 /* Insert zero-sized and grow as we map chunks of it */
584 vfio_link_dma(iommu, dma);
585
586 for (iova = map->iova; iova < end; iova += size, vaddr += size) {
Alex Williamson166fd7d2013-06-21 09:38:02 -0600587 /* Pin a contiguous chunk of memory */
588 npage = vfio_pin_pages(vaddr, (end - iova) >> PAGE_SHIFT,
589 prot, &pfn);
590 if (npage <= 0) {
591 WARN_ON(!npage);
592 ret = (int)npage;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700593 break;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600594 }
Alex Williamson73fa0d12012-07-31 08:16:23 -0600595
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700596 /* Map it! */
597 ret = vfio_iommu_map(iommu, iova, pfn, npage, prot);
Alex Williamson166fd7d2013-06-21 09:38:02 -0600598 if (ret) {
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700599 vfio_unpin_pages(pfn, npage, prot, true);
600 break;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600601 }
602
603 size = npage << PAGE_SHIFT;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700604 dma->size += size;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600605 }
606
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700607 if (ret)
608 vfio_remove_dma(iommu, dma);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600609
Alex Williamson73fa0d12012-07-31 08:16:23 -0600610 mutex_unlock(&iommu->lock);
611 return ret;
612}
613
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700614static int vfio_bus_type(struct device *dev, void *data)
615{
616 struct bus_type **bus = data;
617
618 if (*bus && *bus != dev->bus)
619 return -EINVAL;
620
621 *bus = dev->bus;
622
623 return 0;
624}
625
626static int vfio_iommu_replay(struct vfio_iommu *iommu,
627 struct vfio_domain *domain)
628{
629 struct vfio_domain *d;
630 struct rb_node *n;
631 int ret;
632
633 /* Arbitrarily pick the first domain in the list for lookups */
634 d = list_first_entry(&iommu->domain_list, struct vfio_domain, next);
635 n = rb_first(&iommu->dma_list);
636
637 /* If there's not a domain, there better not be any mappings */
638 if (WARN_ON(n && !d))
639 return -EINVAL;
640
641 for (; n; n = rb_next(n)) {
642 struct vfio_dma *dma;
643 dma_addr_t iova;
644
645 dma = rb_entry(n, struct vfio_dma, node);
646 iova = dma->iova;
647
648 while (iova < dma->iova + dma->size) {
649 phys_addr_t phys = iommu_iova_to_phys(d->domain, iova);
650 size_t size;
651
652 if (WARN_ON(!phys)) {
653 iova += PAGE_SIZE;
654 continue;
655 }
656
657 size = PAGE_SIZE;
658
659 while (iova + size < dma->iova + dma->size &&
660 phys + size == iommu_iova_to_phys(d->domain,
661 iova + size))
662 size += PAGE_SIZE;
663
664 ret = iommu_map(domain->domain, iova, phys,
665 size, dma->prot | domain->prot);
666 if (ret)
667 return ret;
668
669 iova += size;
670 }
671 }
672
673 return 0;
674}
675
Alex Williamson73fa0d12012-07-31 08:16:23 -0600676static int vfio_iommu_type1_attach_group(void *iommu_data,
677 struct iommu_group *iommu_group)
678{
679 struct vfio_iommu *iommu = iommu_data;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700680 struct vfio_group *group, *g;
681 struct vfio_domain *domain, *d;
682 struct bus_type *bus = NULL;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600683 int ret;
684
Alex Williamson73fa0d12012-07-31 08:16:23 -0600685 mutex_lock(&iommu->lock);
686
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700687 list_for_each_entry(d, &iommu->domain_list, next) {
688 list_for_each_entry(g, &d->group_list, next) {
689 if (g->iommu_group != iommu_group)
690 continue;
691
Alex Williamson73fa0d12012-07-31 08:16:23 -0600692 mutex_unlock(&iommu->lock);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600693 return -EINVAL;
694 }
695 }
696
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700697 group = kzalloc(sizeof(*group), GFP_KERNEL);
698 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
699 if (!group || !domain) {
700 ret = -ENOMEM;
701 goto out_free;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600702 }
703
704 group->iommu_group = iommu_group;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700705
706 /* Determine bus_type in order to allocate a domain */
707 ret = iommu_group_for_each_dev(iommu_group, &bus, vfio_bus_type);
708 if (ret)
709 goto out_free;
710
711 domain->domain = iommu_domain_alloc(bus);
712 if (!domain->domain) {
713 ret = -EIO;
714 goto out_free;
715 }
716
717 ret = iommu_attach_group(domain->domain, iommu_group);
718 if (ret)
719 goto out_domain;
720
721 INIT_LIST_HEAD(&domain->group_list);
722 list_add(&group->next, &domain->group_list);
723
724 if (!allow_unsafe_interrupts &&
725 !iommu_domain_has_cap(domain->domain, IOMMU_CAP_INTR_REMAP)) {
726 pr_warn("%s: No interrupt remapping support. Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n",
727 __func__);
728 ret = -EPERM;
729 goto out_detach;
730 }
731
732 if (iommu_domain_has_cap(domain->domain, IOMMU_CAP_CACHE_COHERENCY))
733 domain->prot |= IOMMU_CACHE;
734
735 /*
736 * Try to match an existing compatible domain. We don't want to
737 * preclude an IOMMU driver supporting multiple bus_types and being
738 * able to include different bus_types in the same IOMMU domain, so
739 * we test whether the domains use the same iommu_ops rather than
740 * testing if they're on the same bus_type.
741 */
742 list_for_each_entry(d, &iommu->domain_list, next) {
743 if (d->domain->ops == domain->domain->ops &&
744 d->prot == domain->prot) {
745 iommu_detach_group(domain->domain, iommu_group);
746 if (!iommu_attach_group(d->domain, iommu_group)) {
747 list_add(&group->next, &d->group_list);
748 iommu_domain_free(domain->domain);
749 kfree(domain);
750 mutex_unlock(&iommu->lock);
751 return 0;
752 }
753
754 ret = iommu_attach_group(domain->domain, iommu_group);
755 if (ret)
756 goto out_domain;
757 }
758 }
759
760 /* replay mappings on new domains */
761 ret = vfio_iommu_replay(iommu, domain);
762 if (ret)
763 goto out_detach;
764
765 list_add(&domain->next, &iommu->domain_list);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600766
767 mutex_unlock(&iommu->lock);
768
769 return 0;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700770
771out_detach:
772 iommu_detach_group(domain->domain, iommu_group);
773out_domain:
774 iommu_domain_free(domain->domain);
775out_free:
776 kfree(domain);
777 kfree(group);
778 mutex_unlock(&iommu->lock);
779 return ret;
780}
781
782static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
783{
784 struct rb_node *node;
785
786 while ((node = rb_first(&iommu->dma_list)))
787 vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
Alex Williamson73fa0d12012-07-31 08:16:23 -0600788}
789
790static void vfio_iommu_type1_detach_group(void *iommu_data,
791 struct iommu_group *iommu_group)
792{
793 struct vfio_iommu *iommu = iommu_data;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700794 struct vfio_domain *domain;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600795 struct vfio_group *group;
796
797 mutex_lock(&iommu->lock);
798
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700799 list_for_each_entry(domain, &iommu->domain_list, next) {
800 list_for_each_entry(group, &domain->group_list, next) {
801 if (group->iommu_group != iommu_group)
802 continue;
803
804 iommu_detach_group(domain->domain, iommu_group);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600805 list_del(&group->next);
806 kfree(group);
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700807 /*
808 * Group ownership provides privilege, if the group
809 * list is empty, the domain goes away. If it's the
810 * last domain, then all the mappings go away too.
811 */
812 if (list_empty(&domain->group_list)) {
813 if (list_is_singular(&iommu->domain_list))
814 vfio_iommu_unmap_unpin_all(iommu);
815 iommu_domain_free(domain->domain);
816 list_del(&domain->next);
817 kfree(domain);
818 }
819 goto done;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600820 }
821 }
822
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700823done:
Alex Williamson73fa0d12012-07-31 08:16:23 -0600824 mutex_unlock(&iommu->lock);
825}
826
827static void *vfio_iommu_type1_open(unsigned long arg)
828{
829 struct vfio_iommu *iommu;
830
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700831 if (arg != VFIO_TYPE1_IOMMU && arg != VFIO_TYPE1v2_IOMMU)
Alex Williamson73fa0d12012-07-31 08:16:23 -0600832 return ERR_PTR(-EINVAL);
833
834 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
835 if (!iommu)
836 return ERR_PTR(-ENOMEM);
837
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700838 INIT_LIST_HEAD(&iommu->domain_list);
Alex Williamsoncd9b2262013-06-21 09:37:50 -0600839 iommu->dma_list = RB_ROOT;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600840 mutex_init(&iommu->lock);
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700841 iommu->v2 = (arg == VFIO_TYPE1v2_IOMMU);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600842
843 return iommu;
844}
845
846static void vfio_iommu_type1_release(void *iommu_data)
847{
848 struct vfio_iommu *iommu = iommu_data;
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700849 struct vfio_domain *domain, *domain_tmp;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600850 struct vfio_group *group, *group_tmp;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600851
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700852 vfio_iommu_unmap_unpin_all(iommu);
853
854 list_for_each_entry_safe(domain, domain_tmp,
855 &iommu->domain_list, next) {
856 list_for_each_entry_safe(group, group_tmp,
857 &domain->group_list, next) {
858 iommu_detach_group(domain->domain, group->iommu_group);
859 list_del(&group->next);
860 kfree(group);
861 }
862 iommu_domain_free(domain->domain);
863 list_del(&domain->next);
864 kfree(domain);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600865 }
866
Alex Williamson73fa0d12012-07-31 08:16:23 -0600867 kfree(iommu);
868}
869
Alex Williamsonaa429312014-02-26 11:38:37 -0700870static int vfio_domains_have_iommu_cache(struct vfio_iommu *iommu)
871{
872 struct vfio_domain *domain;
873 int ret = 1;
874
875 mutex_lock(&iommu->lock);
876 list_for_each_entry(domain, &iommu->domain_list, next) {
877 if (!(domain->prot & IOMMU_CACHE)) {
878 ret = 0;
879 break;
880 }
881 }
882 mutex_unlock(&iommu->lock);
883
884 return ret;
885}
886
Alex Williamson73fa0d12012-07-31 08:16:23 -0600887static long vfio_iommu_type1_ioctl(void *iommu_data,
888 unsigned int cmd, unsigned long arg)
889{
890 struct vfio_iommu *iommu = iommu_data;
891 unsigned long minsz;
892
893 if (cmd == VFIO_CHECK_EXTENSION) {
894 switch (arg) {
895 case VFIO_TYPE1_IOMMU:
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700896 case VFIO_TYPE1v2_IOMMU:
Alex Williamson73fa0d12012-07-31 08:16:23 -0600897 return 1;
Alex Williamsonaa429312014-02-26 11:38:37 -0700898 case VFIO_DMA_CC_IOMMU:
899 if (!iommu)
900 return 0;
901 return vfio_domains_have_iommu_cache(iommu);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600902 default:
903 return 0;
904 }
905 } else if (cmd == VFIO_IOMMU_GET_INFO) {
906 struct vfio_iommu_type1_info info;
907
908 minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes);
909
910 if (copy_from_user(&info, (void __user *)arg, minsz))
911 return -EFAULT;
912
913 if (info.argsz < minsz)
914 return -EINVAL;
915
916 info.flags = 0;
917
Alex Williamson1ef3e2b2014-02-26 11:38:36 -0700918 info.iova_pgsizes = vfio_pgsize_bitmap(iommu);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600919
920 return copy_to_user((void __user *)arg, &info, minsz);
921
922 } else if (cmd == VFIO_IOMMU_MAP_DMA) {
923 struct vfio_iommu_type1_dma_map map;
924 uint32_t mask = VFIO_DMA_MAP_FLAG_READ |
925 VFIO_DMA_MAP_FLAG_WRITE;
926
927 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
928
929 if (copy_from_user(&map, (void __user *)arg, minsz))
930 return -EFAULT;
931
932 if (map.argsz < minsz || map.flags & ~mask)
933 return -EINVAL;
934
935 return vfio_dma_do_map(iommu, &map);
936
937 } else if (cmd == VFIO_IOMMU_UNMAP_DMA) {
938 struct vfio_iommu_type1_dma_unmap unmap;
Alex Williamson166fd7d2013-06-21 09:38:02 -0600939 long ret;
Alex Williamson73fa0d12012-07-31 08:16:23 -0600940
941 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
942
943 if (copy_from_user(&unmap, (void __user *)arg, minsz))
944 return -EFAULT;
945
946 if (unmap.argsz < minsz || unmap.flags)
947 return -EINVAL;
948
Alex Williamson166fd7d2013-06-21 09:38:02 -0600949 ret = vfio_dma_do_unmap(iommu, &unmap);
950 if (ret)
951 return ret;
952
953 return copy_to_user((void __user *)arg, &unmap, minsz);
Alex Williamson73fa0d12012-07-31 08:16:23 -0600954 }
955
956 return -ENOTTY;
957}
958
959static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
960 .name = "vfio-iommu-type1",
961 .owner = THIS_MODULE,
962 .open = vfio_iommu_type1_open,
963 .release = vfio_iommu_type1_release,
964 .ioctl = vfio_iommu_type1_ioctl,
965 .attach_group = vfio_iommu_type1_attach_group,
966 .detach_group = vfio_iommu_type1_detach_group,
967};
968
969static int __init vfio_iommu_type1_init(void)
970{
Alex Williamson73fa0d12012-07-31 08:16:23 -0600971 return vfio_register_iommu_driver(&vfio_iommu_driver_ops_type1);
972}
973
974static void __exit vfio_iommu_type1_cleanup(void)
975{
976 vfio_unregister_iommu_driver(&vfio_iommu_driver_ops_type1);
977}
978
979module_init(vfio_iommu_type1_init);
980module_exit(vfio_iommu_type1_cleanup);
981
982MODULE_VERSION(DRIVER_VERSION);
983MODULE_LICENSE("GPL v2");
984MODULE_AUTHOR(DRIVER_AUTHOR);
985MODULE_DESCRIPTION(DRIVER_DESC);