blob: e51d9f9b995f1864faa6921a41cc0809ce69fb8a [file] [log] [blame]
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +03001/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Copyright IBM Corporation, 2008
Avi Kivity221d0592010-05-23 18:37:00 +030019 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
20 *
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030021 * Author: Allen M. Kay <allen.m.kay@intel.com>
22 * Author: Weidong Han <weidong.han@intel.com>
23 * Author: Ben-Ami Yassour <benami@il.ibm.com>
24 */
25
26#include <linux/list.h>
27#include <linux/kvm_host.h>
Paul Gortmaker51441d42011-07-27 21:25:05 -040028#include <linux/module.h>
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030029#include <linux/pci.h>
Paul Gortmaker799fd8b2011-07-27 21:17:59 -040030#include <linux/stat.h>
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030031#include <linux/dmar.h>
Joerg Roedel19de40a2008-12-03 14:43:34 +010032#include <linux/iommu.h>
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030033#include <linux/intel-iommu.h>
34
Rusty Russell90ab5ee2012-01-13 09:32:20 +103035static bool allow_unsafe_assigned_interrupts;
Alex Williamson3f68b032011-07-14 13:27:03 -060036module_param_named(allow_unsafe_assigned_interrupts,
37 allow_unsafe_assigned_interrupts, bool, S_IRUGO | S_IWUSR);
38MODULE_PARM_DESC(allow_unsafe_assigned_interrupts,
39 "Enable device assignment on platforms without interrupt remapping support.");
40
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030041static int kvm_iommu_unmap_memslots(struct kvm *kvm);
42static void kvm_iommu_put_pages(struct kvm *kvm,
43 gfn_t base_gfn, unsigned long npages);
44
Xiao Guangrongd5661042012-07-17 21:56:16 +080045static pfn_t kvm_pin_pages(struct kvm_memory_slot *slot, gfn_t gfn,
46 unsigned long size)
Joerg Roedelfcd95802010-01-11 16:38:18 +010047{
48 gfn_t end_gfn;
49 pfn_t pfn;
50
Xiao Guangrongd5661042012-07-17 21:56:16 +080051 pfn = gfn_to_pfn_memslot(slot, gfn);
Joerg Roedelfcd95802010-01-11 16:38:18 +010052 end_gfn = gfn + (size >> PAGE_SHIFT);
53 gfn += 1;
54
Xiao Guangrong81c52c52012-10-16 20:10:59 +080055 if (is_error_noslot_pfn(pfn))
Joerg Roedelfcd95802010-01-11 16:38:18 +010056 return pfn;
57
58 while (gfn < end_gfn)
Xiao Guangrongd5661042012-07-17 21:56:16 +080059 gfn_to_pfn_memslot(slot, gfn++);
Joerg Roedelfcd95802010-01-11 16:38:18 +010060
61 return pfn;
62}
63
Michael S. Tsirkin350b8bd2014-08-19 19:14:50 +080064static void kvm_unpin_pages(struct kvm *kvm, pfn_t pfn, unsigned long npages)
65{
66 unsigned long i;
67
68 for (i = 0; i < npages; ++i)
69 kvm_release_pfn_clean(pfn + i);
70}
71
Marcelo Tosatti3ad26d82009-12-23 14:35:20 -020072int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030073{
Joerg Roedelfcd95802010-01-11 16:38:18 +010074 gfn_t gfn, end_gfn;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030075 pfn_t pfn;
Joerg Roedelfcd95802010-01-11 16:38:18 +010076 int r = 0;
Joerg Roedel19de40a2008-12-03 14:43:34 +010077 struct iommu_domain *domain = kvm->arch.iommu_domain;
Sheng Yang522c68c2009-04-27 20:35:43 +080078 int flags;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030079
80 /* check if iommu exists and in use */
81 if (!domain)
82 return 0;
83
Joerg Roedelfcd95802010-01-11 16:38:18 +010084 gfn = slot->base_gfn;
85 end_gfn = gfn + slot->npages;
86
Alex Williamsond47510e22013-01-24 15:04:09 -070087 flags = IOMMU_READ;
88 if (!(slot->flags & KVM_MEM_READONLY))
89 flags |= IOMMU_WRITE;
Alex Williamsond96eb2c2013-10-30 11:02:23 -060090 if (!kvm->arch.iommu_noncoherent)
Sheng Yang522c68c2009-04-27 20:35:43 +080091 flags |= IOMMU_CACHE;
92
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +030093
Joerg Roedelfcd95802010-01-11 16:38:18 +010094 while (gfn < end_gfn) {
95 unsigned long page_size;
96
97 /* Check if already mapped */
98 if (iommu_iova_to_phys(domain, gfn_to_gpa(gfn))) {
99 gfn += 1;
100 continue;
101 }
102
103 /* Get the page size we could use to map */
104 page_size = kvm_host_page_size(kvm, gfn);
105
106 /* Make sure the page_size does not exceed the memslot */
107 while ((gfn + (page_size >> PAGE_SHIFT)) > end_gfn)
108 page_size >>= 1;
109
110 /* Make sure gfn is aligned to the page size we want to map */
111 while ((gfn << PAGE_SHIFT) & (page_size - 1))
112 page_size >>= 1;
113
Greg Edwards27ef63c2013-11-04 09:08:12 -0700114 /* Make sure hva is aligned to the page size we want to map */
115 while (__gfn_to_hva_memslot(slot, gfn) & (page_size - 1))
116 page_size >>= 1;
117
Joerg Roedelfcd95802010-01-11 16:38:18 +0100118 /*
119 * Pin all pages we are about to map in memory. This is
120 * important because we unmap and unpin in 4kb steps later.
121 */
Xiao Guangrongd5661042012-07-17 21:56:16 +0800122 pfn = kvm_pin_pages(slot, gfn, page_size);
Xiao Guangrong81c52c52012-10-16 20:10:59 +0800123 if (is_error_noslot_pfn(pfn)) {
Joerg Roedelfcd95802010-01-11 16:38:18 +0100124 gfn += 1;
125 continue;
126 }
127
128 /* Map into IO address space */
129 r = iommu_map(domain, gfn_to_gpa(gfn), pfn_to_hpa(pfn),
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200130 page_size, flags);
Weidong Hane5fcfc82008-09-25 23:32:02 +0800131 if (r) {
Weidong Han260782b2008-12-02 21:03:39 +0800132 printk(KERN_ERR "kvm_iommu_map_address:"
Joerg Roedel5689cc52010-07-01 16:00:12 +0200133 "iommu failed to map pfn=%llx\n", pfn);
Michael S. Tsirkin350b8bd2014-08-19 19:14:50 +0800134 kvm_unpin_pages(kvm, pfn, page_size);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300135 goto unmap_pages;
136 }
Joerg Roedelfcd95802010-01-11 16:38:18 +0100137
138 gfn += page_size >> PAGE_SHIFT;
139
140
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300141 }
Joerg Roedelfcd95802010-01-11 16:38:18 +0100142
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300143 return 0;
144
145unmap_pages:
Michael S. Tsirkin350b8bd2014-08-19 19:14:50 +0800146 kvm_iommu_put_pages(kvm, slot->base_gfn, gfn - slot->base_gfn);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300147 return r;
148}
149
150static int kvm_iommu_map_memslots(struct kvm *kvm)
151{
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800152 int idx, r = 0;
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -0200153 struct kvm_memslots *slots;
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800154 struct kvm_memory_slot *memslot;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300155
Alex Williamsone0f0bbc2013-10-30 11:02:30 -0600156 if (kvm->arch.iommu_noncoherent)
157 kvm_arch_register_noncoherent_dma(kvm);
158
Sheng Yang95c87e22010-07-01 15:00:50 +0800159 idx = srcu_read_lock(&kvm->srcu);
Lai Jiangshan90d83dc2010-04-19 17:41:23 +0800160 slots = kvm_memslots(kvm);
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -0200161
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800162 kvm_for_each_memslot(memslot, slots) {
163 r = kvm_iommu_map_pages(kvm, memslot);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300164 if (r)
165 break;
166 }
Sheng Yang95c87e22010-07-01 15:00:50 +0800167 srcu_read_unlock(&kvm->srcu, idx);
Mark McLoughlin682edb42009-02-05 18:23:46 +0000168
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300169 return r;
170}
171
Weidong Han260782b2008-12-02 21:03:39 +0800172int kvm_assign_device(struct kvm *kvm,
173 struct kvm_assigned_dev_kernel *assigned_dev)
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300174{
175 struct pci_dev *pdev = NULL;
Joerg Roedel19de40a2008-12-03 14:43:34 +0100176 struct iommu_domain *domain = kvm->arch.iommu_domain;
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600177 int r;
178 bool noncoherent;
Weidong Han260782b2008-12-02 21:03:39 +0800179
180 /* check if iommu exists and in use */
181 if (!domain)
182 return 0;
183
184 pdev = assigned_dev->dev;
185 if (pdev == NULL)
186 return -ENODEV;
187
Joerg Roedel19de40a2008-12-03 14:43:34 +0100188 r = iommu_attach_device(domain, &pdev->dev);
Weidong Han260782b2008-12-02 21:03:39 +0800189 if (r) {
Shuah Khand151f632012-10-08 18:36:11 -0600190 dev_err(&pdev->dev, "kvm assign device failed ret %d", r);
Weidong Han260782b2008-12-02 21:03:39 +0800191 return r;
192 }
193
Joerg Roedelee5ba302014-09-05 10:54:09 +0200194 noncoherent = !iommu_capable(&pci_bus_type, IOMMU_CAP_CACHE_COHERENCY);
Sheng Yang522c68c2009-04-27 20:35:43 +0800195
196 /* Check if need to update IOMMU page table for guest memory */
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600197 if (noncoherent != kvm->arch.iommu_noncoherent) {
Sheng Yang522c68c2009-04-27 20:35:43 +0800198 kvm_iommu_unmap_memslots(kvm);
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600199 kvm->arch.iommu_noncoherent = noncoherent;
Sheng Yang522c68c2009-04-27 20:35:43 +0800200 r = kvm_iommu_map_memslots(kvm);
201 if (r)
202 goto out_unmap;
203 }
204
Ethan Zhaoad0d2172014-09-09 10:21:26 +0800205 pci_set_dev_assigned(pdev);
Greg Rose67778292011-07-22 05:46:07 +0000206
Andre Richter29242cb2013-10-02 12:23:26 +0200207 dev_info(&pdev->dev, "kvm assign device\n");
Weidong Han260782b2008-12-02 21:03:39 +0800208
209 return 0;
Sheng Yang522c68c2009-04-27 20:35:43 +0800210out_unmap:
211 kvm_iommu_unmap_memslots(kvm);
212 return r;
Weidong Han260782b2008-12-02 21:03:39 +0800213}
214
Weidong Han0a920352008-12-02 21:24:23 +0800215int kvm_deassign_device(struct kvm *kvm,
216 struct kvm_assigned_dev_kernel *assigned_dev)
217{
Joerg Roedel19de40a2008-12-03 14:43:34 +0100218 struct iommu_domain *domain = kvm->arch.iommu_domain;
Weidong Han0a920352008-12-02 21:24:23 +0800219 struct pci_dev *pdev = NULL;
220
221 /* check if iommu exists and in use */
222 if (!domain)
223 return 0;
224
225 pdev = assigned_dev->dev;
226 if (pdev == NULL)
227 return -ENODEV;
228
Joerg Roedel19de40a2008-12-03 14:43:34 +0100229 iommu_detach_device(domain, &pdev->dev);
Weidong Han0a920352008-12-02 21:24:23 +0800230
Ethan Zhaoad0d2172014-09-09 10:21:26 +0800231 pci_clear_dev_assigned(pdev);
Greg Rose67778292011-07-22 05:46:07 +0000232
Andre Richter29242cb2013-10-02 12:23:26 +0200233 dev_info(&pdev->dev, "kvm deassign device\n");
Weidong Han0a920352008-12-02 21:24:23 +0800234
235 return 0;
236}
237
Weidong Han260782b2008-12-02 21:03:39 +0800238int kvm_iommu_map_guest(struct kvm *kvm)
239{
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300240 int r;
241
Joerg Roedela1b60c12011-09-06 18:46:34 +0200242 if (!iommu_present(&pci_bus_type)) {
Joerg Roedel19de40a2008-12-03 14:43:34 +0100243 printk(KERN_ERR "%s: iommu not found\n", __func__);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300244 return -ENODEV;
245 }
246
Alex Williamson21a14162012-04-17 21:46:44 -0600247 mutex_lock(&kvm->slots_lock);
248
Joerg Roedel905d66c2011-09-06 16:03:26 +0200249 kvm->arch.iommu_domain = iommu_domain_alloc(&pci_bus_type);
Alex Williamson21a14162012-04-17 21:46:44 -0600250 if (!kvm->arch.iommu_domain) {
251 r = -ENOMEM;
252 goto out_unlock;
253 }
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300254
Alex Williamson3f68b032011-07-14 13:27:03 -0600255 if (!allow_unsafe_assigned_interrupts &&
Joerg Roedelee5ba302014-09-05 10:54:09 +0200256 !iommu_capable(&pci_bus_type, IOMMU_CAP_INTR_REMAP)) {
Alex Williamson3f68b032011-07-14 13:27:03 -0600257 printk(KERN_WARNING "%s: No interrupt remapping support,"
258 " disallowing device assignment."
259 " Re-enble with \"allow_unsafe_assigned_interrupts=1\""
260 " module option.\n", __func__);
261 iommu_domain_free(kvm->arch.iommu_domain);
262 kvm->arch.iommu_domain = NULL;
Alex Williamson21a14162012-04-17 21:46:44 -0600263 r = -EPERM;
264 goto out_unlock;
Alex Williamson3f68b032011-07-14 13:27:03 -0600265 }
266
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300267 r = kvm_iommu_map_memslots(kvm);
268 if (r)
Alex Williamson21a14162012-04-17 21:46:44 -0600269 kvm_iommu_unmap_memslots(kvm);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300270
Alex Williamson21a14162012-04-17 21:46:44 -0600271out_unlock:
272 mutex_unlock(&kvm->slots_lock);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300273 return r;
274}
275
276static void kvm_iommu_put_pages(struct kvm *kvm,
Weidong Han260782b2008-12-02 21:03:39 +0800277 gfn_t base_gfn, unsigned long npages)
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300278{
Joerg Roedelfcd95802010-01-11 16:38:18 +0100279 struct iommu_domain *domain;
280 gfn_t end_gfn, gfn;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300281 pfn_t pfn;
Weidong Han260782b2008-12-02 21:03:39 +0800282 u64 phys;
283
Joerg Roedelfcd95802010-01-11 16:38:18 +0100284 domain = kvm->arch.iommu_domain;
285 end_gfn = base_gfn + npages;
286 gfn = base_gfn;
287
Weidong Han260782b2008-12-02 21:03:39 +0800288 /* check if iommu exists and in use */
289 if (!domain)
290 return;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300291
Joerg Roedelfcd95802010-01-11 16:38:18 +0100292 while (gfn < end_gfn) {
293 unsigned long unmap_pages;
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200294 size_t size;
Weidong Han260782b2008-12-02 21:03:39 +0800295
Joerg Roedelfcd95802010-01-11 16:38:18 +0100296 /* Get physical address */
297 phys = iommu_iova_to_phys(domain, gfn_to_gpa(gfn));
Xiao Guangrong16b854c2012-08-03 15:36:52 +0800298
299 if (!phys) {
300 gfn++;
301 continue;
302 }
303
Joerg Roedelfcd95802010-01-11 16:38:18 +0100304 pfn = phys >> PAGE_SHIFT;
305
306 /* Unmap address from IO address space */
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200307 size = iommu_unmap(domain, gfn_to_gpa(gfn), PAGE_SIZE);
308 unmap_pages = 1ULL << get_order(size);
Joerg Roedelfcd95802010-01-11 16:38:18 +0100309
310 /* Unpin all pages we just unmapped to not leak any memory */
311 kvm_unpin_pages(kvm, pfn, unmap_pages);
312
313 gfn += unmap_pages;
314 }
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300315}
316
Alex Williamson32f6daa2012-04-11 09:51:49 -0600317void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
318{
319 kvm_iommu_put_pages(kvm, slot->base_gfn, slot->npages);
320}
321
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300322static int kvm_iommu_unmap_memslots(struct kvm *kvm)
323{
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800324 int idx;
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -0200325 struct kvm_memslots *slots;
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800326 struct kvm_memory_slot *memslot;
Mark McLoughlin682edb42009-02-05 18:23:46 +0000327
Sheng Yang95c87e22010-07-01 15:00:50 +0800328 idx = srcu_read_lock(&kvm->srcu);
Lai Jiangshan90d83dc2010-04-19 17:41:23 +0800329 slots = kvm_memslots(kvm);
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -0200330
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800331 kvm_for_each_memslot(memslot, slots)
Alex Williamson32f6daa2012-04-11 09:51:49 -0600332 kvm_iommu_unmap_pages(kvm, memslot);
Xiao Guangrongbe6ba0f2011-11-24 17:39:18 +0800333
Sheng Yang95c87e22010-07-01 15:00:50 +0800334 srcu_read_unlock(&kvm->srcu, idx);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300335
Alex Williamsone0f0bbc2013-10-30 11:02:30 -0600336 if (kvm->arch.iommu_noncoherent)
337 kvm_arch_unregister_noncoherent_dma(kvm);
338
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300339 return 0;
340}
341
342int kvm_iommu_unmap_guest(struct kvm *kvm)
343{
Joerg Roedel19de40a2008-12-03 14:43:34 +0100344 struct iommu_domain *domain = kvm->arch.iommu_domain;
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300345
346 /* check if iommu exists and in use */
347 if (!domain)
348 return 0;
349
Alex Williamson21a14162012-04-17 21:46:44 -0600350 mutex_lock(&kvm->slots_lock);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300351 kvm_iommu_unmap_memslots(kvm);
Alex Williamson21a14162012-04-17 21:46:44 -0600352 kvm->arch.iommu_domain = NULL;
Alex Williamsond96eb2c2013-10-30 11:02:23 -0600353 kvm->arch.iommu_noncoherent = false;
Alex Williamson21a14162012-04-17 21:46:44 -0600354 mutex_unlock(&kvm->slots_lock);
355
Joerg Roedel19de40a2008-12-03 14:43:34 +0100356 iommu_domain_free(domain);
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300357 return 0;
358}