blob: 6fd6fa5469def0b8624741938d61a9fc1fc02a70 [file] [log] [blame]
Alex Williamsoncba33452012-07-31 08:16:22 -06001/*
2 * VFIO core
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
14 */
15
16#include <linux/cdev.h>
17#include <linux/compat.h>
18#include <linux/device.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <linux/idr.h>
23#include <linux/iommu.h>
24#include <linux/list.h>
Alex Williamsond1099902013-12-19 10:17:13 -070025#include <linux/miscdevice.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060026#include <linux/module.h>
27#include <linux/mutex.h>
Alex Williamson5f096b12015-10-27 14:53:04 -060028#include <linux/pci.h>
Alex Williamson9587f442013-04-25 16:12:38 -060029#include <linux/rwsem.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060030#include <linux/sched.h>
31#include <linux/slab.h>
Alex Williamson664e9382013-04-30 15:42:28 -060032#include <linux/stat.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060033#include <linux/string.h>
34#include <linux/uaccess.h>
35#include <linux/vfio.h>
36#include <linux/wait.h>
37
38#define DRIVER_VERSION "0.3"
39#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
40#define DRIVER_DESC "VFIO - User Level meta-driver"
41
42static struct vfio {
43 struct class *class;
44 struct list_head iommu_drivers_list;
45 struct mutex iommu_drivers_lock;
46 struct list_head group_list;
47 struct idr group_idr;
48 struct mutex group_lock;
49 struct cdev group_cdev;
Alex Williamsond1099902013-12-19 10:17:13 -070050 dev_t group_devt;
Alex Williamsoncba33452012-07-31 08:16:22 -060051 wait_queue_head_t release_q;
52} vfio;
53
54struct vfio_iommu_driver {
55 const struct vfio_iommu_driver_ops *ops;
56 struct list_head vfio_next;
57};
58
59struct vfio_container {
60 struct kref kref;
61 struct list_head group_list;
Alex Williamson9587f442013-04-25 16:12:38 -060062 struct rw_semaphore group_lock;
Alex Williamsoncba33452012-07-31 08:16:22 -060063 struct vfio_iommu_driver *iommu_driver;
64 void *iommu_data;
Alex Williamson03a76b62015-12-21 15:13:33 -070065 bool noiommu;
Alex Williamsoncba33452012-07-31 08:16:22 -060066};
67
Alex Williamson60720a02015-02-06 15:05:06 -070068struct vfio_unbound_dev {
69 struct device *dev;
70 struct list_head unbound_next;
71};
72
Alex Williamsoncba33452012-07-31 08:16:22 -060073struct vfio_group {
74 struct kref kref;
75 int minor;
76 atomic_t container_users;
77 struct iommu_group *iommu_group;
78 struct vfio_container *container;
79 struct list_head device_list;
80 struct mutex device_lock;
81 struct device *dev;
82 struct notifier_block nb;
83 struct list_head vfio_next;
84 struct list_head container_next;
Alex Williamson60720a02015-02-06 15:05:06 -070085 struct list_head unbound_list;
86 struct mutex unbound_lock;
Alex Williamson6d6768c2013-06-25 16:06:54 -060087 atomic_t opened;
Alex Williamson03a76b62015-12-21 15:13:33 -070088 bool noiommu;
Alex Williamsoncba33452012-07-31 08:16:22 -060089};
90
91struct vfio_device {
92 struct kref kref;
93 struct device *dev;
94 const struct vfio_device_ops *ops;
95 struct vfio_group *group;
96 struct list_head group_next;
97 void *device_data;
98};
99
Alex Williamson03a76b62015-12-21 15:13:33 -0700100#ifdef CONFIG_VFIO_NOIOMMU
101static bool noiommu __read_mostly;
102module_param_named(enable_unsafe_noiommu_mode,
103 noiommu, bool, S_IRUGO | S_IWUSR);
104MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)");
105#endif
106
107/*
108 * vfio_iommu_group_{get,put} are only intended for VFIO bus driver probe
109 * and remove functions, any use cases other than acquiring the first
110 * reference for the purpose of calling vfio_add_group_dev() or removing
111 * that symmetric reference after vfio_del_group_dev() should use the raw
112 * iommu_group_{get,put} functions. In particular, vfio_iommu_group_put()
113 * removes the device from the dummy group and cannot be nested.
114 */
115struct iommu_group *vfio_iommu_group_get(struct device *dev)
116{
117 struct iommu_group *group;
118 int __maybe_unused ret;
119
120 group = iommu_group_get(dev);
121
122#ifdef CONFIG_VFIO_NOIOMMU
123 /*
124 * With noiommu enabled, an IOMMU group will be created for a device
125 * that doesn't already have one and doesn't have an iommu_ops on their
Alex Williamson16ab8a52016-01-27 11:22:25 -0700126 * bus. We set iommudata simply to be able to identify these groups
127 * as special use and for reclamation later.
Alex Williamson03a76b62015-12-21 15:13:33 -0700128 */
129 if (group || !noiommu || iommu_present(dev->bus))
130 return group;
131
132 group = iommu_group_alloc();
133 if (IS_ERR(group))
134 return NULL;
135
136 iommu_group_set_name(group, "vfio-noiommu");
Alex Williamson16ab8a52016-01-27 11:22:25 -0700137 iommu_group_set_iommudata(group, &noiommu, NULL);
Alex Williamson03a76b62015-12-21 15:13:33 -0700138 ret = iommu_group_add_device(group, dev);
139 iommu_group_put(group);
140 if (ret)
141 return NULL;
142
143 /*
144 * Where to taint? At this point we've added an IOMMU group for a
145 * device that is not backed by iommu_ops, therefore any iommu_
146 * callback using iommu_ops can legitimately Oops. So, while we may
147 * be about to give a DMA capable device to a user without IOMMU
148 * protection, which is clearly taint-worthy, let's go ahead and do
149 * it here.
150 */
151 add_taint(TAINT_USER, LOCKDEP_STILL_OK);
152 dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
153#endif
154
155 return group;
156}
157EXPORT_SYMBOL_GPL(vfio_iommu_group_get);
158
159void vfio_iommu_group_put(struct iommu_group *group, struct device *dev)
160{
161#ifdef CONFIG_VFIO_NOIOMMU
Alex Williamson16ab8a52016-01-27 11:22:25 -0700162 if (iommu_group_get_iommudata(group) == &noiommu)
Alex Williamson03a76b62015-12-21 15:13:33 -0700163 iommu_group_remove_device(dev);
164#endif
165
166 iommu_group_put(group);
167}
168EXPORT_SYMBOL_GPL(vfio_iommu_group_put);
169
170#ifdef CONFIG_VFIO_NOIOMMU
171static void *vfio_noiommu_open(unsigned long arg)
172{
173 if (arg != VFIO_NOIOMMU_IOMMU)
174 return ERR_PTR(-EINVAL);
175 if (!capable(CAP_SYS_RAWIO))
176 return ERR_PTR(-EPERM);
177
178 return NULL;
179}
180
181static void vfio_noiommu_release(void *iommu_data)
182{
183}
184
185static long vfio_noiommu_ioctl(void *iommu_data,
186 unsigned int cmd, unsigned long arg)
187{
188 if (cmd == VFIO_CHECK_EXTENSION)
189 return noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;
190
191 return -ENOTTY;
192}
193
Alex Williamson03a76b62015-12-21 15:13:33 -0700194static int vfio_noiommu_attach_group(void *iommu_data,
195 struct iommu_group *iommu_group)
196{
Alex Williamson16ab8a52016-01-27 11:22:25 -0700197 return iommu_group_get_iommudata(iommu_group) == &noiommu ? 0 : -EINVAL;
Alex Williamson03a76b62015-12-21 15:13:33 -0700198}
199
200static void vfio_noiommu_detach_group(void *iommu_data,
201 struct iommu_group *iommu_group)
202{
203}
204
205static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
206 .name = "vfio-noiommu",
207 .owner = THIS_MODULE,
208 .open = vfio_noiommu_open,
209 .release = vfio_noiommu_release,
210 .ioctl = vfio_noiommu_ioctl,
211 .attach_group = vfio_noiommu_attach_group,
212 .detach_group = vfio_noiommu_detach_group,
213};
214#endif
215
216
Alex Williamsoncba33452012-07-31 08:16:22 -0600217/**
218 * IOMMU driver registration
219 */
220int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
221{
222 struct vfio_iommu_driver *driver, *tmp;
223
224 driver = kzalloc(sizeof(*driver), GFP_KERNEL);
225 if (!driver)
226 return -ENOMEM;
227
228 driver->ops = ops;
229
230 mutex_lock(&vfio.iommu_drivers_lock);
231
232 /* Check for duplicates */
233 list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
234 if (tmp->ops == ops) {
235 mutex_unlock(&vfio.iommu_drivers_lock);
236 kfree(driver);
237 return -EINVAL;
238 }
239 }
240
241 list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
242
243 mutex_unlock(&vfio.iommu_drivers_lock);
244
245 return 0;
246}
247EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
248
249void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
250{
251 struct vfio_iommu_driver *driver;
252
253 mutex_lock(&vfio.iommu_drivers_lock);
254 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
255 if (driver->ops == ops) {
256 list_del(&driver->vfio_next);
257 mutex_unlock(&vfio.iommu_drivers_lock);
258 kfree(driver);
259 return;
260 }
261 }
262 mutex_unlock(&vfio.iommu_drivers_lock);
263}
264EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
265
266/**
267 * Group minor allocation/free - both called with vfio.group_lock held
268 */
269static int vfio_alloc_group_minor(struct vfio_group *group)
270{
Alex Williamsond1099902013-12-19 10:17:13 -0700271 return idr_alloc(&vfio.group_idr, group, 0, MINORMASK + 1, GFP_KERNEL);
Alex Williamsoncba33452012-07-31 08:16:22 -0600272}
273
274static void vfio_free_group_minor(int minor)
275{
276 idr_remove(&vfio.group_idr, minor);
277}
278
279static int vfio_iommu_group_notifier(struct notifier_block *nb,
280 unsigned long action, void *data);
281static void vfio_group_get(struct vfio_group *group);
282
283/**
284 * Container objects - containers are created when /dev/vfio/vfio is
285 * opened, but their lifecycle extends until the last user is done, so
286 * it's freed via kref. Must support container/group/device being
287 * closed in any order.
288 */
289static void vfio_container_get(struct vfio_container *container)
290{
291 kref_get(&container->kref);
292}
293
294static void vfio_container_release(struct kref *kref)
295{
296 struct vfio_container *container;
297 container = container_of(kref, struct vfio_container, kref);
298
299 kfree(container);
300}
301
302static void vfio_container_put(struct vfio_container *container)
303{
304 kref_put(&container->kref, vfio_container_release);
305}
306
Jiang Liu9df7b252012-12-07 13:43:50 -0700307static void vfio_group_unlock_and_free(struct vfio_group *group)
308{
309 mutex_unlock(&vfio.group_lock);
310 /*
311 * Unregister outside of lock. A spurious callback is harmless now
312 * that the group is no longer in vfio.group_list.
313 */
314 iommu_group_unregister_notifier(group->iommu_group, &group->nb);
315 kfree(group);
316}
317
Alex Williamsoncba33452012-07-31 08:16:22 -0600318/**
319 * Group objects - create, release, get, put, search
320 */
Alex Williamson16ab8a52016-01-27 11:22:25 -0700321static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
Alex Williamsoncba33452012-07-31 08:16:22 -0600322{
323 struct vfio_group *group, *tmp;
324 struct device *dev;
325 int ret, minor;
326
327 group = kzalloc(sizeof(*group), GFP_KERNEL);
328 if (!group)
329 return ERR_PTR(-ENOMEM);
330
331 kref_init(&group->kref);
332 INIT_LIST_HEAD(&group->device_list);
333 mutex_init(&group->device_lock);
Alex Williamson60720a02015-02-06 15:05:06 -0700334 INIT_LIST_HEAD(&group->unbound_list);
335 mutex_init(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600336 atomic_set(&group->container_users, 0);
Alex Williamson6d6768c2013-06-25 16:06:54 -0600337 atomic_set(&group->opened, 0);
Alex Williamsoncba33452012-07-31 08:16:22 -0600338 group->iommu_group = iommu_group;
Alex Williamson16ab8a52016-01-27 11:22:25 -0700339#ifdef CONFIG_VFIO_NOIOMMU
340 group->noiommu = (iommu_group_get_iommudata(iommu_group) == &noiommu);
341#endif
Alex Williamsoncba33452012-07-31 08:16:22 -0600342
343 group->nb.notifier_call = vfio_iommu_group_notifier;
344
345 /*
346 * blocking notifiers acquire a rwsem around registering and hold
347 * it around callback. Therefore, need to register outside of
348 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
349 * do anything unless it can find the group in vfio.group_list, so
350 * no harm in registering early.
351 */
352 ret = iommu_group_register_notifier(iommu_group, &group->nb);
353 if (ret) {
354 kfree(group);
355 return ERR_PTR(ret);
356 }
357
358 mutex_lock(&vfio.group_lock);
359
Alex Williamsoncba33452012-07-31 08:16:22 -0600360 /* Did we race creating this group? */
361 list_for_each_entry(tmp, &vfio.group_list, vfio_next) {
362 if (tmp->iommu_group == iommu_group) {
363 vfio_group_get(tmp);
Jiang Liu9df7b252012-12-07 13:43:50 -0700364 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600365 return tmp;
366 }
367 }
368
Zhen Lei2f51bf42015-03-16 14:08:56 -0600369 minor = vfio_alloc_group_minor(group);
370 if (minor < 0) {
371 vfio_group_unlock_and_free(group);
372 return ERR_PTR(minor);
373 }
374
Alex Williamsond1099902013-12-19 10:17:13 -0700375 dev = device_create(vfio.class, NULL,
376 MKDEV(MAJOR(vfio.group_devt), minor),
Alex Williamson03a76b62015-12-21 15:13:33 -0700377 group, "%s%d", group->noiommu ? "noiommu-" : "",
378 iommu_group_id(iommu_group));
Alex Williamsoncba33452012-07-31 08:16:22 -0600379 if (IS_ERR(dev)) {
380 vfio_free_group_minor(minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700381 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600382 return (struct vfio_group *)dev; /* ERR_PTR */
383 }
384
385 group->minor = minor;
386 group->dev = dev;
387
388 list_add(&group->vfio_next, &vfio.group_list);
389
390 mutex_unlock(&vfio.group_lock);
391
392 return group;
393}
394
Al Viro6d2cd3c2012-08-17 21:27:32 -0400395/* called with vfio.group_lock held */
Alex Williamsoncba33452012-07-31 08:16:22 -0600396static void vfio_group_release(struct kref *kref)
397{
398 struct vfio_group *group = container_of(kref, struct vfio_group, kref);
Alex Williamson60720a02015-02-06 15:05:06 -0700399 struct vfio_unbound_dev *unbound, *tmp;
Alex Williamson4a688102015-02-06 15:05:06 -0700400 struct iommu_group *iommu_group = group->iommu_group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600401
402 WARN_ON(!list_empty(&group->device_list));
403
Alex Williamson60720a02015-02-06 15:05:06 -0700404 list_for_each_entry_safe(unbound, tmp,
405 &group->unbound_list, unbound_next) {
406 list_del(&unbound->unbound_next);
407 kfree(unbound);
408 }
409
Alex Williamsond1099902013-12-19 10:17:13 -0700410 device_destroy(vfio.class, MKDEV(MAJOR(vfio.group_devt), group->minor));
Alex Williamsoncba33452012-07-31 08:16:22 -0600411 list_del(&group->vfio_next);
412 vfio_free_group_minor(group->minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700413 vfio_group_unlock_and_free(group);
Alex Williamson4a688102015-02-06 15:05:06 -0700414 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600415}
416
417static void vfio_group_put(struct vfio_group *group)
418{
Al Viro6d2cd3c2012-08-17 21:27:32 -0400419 kref_put_mutex(&group->kref, vfio_group_release, &vfio.group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600420}
421
422/* Assume group_lock or group reference is held */
423static void vfio_group_get(struct vfio_group *group)
424{
425 kref_get(&group->kref);
426}
427
428/*
429 * Not really a try as we will sleep for mutex, but we need to make
430 * sure the group pointer is valid under lock and get a reference.
431 */
432static struct vfio_group *vfio_group_try_get(struct vfio_group *group)
433{
434 struct vfio_group *target = group;
435
436 mutex_lock(&vfio.group_lock);
437 list_for_each_entry(group, &vfio.group_list, vfio_next) {
438 if (group == target) {
439 vfio_group_get(group);
440 mutex_unlock(&vfio.group_lock);
441 return group;
442 }
443 }
444 mutex_unlock(&vfio.group_lock);
445
446 return NULL;
447}
448
449static
450struct vfio_group *vfio_group_get_from_iommu(struct iommu_group *iommu_group)
451{
452 struct vfio_group *group;
453
454 mutex_lock(&vfio.group_lock);
455 list_for_each_entry(group, &vfio.group_list, vfio_next) {
456 if (group->iommu_group == iommu_group) {
457 vfio_group_get(group);
458 mutex_unlock(&vfio.group_lock);
459 return group;
460 }
461 }
462 mutex_unlock(&vfio.group_lock);
463
464 return NULL;
465}
466
467static struct vfio_group *vfio_group_get_from_minor(int minor)
468{
469 struct vfio_group *group;
470
471 mutex_lock(&vfio.group_lock);
472 group = idr_find(&vfio.group_idr, minor);
473 if (!group) {
474 mutex_unlock(&vfio.group_lock);
475 return NULL;
476 }
477 vfio_group_get(group);
478 mutex_unlock(&vfio.group_lock);
479
480 return group;
481}
482
483/**
484 * Device objects - create, release, get, put, search
485 */
486static
487struct vfio_device *vfio_group_create_device(struct vfio_group *group,
488 struct device *dev,
489 const struct vfio_device_ops *ops,
490 void *device_data)
491{
492 struct vfio_device *device;
Alex Williamsoncba33452012-07-31 08:16:22 -0600493
494 device = kzalloc(sizeof(*device), GFP_KERNEL);
495 if (!device)
496 return ERR_PTR(-ENOMEM);
497
498 kref_init(&device->kref);
499 device->dev = dev;
500 device->group = group;
501 device->ops = ops;
502 device->device_data = device_data;
Jean Delvare8283b492014-04-14 12:55:38 +0200503 dev_set_drvdata(dev, device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600504
505 /* No need to get group_lock, caller has group reference */
506 vfio_group_get(group);
507
508 mutex_lock(&group->device_lock);
509 list_add(&device->group_next, &group->device_list);
510 mutex_unlock(&group->device_lock);
511
512 return device;
513}
514
515static void vfio_device_release(struct kref *kref)
516{
517 struct vfio_device *device = container_of(kref,
518 struct vfio_device, kref);
519 struct vfio_group *group = device->group;
520
Alex Williamsoncba33452012-07-31 08:16:22 -0600521 list_del(&device->group_next);
522 mutex_unlock(&group->device_lock);
523
524 dev_set_drvdata(device->dev, NULL);
525
526 kfree(device);
527
528 /* vfio_del_group_dev may be waiting for this device */
529 wake_up(&vfio.release_q);
530}
531
532/* Device reference always implies a group reference */
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600533void vfio_device_put(struct vfio_device *device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600534{
Al Viro934ad4c2012-08-17 19:49:09 -0400535 struct vfio_group *group = device->group;
Al Viro90b12532012-08-17 21:29:06 -0400536 kref_put_mutex(&device->kref, vfio_device_release, &group->device_lock);
Al Viro934ad4c2012-08-17 19:49:09 -0400537 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600538}
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600539EXPORT_SYMBOL_GPL(vfio_device_put);
Alex Williamsoncba33452012-07-31 08:16:22 -0600540
541static void vfio_device_get(struct vfio_device *device)
542{
543 vfio_group_get(device->group);
544 kref_get(&device->kref);
545}
546
547static struct vfio_device *vfio_group_get_device(struct vfio_group *group,
548 struct device *dev)
549{
550 struct vfio_device *device;
551
552 mutex_lock(&group->device_lock);
553 list_for_each_entry(device, &group->device_list, group_next) {
554 if (device->dev == dev) {
555 vfio_device_get(device);
556 mutex_unlock(&group->device_lock);
557 return device;
558 }
559 }
560 mutex_unlock(&group->device_lock);
561 return NULL;
562}
563
564/*
Alex Williamson5f096b12015-10-27 14:53:04 -0600565 * Some drivers, like pci-stub, are only used to prevent other drivers from
566 * claiming a device and are therefore perfectly legitimate for a user owned
567 * group. The pci-stub driver has no dependencies on DMA or the IOVA mapping
568 * of the device, but it does prevent the user from having direct access to
569 * the device, which is useful in some circumstances.
570 *
571 * We also assume that we can include PCI interconnect devices, ie. bridges.
572 * IOMMU grouping on PCI necessitates that if we lack isolation on a bridge
573 * then all of the downstream devices will be part of the same IOMMU group as
574 * the bridge. Thus, if placing the bridge into the user owned IOVA space
575 * breaks anything, it only does so for user owned devices downstream. Note
576 * that error notification via MSI can be affected for platforms that handle
577 * MSI within the same IOVA space as DMA.
Alex Williamsoncba33452012-07-31 08:16:22 -0600578 */
Alex Williamson5f096b12015-10-27 14:53:04 -0600579static const char * const vfio_driver_whitelist[] = { "pci-stub" };
Alex Williamsoncba33452012-07-31 08:16:22 -0600580
Alex Williamson5f096b12015-10-27 14:53:04 -0600581static bool vfio_dev_whitelisted(struct device *dev, struct device_driver *drv)
Alex Williamsoncba33452012-07-31 08:16:22 -0600582{
583 int i;
584
Alex Williamson5f096b12015-10-27 14:53:04 -0600585 if (dev_is_pci(dev)) {
586 struct pci_dev *pdev = to_pci_dev(dev);
587
588 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
589 return true;
590 }
591
Alex Williamsoncba33452012-07-31 08:16:22 -0600592 for (i = 0; i < ARRAY_SIZE(vfio_driver_whitelist); i++) {
593 if (!strcmp(drv->name, vfio_driver_whitelist[i]))
594 return true;
595 }
596
597 return false;
598}
599
600/*
Alex Williamson60720a02015-02-06 15:05:06 -0700601 * A vfio group is viable for use by userspace if all devices are in
602 * one of the following states:
603 * - driver-less
604 * - bound to a vfio driver
605 * - bound to a whitelisted driver
Alex Williamson5f096b12015-10-27 14:53:04 -0600606 * - a PCI interconnect device
Alex Williamson60720a02015-02-06 15:05:06 -0700607 *
608 * We use two methods to determine whether a device is bound to a vfio
609 * driver. The first is to test whether the device exists in the vfio
610 * group. The second is to test if the device exists on the group
611 * unbound_list, indicating it's in the middle of transitioning from
612 * a vfio driver to driver-less.
Alex Williamsoncba33452012-07-31 08:16:22 -0600613 */
614static int vfio_dev_viable(struct device *dev, void *data)
615{
616 struct vfio_group *group = data;
617 struct vfio_device *device;
Jiang Liude2b3ee2012-12-07 13:43:50 -0700618 struct device_driver *drv = ACCESS_ONCE(dev->driver);
Alex Williamson60720a02015-02-06 15:05:06 -0700619 struct vfio_unbound_dev *unbound;
620 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -0600621
Alex Williamson60720a02015-02-06 15:05:06 -0700622 mutex_lock(&group->unbound_lock);
623 list_for_each_entry(unbound, &group->unbound_list, unbound_next) {
624 if (dev == unbound->dev) {
625 ret = 0;
626 break;
627 }
628 }
629 mutex_unlock(&group->unbound_lock);
630
Alex Williamson5f096b12015-10-27 14:53:04 -0600631 if (!ret || !drv || vfio_dev_whitelisted(dev, drv))
Alex Williamsoncba33452012-07-31 08:16:22 -0600632 return 0;
633
634 device = vfio_group_get_device(group, dev);
635 if (device) {
636 vfio_device_put(device);
637 return 0;
638 }
639
Alex Williamson60720a02015-02-06 15:05:06 -0700640 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -0600641}
642
643/**
644 * Async device support
645 */
646static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev)
647{
648 struct vfio_device *device;
649
650 /* Do we already know about it? We shouldn't */
651 device = vfio_group_get_device(group, dev);
652 if (WARN_ON_ONCE(device)) {
653 vfio_device_put(device);
654 return 0;
655 }
656
657 /* Nothing to do for idle groups */
658 if (!atomic_read(&group->container_users))
659 return 0;
660
661 /* TODO Prevent device auto probing */
Dan Carpenter049af102015-11-21 13:32:21 +0300662 WARN(1, "Device %s added to live group %d!\n", dev_name(dev),
Alex Williamsoncba33452012-07-31 08:16:22 -0600663 iommu_group_id(group->iommu_group));
664
665 return 0;
666}
667
Alex Williamsoncba33452012-07-31 08:16:22 -0600668static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev)
669{
670 /* We don't care what happens when the group isn't in use */
671 if (!atomic_read(&group->container_users))
672 return 0;
673
674 return vfio_dev_viable(dev, group);
675}
676
677static int vfio_iommu_group_notifier(struct notifier_block *nb,
678 unsigned long action, void *data)
679{
680 struct vfio_group *group = container_of(nb, struct vfio_group, nb);
681 struct device *dev = data;
Alex Williamson60720a02015-02-06 15:05:06 -0700682 struct vfio_unbound_dev *unbound;
Alex Williamsoncba33452012-07-31 08:16:22 -0600683
684 /*
Alex Williamsonc6401932013-06-10 16:40:56 -0600685 * Need to go through a group_lock lookup to get a reference or we
686 * risk racing a group being removed. Ignore spurious notifies.
Alex Williamsoncba33452012-07-31 08:16:22 -0600687 */
688 group = vfio_group_try_get(group);
Alex Williamsonc6401932013-06-10 16:40:56 -0600689 if (!group)
Alex Williamsoncba33452012-07-31 08:16:22 -0600690 return NOTIFY_OK;
691
692 switch (action) {
693 case IOMMU_GROUP_NOTIFY_ADD_DEVICE:
694 vfio_group_nb_add_dev(group, dev);
695 break;
696 case IOMMU_GROUP_NOTIFY_DEL_DEVICE:
Alex Williamsonde9c7602013-06-10 16:40:56 -0600697 /*
698 * Nothing to do here. If the device is in use, then the
699 * vfio sub-driver should block the remove callback until
700 * it is unused. If the device is unused or attached to a
701 * stub driver, then it should be released and we don't
702 * care that it will be going away.
703 */
Alex Williamsoncba33452012-07-31 08:16:22 -0600704 break;
705 case IOMMU_GROUP_NOTIFY_BIND_DRIVER:
706 pr_debug("%s: Device %s, group %d binding to driver\n",
707 __func__, dev_name(dev),
708 iommu_group_id(group->iommu_group));
709 break;
710 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER:
711 pr_debug("%s: Device %s, group %d bound to driver %s\n",
712 __func__, dev_name(dev),
713 iommu_group_id(group->iommu_group), dev->driver->name);
714 BUG_ON(vfio_group_nb_verify(group, dev));
715 break;
716 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER:
717 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
718 __func__, dev_name(dev),
719 iommu_group_id(group->iommu_group), dev->driver->name);
720 break;
721 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER:
722 pr_debug("%s: Device %s, group %d unbound from driver\n",
723 __func__, dev_name(dev),
724 iommu_group_id(group->iommu_group));
725 /*
726 * XXX An unbound device in a live group is ok, but we'd
727 * really like to avoid the above BUG_ON by preventing other
728 * drivers from binding to it. Once that occurs, we have to
729 * stop the system to maintain isolation. At a minimum, we'd
730 * want a toggle to disable driver auto probe for this device.
731 */
Alex Williamson60720a02015-02-06 15:05:06 -0700732
733 mutex_lock(&group->unbound_lock);
734 list_for_each_entry(unbound,
735 &group->unbound_list, unbound_next) {
736 if (dev == unbound->dev) {
737 list_del(&unbound->unbound_next);
738 kfree(unbound);
739 break;
740 }
741 }
742 mutex_unlock(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600743 break;
744 }
745
746 vfio_group_put(group);
747 return NOTIFY_OK;
748}
749
750/**
751 * VFIO driver API
752 */
753int vfio_add_group_dev(struct device *dev,
754 const struct vfio_device_ops *ops, void *device_data)
755{
756 struct iommu_group *iommu_group;
757 struct vfio_group *group;
758 struct vfio_device *device;
759
760 iommu_group = iommu_group_get(dev);
761 if (!iommu_group)
762 return -EINVAL;
763
764 group = vfio_group_get_from_iommu(iommu_group);
765 if (!group) {
Alex Williamson16ab8a52016-01-27 11:22:25 -0700766 group = vfio_create_group(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600767 if (IS_ERR(group)) {
768 iommu_group_put(iommu_group);
769 return PTR_ERR(group);
770 }
Alex Williamson4a688102015-02-06 15:05:06 -0700771 } else {
772 /*
773 * A found vfio_group already holds a reference to the
774 * iommu_group. A created vfio_group keeps the reference.
775 */
776 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600777 }
778
779 device = vfio_group_get_device(group, dev);
780 if (device) {
781 WARN(1, "Device %s already exists on group %d\n",
782 dev_name(dev), iommu_group_id(iommu_group));
783 vfio_device_put(device);
784 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600785 return -EBUSY;
786 }
787
788 device = vfio_group_create_device(group, dev, ops, device_data);
789 if (IS_ERR(device)) {
790 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600791 return PTR_ERR(device);
792 }
793
794 /*
Alex Williamson4a688102015-02-06 15:05:06 -0700795 * Drop all but the vfio_device reference. The vfio_device holds
796 * a reference to the vfio_group, which holds a reference to the
797 * iommu_group.
Alex Williamsoncba33452012-07-31 08:16:22 -0600798 */
799 vfio_group_put(group);
800
801 return 0;
802}
803EXPORT_SYMBOL_GPL(vfio_add_group_dev);
804
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600805/**
Alex Williamson20f30012015-06-09 10:08:57 -0600806 * Get a reference to the vfio_device for a device. Even if the
807 * caller thinks they own the device, they could be racing with a
808 * release call path, so we can't trust drvdata for the shortcut.
809 * Go the long way around, from the iommu_group to the vfio_group
810 * to the vfio_device.
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600811 */
812struct vfio_device *vfio_device_get_from_dev(struct device *dev)
813{
Alex Williamson20f30012015-06-09 10:08:57 -0600814 struct iommu_group *iommu_group;
815 struct vfio_group *group;
816 struct vfio_device *device;
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600817
Alex Williamson20f30012015-06-09 10:08:57 -0600818 iommu_group = iommu_group_get(dev);
819 if (!iommu_group)
820 return NULL;
821
822 group = vfio_group_get_from_iommu(iommu_group);
823 iommu_group_put(iommu_group);
824 if (!group)
825 return NULL;
826
827 device = vfio_group_get_device(group, dev);
828 vfio_group_put(group);
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600829
830 return device;
831}
832EXPORT_SYMBOL_GPL(vfio_device_get_from_dev);
833
Alex Williamson4bc94d52015-07-24 15:14:04 -0600834static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
835 char *buf)
836{
Joerg Roedele324fc82015-11-04 13:53:26 +0100837 struct vfio_device *it, *device = NULL;
Alex Williamson4bc94d52015-07-24 15:14:04 -0600838
839 mutex_lock(&group->device_lock);
Joerg Roedele324fc82015-11-04 13:53:26 +0100840 list_for_each_entry(it, &group->device_list, group_next) {
841 if (!strcmp(dev_name(it->dev), buf)) {
842 device = it;
Alex Williamson4bc94d52015-07-24 15:14:04 -0600843 vfio_device_get(device);
844 break;
845 }
846 }
847 mutex_unlock(&group->device_lock);
848
849 return device;
850}
851
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600852/*
853 * Caller must hold a reference to the vfio_device
854 */
855void *vfio_device_data(struct vfio_device *device)
856{
857 return device->device_data;
858}
859EXPORT_SYMBOL_GPL(vfio_device_data);
860
Alex Williamsone014e942013-02-14 14:02:13 -0700861/* Given a referenced group, check if it contains the device */
862static bool vfio_dev_present(struct vfio_group *group, struct device *dev)
Alex Williamsoncba33452012-07-31 08:16:22 -0600863{
Alex Williamsoncba33452012-07-31 08:16:22 -0600864 struct vfio_device *device;
865
Alex Williamsoncba33452012-07-31 08:16:22 -0600866 device = vfio_group_get_device(group, dev);
Alex Williamsone014e942013-02-14 14:02:13 -0700867 if (!device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600868 return false;
Alex Williamsoncba33452012-07-31 08:16:22 -0600869
870 vfio_device_put(device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600871 return true;
872}
873
874/*
875 * Decrement the device reference count and wait for the device to be
876 * removed. Open file descriptors for the device... */
877void *vfio_del_group_dev(struct device *dev)
878{
879 struct vfio_device *device = dev_get_drvdata(dev);
880 struct vfio_group *group = device->group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600881 void *device_data = device->device_data;
Alex Williamson60720a02015-02-06 15:05:06 -0700882 struct vfio_unbound_dev *unbound;
Alex Williamson13060b62015-02-06 15:05:07 -0700883 unsigned int i = 0;
Alex Williamsondb7d4d72015-05-01 16:31:41 -0600884 long ret;
885 bool interrupted = false;
Alex Williamsoncba33452012-07-31 08:16:22 -0600886
Alex Williamsone014e942013-02-14 14:02:13 -0700887 /*
888 * The group exists so long as we have a device reference. Get
889 * a group reference and use it to scan for the device going away.
890 */
891 vfio_group_get(group);
892
Alex Williamson60720a02015-02-06 15:05:06 -0700893 /*
894 * When the device is removed from the group, the group suddenly
895 * becomes non-viable; the device has a driver (until the unbind
896 * completes), but it's not present in the group. This is bad news
897 * for any external users that need to re-acquire a group reference
898 * in order to match and release their existing reference. To
899 * solve this, we track such devices on the unbound_list to bridge
900 * the gap until they're fully unbound.
901 */
902 unbound = kzalloc(sizeof(*unbound), GFP_KERNEL);
903 if (unbound) {
904 unbound->dev = dev;
905 mutex_lock(&group->unbound_lock);
906 list_add(&unbound->unbound_next, &group->unbound_list);
907 mutex_unlock(&group->unbound_lock);
908 }
909 WARN_ON(!unbound);
910
Alex Williamsoncba33452012-07-31 08:16:22 -0600911 vfio_device_put(device);
912
Alex Williamson13060b62015-02-06 15:05:07 -0700913 /*
914 * If the device is still present in the group after the above
915 * 'put', then it is in use and we need to request it from the
916 * bus driver. The driver may in turn need to request the
917 * device from the user. We send the request on an arbitrary
918 * interval with counter to allow the driver to take escalating
919 * measures to release the device if it has the ability to do so.
920 */
921 do {
922 device = vfio_group_get_device(group, dev);
923 if (!device)
924 break;
925
926 if (device->ops->request)
927 device->ops->request(device_data, i++);
928
929 vfio_device_put(device);
930
Alex Williamsondb7d4d72015-05-01 16:31:41 -0600931 if (interrupted) {
932 ret = wait_event_timeout(vfio.release_q,
933 !vfio_dev_present(group, dev), HZ * 10);
934 } else {
935 ret = wait_event_interruptible_timeout(vfio.release_q,
936 !vfio_dev_present(group, dev), HZ * 10);
937 if (ret == -ERESTARTSYS) {
938 interrupted = true;
939 dev_warn(dev,
940 "Device is currently in use, task"
941 " \"%s\" (%d) "
942 "blocked until device is released",
943 current->comm, task_pid_nr(current));
944 }
945 }
946 } while (ret <= 0);
Alex Williamsone014e942013-02-14 14:02:13 -0700947
948 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600949
Alex Williamsoncba33452012-07-31 08:16:22 -0600950 return device_data;
951}
952EXPORT_SYMBOL_GPL(vfio_del_group_dev);
953
954/**
955 * VFIO base fd, /dev/vfio/vfio
956 */
957static long vfio_ioctl_check_extension(struct vfio_container *container,
958 unsigned long arg)
959{
Alex Williamson0b43c082013-04-29 08:41:36 -0600960 struct vfio_iommu_driver *driver;
Alex Williamsoncba33452012-07-31 08:16:22 -0600961 long ret = 0;
962
Alex Williamson0b43c082013-04-29 08:41:36 -0600963 down_read(&container->group_lock);
964
965 driver = container->iommu_driver;
966
Alex Williamsoncba33452012-07-31 08:16:22 -0600967 switch (arg) {
968 /* No base extensions yet */
969 default:
970 /*
971 * If no driver is set, poll all registered drivers for
972 * extensions and return the first positive result. If
973 * a driver is already set, further queries will be passed
974 * only to that driver.
975 */
976 if (!driver) {
977 mutex_lock(&vfio.iommu_drivers_lock);
Alex Williamsonae5515d2015-12-04 08:38:42 -0700978 list_for_each_entry(driver, &vfio.iommu_drivers_list,
979 vfio_next) {
Alex Williamson03a76b62015-12-21 15:13:33 -0700980
981#ifdef CONFIG_VFIO_NOIOMMU
982 if (!list_empty(&container->group_list) &&
983 (container->noiommu !=
984 (driver->ops == &vfio_noiommu_ops)))
985 continue;
986#endif
987
Alex Williamsoncba33452012-07-31 08:16:22 -0600988 if (!try_module_get(driver->ops->owner))
989 continue;
990
991 ret = driver->ops->ioctl(NULL,
992 VFIO_CHECK_EXTENSION,
993 arg);
994 module_put(driver->ops->owner);
995 if (ret > 0)
996 break;
997 }
998 mutex_unlock(&vfio.iommu_drivers_lock);
999 } else
1000 ret = driver->ops->ioctl(container->iommu_data,
1001 VFIO_CHECK_EXTENSION, arg);
1002 }
1003
Alex Williamson0b43c082013-04-29 08:41:36 -06001004 up_read(&container->group_lock);
1005
Alex Williamsoncba33452012-07-31 08:16:22 -06001006 return ret;
1007}
1008
Alex Williamson9587f442013-04-25 16:12:38 -06001009/* hold write lock on container->group_lock */
Alex Williamsoncba33452012-07-31 08:16:22 -06001010static int __vfio_container_attach_groups(struct vfio_container *container,
1011 struct vfio_iommu_driver *driver,
1012 void *data)
1013{
1014 struct vfio_group *group;
1015 int ret = -ENODEV;
1016
1017 list_for_each_entry(group, &container->group_list, container_next) {
1018 ret = driver->ops->attach_group(data, group->iommu_group);
1019 if (ret)
1020 goto unwind;
1021 }
1022
1023 return ret;
1024
1025unwind:
1026 list_for_each_entry_continue_reverse(group, &container->group_list,
1027 container_next) {
1028 driver->ops->detach_group(data, group->iommu_group);
1029 }
1030
1031 return ret;
1032}
1033
1034static long vfio_ioctl_set_iommu(struct vfio_container *container,
1035 unsigned long arg)
1036{
1037 struct vfio_iommu_driver *driver;
1038 long ret = -ENODEV;
1039
Alex Williamson9587f442013-04-25 16:12:38 -06001040 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001041
1042 /*
1043 * The container is designed to be an unprivileged interface while
1044 * the group can be assigned to specific users. Therefore, only by
1045 * adding a group to a container does the user get the privilege of
1046 * enabling the iommu, which may allocate finite resources. There
1047 * is no unset_iommu, but by removing all the groups from a container,
1048 * the container is deprivileged and returns to an unset state.
1049 */
1050 if (list_empty(&container->group_list) || container->iommu_driver) {
Alex Williamson9587f442013-04-25 16:12:38 -06001051 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001052 return -EINVAL;
1053 }
1054
1055 mutex_lock(&vfio.iommu_drivers_lock);
Alex Williamsonae5515d2015-12-04 08:38:42 -07001056 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
Alex Williamsoncba33452012-07-31 08:16:22 -06001057 void *data;
1058
Alex Williamson03a76b62015-12-21 15:13:33 -07001059#ifdef CONFIG_VFIO_NOIOMMU
1060 /*
1061 * Only noiommu containers can use vfio-noiommu and noiommu
1062 * containers can only use vfio-noiommu.
1063 */
1064 if (container->noiommu != (driver->ops == &vfio_noiommu_ops))
1065 continue;
1066#endif
1067
Alex Williamsoncba33452012-07-31 08:16:22 -06001068 if (!try_module_get(driver->ops->owner))
1069 continue;
1070
1071 /*
1072 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
1073 * so test which iommu driver reported support for this
1074 * extension and call open on them. We also pass them the
1075 * magic, allowing a single driver to support multiple
1076 * interfaces if they'd like.
1077 */
1078 if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
1079 module_put(driver->ops->owner);
1080 continue;
1081 }
1082
Alex Williamsoncba33452012-07-31 08:16:22 -06001083 data = driver->ops->open(arg);
1084 if (IS_ERR(data)) {
1085 ret = PTR_ERR(data);
1086 module_put(driver->ops->owner);
Alex Williamson7c435b42016-02-22 16:02:30 -07001087 continue;
Alex Williamsoncba33452012-07-31 08:16:22 -06001088 }
1089
1090 ret = __vfio_container_attach_groups(container, driver, data);
Alex Williamson7c435b42016-02-22 16:02:30 -07001091 if (ret) {
Alex Williamsoncba33452012-07-31 08:16:22 -06001092 driver->ops->release(data);
1093 module_put(driver->ops->owner);
Alex Williamson7c435b42016-02-22 16:02:30 -07001094 continue;
Alex Williamsoncba33452012-07-31 08:16:22 -06001095 }
1096
Alex Williamson7c435b42016-02-22 16:02:30 -07001097 container->iommu_driver = driver;
1098 container->iommu_data = data;
1099 break;
Alex Williamsoncba33452012-07-31 08:16:22 -06001100 }
1101
1102 mutex_unlock(&vfio.iommu_drivers_lock);
Alex Williamson9587f442013-04-25 16:12:38 -06001103 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001104
1105 return ret;
1106}
1107
1108static long vfio_fops_unl_ioctl(struct file *filep,
1109 unsigned int cmd, unsigned long arg)
1110{
1111 struct vfio_container *container = filep->private_data;
1112 struct vfio_iommu_driver *driver;
1113 void *data;
1114 long ret = -EINVAL;
1115
1116 if (!container)
1117 return ret;
1118
Alex Williamsoncba33452012-07-31 08:16:22 -06001119 switch (cmd) {
1120 case VFIO_GET_API_VERSION:
1121 ret = VFIO_API_VERSION;
1122 break;
1123 case VFIO_CHECK_EXTENSION:
1124 ret = vfio_ioctl_check_extension(container, arg);
1125 break;
1126 case VFIO_SET_IOMMU:
1127 ret = vfio_ioctl_set_iommu(container, arg);
1128 break;
1129 default:
Alex Williamson0b43c082013-04-29 08:41:36 -06001130 down_read(&container->group_lock);
1131
1132 driver = container->iommu_driver;
1133 data = container->iommu_data;
1134
Alex Williamsoncba33452012-07-31 08:16:22 -06001135 if (driver) /* passthrough all unrecognized ioctls */
1136 ret = driver->ops->ioctl(data, cmd, arg);
Alex Williamson0b43c082013-04-29 08:41:36 -06001137
1138 up_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001139 }
1140
1141 return ret;
1142}
1143
1144#ifdef CONFIG_COMPAT
1145static long vfio_fops_compat_ioctl(struct file *filep,
1146 unsigned int cmd, unsigned long arg)
1147{
1148 arg = (unsigned long)compat_ptr(arg);
1149 return vfio_fops_unl_ioctl(filep, cmd, arg);
1150}
1151#endif /* CONFIG_COMPAT */
1152
1153static int vfio_fops_open(struct inode *inode, struct file *filep)
1154{
1155 struct vfio_container *container;
1156
1157 container = kzalloc(sizeof(*container), GFP_KERNEL);
1158 if (!container)
1159 return -ENOMEM;
1160
1161 INIT_LIST_HEAD(&container->group_list);
Alex Williamson9587f442013-04-25 16:12:38 -06001162 init_rwsem(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001163 kref_init(&container->kref);
1164
1165 filep->private_data = container;
1166
1167 return 0;
1168}
1169
1170static int vfio_fops_release(struct inode *inode, struct file *filep)
1171{
1172 struct vfio_container *container = filep->private_data;
1173
1174 filep->private_data = NULL;
1175
1176 vfio_container_put(container);
1177
1178 return 0;
1179}
1180
1181/*
1182 * Once an iommu driver is set, we optionally pass read/write/mmap
1183 * on to the driver, allowing management interfaces beyond ioctl.
1184 */
1185static ssize_t vfio_fops_read(struct file *filep, char __user *buf,
1186 size_t count, loff_t *ppos)
1187{
1188 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001189 struct vfio_iommu_driver *driver;
1190 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001191
Alex Williamson0b43c082013-04-29 08:41:36 -06001192 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001193
Alex Williamson0b43c082013-04-29 08:41:36 -06001194 driver = container->iommu_driver;
1195 if (likely(driver && driver->ops->read))
1196 ret = driver->ops->read(container->iommu_data,
1197 buf, count, ppos);
1198
1199 up_read(&container->group_lock);
1200
1201 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001202}
1203
1204static ssize_t vfio_fops_write(struct file *filep, const char __user *buf,
1205 size_t count, loff_t *ppos)
1206{
1207 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001208 struct vfio_iommu_driver *driver;
1209 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001210
Alex Williamson0b43c082013-04-29 08:41:36 -06001211 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001212
Alex Williamson0b43c082013-04-29 08:41:36 -06001213 driver = container->iommu_driver;
1214 if (likely(driver && driver->ops->write))
1215 ret = driver->ops->write(container->iommu_data,
1216 buf, count, ppos);
1217
1218 up_read(&container->group_lock);
1219
1220 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001221}
1222
1223static int vfio_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1224{
1225 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001226 struct vfio_iommu_driver *driver;
1227 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001228
Alex Williamson0b43c082013-04-29 08:41:36 -06001229 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001230
Alex Williamson0b43c082013-04-29 08:41:36 -06001231 driver = container->iommu_driver;
1232 if (likely(driver && driver->ops->mmap))
1233 ret = driver->ops->mmap(container->iommu_data, vma);
1234
1235 up_read(&container->group_lock);
1236
1237 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001238}
1239
1240static const struct file_operations vfio_fops = {
1241 .owner = THIS_MODULE,
1242 .open = vfio_fops_open,
1243 .release = vfio_fops_release,
1244 .read = vfio_fops_read,
1245 .write = vfio_fops_write,
1246 .unlocked_ioctl = vfio_fops_unl_ioctl,
1247#ifdef CONFIG_COMPAT
1248 .compat_ioctl = vfio_fops_compat_ioctl,
1249#endif
1250 .mmap = vfio_fops_mmap,
1251};
1252
1253/**
1254 * VFIO Group fd, /dev/vfio/$GROUP
1255 */
1256static void __vfio_group_unset_container(struct vfio_group *group)
1257{
1258 struct vfio_container *container = group->container;
1259 struct vfio_iommu_driver *driver;
1260
Alex Williamson9587f442013-04-25 16:12:38 -06001261 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001262
1263 driver = container->iommu_driver;
1264 if (driver)
1265 driver->ops->detach_group(container->iommu_data,
1266 group->iommu_group);
1267
1268 group->container = NULL;
1269 list_del(&group->container_next);
1270
1271 /* Detaching the last group deprivileges a container, remove iommu */
1272 if (driver && list_empty(&container->group_list)) {
1273 driver->ops->release(container->iommu_data);
1274 module_put(driver->ops->owner);
1275 container->iommu_driver = NULL;
1276 container->iommu_data = NULL;
1277 }
1278
Alex Williamson9587f442013-04-25 16:12:38 -06001279 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001280
1281 vfio_container_put(container);
1282}
1283
1284/*
1285 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
1286 * if there was no container to unset. Since the ioctl is called on
1287 * the group, we know that still exists, therefore the only valid
1288 * transition here is 1->0.
1289 */
1290static int vfio_group_unset_container(struct vfio_group *group)
1291{
1292 int users = atomic_cmpxchg(&group->container_users, 1, 0);
1293
1294 if (!users)
1295 return -EINVAL;
1296 if (users != 1)
1297 return -EBUSY;
1298
1299 __vfio_group_unset_container(group);
1300
1301 return 0;
1302}
1303
1304/*
1305 * When removing container users, anything that removes the last user
1306 * implicitly removes the group from the container. That is, if the
1307 * group file descriptor is closed, as well as any device file descriptors,
1308 * the group is free.
1309 */
1310static void vfio_group_try_dissolve_container(struct vfio_group *group)
1311{
1312 if (0 == atomic_dec_if_positive(&group->container_users))
1313 __vfio_group_unset_container(group);
1314}
1315
1316static int vfio_group_set_container(struct vfio_group *group, int container_fd)
1317{
Al Viro2903ff02012-08-28 12:52:22 -04001318 struct fd f;
Alex Williamsoncba33452012-07-31 08:16:22 -06001319 struct vfio_container *container;
1320 struct vfio_iommu_driver *driver;
Al Viro2903ff02012-08-28 12:52:22 -04001321 int ret = 0;
Alex Williamsoncba33452012-07-31 08:16:22 -06001322
1323 if (atomic_read(&group->container_users))
1324 return -EINVAL;
1325
Alex Williamson03a76b62015-12-21 15:13:33 -07001326 if (group->noiommu && !capable(CAP_SYS_RAWIO))
1327 return -EPERM;
1328
Al Viro2903ff02012-08-28 12:52:22 -04001329 f = fdget(container_fd);
1330 if (!f.file)
Alex Williamsoncba33452012-07-31 08:16:22 -06001331 return -EBADF;
1332
1333 /* Sanity check, is this really our fd? */
Al Viro2903ff02012-08-28 12:52:22 -04001334 if (f.file->f_op != &vfio_fops) {
1335 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001336 return -EINVAL;
1337 }
1338
Al Viro2903ff02012-08-28 12:52:22 -04001339 container = f.file->private_data;
Alex Williamsoncba33452012-07-31 08:16:22 -06001340 WARN_ON(!container); /* fget ensures we don't race vfio_release */
1341
Alex Williamson9587f442013-04-25 16:12:38 -06001342 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001343
Alex Williamson03a76b62015-12-21 15:13:33 -07001344 /* Real groups and fake groups cannot mix */
1345 if (!list_empty(&container->group_list) &&
1346 container->noiommu != group->noiommu) {
1347 ret = -EPERM;
1348 goto unlock_out;
1349 }
1350
Alex Williamsoncba33452012-07-31 08:16:22 -06001351 driver = container->iommu_driver;
1352 if (driver) {
1353 ret = driver->ops->attach_group(container->iommu_data,
1354 group->iommu_group);
1355 if (ret)
1356 goto unlock_out;
1357 }
1358
1359 group->container = container;
Alex Williamson03a76b62015-12-21 15:13:33 -07001360 container->noiommu = group->noiommu;
Alex Williamsoncba33452012-07-31 08:16:22 -06001361 list_add(&group->container_next, &container->group_list);
1362
1363 /* Get a reference on the container and mark a user within the group */
1364 vfio_container_get(container);
1365 atomic_inc(&group->container_users);
1366
1367unlock_out:
Alex Williamson9587f442013-04-25 16:12:38 -06001368 up_write(&container->group_lock);
Al Viro2903ff02012-08-28 12:52:22 -04001369 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001370 return ret;
1371}
1372
1373static bool vfio_group_viable(struct vfio_group *group)
1374{
1375 return (iommu_group_for_each_dev(group->iommu_group,
1376 group, vfio_dev_viable) == 0);
1377}
1378
1379static const struct file_operations vfio_device_fops;
1380
1381static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
1382{
1383 struct vfio_device *device;
1384 struct file *filep;
Alex Williamson4bc94d52015-07-24 15:14:04 -06001385 int ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001386
1387 if (0 == atomic_read(&group->container_users) ||
1388 !group->container->iommu_driver || !vfio_group_viable(group))
1389 return -EINVAL;
1390
Alex Williamson03a76b62015-12-21 15:13:33 -07001391 if (group->noiommu && !capable(CAP_SYS_RAWIO))
1392 return -EPERM;
1393
Alex Williamson4bc94d52015-07-24 15:14:04 -06001394 device = vfio_device_get_from_name(group, buf);
1395 if (!device)
1396 return -ENODEV;
Alex Williamsoncba33452012-07-31 08:16:22 -06001397
Alex Williamson4bc94d52015-07-24 15:14:04 -06001398 ret = device->ops->open(device->device_data);
1399 if (ret) {
1400 vfio_device_put(device);
1401 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001402 }
Alex Williamson4bc94d52015-07-24 15:14:04 -06001403
1404 /*
1405 * We can't use anon_inode_getfd() because we need to modify
1406 * the f_mode flags directly to allow more than just ioctls
1407 */
1408 ret = get_unused_fd_flags(O_CLOEXEC);
1409 if (ret < 0) {
1410 device->ops->release(device->device_data);
1411 vfio_device_put(device);
1412 return ret;
1413 }
1414
1415 filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
1416 device, O_RDWR);
1417 if (IS_ERR(filep)) {
1418 put_unused_fd(ret);
1419 ret = PTR_ERR(filep);
1420 device->ops->release(device->device_data);
1421 vfio_device_put(device);
1422 return ret;
1423 }
1424
1425 /*
1426 * TODO: add an anon_inode interface to do this.
1427 * Appears to be missing by lack of need rather than
1428 * explicitly prevented. Now there's need.
1429 */
1430 filep->f_mode |= (FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
1431
1432 atomic_inc(&group->container_users);
1433
1434 fd_install(ret, filep);
Alex Williamsoncba33452012-07-31 08:16:22 -06001435
Alex Williamson03a76b62015-12-21 15:13:33 -07001436 if (group->noiommu)
1437 dev_warn(device->dev, "vfio-noiommu device opened by user "
1438 "(%s:%d)\n", current->comm, task_pid_nr(current));
1439
Alex Williamsoncba33452012-07-31 08:16:22 -06001440 return ret;
1441}
1442
1443static long vfio_group_fops_unl_ioctl(struct file *filep,
1444 unsigned int cmd, unsigned long arg)
1445{
1446 struct vfio_group *group = filep->private_data;
1447 long ret = -ENOTTY;
1448
1449 switch (cmd) {
1450 case VFIO_GROUP_GET_STATUS:
1451 {
1452 struct vfio_group_status status;
1453 unsigned long minsz;
1454
1455 minsz = offsetofend(struct vfio_group_status, flags);
1456
1457 if (copy_from_user(&status, (void __user *)arg, minsz))
1458 return -EFAULT;
1459
1460 if (status.argsz < minsz)
1461 return -EINVAL;
1462
1463 status.flags = 0;
1464
1465 if (vfio_group_viable(group))
1466 status.flags |= VFIO_GROUP_FLAGS_VIABLE;
1467
1468 if (group->container)
1469 status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET;
1470
1471 if (copy_to_user((void __user *)arg, &status, minsz))
1472 return -EFAULT;
1473
1474 ret = 0;
1475 break;
1476 }
1477 case VFIO_GROUP_SET_CONTAINER:
1478 {
1479 int fd;
1480
1481 if (get_user(fd, (int __user *)arg))
1482 return -EFAULT;
1483
1484 if (fd < 0)
1485 return -EINVAL;
1486
1487 ret = vfio_group_set_container(group, fd);
1488 break;
1489 }
1490 case VFIO_GROUP_UNSET_CONTAINER:
1491 ret = vfio_group_unset_container(group);
1492 break;
1493 case VFIO_GROUP_GET_DEVICE_FD:
1494 {
1495 char *buf;
1496
1497 buf = strndup_user((const char __user *)arg, PAGE_SIZE);
1498 if (IS_ERR(buf))
1499 return PTR_ERR(buf);
1500
1501 ret = vfio_group_get_device_fd(group, buf);
1502 kfree(buf);
1503 break;
1504 }
1505 }
1506
1507 return ret;
1508}
1509
1510#ifdef CONFIG_COMPAT
1511static long vfio_group_fops_compat_ioctl(struct file *filep,
1512 unsigned int cmd, unsigned long arg)
1513{
1514 arg = (unsigned long)compat_ptr(arg);
1515 return vfio_group_fops_unl_ioctl(filep, cmd, arg);
1516}
1517#endif /* CONFIG_COMPAT */
1518
1519static int vfio_group_fops_open(struct inode *inode, struct file *filep)
1520{
1521 struct vfio_group *group;
Alex Williamson6d6768c2013-06-25 16:06:54 -06001522 int opened;
Alex Williamsoncba33452012-07-31 08:16:22 -06001523
1524 group = vfio_group_get_from_minor(iminor(inode));
1525 if (!group)
1526 return -ENODEV;
1527
Alex Williamson03a76b62015-12-21 15:13:33 -07001528 if (group->noiommu && !capable(CAP_SYS_RAWIO)) {
1529 vfio_group_put(group);
1530 return -EPERM;
1531 }
1532
Alex Williamson6d6768c2013-06-25 16:06:54 -06001533 /* Do we need multiple instances of the group open? Seems not. */
1534 opened = atomic_cmpxchg(&group->opened, 0, 1);
1535 if (opened) {
1536 vfio_group_put(group);
1537 return -EBUSY;
1538 }
1539
1540 /* Is something still in use from a previous open? */
Alex Williamsoncba33452012-07-31 08:16:22 -06001541 if (group->container) {
Alex Williamson6d6768c2013-06-25 16:06:54 -06001542 atomic_dec(&group->opened);
Alex Williamsoncba33452012-07-31 08:16:22 -06001543 vfio_group_put(group);
1544 return -EBUSY;
1545 }
1546
1547 filep->private_data = group;
1548
1549 return 0;
1550}
1551
1552static int vfio_group_fops_release(struct inode *inode, struct file *filep)
1553{
1554 struct vfio_group *group = filep->private_data;
1555
1556 filep->private_data = NULL;
1557
1558 vfio_group_try_dissolve_container(group);
1559
Alex Williamson6d6768c2013-06-25 16:06:54 -06001560 atomic_dec(&group->opened);
1561
Alex Williamsoncba33452012-07-31 08:16:22 -06001562 vfio_group_put(group);
1563
1564 return 0;
1565}
1566
1567static const struct file_operations vfio_group_fops = {
1568 .owner = THIS_MODULE,
1569 .unlocked_ioctl = vfio_group_fops_unl_ioctl,
1570#ifdef CONFIG_COMPAT
1571 .compat_ioctl = vfio_group_fops_compat_ioctl,
1572#endif
1573 .open = vfio_group_fops_open,
1574 .release = vfio_group_fops_release,
1575};
1576
1577/**
1578 * VFIO Device fd
1579 */
1580static int vfio_device_fops_release(struct inode *inode, struct file *filep)
1581{
1582 struct vfio_device *device = filep->private_data;
1583
1584 device->ops->release(device->device_data);
1585
1586 vfio_group_try_dissolve_container(device->group);
1587
1588 vfio_device_put(device);
1589
1590 return 0;
1591}
1592
1593static long vfio_device_fops_unl_ioctl(struct file *filep,
1594 unsigned int cmd, unsigned long arg)
1595{
1596 struct vfio_device *device = filep->private_data;
1597
1598 if (unlikely(!device->ops->ioctl))
1599 return -EINVAL;
1600
1601 return device->ops->ioctl(device->device_data, cmd, arg);
1602}
1603
1604static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1605 size_t count, loff_t *ppos)
1606{
1607 struct vfio_device *device = filep->private_data;
1608
1609 if (unlikely(!device->ops->read))
1610 return -EINVAL;
1611
1612 return device->ops->read(device->device_data, buf, count, ppos);
1613}
1614
1615static ssize_t vfio_device_fops_write(struct file *filep,
1616 const char __user *buf,
1617 size_t count, loff_t *ppos)
1618{
1619 struct vfio_device *device = filep->private_data;
1620
1621 if (unlikely(!device->ops->write))
1622 return -EINVAL;
1623
1624 return device->ops->write(device->device_data, buf, count, ppos);
1625}
1626
1627static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1628{
1629 struct vfio_device *device = filep->private_data;
1630
1631 if (unlikely(!device->ops->mmap))
1632 return -EINVAL;
1633
1634 return device->ops->mmap(device->device_data, vma);
1635}
1636
1637#ifdef CONFIG_COMPAT
1638static long vfio_device_fops_compat_ioctl(struct file *filep,
1639 unsigned int cmd, unsigned long arg)
1640{
1641 arg = (unsigned long)compat_ptr(arg);
1642 return vfio_device_fops_unl_ioctl(filep, cmd, arg);
1643}
1644#endif /* CONFIG_COMPAT */
1645
1646static const struct file_operations vfio_device_fops = {
1647 .owner = THIS_MODULE,
1648 .release = vfio_device_fops_release,
1649 .read = vfio_device_fops_read,
1650 .write = vfio_device_fops_write,
1651 .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1652#ifdef CONFIG_COMPAT
1653 .compat_ioctl = vfio_device_fops_compat_ioctl,
1654#endif
1655 .mmap = vfio_device_fops_mmap,
1656};
1657
1658/**
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001659 * External user API, exported by symbols to be linked dynamically.
1660 *
1661 * The protocol includes:
1662 * 1. do normal VFIO init operation:
1663 * - opening a new container;
1664 * - attaching group(s) to it;
1665 * - setting an IOMMU driver for a container.
1666 * When IOMMU is set for a container, all groups in it are
1667 * considered ready to use by an external user.
1668 *
1669 * 2. User space passes a group fd to an external user.
1670 * The external user calls vfio_group_get_external_user()
1671 * to verify that:
1672 * - the group is initialized;
1673 * - IOMMU is set for it.
1674 * If both checks passed, vfio_group_get_external_user()
1675 * increments the container user counter to prevent
1676 * the VFIO group from disposal before KVM exits.
1677 *
1678 * 3. The external user calls vfio_external_user_iommu_id()
1679 * to know an IOMMU ID.
1680 *
1681 * 4. When the external KVM finishes, it calls
1682 * vfio_group_put_external_user() to release the VFIO group.
1683 * This call decrements the container user counter.
1684 */
1685struct vfio_group *vfio_group_get_external_user(struct file *filep)
1686{
1687 struct vfio_group *group = filep->private_data;
1688
1689 if (filep->f_op != &vfio_group_fops)
1690 return ERR_PTR(-EINVAL);
1691
1692 if (!atomic_inc_not_zero(&group->container_users))
1693 return ERR_PTR(-EINVAL);
1694
Alex Williamson03a76b62015-12-21 15:13:33 -07001695 if (group->noiommu) {
1696 atomic_dec(&group->container_users);
1697 return ERR_PTR(-EPERM);
1698 }
1699
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001700 if (!group->container->iommu_driver ||
1701 !vfio_group_viable(group)) {
1702 atomic_dec(&group->container_users);
1703 return ERR_PTR(-EINVAL);
1704 }
1705
1706 vfio_group_get(group);
1707
1708 return group;
1709}
1710EXPORT_SYMBOL_GPL(vfio_group_get_external_user);
1711
1712void vfio_group_put_external_user(struct vfio_group *group)
1713{
1714 vfio_group_put(group);
1715 vfio_group_try_dissolve_container(group);
1716}
1717EXPORT_SYMBOL_GPL(vfio_group_put_external_user);
1718
1719int vfio_external_user_iommu_id(struct vfio_group *group)
1720{
1721 return iommu_group_id(group->iommu_group);
1722}
1723EXPORT_SYMBOL_GPL(vfio_external_user_iommu_id);
1724
Alex Williamson88d7ab82014-02-26 11:38:39 -07001725long vfio_external_check_extension(struct vfio_group *group, unsigned long arg)
1726{
1727 return vfio_ioctl_check_extension(group->container, arg);
1728}
1729EXPORT_SYMBOL_GPL(vfio_external_check_extension);
1730
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001731/**
Alex Williamsond7a8d5e2016-02-22 16:02:33 -07001732 * Sub-module support
1733 */
1734/*
1735 * Helper for managing a buffer of info chain capabilities, allocate or
1736 * reallocate a buffer with additional @size, filling in @id and @version
1737 * of the capability. A pointer to the new capability is returned.
1738 *
1739 * NB. The chain is based at the head of the buffer, so new entries are
1740 * added to the tail, vfio_info_cap_shift() should be called to fixup the
1741 * next offsets prior to copying to the user buffer.
1742 */
1743struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1744 size_t size, u16 id, u16 version)
1745{
1746 void *buf;
1747 struct vfio_info_cap_header *header, *tmp;
1748
1749 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1750 if (!buf) {
1751 kfree(caps->buf);
1752 caps->size = 0;
1753 return ERR_PTR(-ENOMEM);
1754 }
1755
1756 caps->buf = buf;
1757 header = buf + caps->size;
1758
1759 /* Eventually copied to user buffer, zero */
1760 memset(header, 0, size);
1761
1762 header->id = id;
1763 header->version = version;
1764
1765 /* Add to the end of the capability chain */
1766 for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next)
1767 ; /* nothing */
1768
1769 tmp->next = caps->size;
1770 caps->size += size;
1771
1772 return header;
1773}
1774EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1775
1776void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1777{
1778 struct vfio_info_cap_header *tmp;
1779
1780 for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next - offset)
1781 tmp->next += offset;
1782}
1783EXPORT_SYMBOL_GPL(vfio_info_cap_shift);
1784
1785/**
Alex Williamsoncba33452012-07-31 08:16:22 -06001786 * Module/class support
1787 */
1788static char *vfio_devnode(struct device *dev, umode_t *mode)
1789{
1790 return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
1791}
1792
Alex Williamsond1099902013-12-19 10:17:13 -07001793static struct miscdevice vfio_dev = {
1794 .minor = VFIO_MINOR,
1795 .name = "vfio",
1796 .fops = &vfio_fops,
1797 .nodename = "vfio/vfio",
1798 .mode = S_IRUGO | S_IWUGO,
1799};
1800
Alex Williamsoncba33452012-07-31 08:16:22 -06001801static int __init vfio_init(void)
1802{
1803 int ret;
1804
1805 idr_init(&vfio.group_idr);
1806 mutex_init(&vfio.group_lock);
1807 mutex_init(&vfio.iommu_drivers_lock);
1808 INIT_LIST_HEAD(&vfio.group_list);
1809 INIT_LIST_HEAD(&vfio.iommu_drivers_list);
1810 init_waitqueue_head(&vfio.release_q);
1811
Alex Williamsond1099902013-12-19 10:17:13 -07001812 ret = misc_register(&vfio_dev);
1813 if (ret) {
1814 pr_err("vfio: misc device register failed\n");
1815 return ret;
1816 }
1817
1818 /* /dev/vfio/$GROUP */
Alex Williamsoncba33452012-07-31 08:16:22 -06001819 vfio.class = class_create(THIS_MODULE, "vfio");
1820 if (IS_ERR(vfio.class)) {
1821 ret = PTR_ERR(vfio.class);
1822 goto err_class;
1823 }
1824
1825 vfio.class->devnode = vfio_devnode;
1826
Alex Williamsond1099902013-12-19 10:17:13 -07001827 ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK, "vfio");
Alex Williamsoncba33452012-07-31 08:16:22 -06001828 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001829 goto err_alloc_chrdev;
Alex Williamsoncba33452012-07-31 08:16:22 -06001830
Alex Williamsoncba33452012-07-31 08:16:22 -06001831 cdev_init(&vfio.group_cdev, &vfio_group_fops);
Alex Williamsond1099902013-12-19 10:17:13 -07001832 ret = cdev_add(&vfio.group_cdev, vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001833 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001834 goto err_cdev_add;
Alex Williamsoncba33452012-07-31 08:16:22 -06001835
1836 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1837
Alex Williamson73fa0d12012-07-31 08:16:23 -06001838 /*
1839 * Attempt to load known iommu-drivers. This gives us a working
1840 * environment without the user needing to explicitly load iommu
1841 * drivers.
1842 */
1843 request_module_nowait("vfio_iommu_type1");
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001844 request_module_nowait("vfio_iommu_spapr_tce");
Alex Williamson73fa0d12012-07-31 08:16:23 -06001845
Alex Williamson03a76b62015-12-21 15:13:33 -07001846#ifdef CONFIG_VFIO_NOIOMMU
1847 vfio_register_iommu_driver(&vfio_noiommu_ops);
1848#endif
Alex Williamsoncba33452012-07-31 08:16:22 -06001849 return 0;
1850
Alex Williamsond1099902013-12-19 10:17:13 -07001851err_cdev_add:
1852 unregister_chrdev_region(vfio.group_devt, MINORMASK);
1853err_alloc_chrdev:
Alex Williamsoncba33452012-07-31 08:16:22 -06001854 class_destroy(vfio.class);
1855 vfio.class = NULL;
1856err_class:
Alex Williamsond1099902013-12-19 10:17:13 -07001857 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001858 return ret;
1859}
1860
1861static void __exit vfio_cleanup(void)
1862{
1863 WARN_ON(!list_empty(&vfio.group_list));
1864
Alex Williamson03a76b62015-12-21 15:13:33 -07001865#ifdef CONFIG_VFIO_NOIOMMU
1866 vfio_unregister_iommu_driver(&vfio_noiommu_ops);
1867#endif
Alex Williamsoncba33452012-07-31 08:16:22 -06001868 idr_destroy(&vfio.group_idr);
1869 cdev_del(&vfio.group_cdev);
Alex Williamsond1099902013-12-19 10:17:13 -07001870 unregister_chrdev_region(vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001871 class_destroy(vfio.class);
1872 vfio.class = NULL;
Alex Williamsond1099902013-12-19 10:17:13 -07001873 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001874}
1875
1876module_init(vfio_init);
1877module_exit(vfio_cleanup);
1878
1879MODULE_VERSION(DRIVER_VERSION);
1880MODULE_LICENSE("GPL v2");
1881MODULE_AUTHOR(DRIVER_AUTHOR);
1882MODULE_DESCRIPTION(DRIVER_DESC);
Alex Williamsond1099902013-12-19 10:17:13 -07001883MODULE_ALIAS_MISCDEV(VFIO_MINOR);
1884MODULE_ALIAS("devname:vfio/vfio");