blob: 0d336625ac7113b0e0cc10d4a9d00283cc673766 [file] [log] [blame]
Alex Williamsoncba33452012-07-31 08:16:22 -06001/*
2 * VFIO core
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
14 */
15
16#include <linux/cdev.h>
17#include <linux/compat.h>
18#include <linux/device.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <linux/idr.h>
23#include <linux/iommu.h>
24#include <linux/list.h>
Alex Williamsond1099902013-12-19 10:17:13 -070025#include <linux/miscdevice.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060026#include <linux/module.h>
27#include <linux/mutex.h>
Alex Williamson9587f442013-04-25 16:12:38 -060028#include <linux/rwsem.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060029#include <linux/sched.h>
30#include <linux/slab.h>
Alex Williamson664e9382013-04-30 15:42:28 -060031#include <linux/stat.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060032#include <linux/string.h>
33#include <linux/uaccess.h>
34#include <linux/vfio.h>
35#include <linux/wait.h>
36
37#define DRIVER_VERSION "0.3"
38#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
39#define DRIVER_DESC "VFIO - User Level meta-driver"
40
41static struct vfio {
42 struct class *class;
43 struct list_head iommu_drivers_list;
44 struct mutex iommu_drivers_lock;
45 struct list_head group_list;
46 struct idr group_idr;
47 struct mutex group_lock;
48 struct cdev group_cdev;
Alex Williamsond1099902013-12-19 10:17:13 -070049 dev_t group_devt;
Alex Williamsoncba33452012-07-31 08:16:22 -060050 wait_queue_head_t release_q;
51} vfio;
52
53struct vfio_iommu_driver {
54 const struct vfio_iommu_driver_ops *ops;
55 struct list_head vfio_next;
56};
57
58struct vfio_container {
59 struct kref kref;
60 struct list_head group_list;
Alex Williamson9587f442013-04-25 16:12:38 -060061 struct rw_semaphore group_lock;
Alex Williamsoncba33452012-07-31 08:16:22 -060062 struct vfio_iommu_driver *iommu_driver;
63 void *iommu_data;
64};
65
Alex Williamson60720a02015-02-06 15:05:06 -070066struct vfio_unbound_dev {
67 struct device *dev;
68 struct list_head unbound_next;
69};
70
Alex Williamsoncba33452012-07-31 08:16:22 -060071struct vfio_group {
72 struct kref kref;
73 int minor;
74 atomic_t container_users;
75 struct iommu_group *iommu_group;
76 struct vfio_container *container;
77 struct list_head device_list;
78 struct mutex device_lock;
79 struct device *dev;
80 struct notifier_block nb;
81 struct list_head vfio_next;
82 struct list_head container_next;
Alex Williamson60720a02015-02-06 15:05:06 -070083 struct list_head unbound_list;
84 struct mutex unbound_lock;
Alex Williamson6d6768c2013-06-25 16:06:54 -060085 atomic_t opened;
Alex Williamsoncba33452012-07-31 08:16:22 -060086};
87
88struct vfio_device {
89 struct kref kref;
90 struct device *dev;
91 const struct vfio_device_ops *ops;
92 struct vfio_group *group;
93 struct list_head group_next;
94 void *device_data;
95};
96
97/**
98 * IOMMU driver registration
99 */
100int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
101{
102 struct vfio_iommu_driver *driver, *tmp;
103
104 driver = kzalloc(sizeof(*driver), GFP_KERNEL);
105 if (!driver)
106 return -ENOMEM;
107
108 driver->ops = ops;
109
110 mutex_lock(&vfio.iommu_drivers_lock);
111
112 /* Check for duplicates */
113 list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
114 if (tmp->ops == ops) {
115 mutex_unlock(&vfio.iommu_drivers_lock);
116 kfree(driver);
117 return -EINVAL;
118 }
119 }
120
121 list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
122
123 mutex_unlock(&vfio.iommu_drivers_lock);
124
125 return 0;
126}
127EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
128
129void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
130{
131 struct vfio_iommu_driver *driver;
132
133 mutex_lock(&vfio.iommu_drivers_lock);
134 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
135 if (driver->ops == ops) {
136 list_del(&driver->vfio_next);
137 mutex_unlock(&vfio.iommu_drivers_lock);
138 kfree(driver);
139 return;
140 }
141 }
142 mutex_unlock(&vfio.iommu_drivers_lock);
143}
144EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
145
146/**
147 * Group minor allocation/free - both called with vfio.group_lock held
148 */
149static int vfio_alloc_group_minor(struct vfio_group *group)
150{
Alex Williamsond1099902013-12-19 10:17:13 -0700151 return idr_alloc(&vfio.group_idr, group, 0, MINORMASK + 1, GFP_KERNEL);
Alex Williamsoncba33452012-07-31 08:16:22 -0600152}
153
154static void vfio_free_group_minor(int minor)
155{
156 idr_remove(&vfio.group_idr, minor);
157}
158
159static int vfio_iommu_group_notifier(struct notifier_block *nb,
160 unsigned long action, void *data);
161static void vfio_group_get(struct vfio_group *group);
162
163/**
164 * Container objects - containers are created when /dev/vfio/vfio is
165 * opened, but their lifecycle extends until the last user is done, so
166 * it's freed via kref. Must support container/group/device being
167 * closed in any order.
168 */
169static void vfio_container_get(struct vfio_container *container)
170{
171 kref_get(&container->kref);
172}
173
174static void vfio_container_release(struct kref *kref)
175{
176 struct vfio_container *container;
177 container = container_of(kref, struct vfio_container, kref);
178
179 kfree(container);
180}
181
182static void vfio_container_put(struct vfio_container *container)
183{
184 kref_put(&container->kref, vfio_container_release);
185}
186
Jiang Liu9df7b252012-12-07 13:43:50 -0700187static void vfio_group_unlock_and_free(struct vfio_group *group)
188{
189 mutex_unlock(&vfio.group_lock);
190 /*
191 * Unregister outside of lock. A spurious callback is harmless now
192 * that the group is no longer in vfio.group_list.
193 */
194 iommu_group_unregister_notifier(group->iommu_group, &group->nb);
195 kfree(group);
196}
197
Alex Williamsoncba33452012-07-31 08:16:22 -0600198/**
199 * Group objects - create, release, get, put, search
200 */
201static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
202{
203 struct vfio_group *group, *tmp;
204 struct device *dev;
205 int ret, minor;
206
207 group = kzalloc(sizeof(*group), GFP_KERNEL);
208 if (!group)
209 return ERR_PTR(-ENOMEM);
210
211 kref_init(&group->kref);
212 INIT_LIST_HEAD(&group->device_list);
213 mutex_init(&group->device_lock);
Alex Williamson60720a02015-02-06 15:05:06 -0700214 INIT_LIST_HEAD(&group->unbound_list);
215 mutex_init(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600216 atomic_set(&group->container_users, 0);
Alex Williamson6d6768c2013-06-25 16:06:54 -0600217 atomic_set(&group->opened, 0);
Alex Williamsoncba33452012-07-31 08:16:22 -0600218 group->iommu_group = iommu_group;
219
220 group->nb.notifier_call = vfio_iommu_group_notifier;
221
222 /*
223 * blocking notifiers acquire a rwsem around registering and hold
224 * it around callback. Therefore, need to register outside of
225 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
226 * do anything unless it can find the group in vfio.group_list, so
227 * no harm in registering early.
228 */
229 ret = iommu_group_register_notifier(iommu_group, &group->nb);
230 if (ret) {
231 kfree(group);
232 return ERR_PTR(ret);
233 }
234
235 mutex_lock(&vfio.group_lock);
236
Alex Williamsoncba33452012-07-31 08:16:22 -0600237 /* Did we race creating this group? */
238 list_for_each_entry(tmp, &vfio.group_list, vfio_next) {
239 if (tmp->iommu_group == iommu_group) {
240 vfio_group_get(tmp);
Jiang Liu9df7b252012-12-07 13:43:50 -0700241 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600242 return tmp;
243 }
244 }
245
Zhen Lei2f51bf42015-03-16 14:08:56 -0600246 minor = vfio_alloc_group_minor(group);
247 if (minor < 0) {
248 vfio_group_unlock_and_free(group);
249 return ERR_PTR(minor);
250 }
251
Alex Williamsond1099902013-12-19 10:17:13 -0700252 dev = device_create(vfio.class, NULL,
253 MKDEV(MAJOR(vfio.group_devt), minor),
Alex Williamsoncba33452012-07-31 08:16:22 -0600254 group, "%d", iommu_group_id(iommu_group));
255 if (IS_ERR(dev)) {
256 vfio_free_group_minor(minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700257 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600258 return (struct vfio_group *)dev; /* ERR_PTR */
259 }
260
261 group->minor = minor;
262 group->dev = dev;
263
264 list_add(&group->vfio_next, &vfio.group_list);
265
266 mutex_unlock(&vfio.group_lock);
267
268 return group;
269}
270
Al Viro6d2cd3c2012-08-17 21:27:32 -0400271/* called with vfio.group_lock held */
Alex Williamsoncba33452012-07-31 08:16:22 -0600272static void vfio_group_release(struct kref *kref)
273{
274 struct vfio_group *group = container_of(kref, struct vfio_group, kref);
Alex Williamson60720a02015-02-06 15:05:06 -0700275 struct vfio_unbound_dev *unbound, *tmp;
Alex Williamson4a688102015-02-06 15:05:06 -0700276 struct iommu_group *iommu_group = group->iommu_group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600277
278 WARN_ON(!list_empty(&group->device_list));
279
Alex Williamson60720a02015-02-06 15:05:06 -0700280 list_for_each_entry_safe(unbound, tmp,
281 &group->unbound_list, unbound_next) {
282 list_del(&unbound->unbound_next);
283 kfree(unbound);
284 }
285
Alex Williamsond1099902013-12-19 10:17:13 -0700286 device_destroy(vfio.class, MKDEV(MAJOR(vfio.group_devt), group->minor));
Alex Williamsoncba33452012-07-31 08:16:22 -0600287 list_del(&group->vfio_next);
288 vfio_free_group_minor(group->minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700289 vfio_group_unlock_and_free(group);
Alex Williamson4a688102015-02-06 15:05:06 -0700290 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600291}
292
293static void vfio_group_put(struct vfio_group *group)
294{
Al Viro6d2cd3c2012-08-17 21:27:32 -0400295 kref_put_mutex(&group->kref, vfio_group_release, &vfio.group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600296}
297
298/* Assume group_lock or group reference is held */
299static void vfio_group_get(struct vfio_group *group)
300{
301 kref_get(&group->kref);
302}
303
304/*
305 * Not really a try as we will sleep for mutex, but we need to make
306 * sure the group pointer is valid under lock and get a reference.
307 */
308static struct vfio_group *vfio_group_try_get(struct vfio_group *group)
309{
310 struct vfio_group *target = group;
311
312 mutex_lock(&vfio.group_lock);
313 list_for_each_entry(group, &vfio.group_list, vfio_next) {
314 if (group == target) {
315 vfio_group_get(group);
316 mutex_unlock(&vfio.group_lock);
317 return group;
318 }
319 }
320 mutex_unlock(&vfio.group_lock);
321
322 return NULL;
323}
324
325static
326struct vfio_group *vfio_group_get_from_iommu(struct iommu_group *iommu_group)
327{
328 struct vfio_group *group;
329
330 mutex_lock(&vfio.group_lock);
331 list_for_each_entry(group, &vfio.group_list, vfio_next) {
332 if (group->iommu_group == iommu_group) {
333 vfio_group_get(group);
334 mutex_unlock(&vfio.group_lock);
335 return group;
336 }
337 }
338 mutex_unlock(&vfio.group_lock);
339
340 return NULL;
341}
342
343static struct vfio_group *vfio_group_get_from_minor(int minor)
344{
345 struct vfio_group *group;
346
347 mutex_lock(&vfio.group_lock);
348 group = idr_find(&vfio.group_idr, minor);
349 if (!group) {
350 mutex_unlock(&vfio.group_lock);
351 return NULL;
352 }
353 vfio_group_get(group);
354 mutex_unlock(&vfio.group_lock);
355
356 return group;
357}
358
359/**
360 * Device objects - create, release, get, put, search
361 */
362static
363struct vfio_device *vfio_group_create_device(struct vfio_group *group,
364 struct device *dev,
365 const struct vfio_device_ops *ops,
366 void *device_data)
367{
368 struct vfio_device *device;
Alex Williamsoncba33452012-07-31 08:16:22 -0600369
370 device = kzalloc(sizeof(*device), GFP_KERNEL);
371 if (!device)
372 return ERR_PTR(-ENOMEM);
373
374 kref_init(&device->kref);
375 device->dev = dev;
376 device->group = group;
377 device->ops = ops;
378 device->device_data = device_data;
Jean Delvare8283b492014-04-14 12:55:38 +0200379 dev_set_drvdata(dev, device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600380
381 /* No need to get group_lock, caller has group reference */
382 vfio_group_get(group);
383
384 mutex_lock(&group->device_lock);
385 list_add(&device->group_next, &group->device_list);
386 mutex_unlock(&group->device_lock);
387
388 return device;
389}
390
391static void vfio_device_release(struct kref *kref)
392{
393 struct vfio_device *device = container_of(kref,
394 struct vfio_device, kref);
395 struct vfio_group *group = device->group;
396
Alex Williamsoncba33452012-07-31 08:16:22 -0600397 list_del(&device->group_next);
398 mutex_unlock(&group->device_lock);
399
400 dev_set_drvdata(device->dev, NULL);
401
402 kfree(device);
403
404 /* vfio_del_group_dev may be waiting for this device */
405 wake_up(&vfio.release_q);
406}
407
408/* Device reference always implies a group reference */
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600409void vfio_device_put(struct vfio_device *device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600410{
Al Viro934ad4c2012-08-17 19:49:09 -0400411 struct vfio_group *group = device->group;
Al Viro90b12532012-08-17 21:29:06 -0400412 kref_put_mutex(&device->kref, vfio_device_release, &group->device_lock);
Al Viro934ad4c2012-08-17 19:49:09 -0400413 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600414}
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600415EXPORT_SYMBOL_GPL(vfio_device_put);
Alex Williamsoncba33452012-07-31 08:16:22 -0600416
417static void vfio_device_get(struct vfio_device *device)
418{
419 vfio_group_get(device->group);
420 kref_get(&device->kref);
421}
422
423static struct vfio_device *vfio_group_get_device(struct vfio_group *group,
424 struct device *dev)
425{
426 struct vfio_device *device;
427
428 mutex_lock(&group->device_lock);
429 list_for_each_entry(device, &group->device_list, group_next) {
430 if (device->dev == dev) {
431 vfio_device_get(device);
432 mutex_unlock(&group->device_lock);
433 return device;
434 }
435 }
436 mutex_unlock(&group->device_lock);
437 return NULL;
438}
439
440/*
441 * Whitelist some drivers that we know are safe (no dma) or just sit on
442 * a device. It's not always practical to leave a device within a group
443 * driverless as it could get re-bound to something unsafe.
444 */
Alex Williamson2b489a42013-02-14 14:02:13 -0700445static const char * const vfio_driver_whitelist[] = { "pci-stub", "pcieport" };
Alex Williamsoncba33452012-07-31 08:16:22 -0600446
447static bool vfio_whitelisted_driver(struct device_driver *drv)
448{
449 int i;
450
451 for (i = 0; i < ARRAY_SIZE(vfio_driver_whitelist); i++) {
452 if (!strcmp(drv->name, vfio_driver_whitelist[i]))
453 return true;
454 }
455
456 return false;
457}
458
459/*
Alex Williamson60720a02015-02-06 15:05:06 -0700460 * A vfio group is viable for use by userspace if all devices are in
461 * one of the following states:
462 * - driver-less
463 * - bound to a vfio driver
464 * - bound to a whitelisted driver
465 *
466 * We use two methods to determine whether a device is bound to a vfio
467 * driver. The first is to test whether the device exists in the vfio
468 * group. The second is to test if the device exists on the group
469 * unbound_list, indicating it's in the middle of transitioning from
470 * a vfio driver to driver-less.
Alex Williamsoncba33452012-07-31 08:16:22 -0600471 */
472static int vfio_dev_viable(struct device *dev, void *data)
473{
474 struct vfio_group *group = data;
475 struct vfio_device *device;
Jiang Liude2b3ee2012-12-07 13:43:50 -0700476 struct device_driver *drv = ACCESS_ONCE(dev->driver);
Alex Williamson60720a02015-02-06 15:05:06 -0700477 struct vfio_unbound_dev *unbound;
478 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -0600479
Alex Williamson60720a02015-02-06 15:05:06 -0700480 mutex_lock(&group->unbound_lock);
481 list_for_each_entry(unbound, &group->unbound_list, unbound_next) {
482 if (dev == unbound->dev) {
483 ret = 0;
484 break;
485 }
486 }
487 mutex_unlock(&group->unbound_lock);
488
489 if (!ret || !drv || vfio_whitelisted_driver(drv))
Alex Williamsoncba33452012-07-31 08:16:22 -0600490 return 0;
491
492 device = vfio_group_get_device(group, dev);
493 if (device) {
494 vfio_device_put(device);
495 return 0;
496 }
497
Alex Williamson60720a02015-02-06 15:05:06 -0700498 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -0600499}
500
501/**
502 * Async device support
503 */
504static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev)
505{
506 struct vfio_device *device;
507
508 /* Do we already know about it? We shouldn't */
509 device = vfio_group_get_device(group, dev);
510 if (WARN_ON_ONCE(device)) {
511 vfio_device_put(device);
512 return 0;
513 }
514
515 /* Nothing to do for idle groups */
516 if (!atomic_read(&group->container_users))
517 return 0;
518
519 /* TODO Prevent device auto probing */
520 WARN("Device %s added to live group %d!\n", dev_name(dev),
521 iommu_group_id(group->iommu_group));
522
523 return 0;
524}
525
Alex Williamsoncba33452012-07-31 08:16:22 -0600526static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev)
527{
528 /* We don't care what happens when the group isn't in use */
529 if (!atomic_read(&group->container_users))
530 return 0;
531
532 return vfio_dev_viable(dev, group);
533}
534
535static int vfio_iommu_group_notifier(struct notifier_block *nb,
536 unsigned long action, void *data)
537{
538 struct vfio_group *group = container_of(nb, struct vfio_group, nb);
539 struct device *dev = data;
Alex Williamson60720a02015-02-06 15:05:06 -0700540 struct vfio_unbound_dev *unbound;
Alex Williamsoncba33452012-07-31 08:16:22 -0600541
542 /*
Alex Williamsonc6401932013-06-10 16:40:56 -0600543 * Need to go through a group_lock lookup to get a reference or we
544 * risk racing a group being removed. Ignore spurious notifies.
Alex Williamsoncba33452012-07-31 08:16:22 -0600545 */
546 group = vfio_group_try_get(group);
Alex Williamsonc6401932013-06-10 16:40:56 -0600547 if (!group)
Alex Williamsoncba33452012-07-31 08:16:22 -0600548 return NOTIFY_OK;
549
550 switch (action) {
551 case IOMMU_GROUP_NOTIFY_ADD_DEVICE:
552 vfio_group_nb_add_dev(group, dev);
553 break;
554 case IOMMU_GROUP_NOTIFY_DEL_DEVICE:
Alex Williamsonde9c7602013-06-10 16:40:56 -0600555 /*
556 * Nothing to do here. If the device is in use, then the
557 * vfio sub-driver should block the remove callback until
558 * it is unused. If the device is unused or attached to a
559 * stub driver, then it should be released and we don't
560 * care that it will be going away.
561 */
Alex Williamsoncba33452012-07-31 08:16:22 -0600562 break;
563 case IOMMU_GROUP_NOTIFY_BIND_DRIVER:
564 pr_debug("%s: Device %s, group %d binding to driver\n",
565 __func__, dev_name(dev),
566 iommu_group_id(group->iommu_group));
567 break;
568 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER:
569 pr_debug("%s: Device %s, group %d bound to driver %s\n",
570 __func__, dev_name(dev),
571 iommu_group_id(group->iommu_group), dev->driver->name);
572 BUG_ON(vfio_group_nb_verify(group, dev));
573 break;
574 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER:
575 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
576 __func__, dev_name(dev),
577 iommu_group_id(group->iommu_group), dev->driver->name);
578 break;
579 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER:
580 pr_debug("%s: Device %s, group %d unbound from driver\n",
581 __func__, dev_name(dev),
582 iommu_group_id(group->iommu_group));
583 /*
584 * XXX An unbound device in a live group is ok, but we'd
585 * really like to avoid the above BUG_ON by preventing other
586 * drivers from binding to it. Once that occurs, we have to
587 * stop the system to maintain isolation. At a minimum, we'd
588 * want a toggle to disable driver auto probe for this device.
589 */
Alex Williamson60720a02015-02-06 15:05:06 -0700590
591 mutex_lock(&group->unbound_lock);
592 list_for_each_entry(unbound,
593 &group->unbound_list, unbound_next) {
594 if (dev == unbound->dev) {
595 list_del(&unbound->unbound_next);
596 kfree(unbound);
597 break;
598 }
599 }
600 mutex_unlock(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600601 break;
602 }
603
604 vfio_group_put(group);
605 return NOTIFY_OK;
606}
607
608/**
609 * VFIO driver API
610 */
611int vfio_add_group_dev(struct device *dev,
612 const struct vfio_device_ops *ops, void *device_data)
613{
614 struct iommu_group *iommu_group;
615 struct vfio_group *group;
616 struct vfio_device *device;
617
618 iommu_group = iommu_group_get(dev);
619 if (!iommu_group)
620 return -EINVAL;
621
622 group = vfio_group_get_from_iommu(iommu_group);
623 if (!group) {
624 group = vfio_create_group(iommu_group);
625 if (IS_ERR(group)) {
626 iommu_group_put(iommu_group);
627 return PTR_ERR(group);
628 }
Alex Williamson4a688102015-02-06 15:05:06 -0700629 } else {
630 /*
631 * A found vfio_group already holds a reference to the
632 * iommu_group. A created vfio_group keeps the reference.
633 */
634 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600635 }
636
637 device = vfio_group_get_device(group, dev);
638 if (device) {
639 WARN(1, "Device %s already exists on group %d\n",
640 dev_name(dev), iommu_group_id(iommu_group));
641 vfio_device_put(device);
642 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600643 return -EBUSY;
644 }
645
646 device = vfio_group_create_device(group, dev, ops, device_data);
647 if (IS_ERR(device)) {
648 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600649 return PTR_ERR(device);
650 }
651
652 /*
Alex Williamson4a688102015-02-06 15:05:06 -0700653 * Drop all but the vfio_device reference. The vfio_device holds
654 * a reference to the vfio_group, which holds a reference to the
655 * iommu_group.
Alex Williamsoncba33452012-07-31 08:16:22 -0600656 */
657 vfio_group_put(group);
658
659 return 0;
660}
661EXPORT_SYMBOL_GPL(vfio_add_group_dev);
662
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600663/**
664 * Get a reference to the vfio_device for a device that is known to
665 * be bound to a vfio driver. The driver implicitly holds a
666 * vfio_device reference between vfio_add_group_dev and
667 * vfio_del_group_dev. We can therefore use drvdata to increment
668 * that reference from the struct device. This additional
669 * reference must be released by calling vfio_device_put.
670 */
671struct vfio_device *vfio_device_get_from_dev(struct device *dev)
672{
673 struct vfio_device *device = dev_get_drvdata(dev);
674
675 vfio_device_get(device);
676
677 return device;
678}
679EXPORT_SYMBOL_GPL(vfio_device_get_from_dev);
680
681/*
682 * Caller must hold a reference to the vfio_device
683 */
684void *vfio_device_data(struct vfio_device *device)
685{
686 return device->device_data;
687}
688EXPORT_SYMBOL_GPL(vfio_device_data);
689
Alex Williamsone014e942013-02-14 14:02:13 -0700690/* Given a referenced group, check if it contains the device */
691static bool vfio_dev_present(struct vfio_group *group, struct device *dev)
Alex Williamsoncba33452012-07-31 08:16:22 -0600692{
Alex Williamsoncba33452012-07-31 08:16:22 -0600693 struct vfio_device *device;
694
Alex Williamsoncba33452012-07-31 08:16:22 -0600695 device = vfio_group_get_device(group, dev);
Alex Williamsone014e942013-02-14 14:02:13 -0700696 if (!device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600697 return false;
Alex Williamsoncba33452012-07-31 08:16:22 -0600698
699 vfio_device_put(device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600700 return true;
701}
702
703/*
704 * Decrement the device reference count and wait for the device to be
705 * removed. Open file descriptors for the device... */
706void *vfio_del_group_dev(struct device *dev)
707{
708 struct vfio_device *device = dev_get_drvdata(dev);
709 struct vfio_group *group = device->group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600710 void *device_data = device->device_data;
Alex Williamson60720a02015-02-06 15:05:06 -0700711 struct vfio_unbound_dev *unbound;
Alex Williamson13060b62015-02-06 15:05:07 -0700712 unsigned int i = 0;
Alex Williamsoncba33452012-07-31 08:16:22 -0600713
Alex Williamsone014e942013-02-14 14:02:13 -0700714 /*
715 * The group exists so long as we have a device reference. Get
716 * a group reference and use it to scan for the device going away.
717 */
718 vfio_group_get(group);
719
Alex Williamson60720a02015-02-06 15:05:06 -0700720 /*
721 * When the device is removed from the group, the group suddenly
722 * becomes non-viable; the device has a driver (until the unbind
723 * completes), but it's not present in the group. This is bad news
724 * for any external users that need to re-acquire a group reference
725 * in order to match and release their existing reference. To
726 * solve this, we track such devices on the unbound_list to bridge
727 * the gap until they're fully unbound.
728 */
729 unbound = kzalloc(sizeof(*unbound), GFP_KERNEL);
730 if (unbound) {
731 unbound->dev = dev;
732 mutex_lock(&group->unbound_lock);
733 list_add(&unbound->unbound_next, &group->unbound_list);
734 mutex_unlock(&group->unbound_lock);
735 }
736 WARN_ON(!unbound);
737
Alex Williamsoncba33452012-07-31 08:16:22 -0600738 vfio_device_put(device);
739
Alex Williamson13060b62015-02-06 15:05:07 -0700740 /*
741 * If the device is still present in the group after the above
742 * 'put', then it is in use and we need to request it from the
743 * bus driver. The driver may in turn need to request the
744 * device from the user. We send the request on an arbitrary
745 * interval with counter to allow the driver to take escalating
746 * measures to release the device if it has the ability to do so.
747 */
748 do {
749 device = vfio_group_get_device(group, dev);
750 if (!device)
751 break;
752
753 if (device->ops->request)
754 device->ops->request(device_data, i++);
755
756 vfio_device_put(device);
757
758 } while (wait_event_interruptible_timeout(vfio.release_q,
759 !vfio_dev_present(group, dev),
760 HZ * 10) <= 0);
Alex Williamsone014e942013-02-14 14:02:13 -0700761
762 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600763
Alex Williamsoncba33452012-07-31 08:16:22 -0600764 return device_data;
765}
766EXPORT_SYMBOL_GPL(vfio_del_group_dev);
767
768/**
769 * VFIO base fd, /dev/vfio/vfio
770 */
771static long vfio_ioctl_check_extension(struct vfio_container *container,
772 unsigned long arg)
773{
Alex Williamson0b43c082013-04-29 08:41:36 -0600774 struct vfio_iommu_driver *driver;
Alex Williamsoncba33452012-07-31 08:16:22 -0600775 long ret = 0;
776
Alex Williamson0b43c082013-04-29 08:41:36 -0600777 down_read(&container->group_lock);
778
779 driver = container->iommu_driver;
780
Alex Williamsoncba33452012-07-31 08:16:22 -0600781 switch (arg) {
782 /* No base extensions yet */
783 default:
784 /*
785 * If no driver is set, poll all registered drivers for
786 * extensions and return the first positive result. If
787 * a driver is already set, further queries will be passed
788 * only to that driver.
789 */
790 if (!driver) {
791 mutex_lock(&vfio.iommu_drivers_lock);
792 list_for_each_entry(driver, &vfio.iommu_drivers_list,
793 vfio_next) {
794 if (!try_module_get(driver->ops->owner))
795 continue;
796
797 ret = driver->ops->ioctl(NULL,
798 VFIO_CHECK_EXTENSION,
799 arg);
800 module_put(driver->ops->owner);
801 if (ret > 0)
802 break;
803 }
804 mutex_unlock(&vfio.iommu_drivers_lock);
805 } else
806 ret = driver->ops->ioctl(container->iommu_data,
807 VFIO_CHECK_EXTENSION, arg);
808 }
809
Alex Williamson0b43c082013-04-29 08:41:36 -0600810 up_read(&container->group_lock);
811
Alex Williamsoncba33452012-07-31 08:16:22 -0600812 return ret;
813}
814
Alex Williamson9587f442013-04-25 16:12:38 -0600815/* hold write lock on container->group_lock */
Alex Williamsoncba33452012-07-31 08:16:22 -0600816static int __vfio_container_attach_groups(struct vfio_container *container,
817 struct vfio_iommu_driver *driver,
818 void *data)
819{
820 struct vfio_group *group;
821 int ret = -ENODEV;
822
823 list_for_each_entry(group, &container->group_list, container_next) {
824 ret = driver->ops->attach_group(data, group->iommu_group);
825 if (ret)
826 goto unwind;
827 }
828
829 return ret;
830
831unwind:
832 list_for_each_entry_continue_reverse(group, &container->group_list,
833 container_next) {
834 driver->ops->detach_group(data, group->iommu_group);
835 }
836
837 return ret;
838}
839
840static long vfio_ioctl_set_iommu(struct vfio_container *container,
841 unsigned long arg)
842{
843 struct vfio_iommu_driver *driver;
844 long ret = -ENODEV;
845
Alex Williamson9587f442013-04-25 16:12:38 -0600846 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600847
848 /*
849 * The container is designed to be an unprivileged interface while
850 * the group can be assigned to specific users. Therefore, only by
851 * adding a group to a container does the user get the privilege of
852 * enabling the iommu, which may allocate finite resources. There
853 * is no unset_iommu, but by removing all the groups from a container,
854 * the container is deprivileged and returns to an unset state.
855 */
856 if (list_empty(&container->group_list) || container->iommu_driver) {
Alex Williamson9587f442013-04-25 16:12:38 -0600857 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600858 return -EINVAL;
859 }
860
861 mutex_lock(&vfio.iommu_drivers_lock);
862 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
863 void *data;
864
865 if (!try_module_get(driver->ops->owner))
866 continue;
867
868 /*
869 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
870 * so test which iommu driver reported support for this
871 * extension and call open on them. We also pass them the
872 * magic, allowing a single driver to support multiple
873 * interfaces if they'd like.
874 */
875 if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
876 module_put(driver->ops->owner);
877 continue;
878 }
879
880 /* module reference holds the driver we're working on */
881 mutex_unlock(&vfio.iommu_drivers_lock);
882
883 data = driver->ops->open(arg);
884 if (IS_ERR(data)) {
885 ret = PTR_ERR(data);
886 module_put(driver->ops->owner);
887 goto skip_drivers_unlock;
888 }
889
890 ret = __vfio_container_attach_groups(container, driver, data);
891 if (!ret) {
892 container->iommu_driver = driver;
893 container->iommu_data = data;
894 } else {
895 driver->ops->release(data);
896 module_put(driver->ops->owner);
897 }
898
899 goto skip_drivers_unlock;
900 }
901
902 mutex_unlock(&vfio.iommu_drivers_lock);
903skip_drivers_unlock:
Alex Williamson9587f442013-04-25 16:12:38 -0600904 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600905
906 return ret;
907}
908
909static long vfio_fops_unl_ioctl(struct file *filep,
910 unsigned int cmd, unsigned long arg)
911{
912 struct vfio_container *container = filep->private_data;
913 struct vfio_iommu_driver *driver;
914 void *data;
915 long ret = -EINVAL;
916
917 if (!container)
918 return ret;
919
Alex Williamsoncba33452012-07-31 08:16:22 -0600920 switch (cmd) {
921 case VFIO_GET_API_VERSION:
922 ret = VFIO_API_VERSION;
923 break;
924 case VFIO_CHECK_EXTENSION:
925 ret = vfio_ioctl_check_extension(container, arg);
926 break;
927 case VFIO_SET_IOMMU:
928 ret = vfio_ioctl_set_iommu(container, arg);
929 break;
930 default:
Alex Williamson0b43c082013-04-29 08:41:36 -0600931 down_read(&container->group_lock);
932
933 driver = container->iommu_driver;
934 data = container->iommu_data;
935
Alex Williamsoncba33452012-07-31 08:16:22 -0600936 if (driver) /* passthrough all unrecognized ioctls */
937 ret = driver->ops->ioctl(data, cmd, arg);
Alex Williamson0b43c082013-04-29 08:41:36 -0600938
939 up_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600940 }
941
942 return ret;
943}
944
945#ifdef CONFIG_COMPAT
946static long vfio_fops_compat_ioctl(struct file *filep,
947 unsigned int cmd, unsigned long arg)
948{
949 arg = (unsigned long)compat_ptr(arg);
950 return vfio_fops_unl_ioctl(filep, cmd, arg);
951}
952#endif /* CONFIG_COMPAT */
953
954static int vfio_fops_open(struct inode *inode, struct file *filep)
955{
956 struct vfio_container *container;
957
958 container = kzalloc(sizeof(*container), GFP_KERNEL);
959 if (!container)
960 return -ENOMEM;
961
962 INIT_LIST_HEAD(&container->group_list);
Alex Williamson9587f442013-04-25 16:12:38 -0600963 init_rwsem(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600964 kref_init(&container->kref);
965
966 filep->private_data = container;
967
968 return 0;
969}
970
971static int vfio_fops_release(struct inode *inode, struct file *filep)
972{
973 struct vfio_container *container = filep->private_data;
974
975 filep->private_data = NULL;
976
977 vfio_container_put(container);
978
979 return 0;
980}
981
982/*
983 * Once an iommu driver is set, we optionally pass read/write/mmap
984 * on to the driver, allowing management interfaces beyond ioctl.
985 */
986static ssize_t vfio_fops_read(struct file *filep, char __user *buf,
987 size_t count, loff_t *ppos)
988{
989 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -0600990 struct vfio_iommu_driver *driver;
991 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -0600992
Alex Williamson0b43c082013-04-29 08:41:36 -0600993 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600994
Alex Williamson0b43c082013-04-29 08:41:36 -0600995 driver = container->iommu_driver;
996 if (likely(driver && driver->ops->read))
997 ret = driver->ops->read(container->iommu_data,
998 buf, count, ppos);
999
1000 up_read(&container->group_lock);
1001
1002 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001003}
1004
1005static ssize_t vfio_fops_write(struct file *filep, const char __user *buf,
1006 size_t count, loff_t *ppos)
1007{
1008 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001009 struct vfio_iommu_driver *driver;
1010 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001011
Alex Williamson0b43c082013-04-29 08:41:36 -06001012 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001013
Alex Williamson0b43c082013-04-29 08:41:36 -06001014 driver = container->iommu_driver;
1015 if (likely(driver && driver->ops->write))
1016 ret = driver->ops->write(container->iommu_data,
1017 buf, count, ppos);
1018
1019 up_read(&container->group_lock);
1020
1021 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001022}
1023
1024static int vfio_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1025{
1026 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001027 struct vfio_iommu_driver *driver;
1028 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001029
Alex Williamson0b43c082013-04-29 08:41:36 -06001030 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001031
Alex Williamson0b43c082013-04-29 08:41:36 -06001032 driver = container->iommu_driver;
1033 if (likely(driver && driver->ops->mmap))
1034 ret = driver->ops->mmap(container->iommu_data, vma);
1035
1036 up_read(&container->group_lock);
1037
1038 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001039}
1040
1041static const struct file_operations vfio_fops = {
1042 .owner = THIS_MODULE,
1043 .open = vfio_fops_open,
1044 .release = vfio_fops_release,
1045 .read = vfio_fops_read,
1046 .write = vfio_fops_write,
1047 .unlocked_ioctl = vfio_fops_unl_ioctl,
1048#ifdef CONFIG_COMPAT
1049 .compat_ioctl = vfio_fops_compat_ioctl,
1050#endif
1051 .mmap = vfio_fops_mmap,
1052};
1053
1054/**
1055 * VFIO Group fd, /dev/vfio/$GROUP
1056 */
1057static void __vfio_group_unset_container(struct vfio_group *group)
1058{
1059 struct vfio_container *container = group->container;
1060 struct vfio_iommu_driver *driver;
1061
Alex Williamson9587f442013-04-25 16:12:38 -06001062 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001063
1064 driver = container->iommu_driver;
1065 if (driver)
1066 driver->ops->detach_group(container->iommu_data,
1067 group->iommu_group);
1068
1069 group->container = NULL;
1070 list_del(&group->container_next);
1071
1072 /* Detaching the last group deprivileges a container, remove iommu */
1073 if (driver && list_empty(&container->group_list)) {
1074 driver->ops->release(container->iommu_data);
1075 module_put(driver->ops->owner);
1076 container->iommu_driver = NULL;
1077 container->iommu_data = NULL;
1078 }
1079
Alex Williamson9587f442013-04-25 16:12:38 -06001080 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001081
1082 vfio_container_put(container);
1083}
1084
1085/*
1086 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
1087 * if there was no container to unset. Since the ioctl is called on
1088 * the group, we know that still exists, therefore the only valid
1089 * transition here is 1->0.
1090 */
1091static int vfio_group_unset_container(struct vfio_group *group)
1092{
1093 int users = atomic_cmpxchg(&group->container_users, 1, 0);
1094
1095 if (!users)
1096 return -EINVAL;
1097 if (users != 1)
1098 return -EBUSY;
1099
1100 __vfio_group_unset_container(group);
1101
1102 return 0;
1103}
1104
1105/*
1106 * When removing container users, anything that removes the last user
1107 * implicitly removes the group from the container. That is, if the
1108 * group file descriptor is closed, as well as any device file descriptors,
1109 * the group is free.
1110 */
1111static void vfio_group_try_dissolve_container(struct vfio_group *group)
1112{
1113 if (0 == atomic_dec_if_positive(&group->container_users))
1114 __vfio_group_unset_container(group);
1115}
1116
1117static int vfio_group_set_container(struct vfio_group *group, int container_fd)
1118{
Al Viro2903ff02012-08-28 12:52:22 -04001119 struct fd f;
Alex Williamsoncba33452012-07-31 08:16:22 -06001120 struct vfio_container *container;
1121 struct vfio_iommu_driver *driver;
Al Viro2903ff02012-08-28 12:52:22 -04001122 int ret = 0;
Alex Williamsoncba33452012-07-31 08:16:22 -06001123
1124 if (atomic_read(&group->container_users))
1125 return -EINVAL;
1126
Al Viro2903ff02012-08-28 12:52:22 -04001127 f = fdget(container_fd);
1128 if (!f.file)
Alex Williamsoncba33452012-07-31 08:16:22 -06001129 return -EBADF;
1130
1131 /* Sanity check, is this really our fd? */
Al Viro2903ff02012-08-28 12:52:22 -04001132 if (f.file->f_op != &vfio_fops) {
1133 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001134 return -EINVAL;
1135 }
1136
Al Viro2903ff02012-08-28 12:52:22 -04001137 container = f.file->private_data;
Alex Williamsoncba33452012-07-31 08:16:22 -06001138 WARN_ON(!container); /* fget ensures we don't race vfio_release */
1139
Alex Williamson9587f442013-04-25 16:12:38 -06001140 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001141
1142 driver = container->iommu_driver;
1143 if (driver) {
1144 ret = driver->ops->attach_group(container->iommu_data,
1145 group->iommu_group);
1146 if (ret)
1147 goto unlock_out;
1148 }
1149
1150 group->container = container;
1151 list_add(&group->container_next, &container->group_list);
1152
1153 /* Get a reference on the container and mark a user within the group */
1154 vfio_container_get(container);
1155 atomic_inc(&group->container_users);
1156
1157unlock_out:
Alex Williamson9587f442013-04-25 16:12:38 -06001158 up_write(&container->group_lock);
Al Viro2903ff02012-08-28 12:52:22 -04001159 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001160 return ret;
1161}
1162
1163static bool vfio_group_viable(struct vfio_group *group)
1164{
1165 return (iommu_group_for_each_dev(group->iommu_group,
1166 group, vfio_dev_viable) == 0);
1167}
1168
1169static const struct file_operations vfio_device_fops;
1170
1171static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
1172{
1173 struct vfio_device *device;
1174 struct file *filep;
1175 int ret = -ENODEV;
1176
1177 if (0 == atomic_read(&group->container_users) ||
1178 !group->container->iommu_driver || !vfio_group_viable(group))
1179 return -EINVAL;
1180
1181 mutex_lock(&group->device_lock);
1182 list_for_each_entry(device, &group->device_list, group_next) {
1183 if (strcmp(dev_name(device->dev), buf))
1184 continue;
1185
1186 ret = device->ops->open(device->device_data);
1187 if (ret)
1188 break;
1189 /*
1190 * We can't use anon_inode_getfd() because we need to modify
1191 * the f_mode flags directly to allow more than just ioctls
1192 */
Alex Williamson5d042fb2013-08-22 10:33:41 -06001193 ret = get_unused_fd_flags(O_CLOEXEC);
Alex Williamsoncba33452012-07-31 08:16:22 -06001194 if (ret < 0) {
1195 device->ops->release(device->device_data);
1196 break;
1197 }
1198
1199 filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
1200 device, O_RDWR);
1201 if (IS_ERR(filep)) {
1202 put_unused_fd(ret);
1203 ret = PTR_ERR(filep);
1204 device->ops->release(device->device_data);
1205 break;
1206 }
1207
1208 /*
1209 * TODO: add an anon_inode interface to do this.
1210 * Appears to be missing by lack of need rather than
1211 * explicitly prevented. Now there's need.
1212 */
1213 filep->f_mode |= (FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
1214
Alex Williamsoncba33452012-07-31 08:16:22 -06001215 vfio_device_get(device);
1216 atomic_inc(&group->container_users);
Al Viro31605de2012-08-17 21:32:56 -04001217
1218 fd_install(ret, filep);
Alex Williamsoncba33452012-07-31 08:16:22 -06001219 break;
1220 }
1221 mutex_unlock(&group->device_lock);
1222
1223 return ret;
1224}
1225
1226static long vfio_group_fops_unl_ioctl(struct file *filep,
1227 unsigned int cmd, unsigned long arg)
1228{
1229 struct vfio_group *group = filep->private_data;
1230 long ret = -ENOTTY;
1231
1232 switch (cmd) {
1233 case VFIO_GROUP_GET_STATUS:
1234 {
1235 struct vfio_group_status status;
1236 unsigned long minsz;
1237
1238 minsz = offsetofend(struct vfio_group_status, flags);
1239
1240 if (copy_from_user(&status, (void __user *)arg, minsz))
1241 return -EFAULT;
1242
1243 if (status.argsz < minsz)
1244 return -EINVAL;
1245
1246 status.flags = 0;
1247
1248 if (vfio_group_viable(group))
1249 status.flags |= VFIO_GROUP_FLAGS_VIABLE;
1250
1251 if (group->container)
1252 status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET;
1253
1254 if (copy_to_user((void __user *)arg, &status, minsz))
1255 return -EFAULT;
1256
1257 ret = 0;
1258 break;
1259 }
1260 case VFIO_GROUP_SET_CONTAINER:
1261 {
1262 int fd;
1263
1264 if (get_user(fd, (int __user *)arg))
1265 return -EFAULT;
1266
1267 if (fd < 0)
1268 return -EINVAL;
1269
1270 ret = vfio_group_set_container(group, fd);
1271 break;
1272 }
1273 case VFIO_GROUP_UNSET_CONTAINER:
1274 ret = vfio_group_unset_container(group);
1275 break;
1276 case VFIO_GROUP_GET_DEVICE_FD:
1277 {
1278 char *buf;
1279
1280 buf = strndup_user((const char __user *)arg, PAGE_SIZE);
1281 if (IS_ERR(buf))
1282 return PTR_ERR(buf);
1283
1284 ret = vfio_group_get_device_fd(group, buf);
1285 kfree(buf);
1286 break;
1287 }
1288 }
1289
1290 return ret;
1291}
1292
1293#ifdef CONFIG_COMPAT
1294static long vfio_group_fops_compat_ioctl(struct file *filep,
1295 unsigned int cmd, unsigned long arg)
1296{
1297 arg = (unsigned long)compat_ptr(arg);
1298 return vfio_group_fops_unl_ioctl(filep, cmd, arg);
1299}
1300#endif /* CONFIG_COMPAT */
1301
1302static int vfio_group_fops_open(struct inode *inode, struct file *filep)
1303{
1304 struct vfio_group *group;
Alex Williamson6d6768c2013-06-25 16:06:54 -06001305 int opened;
Alex Williamsoncba33452012-07-31 08:16:22 -06001306
1307 group = vfio_group_get_from_minor(iminor(inode));
1308 if (!group)
1309 return -ENODEV;
1310
Alex Williamson6d6768c2013-06-25 16:06:54 -06001311 /* Do we need multiple instances of the group open? Seems not. */
1312 opened = atomic_cmpxchg(&group->opened, 0, 1);
1313 if (opened) {
1314 vfio_group_put(group);
1315 return -EBUSY;
1316 }
1317
1318 /* Is something still in use from a previous open? */
Alex Williamsoncba33452012-07-31 08:16:22 -06001319 if (group->container) {
Alex Williamson6d6768c2013-06-25 16:06:54 -06001320 atomic_dec(&group->opened);
Alex Williamsoncba33452012-07-31 08:16:22 -06001321 vfio_group_put(group);
1322 return -EBUSY;
1323 }
1324
1325 filep->private_data = group;
1326
1327 return 0;
1328}
1329
1330static int vfio_group_fops_release(struct inode *inode, struct file *filep)
1331{
1332 struct vfio_group *group = filep->private_data;
1333
1334 filep->private_data = NULL;
1335
1336 vfio_group_try_dissolve_container(group);
1337
Alex Williamson6d6768c2013-06-25 16:06:54 -06001338 atomic_dec(&group->opened);
1339
Alex Williamsoncba33452012-07-31 08:16:22 -06001340 vfio_group_put(group);
1341
1342 return 0;
1343}
1344
1345static const struct file_operations vfio_group_fops = {
1346 .owner = THIS_MODULE,
1347 .unlocked_ioctl = vfio_group_fops_unl_ioctl,
1348#ifdef CONFIG_COMPAT
1349 .compat_ioctl = vfio_group_fops_compat_ioctl,
1350#endif
1351 .open = vfio_group_fops_open,
1352 .release = vfio_group_fops_release,
1353};
1354
1355/**
1356 * VFIO Device fd
1357 */
1358static int vfio_device_fops_release(struct inode *inode, struct file *filep)
1359{
1360 struct vfio_device *device = filep->private_data;
1361
1362 device->ops->release(device->device_data);
1363
1364 vfio_group_try_dissolve_container(device->group);
1365
1366 vfio_device_put(device);
1367
1368 return 0;
1369}
1370
1371static long vfio_device_fops_unl_ioctl(struct file *filep,
1372 unsigned int cmd, unsigned long arg)
1373{
1374 struct vfio_device *device = filep->private_data;
1375
1376 if (unlikely(!device->ops->ioctl))
1377 return -EINVAL;
1378
1379 return device->ops->ioctl(device->device_data, cmd, arg);
1380}
1381
1382static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1383 size_t count, loff_t *ppos)
1384{
1385 struct vfio_device *device = filep->private_data;
1386
1387 if (unlikely(!device->ops->read))
1388 return -EINVAL;
1389
1390 return device->ops->read(device->device_data, buf, count, ppos);
1391}
1392
1393static ssize_t vfio_device_fops_write(struct file *filep,
1394 const char __user *buf,
1395 size_t count, loff_t *ppos)
1396{
1397 struct vfio_device *device = filep->private_data;
1398
1399 if (unlikely(!device->ops->write))
1400 return -EINVAL;
1401
1402 return device->ops->write(device->device_data, buf, count, ppos);
1403}
1404
1405static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1406{
1407 struct vfio_device *device = filep->private_data;
1408
1409 if (unlikely(!device->ops->mmap))
1410 return -EINVAL;
1411
1412 return device->ops->mmap(device->device_data, vma);
1413}
1414
1415#ifdef CONFIG_COMPAT
1416static long vfio_device_fops_compat_ioctl(struct file *filep,
1417 unsigned int cmd, unsigned long arg)
1418{
1419 arg = (unsigned long)compat_ptr(arg);
1420 return vfio_device_fops_unl_ioctl(filep, cmd, arg);
1421}
1422#endif /* CONFIG_COMPAT */
1423
1424static const struct file_operations vfio_device_fops = {
1425 .owner = THIS_MODULE,
1426 .release = vfio_device_fops_release,
1427 .read = vfio_device_fops_read,
1428 .write = vfio_device_fops_write,
1429 .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1430#ifdef CONFIG_COMPAT
1431 .compat_ioctl = vfio_device_fops_compat_ioctl,
1432#endif
1433 .mmap = vfio_device_fops_mmap,
1434};
1435
1436/**
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001437 * External user API, exported by symbols to be linked dynamically.
1438 *
1439 * The protocol includes:
1440 * 1. do normal VFIO init operation:
1441 * - opening a new container;
1442 * - attaching group(s) to it;
1443 * - setting an IOMMU driver for a container.
1444 * When IOMMU is set for a container, all groups in it are
1445 * considered ready to use by an external user.
1446 *
1447 * 2. User space passes a group fd to an external user.
1448 * The external user calls vfio_group_get_external_user()
1449 * to verify that:
1450 * - the group is initialized;
1451 * - IOMMU is set for it.
1452 * If both checks passed, vfio_group_get_external_user()
1453 * increments the container user counter to prevent
1454 * the VFIO group from disposal before KVM exits.
1455 *
1456 * 3. The external user calls vfio_external_user_iommu_id()
1457 * to know an IOMMU ID.
1458 *
1459 * 4. When the external KVM finishes, it calls
1460 * vfio_group_put_external_user() to release the VFIO group.
1461 * This call decrements the container user counter.
1462 */
1463struct vfio_group *vfio_group_get_external_user(struct file *filep)
1464{
1465 struct vfio_group *group = filep->private_data;
1466
1467 if (filep->f_op != &vfio_group_fops)
1468 return ERR_PTR(-EINVAL);
1469
1470 if (!atomic_inc_not_zero(&group->container_users))
1471 return ERR_PTR(-EINVAL);
1472
1473 if (!group->container->iommu_driver ||
1474 !vfio_group_viable(group)) {
1475 atomic_dec(&group->container_users);
1476 return ERR_PTR(-EINVAL);
1477 }
1478
1479 vfio_group_get(group);
1480
1481 return group;
1482}
1483EXPORT_SYMBOL_GPL(vfio_group_get_external_user);
1484
1485void vfio_group_put_external_user(struct vfio_group *group)
1486{
1487 vfio_group_put(group);
1488 vfio_group_try_dissolve_container(group);
1489}
1490EXPORT_SYMBOL_GPL(vfio_group_put_external_user);
1491
1492int vfio_external_user_iommu_id(struct vfio_group *group)
1493{
1494 return iommu_group_id(group->iommu_group);
1495}
1496EXPORT_SYMBOL_GPL(vfio_external_user_iommu_id);
1497
Alex Williamson88d7ab82014-02-26 11:38:39 -07001498long vfio_external_check_extension(struct vfio_group *group, unsigned long arg)
1499{
1500 return vfio_ioctl_check_extension(group->container, arg);
1501}
1502EXPORT_SYMBOL_GPL(vfio_external_check_extension);
1503
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001504/**
Alex Williamsoncba33452012-07-31 08:16:22 -06001505 * Module/class support
1506 */
1507static char *vfio_devnode(struct device *dev, umode_t *mode)
1508{
1509 return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
1510}
1511
Alex Williamsond1099902013-12-19 10:17:13 -07001512static struct miscdevice vfio_dev = {
1513 .minor = VFIO_MINOR,
1514 .name = "vfio",
1515 .fops = &vfio_fops,
1516 .nodename = "vfio/vfio",
1517 .mode = S_IRUGO | S_IWUGO,
1518};
1519
Alex Williamsoncba33452012-07-31 08:16:22 -06001520static int __init vfio_init(void)
1521{
1522 int ret;
1523
1524 idr_init(&vfio.group_idr);
1525 mutex_init(&vfio.group_lock);
1526 mutex_init(&vfio.iommu_drivers_lock);
1527 INIT_LIST_HEAD(&vfio.group_list);
1528 INIT_LIST_HEAD(&vfio.iommu_drivers_list);
1529 init_waitqueue_head(&vfio.release_q);
1530
Alex Williamsond1099902013-12-19 10:17:13 -07001531 ret = misc_register(&vfio_dev);
1532 if (ret) {
1533 pr_err("vfio: misc device register failed\n");
1534 return ret;
1535 }
1536
1537 /* /dev/vfio/$GROUP */
Alex Williamsoncba33452012-07-31 08:16:22 -06001538 vfio.class = class_create(THIS_MODULE, "vfio");
1539 if (IS_ERR(vfio.class)) {
1540 ret = PTR_ERR(vfio.class);
1541 goto err_class;
1542 }
1543
1544 vfio.class->devnode = vfio_devnode;
1545
Alex Williamsond1099902013-12-19 10:17:13 -07001546 ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK, "vfio");
Alex Williamsoncba33452012-07-31 08:16:22 -06001547 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001548 goto err_alloc_chrdev;
Alex Williamsoncba33452012-07-31 08:16:22 -06001549
Alex Williamsoncba33452012-07-31 08:16:22 -06001550 cdev_init(&vfio.group_cdev, &vfio_group_fops);
Alex Williamsond1099902013-12-19 10:17:13 -07001551 ret = cdev_add(&vfio.group_cdev, vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001552 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001553 goto err_cdev_add;
Alex Williamsoncba33452012-07-31 08:16:22 -06001554
1555 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1556
Alex Williamson73fa0d12012-07-31 08:16:23 -06001557 /*
1558 * Attempt to load known iommu-drivers. This gives us a working
1559 * environment without the user needing to explicitly load iommu
1560 * drivers.
1561 */
1562 request_module_nowait("vfio_iommu_type1");
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001563 request_module_nowait("vfio_iommu_spapr_tce");
Alex Williamson73fa0d12012-07-31 08:16:23 -06001564
Alex Williamsoncba33452012-07-31 08:16:22 -06001565 return 0;
1566
Alex Williamsond1099902013-12-19 10:17:13 -07001567err_cdev_add:
1568 unregister_chrdev_region(vfio.group_devt, MINORMASK);
1569err_alloc_chrdev:
Alex Williamsoncba33452012-07-31 08:16:22 -06001570 class_destroy(vfio.class);
1571 vfio.class = NULL;
1572err_class:
Alex Williamsond1099902013-12-19 10:17:13 -07001573 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001574 return ret;
1575}
1576
1577static void __exit vfio_cleanup(void)
1578{
1579 WARN_ON(!list_empty(&vfio.group_list));
1580
1581 idr_destroy(&vfio.group_idr);
1582 cdev_del(&vfio.group_cdev);
Alex Williamsond1099902013-12-19 10:17:13 -07001583 unregister_chrdev_region(vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001584 class_destroy(vfio.class);
1585 vfio.class = NULL;
Alex Williamsond1099902013-12-19 10:17:13 -07001586 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001587}
1588
1589module_init(vfio_init);
1590module_exit(vfio_cleanup);
1591
1592MODULE_VERSION(DRIVER_VERSION);
1593MODULE_LICENSE("GPL v2");
1594MODULE_AUTHOR(DRIVER_AUTHOR);
1595MODULE_DESCRIPTION(DRIVER_DESC);
Alex Williamsond1099902013-12-19 10:17:13 -07001596MODULE_ALIAS_MISCDEV(VFIO_MINOR);
1597MODULE_ALIAS("devname:vfio/vfio");