blob: 563c510f285c47d2a7362a4da8729fdc38e1dee0 [file] [log] [blame]
Alex Williamsoncba33452012-07-31 08:16:22 -06001/*
2 * VFIO core
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
14 */
15
16#include <linux/cdev.h>
17#include <linux/compat.h>
18#include <linux/device.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <linux/idr.h>
23#include <linux/iommu.h>
24#include <linux/list.h>
Alex Williamsond1099902013-12-19 10:17:13 -070025#include <linux/miscdevice.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060026#include <linux/module.h>
27#include <linux/mutex.h>
Alex Williamson9587f442013-04-25 16:12:38 -060028#include <linux/rwsem.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060029#include <linux/sched.h>
30#include <linux/slab.h>
Alex Williamson664e9382013-04-30 15:42:28 -060031#include <linux/stat.h>
Alex Williamsoncba33452012-07-31 08:16:22 -060032#include <linux/string.h>
33#include <linux/uaccess.h>
34#include <linux/vfio.h>
35#include <linux/wait.h>
36
37#define DRIVER_VERSION "0.3"
38#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
39#define DRIVER_DESC "VFIO - User Level meta-driver"
40
41static struct vfio {
42 struct class *class;
43 struct list_head iommu_drivers_list;
44 struct mutex iommu_drivers_lock;
45 struct list_head group_list;
46 struct idr group_idr;
47 struct mutex group_lock;
48 struct cdev group_cdev;
Alex Williamsond1099902013-12-19 10:17:13 -070049 dev_t group_devt;
Alex Williamsoncba33452012-07-31 08:16:22 -060050 wait_queue_head_t release_q;
51} vfio;
52
53struct vfio_iommu_driver {
54 const struct vfio_iommu_driver_ops *ops;
55 struct list_head vfio_next;
56};
57
58struct vfio_container {
59 struct kref kref;
60 struct list_head group_list;
Alex Williamson9587f442013-04-25 16:12:38 -060061 struct rw_semaphore group_lock;
Alex Williamsoncba33452012-07-31 08:16:22 -060062 struct vfio_iommu_driver *iommu_driver;
63 void *iommu_data;
64};
65
Alex Williamson60720a02015-02-06 15:05:06 -070066struct vfio_unbound_dev {
67 struct device *dev;
68 struct list_head unbound_next;
69};
70
Alex Williamsoncba33452012-07-31 08:16:22 -060071struct vfio_group {
72 struct kref kref;
73 int minor;
74 atomic_t container_users;
75 struct iommu_group *iommu_group;
76 struct vfio_container *container;
77 struct list_head device_list;
78 struct mutex device_lock;
79 struct device *dev;
80 struct notifier_block nb;
81 struct list_head vfio_next;
82 struct list_head container_next;
Alex Williamson60720a02015-02-06 15:05:06 -070083 struct list_head unbound_list;
84 struct mutex unbound_lock;
Alex Williamson6d6768c2013-06-25 16:06:54 -060085 atomic_t opened;
Alex Williamsoncba33452012-07-31 08:16:22 -060086};
87
88struct vfio_device {
89 struct kref kref;
90 struct device *dev;
91 const struct vfio_device_ops *ops;
92 struct vfio_group *group;
93 struct list_head group_next;
94 void *device_data;
95};
96
97/**
98 * IOMMU driver registration
99 */
100int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
101{
102 struct vfio_iommu_driver *driver, *tmp;
103
104 driver = kzalloc(sizeof(*driver), GFP_KERNEL);
105 if (!driver)
106 return -ENOMEM;
107
108 driver->ops = ops;
109
110 mutex_lock(&vfio.iommu_drivers_lock);
111
112 /* Check for duplicates */
113 list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
114 if (tmp->ops == ops) {
115 mutex_unlock(&vfio.iommu_drivers_lock);
116 kfree(driver);
117 return -EINVAL;
118 }
119 }
120
121 list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
122
123 mutex_unlock(&vfio.iommu_drivers_lock);
124
125 return 0;
126}
127EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
128
129void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
130{
131 struct vfio_iommu_driver *driver;
132
133 mutex_lock(&vfio.iommu_drivers_lock);
134 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
135 if (driver->ops == ops) {
136 list_del(&driver->vfio_next);
137 mutex_unlock(&vfio.iommu_drivers_lock);
138 kfree(driver);
139 return;
140 }
141 }
142 mutex_unlock(&vfio.iommu_drivers_lock);
143}
144EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
145
146/**
147 * Group minor allocation/free - both called with vfio.group_lock held
148 */
149static int vfio_alloc_group_minor(struct vfio_group *group)
150{
Alex Williamsond1099902013-12-19 10:17:13 -0700151 return idr_alloc(&vfio.group_idr, group, 0, MINORMASK + 1, GFP_KERNEL);
Alex Williamsoncba33452012-07-31 08:16:22 -0600152}
153
154static void vfio_free_group_minor(int minor)
155{
156 idr_remove(&vfio.group_idr, minor);
157}
158
159static int vfio_iommu_group_notifier(struct notifier_block *nb,
160 unsigned long action, void *data);
161static void vfio_group_get(struct vfio_group *group);
162
163/**
164 * Container objects - containers are created when /dev/vfio/vfio is
165 * opened, but their lifecycle extends until the last user is done, so
166 * it's freed via kref. Must support container/group/device being
167 * closed in any order.
168 */
169static void vfio_container_get(struct vfio_container *container)
170{
171 kref_get(&container->kref);
172}
173
174static void vfio_container_release(struct kref *kref)
175{
176 struct vfio_container *container;
177 container = container_of(kref, struct vfio_container, kref);
178
179 kfree(container);
180}
181
182static void vfio_container_put(struct vfio_container *container)
183{
184 kref_put(&container->kref, vfio_container_release);
185}
186
Jiang Liu9df7b252012-12-07 13:43:50 -0700187static void vfio_group_unlock_and_free(struct vfio_group *group)
188{
189 mutex_unlock(&vfio.group_lock);
190 /*
191 * Unregister outside of lock. A spurious callback is harmless now
192 * that the group is no longer in vfio.group_list.
193 */
194 iommu_group_unregister_notifier(group->iommu_group, &group->nb);
195 kfree(group);
196}
197
Alex Williamsoncba33452012-07-31 08:16:22 -0600198/**
199 * Group objects - create, release, get, put, search
200 */
201static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
202{
203 struct vfio_group *group, *tmp;
204 struct device *dev;
205 int ret, minor;
206
207 group = kzalloc(sizeof(*group), GFP_KERNEL);
208 if (!group)
209 return ERR_PTR(-ENOMEM);
210
211 kref_init(&group->kref);
212 INIT_LIST_HEAD(&group->device_list);
213 mutex_init(&group->device_lock);
Alex Williamson60720a02015-02-06 15:05:06 -0700214 INIT_LIST_HEAD(&group->unbound_list);
215 mutex_init(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600216 atomic_set(&group->container_users, 0);
Alex Williamson6d6768c2013-06-25 16:06:54 -0600217 atomic_set(&group->opened, 0);
Alex Williamsoncba33452012-07-31 08:16:22 -0600218 group->iommu_group = iommu_group;
219
220 group->nb.notifier_call = vfio_iommu_group_notifier;
221
222 /*
223 * blocking notifiers acquire a rwsem around registering and hold
224 * it around callback. Therefore, need to register outside of
225 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
226 * do anything unless it can find the group in vfio.group_list, so
227 * no harm in registering early.
228 */
229 ret = iommu_group_register_notifier(iommu_group, &group->nb);
230 if (ret) {
231 kfree(group);
232 return ERR_PTR(ret);
233 }
234
235 mutex_lock(&vfio.group_lock);
236
Alex Williamsoncba33452012-07-31 08:16:22 -0600237 /* Did we race creating this group? */
238 list_for_each_entry(tmp, &vfio.group_list, vfio_next) {
239 if (tmp->iommu_group == iommu_group) {
240 vfio_group_get(tmp);
Jiang Liu9df7b252012-12-07 13:43:50 -0700241 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600242 return tmp;
243 }
244 }
245
Zhen Lei2f51bf42015-03-16 14:08:56 -0600246 minor = vfio_alloc_group_minor(group);
247 if (minor < 0) {
248 vfio_group_unlock_and_free(group);
249 return ERR_PTR(minor);
250 }
251
Alex Williamsond1099902013-12-19 10:17:13 -0700252 dev = device_create(vfio.class, NULL,
253 MKDEV(MAJOR(vfio.group_devt), minor),
Alex Williamsoncba33452012-07-31 08:16:22 -0600254 group, "%d", iommu_group_id(iommu_group));
255 if (IS_ERR(dev)) {
256 vfio_free_group_minor(minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700257 vfio_group_unlock_and_free(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600258 return (struct vfio_group *)dev; /* ERR_PTR */
259 }
260
261 group->minor = minor;
262 group->dev = dev;
263
264 list_add(&group->vfio_next, &vfio.group_list);
265
266 mutex_unlock(&vfio.group_lock);
267
268 return group;
269}
270
Al Viro6d2cd3c2012-08-17 21:27:32 -0400271/* called with vfio.group_lock held */
Alex Williamsoncba33452012-07-31 08:16:22 -0600272static void vfio_group_release(struct kref *kref)
273{
274 struct vfio_group *group = container_of(kref, struct vfio_group, kref);
Alex Williamson60720a02015-02-06 15:05:06 -0700275 struct vfio_unbound_dev *unbound, *tmp;
Alex Williamson4a688102015-02-06 15:05:06 -0700276 struct iommu_group *iommu_group = group->iommu_group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600277
278 WARN_ON(!list_empty(&group->device_list));
279
Alex Williamson60720a02015-02-06 15:05:06 -0700280 list_for_each_entry_safe(unbound, tmp,
281 &group->unbound_list, unbound_next) {
282 list_del(&unbound->unbound_next);
283 kfree(unbound);
284 }
285
Alex Williamsond1099902013-12-19 10:17:13 -0700286 device_destroy(vfio.class, MKDEV(MAJOR(vfio.group_devt), group->minor));
Alex Williamsoncba33452012-07-31 08:16:22 -0600287 list_del(&group->vfio_next);
288 vfio_free_group_minor(group->minor);
Jiang Liu9df7b252012-12-07 13:43:50 -0700289 vfio_group_unlock_and_free(group);
Alex Williamson4a688102015-02-06 15:05:06 -0700290 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600291}
292
293static void vfio_group_put(struct vfio_group *group)
294{
Al Viro6d2cd3c2012-08-17 21:27:32 -0400295 kref_put_mutex(&group->kref, vfio_group_release, &vfio.group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600296}
297
298/* Assume group_lock or group reference is held */
299static void vfio_group_get(struct vfio_group *group)
300{
301 kref_get(&group->kref);
302}
303
304/*
305 * Not really a try as we will sleep for mutex, but we need to make
306 * sure the group pointer is valid under lock and get a reference.
307 */
308static struct vfio_group *vfio_group_try_get(struct vfio_group *group)
309{
310 struct vfio_group *target = group;
311
312 mutex_lock(&vfio.group_lock);
313 list_for_each_entry(group, &vfio.group_list, vfio_next) {
314 if (group == target) {
315 vfio_group_get(group);
316 mutex_unlock(&vfio.group_lock);
317 return group;
318 }
319 }
320 mutex_unlock(&vfio.group_lock);
321
322 return NULL;
323}
324
325static
326struct vfio_group *vfio_group_get_from_iommu(struct iommu_group *iommu_group)
327{
328 struct vfio_group *group;
329
330 mutex_lock(&vfio.group_lock);
331 list_for_each_entry(group, &vfio.group_list, vfio_next) {
332 if (group->iommu_group == iommu_group) {
333 vfio_group_get(group);
334 mutex_unlock(&vfio.group_lock);
335 return group;
336 }
337 }
338 mutex_unlock(&vfio.group_lock);
339
340 return NULL;
341}
342
343static struct vfio_group *vfio_group_get_from_minor(int minor)
344{
345 struct vfio_group *group;
346
347 mutex_lock(&vfio.group_lock);
348 group = idr_find(&vfio.group_idr, minor);
349 if (!group) {
350 mutex_unlock(&vfio.group_lock);
351 return NULL;
352 }
353 vfio_group_get(group);
354 mutex_unlock(&vfio.group_lock);
355
356 return group;
357}
358
359/**
360 * Device objects - create, release, get, put, search
361 */
362static
363struct vfio_device *vfio_group_create_device(struct vfio_group *group,
364 struct device *dev,
365 const struct vfio_device_ops *ops,
366 void *device_data)
367{
368 struct vfio_device *device;
Alex Williamsoncba33452012-07-31 08:16:22 -0600369
370 device = kzalloc(sizeof(*device), GFP_KERNEL);
371 if (!device)
372 return ERR_PTR(-ENOMEM);
373
374 kref_init(&device->kref);
375 device->dev = dev;
376 device->group = group;
377 device->ops = ops;
378 device->device_data = device_data;
Jean Delvare8283b492014-04-14 12:55:38 +0200379 dev_set_drvdata(dev, device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600380
381 /* No need to get group_lock, caller has group reference */
382 vfio_group_get(group);
383
384 mutex_lock(&group->device_lock);
385 list_add(&device->group_next, &group->device_list);
386 mutex_unlock(&group->device_lock);
387
388 return device;
389}
390
391static void vfio_device_release(struct kref *kref)
392{
393 struct vfio_device *device = container_of(kref,
394 struct vfio_device, kref);
395 struct vfio_group *group = device->group;
396
Alex Williamsoncba33452012-07-31 08:16:22 -0600397 list_del(&device->group_next);
398 mutex_unlock(&group->device_lock);
399
400 dev_set_drvdata(device->dev, NULL);
401
402 kfree(device);
403
404 /* vfio_del_group_dev may be waiting for this device */
405 wake_up(&vfio.release_q);
406}
407
408/* Device reference always implies a group reference */
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600409void vfio_device_put(struct vfio_device *device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600410{
Al Viro934ad4c2012-08-17 19:49:09 -0400411 struct vfio_group *group = device->group;
Al Viro90b12532012-08-17 21:29:06 -0400412 kref_put_mutex(&device->kref, vfio_device_release, &group->device_lock);
Al Viro934ad4c2012-08-17 19:49:09 -0400413 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600414}
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600415EXPORT_SYMBOL_GPL(vfio_device_put);
Alex Williamsoncba33452012-07-31 08:16:22 -0600416
417static void vfio_device_get(struct vfio_device *device)
418{
419 vfio_group_get(device->group);
420 kref_get(&device->kref);
421}
422
423static struct vfio_device *vfio_group_get_device(struct vfio_group *group,
424 struct device *dev)
425{
426 struct vfio_device *device;
427
428 mutex_lock(&group->device_lock);
429 list_for_each_entry(device, &group->device_list, group_next) {
430 if (device->dev == dev) {
431 vfio_device_get(device);
432 mutex_unlock(&group->device_lock);
433 return device;
434 }
435 }
436 mutex_unlock(&group->device_lock);
437 return NULL;
438}
439
440/*
441 * Whitelist some drivers that we know are safe (no dma) or just sit on
442 * a device. It's not always practical to leave a device within a group
443 * driverless as it could get re-bound to something unsafe.
444 */
Alex Williamson2b489a42013-02-14 14:02:13 -0700445static const char * const vfio_driver_whitelist[] = { "pci-stub", "pcieport" };
Alex Williamsoncba33452012-07-31 08:16:22 -0600446
447static bool vfio_whitelisted_driver(struct device_driver *drv)
448{
449 int i;
450
451 for (i = 0; i < ARRAY_SIZE(vfio_driver_whitelist); i++) {
452 if (!strcmp(drv->name, vfio_driver_whitelist[i]))
453 return true;
454 }
455
456 return false;
457}
458
459/*
Alex Williamson60720a02015-02-06 15:05:06 -0700460 * A vfio group is viable for use by userspace if all devices are in
461 * one of the following states:
462 * - driver-less
463 * - bound to a vfio driver
464 * - bound to a whitelisted driver
465 *
466 * We use two methods to determine whether a device is bound to a vfio
467 * driver. The first is to test whether the device exists in the vfio
468 * group. The second is to test if the device exists on the group
469 * unbound_list, indicating it's in the middle of transitioning from
470 * a vfio driver to driver-less.
Alex Williamsoncba33452012-07-31 08:16:22 -0600471 */
472static int vfio_dev_viable(struct device *dev, void *data)
473{
474 struct vfio_group *group = data;
475 struct vfio_device *device;
Jiang Liude2b3ee2012-12-07 13:43:50 -0700476 struct device_driver *drv = ACCESS_ONCE(dev->driver);
Alex Williamson60720a02015-02-06 15:05:06 -0700477 struct vfio_unbound_dev *unbound;
478 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -0600479
Alex Williamson60720a02015-02-06 15:05:06 -0700480 mutex_lock(&group->unbound_lock);
481 list_for_each_entry(unbound, &group->unbound_list, unbound_next) {
482 if (dev == unbound->dev) {
483 ret = 0;
484 break;
485 }
486 }
487 mutex_unlock(&group->unbound_lock);
488
489 if (!ret || !drv || vfio_whitelisted_driver(drv))
Alex Williamsoncba33452012-07-31 08:16:22 -0600490 return 0;
491
492 device = vfio_group_get_device(group, dev);
493 if (device) {
494 vfio_device_put(device);
495 return 0;
496 }
497
Alex Williamson60720a02015-02-06 15:05:06 -0700498 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -0600499}
500
501/**
502 * Async device support
503 */
504static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev)
505{
506 struct vfio_device *device;
507
508 /* Do we already know about it? We shouldn't */
509 device = vfio_group_get_device(group, dev);
510 if (WARN_ON_ONCE(device)) {
511 vfio_device_put(device);
512 return 0;
513 }
514
515 /* Nothing to do for idle groups */
516 if (!atomic_read(&group->container_users))
517 return 0;
518
519 /* TODO Prevent device auto probing */
520 WARN("Device %s added to live group %d!\n", dev_name(dev),
521 iommu_group_id(group->iommu_group));
522
523 return 0;
524}
525
Alex Williamsoncba33452012-07-31 08:16:22 -0600526static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev)
527{
528 /* We don't care what happens when the group isn't in use */
529 if (!atomic_read(&group->container_users))
530 return 0;
531
532 return vfio_dev_viable(dev, group);
533}
534
535static int vfio_iommu_group_notifier(struct notifier_block *nb,
536 unsigned long action, void *data)
537{
538 struct vfio_group *group = container_of(nb, struct vfio_group, nb);
539 struct device *dev = data;
Alex Williamson60720a02015-02-06 15:05:06 -0700540 struct vfio_unbound_dev *unbound;
Alex Williamsoncba33452012-07-31 08:16:22 -0600541
542 /*
Alex Williamsonc6401932013-06-10 16:40:56 -0600543 * Need to go through a group_lock lookup to get a reference or we
544 * risk racing a group being removed. Ignore spurious notifies.
Alex Williamsoncba33452012-07-31 08:16:22 -0600545 */
546 group = vfio_group_try_get(group);
Alex Williamsonc6401932013-06-10 16:40:56 -0600547 if (!group)
Alex Williamsoncba33452012-07-31 08:16:22 -0600548 return NOTIFY_OK;
549
550 switch (action) {
551 case IOMMU_GROUP_NOTIFY_ADD_DEVICE:
552 vfio_group_nb_add_dev(group, dev);
553 break;
554 case IOMMU_GROUP_NOTIFY_DEL_DEVICE:
Alex Williamsonde9c7602013-06-10 16:40:56 -0600555 /*
556 * Nothing to do here. If the device is in use, then the
557 * vfio sub-driver should block the remove callback until
558 * it is unused. If the device is unused or attached to a
559 * stub driver, then it should be released and we don't
560 * care that it will be going away.
561 */
Alex Williamsoncba33452012-07-31 08:16:22 -0600562 break;
563 case IOMMU_GROUP_NOTIFY_BIND_DRIVER:
564 pr_debug("%s: Device %s, group %d binding to driver\n",
565 __func__, dev_name(dev),
566 iommu_group_id(group->iommu_group));
567 break;
568 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER:
569 pr_debug("%s: Device %s, group %d bound to driver %s\n",
570 __func__, dev_name(dev),
571 iommu_group_id(group->iommu_group), dev->driver->name);
572 BUG_ON(vfio_group_nb_verify(group, dev));
573 break;
574 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER:
575 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
576 __func__, dev_name(dev),
577 iommu_group_id(group->iommu_group), dev->driver->name);
578 break;
579 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER:
580 pr_debug("%s: Device %s, group %d unbound from driver\n",
581 __func__, dev_name(dev),
582 iommu_group_id(group->iommu_group));
583 /*
584 * XXX An unbound device in a live group is ok, but we'd
585 * really like to avoid the above BUG_ON by preventing other
586 * drivers from binding to it. Once that occurs, we have to
587 * stop the system to maintain isolation. At a minimum, we'd
588 * want a toggle to disable driver auto probe for this device.
589 */
Alex Williamson60720a02015-02-06 15:05:06 -0700590
591 mutex_lock(&group->unbound_lock);
592 list_for_each_entry(unbound,
593 &group->unbound_list, unbound_next) {
594 if (dev == unbound->dev) {
595 list_del(&unbound->unbound_next);
596 kfree(unbound);
597 break;
598 }
599 }
600 mutex_unlock(&group->unbound_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600601 break;
602 }
603
604 vfio_group_put(group);
605 return NOTIFY_OK;
606}
607
608/**
609 * VFIO driver API
610 */
611int vfio_add_group_dev(struct device *dev,
612 const struct vfio_device_ops *ops, void *device_data)
613{
614 struct iommu_group *iommu_group;
615 struct vfio_group *group;
616 struct vfio_device *device;
617
618 iommu_group = iommu_group_get(dev);
619 if (!iommu_group)
620 return -EINVAL;
621
622 group = vfio_group_get_from_iommu(iommu_group);
623 if (!group) {
624 group = vfio_create_group(iommu_group);
625 if (IS_ERR(group)) {
626 iommu_group_put(iommu_group);
627 return PTR_ERR(group);
628 }
Alex Williamson4a688102015-02-06 15:05:06 -0700629 } else {
630 /*
631 * A found vfio_group already holds a reference to the
632 * iommu_group. A created vfio_group keeps the reference.
633 */
634 iommu_group_put(iommu_group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600635 }
636
637 device = vfio_group_get_device(group, dev);
638 if (device) {
639 WARN(1, "Device %s already exists on group %d\n",
640 dev_name(dev), iommu_group_id(iommu_group));
641 vfio_device_put(device);
642 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600643 return -EBUSY;
644 }
645
646 device = vfio_group_create_device(group, dev, ops, device_data);
647 if (IS_ERR(device)) {
648 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600649 return PTR_ERR(device);
650 }
651
652 /*
Alex Williamson4a688102015-02-06 15:05:06 -0700653 * Drop all but the vfio_device reference. The vfio_device holds
654 * a reference to the vfio_group, which holds a reference to the
655 * iommu_group.
Alex Williamsoncba33452012-07-31 08:16:22 -0600656 */
657 vfio_group_put(group);
658
659 return 0;
660}
661EXPORT_SYMBOL_GPL(vfio_add_group_dev);
662
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600663/**
Alex Williamson20f30012015-06-09 10:08:57 -0600664 * Get a reference to the vfio_device for a device. Even if the
665 * caller thinks they own the device, they could be racing with a
666 * release call path, so we can't trust drvdata for the shortcut.
667 * Go the long way around, from the iommu_group to the vfio_group
668 * to the vfio_device.
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600669 */
670struct vfio_device *vfio_device_get_from_dev(struct device *dev)
671{
Alex Williamson20f30012015-06-09 10:08:57 -0600672 struct iommu_group *iommu_group;
673 struct vfio_group *group;
674 struct vfio_device *device;
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600675
Alex Williamson20f30012015-06-09 10:08:57 -0600676 iommu_group = iommu_group_get(dev);
677 if (!iommu_group)
678 return NULL;
679
680 group = vfio_group_get_from_iommu(iommu_group);
681 iommu_group_put(iommu_group);
682 if (!group)
683 return NULL;
684
685 device = vfio_group_get_device(group, dev);
686 vfio_group_put(group);
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600687
688 return device;
689}
690EXPORT_SYMBOL_GPL(vfio_device_get_from_dev);
691
Alex Williamson4bc94d52015-07-24 15:14:04 -0600692static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
693 char *buf)
694{
695 struct vfio_device *device;
696
697 mutex_lock(&group->device_lock);
698 list_for_each_entry(device, &group->device_list, group_next) {
699 if (!strcmp(dev_name(device->dev), buf)) {
700 vfio_device_get(device);
701 break;
702 }
703 }
704 mutex_unlock(&group->device_lock);
705
706 return device;
707}
708
Vijay Mohan Pandarathil44f50712013-03-11 09:28:44 -0600709/*
710 * Caller must hold a reference to the vfio_device
711 */
712void *vfio_device_data(struct vfio_device *device)
713{
714 return device->device_data;
715}
716EXPORT_SYMBOL_GPL(vfio_device_data);
717
Alex Williamsone014e942013-02-14 14:02:13 -0700718/* Given a referenced group, check if it contains the device */
719static bool vfio_dev_present(struct vfio_group *group, struct device *dev)
Alex Williamsoncba33452012-07-31 08:16:22 -0600720{
Alex Williamsoncba33452012-07-31 08:16:22 -0600721 struct vfio_device *device;
722
Alex Williamsoncba33452012-07-31 08:16:22 -0600723 device = vfio_group_get_device(group, dev);
Alex Williamsone014e942013-02-14 14:02:13 -0700724 if (!device)
Alex Williamsoncba33452012-07-31 08:16:22 -0600725 return false;
Alex Williamsoncba33452012-07-31 08:16:22 -0600726
727 vfio_device_put(device);
Alex Williamsoncba33452012-07-31 08:16:22 -0600728 return true;
729}
730
731/*
732 * Decrement the device reference count and wait for the device to be
733 * removed. Open file descriptors for the device... */
734void *vfio_del_group_dev(struct device *dev)
735{
736 struct vfio_device *device = dev_get_drvdata(dev);
737 struct vfio_group *group = device->group;
Alex Williamsoncba33452012-07-31 08:16:22 -0600738 void *device_data = device->device_data;
Alex Williamson60720a02015-02-06 15:05:06 -0700739 struct vfio_unbound_dev *unbound;
Alex Williamson13060b62015-02-06 15:05:07 -0700740 unsigned int i = 0;
Alex Williamsondb7d4d72015-05-01 16:31:41 -0600741 long ret;
742 bool interrupted = false;
Alex Williamsoncba33452012-07-31 08:16:22 -0600743
Alex Williamsone014e942013-02-14 14:02:13 -0700744 /*
745 * The group exists so long as we have a device reference. Get
746 * a group reference and use it to scan for the device going away.
747 */
748 vfio_group_get(group);
749
Alex Williamson60720a02015-02-06 15:05:06 -0700750 /*
751 * When the device is removed from the group, the group suddenly
752 * becomes non-viable; the device has a driver (until the unbind
753 * completes), but it's not present in the group. This is bad news
754 * for any external users that need to re-acquire a group reference
755 * in order to match and release their existing reference. To
756 * solve this, we track such devices on the unbound_list to bridge
757 * the gap until they're fully unbound.
758 */
759 unbound = kzalloc(sizeof(*unbound), GFP_KERNEL);
760 if (unbound) {
761 unbound->dev = dev;
762 mutex_lock(&group->unbound_lock);
763 list_add(&unbound->unbound_next, &group->unbound_list);
764 mutex_unlock(&group->unbound_lock);
765 }
766 WARN_ON(!unbound);
767
Alex Williamsoncba33452012-07-31 08:16:22 -0600768 vfio_device_put(device);
769
Alex Williamson13060b62015-02-06 15:05:07 -0700770 /*
771 * If the device is still present in the group after the above
772 * 'put', then it is in use and we need to request it from the
773 * bus driver. The driver may in turn need to request the
774 * device from the user. We send the request on an arbitrary
775 * interval with counter to allow the driver to take escalating
776 * measures to release the device if it has the ability to do so.
777 */
778 do {
779 device = vfio_group_get_device(group, dev);
780 if (!device)
781 break;
782
783 if (device->ops->request)
784 device->ops->request(device_data, i++);
785
786 vfio_device_put(device);
787
Alex Williamsondb7d4d72015-05-01 16:31:41 -0600788 if (interrupted) {
789 ret = wait_event_timeout(vfio.release_q,
790 !vfio_dev_present(group, dev), HZ * 10);
791 } else {
792 ret = wait_event_interruptible_timeout(vfio.release_q,
793 !vfio_dev_present(group, dev), HZ * 10);
794 if (ret == -ERESTARTSYS) {
795 interrupted = true;
796 dev_warn(dev,
797 "Device is currently in use, task"
798 " \"%s\" (%d) "
799 "blocked until device is released",
800 current->comm, task_pid_nr(current));
801 }
802 }
803 } while (ret <= 0);
Alex Williamsone014e942013-02-14 14:02:13 -0700804
805 vfio_group_put(group);
Alex Williamsoncba33452012-07-31 08:16:22 -0600806
Alex Williamsoncba33452012-07-31 08:16:22 -0600807 return device_data;
808}
809EXPORT_SYMBOL_GPL(vfio_del_group_dev);
810
811/**
812 * VFIO base fd, /dev/vfio/vfio
813 */
814static long vfio_ioctl_check_extension(struct vfio_container *container,
815 unsigned long arg)
816{
Alex Williamson0b43c082013-04-29 08:41:36 -0600817 struct vfio_iommu_driver *driver;
Alex Williamsoncba33452012-07-31 08:16:22 -0600818 long ret = 0;
819
Alex Williamson0b43c082013-04-29 08:41:36 -0600820 down_read(&container->group_lock);
821
822 driver = container->iommu_driver;
823
Alex Williamsoncba33452012-07-31 08:16:22 -0600824 switch (arg) {
825 /* No base extensions yet */
826 default:
827 /*
828 * If no driver is set, poll all registered drivers for
829 * extensions and return the first positive result. If
830 * a driver is already set, further queries will be passed
831 * only to that driver.
832 */
833 if (!driver) {
834 mutex_lock(&vfio.iommu_drivers_lock);
835 list_for_each_entry(driver, &vfio.iommu_drivers_list,
836 vfio_next) {
837 if (!try_module_get(driver->ops->owner))
838 continue;
839
840 ret = driver->ops->ioctl(NULL,
841 VFIO_CHECK_EXTENSION,
842 arg);
843 module_put(driver->ops->owner);
844 if (ret > 0)
845 break;
846 }
847 mutex_unlock(&vfio.iommu_drivers_lock);
848 } else
849 ret = driver->ops->ioctl(container->iommu_data,
850 VFIO_CHECK_EXTENSION, arg);
851 }
852
Alex Williamson0b43c082013-04-29 08:41:36 -0600853 up_read(&container->group_lock);
854
Alex Williamsoncba33452012-07-31 08:16:22 -0600855 return ret;
856}
857
Alex Williamson9587f442013-04-25 16:12:38 -0600858/* hold write lock on container->group_lock */
Alex Williamsoncba33452012-07-31 08:16:22 -0600859static int __vfio_container_attach_groups(struct vfio_container *container,
860 struct vfio_iommu_driver *driver,
861 void *data)
862{
863 struct vfio_group *group;
864 int ret = -ENODEV;
865
866 list_for_each_entry(group, &container->group_list, container_next) {
867 ret = driver->ops->attach_group(data, group->iommu_group);
868 if (ret)
869 goto unwind;
870 }
871
872 return ret;
873
874unwind:
875 list_for_each_entry_continue_reverse(group, &container->group_list,
876 container_next) {
877 driver->ops->detach_group(data, group->iommu_group);
878 }
879
880 return ret;
881}
882
883static long vfio_ioctl_set_iommu(struct vfio_container *container,
884 unsigned long arg)
885{
886 struct vfio_iommu_driver *driver;
887 long ret = -ENODEV;
888
Alex Williamson9587f442013-04-25 16:12:38 -0600889 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600890
891 /*
892 * The container is designed to be an unprivileged interface while
893 * the group can be assigned to specific users. Therefore, only by
894 * adding a group to a container does the user get the privilege of
895 * enabling the iommu, which may allocate finite resources. There
896 * is no unset_iommu, but by removing all the groups from a container,
897 * the container is deprivileged and returns to an unset state.
898 */
899 if (list_empty(&container->group_list) || container->iommu_driver) {
Alex Williamson9587f442013-04-25 16:12:38 -0600900 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600901 return -EINVAL;
902 }
903
904 mutex_lock(&vfio.iommu_drivers_lock);
905 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
906 void *data;
907
908 if (!try_module_get(driver->ops->owner))
909 continue;
910
911 /*
912 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
913 * so test which iommu driver reported support for this
914 * extension and call open on them. We also pass them the
915 * magic, allowing a single driver to support multiple
916 * interfaces if they'd like.
917 */
918 if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
919 module_put(driver->ops->owner);
920 continue;
921 }
922
923 /* module reference holds the driver we're working on */
924 mutex_unlock(&vfio.iommu_drivers_lock);
925
926 data = driver->ops->open(arg);
927 if (IS_ERR(data)) {
928 ret = PTR_ERR(data);
929 module_put(driver->ops->owner);
930 goto skip_drivers_unlock;
931 }
932
933 ret = __vfio_container_attach_groups(container, driver, data);
934 if (!ret) {
935 container->iommu_driver = driver;
936 container->iommu_data = data;
937 } else {
938 driver->ops->release(data);
939 module_put(driver->ops->owner);
940 }
941
942 goto skip_drivers_unlock;
943 }
944
945 mutex_unlock(&vfio.iommu_drivers_lock);
946skip_drivers_unlock:
Alex Williamson9587f442013-04-25 16:12:38 -0600947 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600948
949 return ret;
950}
951
952static long vfio_fops_unl_ioctl(struct file *filep,
953 unsigned int cmd, unsigned long arg)
954{
955 struct vfio_container *container = filep->private_data;
956 struct vfio_iommu_driver *driver;
957 void *data;
958 long ret = -EINVAL;
959
960 if (!container)
961 return ret;
962
Alex Williamsoncba33452012-07-31 08:16:22 -0600963 switch (cmd) {
964 case VFIO_GET_API_VERSION:
965 ret = VFIO_API_VERSION;
966 break;
967 case VFIO_CHECK_EXTENSION:
968 ret = vfio_ioctl_check_extension(container, arg);
969 break;
970 case VFIO_SET_IOMMU:
971 ret = vfio_ioctl_set_iommu(container, arg);
972 break;
973 default:
Alex Williamson0b43c082013-04-29 08:41:36 -0600974 down_read(&container->group_lock);
975
976 driver = container->iommu_driver;
977 data = container->iommu_data;
978
Alex Williamsoncba33452012-07-31 08:16:22 -0600979 if (driver) /* passthrough all unrecognized ioctls */
980 ret = driver->ops->ioctl(data, cmd, arg);
Alex Williamson0b43c082013-04-29 08:41:36 -0600981
982 up_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -0600983 }
984
985 return ret;
986}
987
988#ifdef CONFIG_COMPAT
989static long vfio_fops_compat_ioctl(struct file *filep,
990 unsigned int cmd, unsigned long arg)
991{
992 arg = (unsigned long)compat_ptr(arg);
993 return vfio_fops_unl_ioctl(filep, cmd, arg);
994}
995#endif /* CONFIG_COMPAT */
996
997static int vfio_fops_open(struct inode *inode, struct file *filep)
998{
999 struct vfio_container *container;
1000
1001 container = kzalloc(sizeof(*container), GFP_KERNEL);
1002 if (!container)
1003 return -ENOMEM;
1004
1005 INIT_LIST_HEAD(&container->group_list);
Alex Williamson9587f442013-04-25 16:12:38 -06001006 init_rwsem(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001007 kref_init(&container->kref);
1008
1009 filep->private_data = container;
1010
1011 return 0;
1012}
1013
1014static int vfio_fops_release(struct inode *inode, struct file *filep)
1015{
1016 struct vfio_container *container = filep->private_data;
1017
1018 filep->private_data = NULL;
1019
1020 vfio_container_put(container);
1021
1022 return 0;
1023}
1024
1025/*
1026 * Once an iommu driver is set, we optionally pass read/write/mmap
1027 * on to the driver, allowing management interfaces beyond ioctl.
1028 */
1029static ssize_t vfio_fops_read(struct file *filep, char __user *buf,
1030 size_t count, loff_t *ppos)
1031{
1032 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001033 struct vfio_iommu_driver *driver;
1034 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001035
Alex Williamson0b43c082013-04-29 08:41:36 -06001036 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001037
Alex Williamson0b43c082013-04-29 08:41:36 -06001038 driver = container->iommu_driver;
1039 if (likely(driver && driver->ops->read))
1040 ret = driver->ops->read(container->iommu_data,
1041 buf, count, ppos);
1042
1043 up_read(&container->group_lock);
1044
1045 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001046}
1047
1048static ssize_t vfio_fops_write(struct file *filep, const char __user *buf,
1049 size_t count, loff_t *ppos)
1050{
1051 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001052 struct vfio_iommu_driver *driver;
1053 ssize_t ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001054
Alex Williamson0b43c082013-04-29 08:41:36 -06001055 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001056
Alex Williamson0b43c082013-04-29 08:41:36 -06001057 driver = container->iommu_driver;
1058 if (likely(driver && driver->ops->write))
1059 ret = driver->ops->write(container->iommu_data,
1060 buf, count, ppos);
1061
1062 up_read(&container->group_lock);
1063
1064 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001065}
1066
1067static int vfio_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1068{
1069 struct vfio_container *container = filep->private_data;
Alex Williamson0b43c082013-04-29 08:41:36 -06001070 struct vfio_iommu_driver *driver;
1071 int ret = -EINVAL;
Alex Williamsoncba33452012-07-31 08:16:22 -06001072
Alex Williamson0b43c082013-04-29 08:41:36 -06001073 down_read(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001074
Alex Williamson0b43c082013-04-29 08:41:36 -06001075 driver = container->iommu_driver;
1076 if (likely(driver && driver->ops->mmap))
1077 ret = driver->ops->mmap(container->iommu_data, vma);
1078
1079 up_read(&container->group_lock);
1080
1081 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001082}
1083
1084static const struct file_operations vfio_fops = {
1085 .owner = THIS_MODULE,
1086 .open = vfio_fops_open,
1087 .release = vfio_fops_release,
1088 .read = vfio_fops_read,
1089 .write = vfio_fops_write,
1090 .unlocked_ioctl = vfio_fops_unl_ioctl,
1091#ifdef CONFIG_COMPAT
1092 .compat_ioctl = vfio_fops_compat_ioctl,
1093#endif
1094 .mmap = vfio_fops_mmap,
1095};
1096
1097/**
1098 * VFIO Group fd, /dev/vfio/$GROUP
1099 */
1100static void __vfio_group_unset_container(struct vfio_group *group)
1101{
1102 struct vfio_container *container = group->container;
1103 struct vfio_iommu_driver *driver;
1104
Alex Williamson9587f442013-04-25 16:12:38 -06001105 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001106
1107 driver = container->iommu_driver;
1108 if (driver)
1109 driver->ops->detach_group(container->iommu_data,
1110 group->iommu_group);
1111
1112 group->container = NULL;
1113 list_del(&group->container_next);
1114
1115 /* Detaching the last group deprivileges a container, remove iommu */
1116 if (driver && list_empty(&container->group_list)) {
1117 driver->ops->release(container->iommu_data);
1118 module_put(driver->ops->owner);
1119 container->iommu_driver = NULL;
1120 container->iommu_data = NULL;
1121 }
1122
Alex Williamson9587f442013-04-25 16:12:38 -06001123 up_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001124
1125 vfio_container_put(container);
1126}
1127
1128/*
1129 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
1130 * if there was no container to unset. Since the ioctl is called on
1131 * the group, we know that still exists, therefore the only valid
1132 * transition here is 1->0.
1133 */
1134static int vfio_group_unset_container(struct vfio_group *group)
1135{
1136 int users = atomic_cmpxchg(&group->container_users, 1, 0);
1137
1138 if (!users)
1139 return -EINVAL;
1140 if (users != 1)
1141 return -EBUSY;
1142
1143 __vfio_group_unset_container(group);
1144
1145 return 0;
1146}
1147
1148/*
1149 * When removing container users, anything that removes the last user
1150 * implicitly removes the group from the container. That is, if the
1151 * group file descriptor is closed, as well as any device file descriptors,
1152 * the group is free.
1153 */
1154static void vfio_group_try_dissolve_container(struct vfio_group *group)
1155{
1156 if (0 == atomic_dec_if_positive(&group->container_users))
1157 __vfio_group_unset_container(group);
1158}
1159
1160static int vfio_group_set_container(struct vfio_group *group, int container_fd)
1161{
Al Viro2903ff02012-08-28 12:52:22 -04001162 struct fd f;
Alex Williamsoncba33452012-07-31 08:16:22 -06001163 struct vfio_container *container;
1164 struct vfio_iommu_driver *driver;
Al Viro2903ff02012-08-28 12:52:22 -04001165 int ret = 0;
Alex Williamsoncba33452012-07-31 08:16:22 -06001166
1167 if (atomic_read(&group->container_users))
1168 return -EINVAL;
1169
Al Viro2903ff02012-08-28 12:52:22 -04001170 f = fdget(container_fd);
1171 if (!f.file)
Alex Williamsoncba33452012-07-31 08:16:22 -06001172 return -EBADF;
1173
1174 /* Sanity check, is this really our fd? */
Al Viro2903ff02012-08-28 12:52:22 -04001175 if (f.file->f_op != &vfio_fops) {
1176 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001177 return -EINVAL;
1178 }
1179
Al Viro2903ff02012-08-28 12:52:22 -04001180 container = f.file->private_data;
Alex Williamsoncba33452012-07-31 08:16:22 -06001181 WARN_ON(!container); /* fget ensures we don't race vfio_release */
1182
Alex Williamson9587f442013-04-25 16:12:38 -06001183 down_write(&container->group_lock);
Alex Williamsoncba33452012-07-31 08:16:22 -06001184
1185 driver = container->iommu_driver;
1186 if (driver) {
1187 ret = driver->ops->attach_group(container->iommu_data,
1188 group->iommu_group);
1189 if (ret)
1190 goto unlock_out;
1191 }
1192
1193 group->container = container;
1194 list_add(&group->container_next, &container->group_list);
1195
1196 /* Get a reference on the container and mark a user within the group */
1197 vfio_container_get(container);
1198 atomic_inc(&group->container_users);
1199
1200unlock_out:
Alex Williamson9587f442013-04-25 16:12:38 -06001201 up_write(&container->group_lock);
Al Viro2903ff02012-08-28 12:52:22 -04001202 fdput(f);
Alex Williamsoncba33452012-07-31 08:16:22 -06001203 return ret;
1204}
1205
1206static bool vfio_group_viable(struct vfio_group *group)
1207{
1208 return (iommu_group_for_each_dev(group->iommu_group,
1209 group, vfio_dev_viable) == 0);
1210}
1211
1212static const struct file_operations vfio_device_fops;
1213
1214static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
1215{
1216 struct vfio_device *device;
1217 struct file *filep;
Alex Williamson4bc94d52015-07-24 15:14:04 -06001218 int ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001219
1220 if (0 == atomic_read(&group->container_users) ||
1221 !group->container->iommu_driver || !vfio_group_viable(group))
1222 return -EINVAL;
1223
Alex Williamson4bc94d52015-07-24 15:14:04 -06001224 device = vfio_device_get_from_name(group, buf);
1225 if (!device)
1226 return -ENODEV;
Alex Williamsoncba33452012-07-31 08:16:22 -06001227
Alex Williamson4bc94d52015-07-24 15:14:04 -06001228 ret = device->ops->open(device->device_data);
1229 if (ret) {
1230 vfio_device_put(device);
1231 return ret;
Alex Williamsoncba33452012-07-31 08:16:22 -06001232 }
Alex Williamson4bc94d52015-07-24 15:14:04 -06001233
1234 /*
1235 * We can't use anon_inode_getfd() because we need to modify
1236 * the f_mode flags directly to allow more than just ioctls
1237 */
1238 ret = get_unused_fd_flags(O_CLOEXEC);
1239 if (ret < 0) {
1240 device->ops->release(device->device_data);
1241 vfio_device_put(device);
1242 return ret;
1243 }
1244
1245 filep = anon_inode_getfile("[vfio-device]", &vfio_device_fops,
1246 device, O_RDWR);
1247 if (IS_ERR(filep)) {
1248 put_unused_fd(ret);
1249 ret = PTR_ERR(filep);
1250 device->ops->release(device->device_data);
1251 vfio_device_put(device);
1252 return ret;
1253 }
1254
1255 /*
1256 * TODO: add an anon_inode interface to do this.
1257 * Appears to be missing by lack of need rather than
1258 * explicitly prevented. Now there's need.
1259 */
1260 filep->f_mode |= (FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
1261
1262 atomic_inc(&group->container_users);
1263
1264 fd_install(ret, filep);
Alex Williamsoncba33452012-07-31 08:16:22 -06001265
1266 return ret;
1267}
1268
1269static long vfio_group_fops_unl_ioctl(struct file *filep,
1270 unsigned int cmd, unsigned long arg)
1271{
1272 struct vfio_group *group = filep->private_data;
1273 long ret = -ENOTTY;
1274
1275 switch (cmd) {
1276 case VFIO_GROUP_GET_STATUS:
1277 {
1278 struct vfio_group_status status;
1279 unsigned long minsz;
1280
1281 minsz = offsetofend(struct vfio_group_status, flags);
1282
1283 if (copy_from_user(&status, (void __user *)arg, minsz))
1284 return -EFAULT;
1285
1286 if (status.argsz < minsz)
1287 return -EINVAL;
1288
1289 status.flags = 0;
1290
1291 if (vfio_group_viable(group))
1292 status.flags |= VFIO_GROUP_FLAGS_VIABLE;
1293
1294 if (group->container)
1295 status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET;
1296
1297 if (copy_to_user((void __user *)arg, &status, minsz))
1298 return -EFAULT;
1299
1300 ret = 0;
1301 break;
1302 }
1303 case VFIO_GROUP_SET_CONTAINER:
1304 {
1305 int fd;
1306
1307 if (get_user(fd, (int __user *)arg))
1308 return -EFAULT;
1309
1310 if (fd < 0)
1311 return -EINVAL;
1312
1313 ret = vfio_group_set_container(group, fd);
1314 break;
1315 }
1316 case VFIO_GROUP_UNSET_CONTAINER:
1317 ret = vfio_group_unset_container(group);
1318 break;
1319 case VFIO_GROUP_GET_DEVICE_FD:
1320 {
1321 char *buf;
1322
1323 buf = strndup_user((const char __user *)arg, PAGE_SIZE);
1324 if (IS_ERR(buf))
1325 return PTR_ERR(buf);
1326
1327 ret = vfio_group_get_device_fd(group, buf);
1328 kfree(buf);
1329 break;
1330 }
1331 }
1332
1333 return ret;
1334}
1335
1336#ifdef CONFIG_COMPAT
1337static long vfio_group_fops_compat_ioctl(struct file *filep,
1338 unsigned int cmd, unsigned long arg)
1339{
1340 arg = (unsigned long)compat_ptr(arg);
1341 return vfio_group_fops_unl_ioctl(filep, cmd, arg);
1342}
1343#endif /* CONFIG_COMPAT */
1344
1345static int vfio_group_fops_open(struct inode *inode, struct file *filep)
1346{
1347 struct vfio_group *group;
Alex Williamson6d6768c2013-06-25 16:06:54 -06001348 int opened;
Alex Williamsoncba33452012-07-31 08:16:22 -06001349
1350 group = vfio_group_get_from_minor(iminor(inode));
1351 if (!group)
1352 return -ENODEV;
1353
Alex Williamson6d6768c2013-06-25 16:06:54 -06001354 /* Do we need multiple instances of the group open? Seems not. */
1355 opened = atomic_cmpxchg(&group->opened, 0, 1);
1356 if (opened) {
1357 vfio_group_put(group);
1358 return -EBUSY;
1359 }
1360
1361 /* Is something still in use from a previous open? */
Alex Williamsoncba33452012-07-31 08:16:22 -06001362 if (group->container) {
Alex Williamson6d6768c2013-06-25 16:06:54 -06001363 atomic_dec(&group->opened);
Alex Williamsoncba33452012-07-31 08:16:22 -06001364 vfio_group_put(group);
1365 return -EBUSY;
1366 }
1367
1368 filep->private_data = group;
1369
1370 return 0;
1371}
1372
1373static int vfio_group_fops_release(struct inode *inode, struct file *filep)
1374{
1375 struct vfio_group *group = filep->private_data;
1376
1377 filep->private_data = NULL;
1378
1379 vfio_group_try_dissolve_container(group);
1380
Alex Williamson6d6768c2013-06-25 16:06:54 -06001381 atomic_dec(&group->opened);
1382
Alex Williamsoncba33452012-07-31 08:16:22 -06001383 vfio_group_put(group);
1384
1385 return 0;
1386}
1387
1388static const struct file_operations vfio_group_fops = {
1389 .owner = THIS_MODULE,
1390 .unlocked_ioctl = vfio_group_fops_unl_ioctl,
1391#ifdef CONFIG_COMPAT
1392 .compat_ioctl = vfio_group_fops_compat_ioctl,
1393#endif
1394 .open = vfio_group_fops_open,
1395 .release = vfio_group_fops_release,
1396};
1397
1398/**
1399 * VFIO Device fd
1400 */
1401static int vfio_device_fops_release(struct inode *inode, struct file *filep)
1402{
1403 struct vfio_device *device = filep->private_data;
1404
1405 device->ops->release(device->device_data);
1406
1407 vfio_group_try_dissolve_container(device->group);
1408
1409 vfio_device_put(device);
1410
1411 return 0;
1412}
1413
1414static long vfio_device_fops_unl_ioctl(struct file *filep,
1415 unsigned int cmd, unsigned long arg)
1416{
1417 struct vfio_device *device = filep->private_data;
1418
1419 if (unlikely(!device->ops->ioctl))
1420 return -EINVAL;
1421
1422 return device->ops->ioctl(device->device_data, cmd, arg);
1423}
1424
1425static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1426 size_t count, loff_t *ppos)
1427{
1428 struct vfio_device *device = filep->private_data;
1429
1430 if (unlikely(!device->ops->read))
1431 return -EINVAL;
1432
1433 return device->ops->read(device->device_data, buf, count, ppos);
1434}
1435
1436static ssize_t vfio_device_fops_write(struct file *filep,
1437 const char __user *buf,
1438 size_t count, loff_t *ppos)
1439{
1440 struct vfio_device *device = filep->private_data;
1441
1442 if (unlikely(!device->ops->write))
1443 return -EINVAL;
1444
1445 return device->ops->write(device->device_data, buf, count, ppos);
1446}
1447
1448static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1449{
1450 struct vfio_device *device = filep->private_data;
1451
1452 if (unlikely(!device->ops->mmap))
1453 return -EINVAL;
1454
1455 return device->ops->mmap(device->device_data, vma);
1456}
1457
1458#ifdef CONFIG_COMPAT
1459static long vfio_device_fops_compat_ioctl(struct file *filep,
1460 unsigned int cmd, unsigned long arg)
1461{
1462 arg = (unsigned long)compat_ptr(arg);
1463 return vfio_device_fops_unl_ioctl(filep, cmd, arg);
1464}
1465#endif /* CONFIG_COMPAT */
1466
1467static const struct file_operations vfio_device_fops = {
1468 .owner = THIS_MODULE,
1469 .release = vfio_device_fops_release,
1470 .read = vfio_device_fops_read,
1471 .write = vfio_device_fops_write,
1472 .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1473#ifdef CONFIG_COMPAT
1474 .compat_ioctl = vfio_device_fops_compat_ioctl,
1475#endif
1476 .mmap = vfio_device_fops_mmap,
1477};
1478
1479/**
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001480 * External user API, exported by symbols to be linked dynamically.
1481 *
1482 * The protocol includes:
1483 * 1. do normal VFIO init operation:
1484 * - opening a new container;
1485 * - attaching group(s) to it;
1486 * - setting an IOMMU driver for a container.
1487 * When IOMMU is set for a container, all groups in it are
1488 * considered ready to use by an external user.
1489 *
1490 * 2. User space passes a group fd to an external user.
1491 * The external user calls vfio_group_get_external_user()
1492 * to verify that:
1493 * - the group is initialized;
1494 * - IOMMU is set for it.
1495 * If both checks passed, vfio_group_get_external_user()
1496 * increments the container user counter to prevent
1497 * the VFIO group from disposal before KVM exits.
1498 *
1499 * 3. The external user calls vfio_external_user_iommu_id()
1500 * to know an IOMMU ID.
1501 *
1502 * 4. When the external KVM finishes, it calls
1503 * vfio_group_put_external_user() to release the VFIO group.
1504 * This call decrements the container user counter.
1505 */
1506struct vfio_group *vfio_group_get_external_user(struct file *filep)
1507{
1508 struct vfio_group *group = filep->private_data;
1509
1510 if (filep->f_op != &vfio_group_fops)
1511 return ERR_PTR(-EINVAL);
1512
1513 if (!atomic_inc_not_zero(&group->container_users))
1514 return ERR_PTR(-EINVAL);
1515
1516 if (!group->container->iommu_driver ||
1517 !vfio_group_viable(group)) {
1518 atomic_dec(&group->container_users);
1519 return ERR_PTR(-EINVAL);
1520 }
1521
1522 vfio_group_get(group);
1523
1524 return group;
1525}
1526EXPORT_SYMBOL_GPL(vfio_group_get_external_user);
1527
1528void vfio_group_put_external_user(struct vfio_group *group)
1529{
1530 vfio_group_put(group);
1531 vfio_group_try_dissolve_container(group);
1532}
1533EXPORT_SYMBOL_GPL(vfio_group_put_external_user);
1534
1535int vfio_external_user_iommu_id(struct vfio_group *group)
1536{
1537 return iommu_group_id(group->iommu_group);
1538}
1539EXPORT_SYMBOL_GPL(vfio_external_user_iommu_id);
1540
Alex Williamson88d7ab82014-02-26 11:38:39 -07001541long vfio_external_check_extension(struct vfio_group *group, unsigned long arg)
1542{
1543 return vfio_ioctl_check_extension(group->container, arg);
1544}
1545EXPORT_SYMBOL_GPL(vfio_external_check_extension);
1546
Alexey Kardashevskiy6cdd9782013-08-05 10:52:36 -06001547/**
Alex Williamsoncba33452012-07-31 08:16:22 -06001548 * Module/class support
1549 */
1550static char *vfio_devnode(struct device *dev, umode_t *mode)
1551{
1552 return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
1553}
1554
Alex Williamsond1099902013-12-19 10:17:13 -07001555static struct miscdevice vfio_dev = {
1556 .minor = VFIO_MINOR,
1557 .name = "vfio",
1558 .fops = &vfio_fops,
1559 .nodename = "vfio/vfio",
1560 .mode = S_IRUGO | S_IWUGO,
1561};
1562
Alex Williamsoncba33452012-07-31 08:16:22 -06001563static int __init vfio_init(void)
1564{
1565 int ret;
1566
1567 idr_init(&vfio.group_idr);
1568 mutex_init(&vfio.group_lock);
1569 mutex_init(&vfio.iommu_drivers_lock);
1570 INIT_LIST_HEAD(&vfio.group_list);
1571 INIT_LIST_HEAD(&vfio.iommu_drivers_list);
1572 init_waitqueue_head(&vfio.release_q);
1573
Alex Williamsond1099902013-12-19 10:17:13 -07001574 ret = misc_register(&vfio_dev);
1575 if (ret) {
1576 pr_err("vfio: misc device register failed\n");
1577 return ret;
1578 }
1579
1580 /* /dev/vfio/$GROUP */
Alex Williamsoncba33452012-07-31 08:16:22 -06001581 vfio.class = class_create(THIS_MODULE, "vfio");
1582 if (IS_ERR(vfio.class)) {
1583 ret = PTR_ERR(vfio.class);
1584 goto err_class;
1585 }
1586
1587 vfio.class->devnode = vfio_devnode;
1588
Alex Williamsond1099902013-12-19 10:17:13 -07001589 ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK, "vfio");
Alex Williamsoncba33452012-07-31 08:16:22 -06001590 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001591 goto err_alloc_chrdev;
Alex Williamsoncba33452012-07-31 08:16:22 -06001592
Alex Williamsoncba33452012-07-31 08:16:22 -06001593 cdev_init(&vfio.group_cdev, &vfio_group_fops);
Alex Williamsond1099902013-12-19 10:17:13 -07001594 ret = cdev_add(&vfio.group_cdev, vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001595 if (ret)
Alex Williamsond1099902013-12-19 10:17:13 -07001596 goto err_cdev_add;
Alex Williamsoncba33452012-07-31 08:16:22 -06001597
1598 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1599
Alex Williamson73fa0d12012-07-31 08:16:23 -06001600 /*
1601 * Attempt to load known iommu-drivers. This gives us a working
1602 * environment without the user needing to explicitly load iommu
1603 * drivers.
1604 */
1605 request_module_nowait("vfio_iommu_type1");
Alexey Kardashevskiy5ffd2292013-05-21 13:33:10 +10001606 request_module_nowait("vfio_iommu_spapr_tce");
Alex Williamson73fa0d12012-07-31 08:16:23 -06001607
Alex Williamsoncba33452012-07-31 08:16:22 -06001608 return 0;
1609
Alex Williamsond1099902013-12-19 10:17:13 -07001610err_cdev_add:
1611 unregister_chrdev_region(vfio.group_devt, MINORMASK);
1612err_alloc_chrdev:
Alex Williamsoncba33452012-07-31 08:16:22 -06001613 class_destroy(vfio.class);
1614 vfio.class = NULL;
1615err_class:
Alex Williamsond1099902013-12-19 10:17:13 -07001616 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001617 return ret;
1618}
1619
1620static void __exit vfio_cleanup(void)
1621{
1622 WARN_ON(!list_empty(&vfio.group_list));
1623
1624 idr_destroy(&vfio.group_idr);
1625 cdev_del(&vfio.group_cdev);
Alex Williamsond1099902013-12-19 10:17:13 -07001626 unregister_chrdev_region(vfio.group_devt, MINORMASK);
Alex Williamsoncba33452012-07-31 08:16:22 -06001627 class_destroy(vfio.class);
1628 vfio.class = NULL;
Alex Williamsond1099902013-12-19 10:17:13 -07001629 misc_deregister(&vfio_dev);
Alex Williamsoncba33452012-07-31 08:16:22 -06001630}
1631
1632module_init(vfio_init);
1633module_exit(vfio_cleanup);
1634
1635MODULE_VERSION(DRIVER_VERSION);
1636MODULE_LICENSE("GPL v2");
1637MODULE_AUTHOR(DRIVER_AUTHOR);
1638MODULE_DESCRIPTION(DRIVER_DESC);
Alex Williamsond1099902013-12-19 10:17:13 -07001639MODULE_ALIAS_MISCDEV(VFIO_MINOR);
1640MODULE_ALIAS("devname:vfio/vfio");