blob: b6307545124c8570498282544a76720b9074a483 [file] [log] [blame]
Joerg Roedelfc2100e2008-11-26 17:21:24 +01001/*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +020019#define pr_fmt(fmt) "%s: " fmt, __func__
20
Joerg Roedel905d66c2011-09-06 16:03:26 +020021#include <linux/device.h>
Ohad Ben-Cohen40998182011-09-02 13:32:32 -040022#include <linux/kernel.h>
Joerg Roedelfc2100e2008-11-26 17:21:24 +010023#include <linux/bug.h>
24#include <linux/types.h>
Andrew Morton60db4022009-05-06 16:03:07 -070025#include <linux/module.h>
26#include <linux/slab.h>
Joerg Roedelfc2100e2008-11-26 17:21:24 +010027#include <linux/errno.h>
28#include <linux/iommu.h>
Alex Williamsond72e31c2012-05-30 14:18:53 -060029#include <linux/idr.h>
30#include <linux/notifier.h>
31#include <linux/err.h>
Shuah Khan7f6db172013-08-15 11:59:23 -060032#include <trace/events/iommu.h>
Joerg Roedelfc2100e2008-11-26 17:21:24 +010033
Alex Williamsond72e31c2012-05-30 14:18:53 -060034static struct kset *iommu_group_kset;
35static struct ida iommu_group_ida;
36static struct mutex iommu_group_mutex;
37
38struct iommu_group {
39 struct kobject kobj;
40 struct kobject *devices_kobj;
41 struct list_head devices;
42 struct mutex mutex;
43 struct blocking_notifier_head notifier;
44 void *iommu_data;
45 void (*iommu_data_release)(void *iommu_data);
46 char *name;
47 int id;
48};
49
50struct iommu_device {
51 struct list_head list;
52 struct device *dev;
53 char *name;
54};
55
56struct iommu_group_attribute {
57 struct attribute attr;
58 ssize_t (*show)(struct iommu_group *group, char *buf);
59 ssize_t (*store)(struct iommu_group *group,
60 const char *buf, size_t count);
61};
62
63#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
64struct iommu_group_attribute iommu_group_attr_##_name = \
65 __ATTR(_name, _mode, _show, _store)
66
67#define to_iommu_group_attr(_attr) \
68 container_of(_attr, struct iommu_group_attribute, attr)
69#define to_iommu_group(_kobj) \
70 container_of(_kobj, struct iommu_group, kobj)
71
72static ssize_t iommu_group_attr_show(struct kobject *kobj,
73 struct attribute *__attr, char *buf)
Alex Williamson14604322011-10-21 15:56:05 -040074{
Alex Williamsond72e31c2012-05-30 14:18:53 -060075 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
76 struct iommu_group *group = to_iommu_group(kobj);
77 ssize_t ret = -EIO;
Alex Williamson14604322011-10-21 15:56:05 -040078
Alex Williamsond72e31c2012-05-30 14:18:53 -060079 if (attr->show)
80 ret = attr->show(group, buf);
81 return ret;
Alex Williamson14604322011-10-21 15:56:05 -040082}
Alex Williamsond72e31c2012-05-30 14:18:53 -060083
84static ssize_t iommu_group_attr_store(struct kobject *kobj,
85 struct attribute *__attr,
86 const char *buf, size_t count)
87{
88 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
89 struct iommu_group *group = to_iommu_group(kobj);
90 ssize_t ret = -EIO;
91
92 if (attr->store)
93 ret = attr->store(group, buf, count);
94 return ret;
95}
96
97static const struct sysfs_ops iommu_group_sysfs_ops = {
98 .show = iommu_group_attr_show,
99 .store = iommu_group_attr_store,
100};
101
102static int iommu_group_create_file(struct iommu_group *group,
103 struct iommu_group_attribute *attr)
104{
105 return sysfs_create_file(&group->kobj, &attr->attr);
106}
107
108static void iommu_group_remove_file(struct iommu_group *group,
109 struct iommu_group_attribute *attr)
110{
111 sysfs_remove_file(&group->kobj, &attr->attr);
112}
113
114static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
115{
116 return sprintf(buf, "%s\n", group->name);
117}
118
119static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
120
121static void iommu_group_release(struct kobject *kobj)
122{
123 struct iommu_group *group = to_iommu_group(kobj);
124
125 if (group->iommu_data_release)
126 group->iommu_data_release(group->iommu_data);
127
128 mutex_lock(&iommu_group_mutex);
129 ida_remove(&iommu_group_ida, group->id);
130 mutex_unlock(&iommu_group_mutex);
131
132 kfree(group->name);
133 kfree(group);
134}
135
136static struct kobj_type iommu_group_ktype = {
137 .sysfs_ops = &iommu_group_sysfs_ops,
138 .release = iommu_group_release,
139};
140
141/**
142 * iommu_group_alloc - Allocate a new group
143 * @name: Optional name to associate with group, visible in sysfs
144 *
145 * This function is called by an iommu driver to allocate a new iommu
146 * group. The iommu group represents the minimum granularity of the iommu.
147 * Upon successful return, the caller holds a reference to the supplied
148 * group in order to hold the group until devices are added. Use
149 * iommu_group_put() to release this extra reference count, allowing the
150 * group to be automatically reclaimed once it has no devices or external
151 * references.
152 */
153struct iommu_group *iommu_group_alloc(void)
154{
155 struct iommu_group *group;
156 int ret;
157
158 group = kzalloc(sizeof(*group), GFP_KERNEL);
159 if (!group)
160 return ERR_PTR(-ENOMEM);
161
162 group->kobj.kset = iommu_group_kset;
163 mutex_init(&group->mutex);
164 INIT_LIST_HEAD(&group->devices);
165 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
166
167 mutex_lock(&iommu_group_mutex);
168
169again:
170 if (unlikely(0 == ida_pre_get(&iommu_group_ida, GFP_KERNEL))) {
171 kfree(group);
172 mutex_unlock(&iommu_group_mutex);
173 return ERR_PTR(-ENOMEM);
174 }
175
176 if (-EAGAIN == ida_get_new(&iommu_group_ida, &group->id))
177 goto again;
178
179 mutex_unlock(&iommu_group_mutex);
180
181 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
182 NULL, "%d", group->id);
183 if (ret) {
184 mutex_lock(&iommu_group_mutex);
185 ida_remove(&iommu_group_ida, group->id);
186 mutex_unlock(&iommu_group_mutex);
187 kfree(group);
188 return ERR_PTR(ret);
189 }
190
191 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
192 if (!group->devices_kobj) {
193 kobject_put(&group->kobj); /* triggers .release & free */
194 return ERR_PTR(-ENOMEM);
195 }
196
197 /*
198 * The devices_kobj holds a reference on the group kobject, so
199 * as long as that exists so will the group. We can therefore
200 * use the devices_kobj for reference counting.
201 */
202 kobject_put(&group->kobj);
203
204 return group;
205}
206EXPORT_SYMBOL_GPL(iommu_group_alloc);
207
Alexey Kardashevskiyaa16bea2013-03-25 10:23:49 +1100208struct iommu_group *iommu_group_get_by_id(int id)
209{
210 struct kobject *group_kobj;
211 struct iommu_group *group;
212 const char *name;
213
214 if (!iommu_group_kset)
215 return NULL;
216
217 name = kasprintf(GFP_KERNEL, "%d", id);
218 if (!name)
219 return NULL;
220
221 group_kobj = kset_find_obj(iommu_group_kset, name);
222 kfree(name);
223
224 if (!group_kobj)
225 return NULL;
226
227 group = container_of(group_kobj, struct iommu_group, kobj);
228 BUG_ON(group->id != id);
229
230 kobject_get(group->devices_kobj);
231 kobject_put(&group->kobj);
232
233 return group;
234}
235EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
236
Alex Williamsond72e31c2012-05-30 14:18:53 -0600237/**
238 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
239 * @group: the group
240 *
241 * iommu drivers can store data in the group for use when doing iommu
242 * operations. This function provides a way to retrieve it. Caller
243 * should hold a group reference.
244 */
245void *iommu_group_get_iommudata(struct iommu_group *group)
246{
247 return group->iommu_data;
248}
249EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
250
251/**
252 * iommu_group_set_iommudata - set iommu_data for a group
253 * @group: the group
254 * @iommu_data: new data
255 * @release: release function for iommu_data
256 *
257 * iommu drivers can store data in the group for use when doing iommu
258 * operations. This function provides a way to set the data after
259 * the group has been allocated. Caller should hold a group reference.
260 */
261void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
262 void (*release)(void *iommu_data))
263{
264 group->iommu_data = iommu_data;
265 group->iommu_data_release = release;
266}
267EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
268
269/**
270 * iommu_group_set_name - set name for a group
271 * @group: the group
272 * @name: name
273 *
274 * Allow iommu driver to set a name for a group. When set it will
275 * appear in a name attribute file under the group in sysfs.
276 */
277int iommu_group_set_name(struct iommu_group *group, const char *name)
278{
279 int ret;
280
281 if (group->name) {
282 iommu_group_remove_file(group, &iommu_group_attr_name);
283 kfree(group->name);
284 group->name = NULL;
285 if (!name)
286 return 0;
287 }
288
289 group->name = kstrdup(name, GFP_KERNEL);
290 if (!group->name)
291 return -ENOMEM;
292
293 ret = iommu_group_create_file(group, &iommu_group_attr_name);
294 if (ret) {
295 kfree(group->name);
296 group->name = NULL;
297 return ret;
298 }
299
300 return 0;
301}
302EXPORT_SYMBOL_GPL(iommu_group_set_name);
303
304/**
305 * iommu_group_add_device - add a device to an iommu group
306 * @group: the group into which to add the device (reference should be held)
307 * @dev: the device
308 *
309 * This function is called by an iommu driver to add a device into a
310 * group. Adding a device increments the group reference count.
311 */
312int iommu_group_add_device(struct iommu_group *group, struct device *dev)
313{
314 int ret, i = 0;
315 struct iommu_device *device;
316
317 device = kzalloc(sizeof(*device), GFP_KERNEL);
318 if (!device)
319 return -ENOMEM;
320
321 device->dev = dev;
322
323 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
324 if (ret) {
325 kfree(device);
326 return ret;
327 }
328
329 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
330rename:
331 if (!device->name) {
332 sysfs_remove_link(&dev->kobj, "iommu_group");
333 kfree(device);
334 return -ENOMEM;
335 }
336
337 ret = sysfs_create_link_nowarn(group->devices_kobj,
338 &dev->kobj, device->name);
339 if (ret) {
340 kfree(device->name);
341 if (ret == -EEXIST && i >= 0) {
342 /*
343 * Account for the slim chance of collision
344 * and append an instance to the name.
345 */
346 device->name = kasprintf(GFP_KERNEL, "%s.%d",
347 kobject_name(&dev->kobj), i++);
348 goto rename;
349 }
350
351 sysfs_remove_link(&dev->kobj, "iommu_group");
352 kfree(device);
353 return ret;
354 }
355
356 kobject_get(group->devices_kobj);
357
358 dev->iommu_group = group;
359
360 mutex_lock(&group->mutex);
361 list_add_tail(&device->list, &group->devices);
362 mutex_unlock(&group->mutex);
363
364 /* Notify any listeners about change to group. */
365 blocking_notifier_call_chain(&group->notifier,
366 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
Shuah Khand1cf7e82013-08-15 11:59:24 -0600367
368 trace_add_device_to_group(group->id, dev);
Alex Williamsond72e31c2012-05-30 14:18:53 -0600369 return 0;
370}
371EXPORT_SYMBOL_GPL(iommu_group_add_device);
372
373/**
374 * iommu_group_remove_device - remove a device from it's current group
375 * @dev: device to be removed
376 *
377 * This function is called by an iommu driver to remove the device from
378 * it's current group. This decrements the iommu group reference count.
379 */
380void iommu_group_remove_device(struct device *dev)
381{
382 struct iommu_group *group = dev->iommu_group;
383 struct iommu_device *tmp_device, *device = NULL;
384
385 /* Pre-notify listeners that a device is being removed. */
386 blocking_notifier_call_chain(&group->notifier,
387 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
388
389 mutex_lock(&group->mutex);
390 list_for_each_entry(tmp_device, &group->devices, list) {
391 if (tmp_device->dev == dev) {
392 device = tmp_device;
393 list_del(&device->list);
394 break;
395 }
396 }
397 mutex_unlock(&group->mutex);
398
399 if (!device)
400 return;
401
402 sysfs_remove_link(group->devices_kobj, device->name);
403 sysfs_remove_link(&dev->kobj, "iommu_group");
404
Shuah Khan2e757082013-08-15 11:59:25 -0600405 trace_remove_device_from_group(group->id, dev);
406
Alex Williamsond72e31c2012-05-30 14:18:53 -0600407 kfree(device->name);
408 kfree(device);
409 dev->iommu_group = NULL;
410 kobject_put(group->devices_kobj);
411}
412EXPORT_SYMBOL_GPL(iommu_group_remove_device);
413
414/**
415 * iommu_group_for_each_dev - iterate over each device in the group
416 * @group: the group
417 * @data: caller opaque data to be passed to callback function
418 * @fn: caller supplied callback function
419 *
420 * This function is called by group users to iterate over group devices.
421 * Callers should hold a reference count to the group during callback.
422 * The group->mutex is held across callbacks, which will block calls to
423 * iommu_group_add/remove_device.
424 */
425int iommu_group_for_each_dev(struct iommu_group *group, void *data,
426 int (*fn)(struct device *, void *))
427{
428 struct iommu_device *device;
429 int ret = 0;
430
431 mutex_lock(&group->mutex);
432 list_for_each_entry(device, &group->devices, list) {
433 ret = fn(device->dev, data);
434 if (ret)
435 break;
436 }
437 mutex_unlock(&group->mutex);
438 return ret;
439}
440EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
441
442/**
443 * iommu_group_get - Return the group for a device and increment reference
444 * @dev: get the group that this device belongs to
445 *
446 * This function is called by iommu drivers and users to get the group
447 * for the specified device. If found, the group is returned and the group
448 * reference in incremented, else NULL.
449 */
450struct iommu_group *iommu_group_get(struct device *dev)
451{
452 struct iommu_group *group = dev->iommu_group;
453
454 if (group)
455 kobject_get(group->devices_kobj);
456
457 return group;
458}
459EXPORT_SYMBOL_GPL(iommu_group_get);
460
461/**
462 * iommu_group_put - Decrement group reference
463 * @group: the group to use
464 *
465 * This function is called by iommu drivers and users to release the
466 * iommu group. Once the reference count is zero, the group is released.
467 */
468void iommu_group_put(struct iommu_group *group)
469{
470 if (group)
471 kobject_put(group->devices_kobj);
472}
473EXPORT_SYMBOL_GPL(iommu_group_put);
474
475/**
476 * iommu_group_register_notifier - Register a notifier for group changes
477 * @group: the group to watch
478 * @nb: notifier block to signal
479 *
480 * This function allows iommu group users to track changes in a group.
481 * See include/linux/iommu.h for actions sent via this notifier. Caller
482 * should hold a reference to the group throughout notifier registration.
483 */
484int iommu_group_register_notifier(struct iommu_group *group,
485 struct notifier_block *nb)
486{
487 return blocking_notifier_chain_register(&group->notifier, nb);
488}
489EXPORT_SYMBOL_GPL(iommu_group_register_notifier);
490
491/**
492 * iommu_group_unregister_notifier - Unregister a notifier
493 * @group: the group to watch
494 * @nb: notifier block to signal
495 *
496 * Unregister a previously registered group notifier block.
497 */
498int iommu_group_unregister_notifier(struct iommu_group *group,
499 struct notifier_block *nb)
500{
501 return blocking_notifier_chain_unregister(&group->notifier, nb);
502}
503EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
504
505/**
506 * iommu_group_id - Return ID for a group
507 * @group: the group to ID
508 *
509 * Return the unique ID for the group matching the sysfs group number.
510 */
511int iommu_group_id(struct iommu_group *group)
512{
513 return group->id;
514}
515EXPORT_SYMBOL_GPL(iommu_group_id);
Alex Williamson14604322011-10-21 15:56:05 -0400516
517static int add_iommu_group(struct device *dev, void *data)
518{
Alex Williamsond72e31c2012-05-30 14:18:53 -0600519 struct iommu_ops *ops = data;
Alex Williamson14604322011-10-21 15:56:05 -0400520
Alex Williamsond72e31c2012-05-30 14:18:53 -0600521 if (!ops->add_device)
522 return -ENODEV;
523
524 WARN_ON(dev->iommu_group);
525
526 ops->add_device(dev);
Alex Williamson14604322011-10-21 15:56:05 -0400527
528 return 0;
529}
530
Alex Williamsond72e31c2012-05-30 14:18:53 -0600531static int iommu_bus_notifier(struct notifier_block *nb,
532 unsigned long action, void *data)
Alex Williamson14604322011-10-21 15:56:05 -0400533{
534 struct device *dev = data;
Alex Williamsond72e31c2012-05-30 14:18:53 -0600535 struct iommu_ops *ops = dev->bus->iommu_ops;
536 struct iommu_group *group;
537 unsigned long group_action = 0;
Alex Williamson14604322011-10-21 15:56:05 -0400538
Alex Williamsond72e31c2012-05-30 14:18:53 -0600539 /*
540 * ADD/DEL call into iommu driver ops if provided, which may
541 * result in ADD/DEL notifiers to group->notifier
542 */
543 if (action == BUS_NOTIFY_ADD_DEVICE) {
544 if (ops->add_device)
545 return ops->add_device(dev);
546 } else if (action == BUS_NOTIFY_DEL_DEVICE) {
547 if (ops->remove_device && dev->iommu_group) {
548 ops->remove_device(dev);
549 return 0;
550 }
551 }
Alex Williamson14604322011-10-21 15:56:05 -0400552
Alex Williamsond72e31c2012-05-30 14:18:53 -0600553 /*
554 * Remaining BUS_NOTIFYs get filtered and republished to the
555 * group, if anyone is listening
556 */
557 group = iommu_group_get(dev);
558 if (!group)
559 return 0;
560
561 switch (action) {
562 case BUS_NOTIFY_BIND_DRIVER:
563 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER;
564 break;
565 case BUS_NOTIFY_BOUND_DRIVER:
566 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER;
567 break;
568 case BUS_NOTIFY_UNBIND_DRIVER:
569 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER;
570 break;
571 case BUS_NOTIFY_UNBOUND_DRIVER:
572 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER;
573 break;
574 }
575
576 if (group_action)
577 blocking_notifier_call_chain(&group->notifier,
578 group_action, dev);
579
580 iommu_group_put(group);
Alex Williamson14604322011-10-21 15:56:05 -0400581 return 0;
582}
583
Alex Williamsond72e31c2012-05-30 14:18:53 -0600584static struct notifier_block iommu_bus_nb = {
585 .notifier_call = iommu_bus_notifier,
Alex Williamson14604322011-10-21 15:56:05 -0400586};
587
Joerg Roedelff217762011-08-26 16:48:26 +0200588static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100589{
Alex Williamsond72e31c2012-05-30 14:18:53 -0600590 bus_register_notifier(bus, &iommu_bus_nb);
591 bus_for_each_dev(bus, NULL, ops, add_iommu_group);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100592}
593
Joerg Roedelff217762011-08-26 16:48:26 +0200594/**
595 * bus_set_iommu - set iommu-callbacks for the bus
596 * @bus: bus.
597 * @ops: the callbacks provided by the iommu-driver
598 *
599 * This function is called by an iommu driver to set the iommu methods
600 * used for a particular bus. Drivers for devices on that bus can use
601 * the iommu-api after these ops are registered.
602 * This special function is needed because IOMMUs are usually devices on
603 * the bus itself, so the iommu drivers are not initialized when the bus
604 * is set up. With this function the iommu-driver can set the iommu-ops
605 * afterwards.
606 */
607int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100608{
Joerg Roedelff217762011-08-26 16:48:26 +0200609 if (bus->iommu_ops != NULL)
610 return -EBUSY;
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100611
Joerg Roedelff217762011-08-26 16:48:26 +0200612 bus->iommu_ops = ops;
613
614 /* Do IOMMU specific setup for this bus-type */
615 iommu_bus_init(bus, ops);
616
617 return 0;
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100618}
Joerg Roedelff217762011-08-26 16:48:26 +0200619EXPORT_SYMBOL_GPL(bus_set_iommu);
620
Joerg Roedela1b60c12011-09-06 18:46:34 +0200621bool iommu_present(struct bus_type *bus)
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100622{
Joerg Roedel94441c32011-09-06 18:58:54 +0200623 return bus->iommu_ops != NULL;
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100624}
Joerg Roedela1b60c12011-09-06 18:46:34 +0200625EXPORT_SYMBOL_GPL(iommu_present);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100626
Ohad Ben-Cohen4f3f8d92011-09-13 15:25:23 -0400627/**
628 * iommu_set_fault_handler() - set a fault handler for an iommu domain
629 * @domain: iommu domain
630 * @handler: fault handler
Ohad Ben-Cohen77ca2332012-05-21 20:20:05 +0300631 * @token: user data, will be passed back to the fault handler
Ohad Ben-Cohen0ed6d2d2011-09-27 07:36:40 -0400632 *
633 * This function should be used by IOMMU users which want to be notified
634 * whenever an IOMMU fault happens.
635 *
636 * The fault handler itself should return 0 on success, and an appropriate
637 * error code otherwise.
Ohad Ben-Cohen4f3f8d92011-09-13 15:25:23 -0400638 */
639void iommu_set_fault_handler(struct iommu_domain *domain,
Ohad Ben-Cohen77ca2332012-05-21 20:20:05 +0300640 iommu_fault_handler_t handler,
641 void *token)
Ohad Ben-Cohen4f3f8d92011-09-13 15:25:23 -0400642{
643 BUG_ON(!domain);
644
645 domain->handler = handler;
Ohad Ben-Cohen77ca2332012-05-21 20:20:05 +0300646 domain->handler_token = token;
Ohad Ben-Cohen4f3f8d92011-09-13 15:25:23 -0400647}
Ohad Ben-Cohen30bd9182011-09-26 09:11:46 -0400648EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
Ohad Ben-Cohen4f3f8d92011-09-13 15:25:23 -0400649
Joerg Roedel905d66c2011-09-06 16:03:26 +0200650struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100651{
652 struct iommu_domain *domain;
653 int ret;
654
Joerg Roedel94441c32011-09-06 18:58:54 +0200655 if (bus == NULL || bus->iommu_ops == NULL)
Joerg Roedel905d66c2011-09-06 16:03:26 +0200656 return NULL;
657
KyongHo Cho8bd69602011-12-16 21:38:25 +0900658 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100659 if (!domain)
660 return NULL;
661
Joerg Roedel94441c32011-09-06 18:58:54 +0200662 domain->ops = bus->iommu_ops;
Joerg Roedel905d66c2011-09-06 16:03:26 +0200663
Joerg Roedel94441c32011-09-06 18:58:54 +0200664 ret = domain->ops->domain_init(domain);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100665 if (ret)
666 goto out_free;
667
668 return domain;
669
670out_free:
671 kfree(domain);
672
673 return NULL;
674}
675EXPORT_SYMBOL_GPL(iommu_domain_alloc);
676
677void iommu_domain_free(struct iommu_domain *domain)
678{
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200679 if (likely(domain->ops->domain_destroy != NULL))
680 domain->ops->domain_destroy(domain);
681
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100682 kfree(domain);
683}
684EXPORT_SYMBOL_GPL(iommu_domain_free);
685
686int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
687{
Shuah Khanb54db772013-08-15 11:59:26 -0600688 int ret;
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200689 if (unlikely(domain->ops->attach_dev == NULL))
690 return -ENODEV;
691
Shuah Khanb54db772013-08-15 11:59:26 -0600692 ret = domain->ops->attach_dev(domain, dev);
693 if (!ret)
694 trace_attach_device_to_domain(dev);
695 return ret;
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100696}
697EXPORT_SYMBOL_GPL(iommu_attach_device);
698
699void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
700{
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200701 if (unlikely(domain->ops->detach_dev == NULL))
702 return;
703
704 domain->ops->detach_dev(domain, dev);
Shuah Khan69980632013-08-15 11:59:27 -0600705 trace_detach_device_from_domain(dev);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100706}
707EXPORT_SYMBOL_GPL(iommu_detach_device);
708
Alex Williamsond72e31c2012-05-30 14:18:53 -0600709/*
710 * IOMMU groups are really the natrual working unit of the IOMMU, but
711 * the IOMMU API works on domains and devices. Bridge that gap by
712 * iterating over the devices in a group. Ideally we'd have a single
713 * device which represents the requestor ID of the group, but we also
714 * allow IOMMU drivers to create policy defined minimum sets, where
715 * the physical hardware may be able to distiguish members, but we
716 * wish to group them at a higher level (ex. untrusted multi-function
717 * PCI devices). Thus we attach each device.
718 */
719static int iommu_group_do_attach_device(struct device *dev, void *data)
720{
721 struct iommu_domain *domain = data;
722
723 return iommu_attach_device(domain, dev);
724}
725
726int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
727{
728 return iommu_group_for_each_dev(group, domain,
729 iommu_group_do_attach_device);
730}
731EXPORT_SYMBOL_GPL(iommu_attach_group);
732
733static int iommu_group_do_detach_device(struct device *dev, void *data)
734{
735 struct iommu_domain *domain = data;
736
737 iommu_detach_device(domain, dev);
738
739 return 0;
740}
741
742void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
743{
744 iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device);
745}
746EXPORT_SYMBOL_GPL(iommu_detach_group);
747
Varun Sethibb5547ac2013-03-29 01:23:58 +0530748phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100749{
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200750 if (unlikely(domain->ops->iova_to_phys == NULL))
751 return 0;
752
753 return domain->ops->iova_to_phys(domain, iova);
Joerg Roedelfc2100e2008-11-26 17:21:24 +0100754}
755EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
Sheng Yangdbb9fd82009-03-18 15:33:06 +0800756
757int iommu_domain_has_cap(struct iommu_domain *domain,
758 unsigned long cap)
759{
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200760 if (unlikely(domain->ops->domain_has_cap == NULL))
761 return 0;
762
763 return domain->ops->domain_has_cap(domain, cap);
Sheng Yangdbb9fd82009-03-18 15:33:06 +0800764}
765EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100766
Alex Williamsonbd139692013-06-17 19:57:34 -0600767static size_t iommu_pgsize(struct iommu_domain *domain,
768 unsigned long addr_merge, size_t size)
769{
770 unsigned int pgsize_idx;
771 size_t pgsize;
772
773 /* Max page size that still fits into 'size' */
774 pgsize_idx = __fls(size);
775
776 /* need to consider alignment requirements ? */
777 if (likely(addr_merge)) {
778 /* Max page size allowed by address */
779 unsigned int align_pgsize_idx = __ffs(addr_merge);
780 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
781 }
782
783 /* build a mask of acceptable page sizes */
784 pgsize = (1UL << (pgsize_idx + 1)) - 1;
785
786 /* throw away page sizes not supported by the hardware */
787 pgsize &= domain->ops->pgsize_bitmap;
788
789 /* make sure we're still sane */
790 BUG_ON(!pgsize);
791
792 /* pick the biggest page */
793 pgsize_idx = __fls(pgsize);
794 pgsize = 1UL << pgsize_idx;
795
796 return pgsize;
797}
798
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100799int iommu_map(struct iommu_domain *domain, unsigned long iova,
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200800 phys_addr_t paddr, size_t size, int prot)
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100801{
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200802 unsigned long orig_iova = iova;
803 unsigned int min_pagesz;
804 size_t orig_size = size;
805 int ret = 0;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100806
Joerg Roedel57886512013-01-29 13:41:09 +0100807 if (unlikely(domain->ops->unmap == NULL ||
808 domain->ops->pgsize_bitmap == 0UL))
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200809 return -ENODEV;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100810
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200811 /* find out the minimum page size supported */
812 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100813
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200814 /*
815 * both the virtual address and the physical one, as well as
816 * the size of the mapping, must be aligned (at least) to the
817 * size of the smallest page supported by the hardware
818 */
819 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
Joe Perches6197ca82013-06-23 12:29:04 -0700820 pr_err("unaligned: iova 0x%lx pa 0x%pa size 0x%zx min_pagesz 0x%x\n",
821 iova, &paddr, size, min_pagesz);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200822 return -EINVAL;
823 }
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100824
Joe Perches6197ca82013-06-23 12:29:04 -0700825 pr_debug("map: iova 0x%lx pa 0x%pa size 0x%zx\n", iova, &paddr, size);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200826
827 while (size) {
Alex Williamsonbd139692013-06-17 19:57:34 -0600828 size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200829
Joe Perches6197ca82013-06-23 12:29:04 -0700830 pr_debug("mapping: iova 0x%lx pa 0x%pa pgsize 0x%zx\n",
831 iova, &paddr, pgsize);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200832
833 ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
834 if (ret)
835 break;
836
837 iova += pgsize;
838 paddr += pgsize;
839 size -= pgsize;
840 }
841
842 /* unroll mapping in case something went wrong */
843 if (ret)
844 iommu_unmap(domain, orig_iova, orig_size - size);
845
846 return ret;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100847}
848EXPORT_SYMBOL_GPL(iommu_map);
849
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200850size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100851{
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200852 size_t unmapped_page, unmapped = 0;
853 unsigned int min_pagesz;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100854
Joerg Roedel57886512013-01-29 13:41:09 +0100855 if (unlikely(domain->ops->unmap == NULL ||
856 domain->ops->pgsize_bitmap == 0UL))
Joerg Roedele5aa7f02011-09-06 16:44:29 +0200857 return -ENODEV;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100858
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200859 /* find out the minimum page size supported */
860 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100861
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200862 /*
863 * The virtual address, as well as the size of the mapping, must be
864 * aligned (at least) to the size of the smallest page supported
865 * by the hardware
866 */
867 if (!IS_ALIGNED(iova | size, min_pagesz)) {
Joe Perches6197ca82013-06-23 12:29:04 -0700868 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
869 iova, size, min_pagesz);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200870 return -EINVAL;
871 }
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100872
Joe Perches6197ca82013-06-23 12:29:04 -0700873 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
Ohad Ben-Cohen50090652011-11-10 11:32:25 +0200874
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200875 /*
876 * Keep iterating until we either unmap 'size' bytes (or more)
877 * or we hit an area that isn't mapped.
878 */
879 while (unmapped < size) {
Alex Williamsonbd139692013-06-17 19:57:34 -0600880 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200881
Alex Williamsonbd139692013-06-17 19:57:34 -0600882 unmapped_page = domain->ops->unmap(domain, iova, pgsize);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200883 if (!unmapped_page)
884 break;
885
Joe Perches6197ca82013-06-23 12:29:04 -0700886 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
887 iova, unmapped_page);
Ohad Ben-Cohen7d3002c2011-11-10 11:32:26 +0200888
889 iova += unmapped_page;
890 unmapped += unmapped_page;
891 }
892
893 return unmapped;
Joerg Roedelcefc53c2010-01-08 13:35:09 +0100894}
895EXPORT_SYMBOL_GPL(iommu_unmap);
Alex Williamson14604322011-10-21 15:56:05 -0400896
Joerg Roedeld7787d52013-01-29 14:26:20 +0100897
898int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
Varun Sethi80f97f02013-03-29 01:24:00 +0530899 phys_addr_t paddr, u64 size, int prot)
Joerg Roedeld7787d52013-01-29 14:26:20 +0100900{
901 if (unlikely(domain->ops->domain_window_enable == NULL))
902 return -ENODEV;
903
Varun Sethi80f97f02013-03-29 01:24:00 +0530904 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size,
905 prot);
Joerg Roedeld7787d52013-01-29 14:26:20 +0100906}
907EXPORT_SYMBOL_GPL(iommu_domain_window_enable);
908
909void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
910{
911 if (unlikely(domain->ops->domain_window_disable == NULL))
912 return;
913
914 return domain->ops->domain_window_disable(domain, wnd_nr);
915}
916EXPORT_SYMBOL_GPL(iommu_domain_window_disable);
917
Alex Williamsond72e31c2012-05-30 14:18:53 -0600918static int __init iommu_init(void)
Alex Williamson14604322011-10-21 15:56:05 -0400919{
Alex Williamsond72e31c2012-05-30 14:18:53 -0600920 iommu_group_kset = kset_create_and_add("iommu_groups",
921 NULL, kernel_kobj);
922 ida_init(&iommu_group_ida);
923 mutex_init(&iommu_group_mutex);
Alex Williamson14604322011-10-21 15:56:05 -0400924
Alex Williamsond72e31c2012-05-30 14:18:53 -0600925 BUG_ON(!iommu_group_kset);
926
927 return 0;
Alex Williamson14604322011-10-21 15:56:05 -0400928}
Alexey Kardashevskiy097e3632013-01-07 18:51:52 +1100929arch_initcall(iommu_init);
Joerg Roedel0cd76dd2012-01-26 19:40:52 +0100930
931int iommu_domain_get_attr(struct iommu_domain *domain,
932 enum iommu_attr attr, void *data)
933{
Joerg Roedel0ff64f82012-01-26 19:40:53 +0100934 struct iommu_domain_geometry *geometry;
Joerg Roedeld2e12162013-01-29 13:49:04 +0100935 bool *paging;
Joerg Roedel0ff64f82012-01-26 19:40:53 +0100936 int ret = 0;
Joerg Roedel69356712013-02-04 14:00:01 +0100937 u32 *count;
Joerg Roedel0cd76dd2012-01-26 19:40:52 +0100938
Joerg Roedel0ff64f82012-01-26 19:40:53 +0100939 switch (attr) {
940 case DOMAIN_ATTR_GEOMETRY:
941 geometry = data;
942 *geometry = domain->geometry;
943
944 break;
Joerg Roedeld2e12162013-01-29 13:49:04 +0100945 case DOMAIN_ATTR_PAGING:
946 paging = data;
947 *paging = (domain->ops->pgsize_bitmap != 0UL);
948 break;
Joerg Roedel69356712013-02-04 14:00:01 +0100949 case DOMAIN_ATTR_WINDOWS:
950 count = data;
951
952 if (domain->ops->domain_get_windows != NULL)
953 *count = domain->ops->domain_get_windows(domain);
954 else
955 ret = -ENODEV;
956
957 break;
Joerg Roedel0ff64f82012-01-26 19:40:53 +0100958 default:
959 if (!domain->ops->domain_get_attr)
960 return -EINVAL;
961
962 ret = domain->ops->domain_get_attr(domain, attr, data);
963 }
964
965 return ret;
Joerg Roedel0cd76dd2012-01-26 19:40:52 +0100966}
967EXPORT_SYMBOL_GPL(iommu_domain_get_attr);
968
969int iommu_domain_set_attr(struct iommu_domain *domain,
970 enum iommu_attr attr, void *data)
971{
Joerg Roedel69356712013-02-04 14:00:01 +0100972 int ret = 0;
973 u32 *count;
Joerg Roedel0cd76dd2012-01-26 19:40:52 +0100974
Joerg Roedel69356712013-02-04 14:00:01 +0100975 switch (attr) {
976 case DOMAIN_ATTR_WINDOWS:
977 count = data;
978
979 if (domain->ops->domain_set_windows != NULL)
980 ret = domain->ops->domain_set_windows(domain, *count);
981 else
982 ret = -ENODEV;
983
984 break;
985 default:
986 if (domain->ops->domain_set_attr == NULL)
987 return -EINVAL;
988
989 ret = domain->ops->domain_set_attr(domain, attr, data);
990 }
991
992 return ret;
Joerg Roedel0cd76dd2012-01-26 19:40:52 +0100993}
994EXPORT_SYMBOL_GPL(iommu_domain_set_attr);