Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Generic page table allocator for IOMMUs. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License version 2 as |
| 6 | * published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 11 | * GNU General Public License for more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License |
| 14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 15 | * |
| 16 | * Copyright (C) 2014 ARM Limited |
| 17 | * |
| 18 | * Author: Will Deacon <will.deacon@arm.com> |
| 19 | */ |
| 20 | |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 21 | #define pr_fmt(fmt) "io-pgtable: " fmt |
| 22 | |
Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 23 | #include <linux/bug.h> |
| 24 | #include <linux/kernel.h> |
| 25 | #include <linux/types.h> |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 26 | #include <linux/iommu.h> |
| 27 | #include <linux/debugfs.h> |
| 28 | #include <linux/atomic.h> |
| 29 | #include <linux/module.h> |
Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 30 | |
| 31 | #include "io-pgtable.h" |
| 32 | |
| 33 | static const struct io_pgtable_init_fns * |
Cosmin-Gabriel Samoila | 54c6d24 | 2016-03-13 02:17:27 +0200 | [diff] [blame] | 34 | io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { |
Will Deacon | e1d3c0f | 2014-11-14 17:18:23 +0000 | [diff] [blame] | 35 | #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE |
| 36 | [ARM_32_LPAE_S1] = &io_pgtable_arm_32_lpae_s1_init_fns, |
| 37 | [ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns, |
| 38 | [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, |
| 39 | [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, |
| 40 | #endif |
Robin Murphy | e5fc975 | 2016-01-26 17:13:13 +0000 | [diff] [blame] | 41 | #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S |
| 42 | [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, |
| 43 | #endif |
Mitchel Humpherys | 86a560e | 2015-09-30 14:23:58 -0700 | [diff] [blame] | 44 | #ifdef CONFIG_IOMMU_IO_PGTABLE_FAST |
| 45 | [ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns, |
| 46 | #endif |
Charan Teja Reddy | 35144b0 | 2017-09-05 16:20:46 +0530 | [diff] [blame] | 47 | #ifdef CONFIG_MSM_TZ_SMMU |
| 48 | [ARM_MSM_SECURE] = &io_pgtable_arm_msm_secure_init_fns, |
| 49 | #endif |
Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 50 | }; |
| 51 | |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 52 | static struct dentry *io_pgtable_top; |
| 53 | |
Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 54 | struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, |
| 55 | struct io_pgtable_cfg *cfg, |
| 56 | void *cookie) |
| 57 | { |
| 58 | struct io_pgtable *iop; |
| 59 | const struct io_pgtable_init_fns *fns; |
| 60 | |
| 61 | if (fmt >= IO_PGTABLE_NUM_FMTS) |
| 62 | return NULL; |
| 63 | |
| 64 | fns = io_pgtable_init_table[fmt]; |
| 65 | if (!fns) |
| 66 | return NULL; |
| 67 | |
| 68 | iop = fns->alloc(cfg, cookie); |
| 69 | if (!iop) |
| 70 | return NULL; |
| 71 | |
| 72 | iop->fmt = fmt; |
| 73 | iop->cookie = cookie; |
| 74 | iop->cfg = *cfg; |
| 75 | |
| 76 | return &iop->ops; |
| 77 | } |
| 78 | |
| 79 | /* |
| 80 | * It is the IOMMU driver's responsibility to ensure that the page table |
| 81 | * is no longer accessible to the walker by this point. |
| 82 | */ |
| 83 | void free_io_pgtable_ops(struct io_pgtable_ops *ops) |
| 84 | { |
| 85 | struct io_pgtable *iop; |
| 86 | |
| 87 | if (!ops) |
| 88 | return; |
| 89 | |
| 90 | iop = container_of(ops, struct io_pgtable, ops); |
Robin Murphy | 507e4c9 | 2016-01-26 17:13:14 +0000 | [diff] [blame] | 91 | io_pgtable_tlb_flush_all(iop); |
Will Deacon | fdb1d7b | 2014-11-14 17:16:49 +0000 | [diff] [blame] | 92 | io_pgtable_init_table[iop->fmt]->free(iop); |
| 93 | } |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 94 | |
| 95 | static atomic_t pages_allocated; |
| 96 | |
Patrick Daly | c11d108 | 2016-09-01 15:52:44 -0700 | [diff] [blame] | 97 | void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, |
| 98 | size_t size, gfp_t gfp_mask) |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 99 | { |
Patrick Daly | c11d108 | 2016-09-01 15:52:44 -0700 | [diff] [blame] | 100 | void *ret; |
| 101 | |
| 102 | if (cfg->tlb->alloc_pages_exact) |
| 103 | ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask); |
| 104 | else |
| 105 | ret = alloc_pages_exact(size, gfp_mask); |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 106 | |
| 107 | if (likely(ret)) |
| 108 | atomic_add(1 << get_order(size), &pages_allocated); |
Patrick Daly | c11d108 | 2016-09-01 15:52:44 -0700 | [diff] [blame] | 109 | |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 110 | return ret; |
| 111 | } |
| 112 | |
Patrick Daly | c11d108 | 2016-09-01 15:52:44 -0700 | [diff] [blame] | 113 | void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, |
| 114 | void *virt, size_t size) |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 115 | { |
Patrick Daly | c11d108 | 2016-09-01 15:52:44 -0700 | [diff] [blame] | 116 | if (cfg->tlb->free_pages_exact) |
| 117 | cfg->tlb->free_pages_exact(cookie, virt, size); |
| 118 | else |
| 119 | free_pages_exact(virt, size); |
| 120 | |
Mitchel Humpherys | f01d6e3 | 2015-07-15 18:25:07 -0700 | [diff] [blame] | 121 | atomic_sub(1 << get_order(size), &pages_allocated); |
| 122 | } |
| 123 | |
| 124 | static int io_pgtable_init(void) |
| 125 | { |
| 126 | io_pgtable_top = debugfs_create_dir("io-pgtable", iommu_debugfs_top); |
| 127 | |
| 128 | if (!io_pgtable_top) |
| 129 | return -ENODEV; |
| 130 | |
| 131 | if (!debugfs_create_atomic_t("pages", 0600, |
| 132 | io_pgtable_top, &pages_allocated)) { |
| 133 | debugfs_remove_recursive(io_pgtable_top); |
| 134 | return -ENODEV; |
| 135 | } |
| 136 | |
| 137 | return 0; |
| 138 | } |
| 139 | |
| 140 | static void io_pgtable_exit(void) |
| 141 | { |
| 142 | debugfs_remove_recursive(io_pgtable_top); |
| 143 | } |
| 144 | |
| 145 | module_init(io_pgtable_init); |
| 146 | module_exit(io_pgtable_exit); |