Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 1 | /* |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 2 | * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * Licensed under the GPL |
| 4 | */ |
| 5 | |
Jeff Dike | 8192ab4 | 2008-02-04 22:30:53 -0800 | [diff] [blame] | 6 | #include <linux/mm.h> |
Al Viro | 73395a0 | 2011-08-18 20:14:10 +0100 | [diff] [blame] | 7 | #include <linux/module.h> |
Jeff Dike | 8192ab4 | 2008-02-04 22:30:53 -0800 | [diff] [blame] | 8 | #include <linux/sched.h> |
| 9 | #include <asm/pgtable.h> |
| 10 | #include <asm/tlbflush.h> |
Al Viro | 37185b3 | 2012-10-08 03:27:32 +0100 | [diff] [blame] | 11 | #include <as-layout.h> |
| 12 | #include <mem_user.h> |
| 13 | #include <os.h> |
| 14 | #include <skas.h> |
Richard Weinberger | 468f659 | 2014-07-20 13:16:20 +0200 | [diff] [blame] | 15 | #include <kern_util.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 17 | struct host_vm_change { |
| 18 | struct host_vm_op { |
| 19 | enum { NONE, MMAP, MUNMAP, MPROTECT } type; |
| 20 | union { |
| 21 | struct { |
| 22 | unsigned long addr; |
| 23 | unsigned long len; |
| 24 | unsigned int prot; |
| 25 | int fd; |
| 26 | __u64 offset; |
| 27 | } mmap; |
| 28 | struct { |
| 29 | unsigned long addr; |
| 30 | unsigned long len; |
| 31 | } munmap; |
| 32 | struct { |
| 33 | unsigned long addr; |
| 34 | unsigned long len; |
| 35 | unsigned int prot; |
| 36 | } mprotect; |
| 37 | } u; |
| 38 | } ops[1]; |
| 39 | int index; |
| 40 | struct mm_id *id; |
| 41 | void *data; |
| 42 | int force; |
| 43 | }; |
| 44 | |
| 45 | #define INIT_HVC(mm, force) \ |
| 46 | ((struct host_vm_change) \ |
| 47 | { .ops = { { .type = NONE } }, \ |
| 48 | .id = &mm->context.id, \ |
| 49 | .data = NULL, \ |
| 50 | .index = 0, \ |
| 51 | .force = force }) |
| 52 | |
Richard Weinberger | 70c8205 | 2015-10-25 22:26:09 +0100 | [diff] [blame] | 53 | static void report_enomem(void) |
| 54 | { |
| 55 | printk(KERN_ERR "UML ran out of memory on the host side! " |
| 56 | "This can happen due to a memory limitation or " |
| 57 | "vm.max_map_count has been reached.\n"); |
| 58 | } |
| 59 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 60 | static int do_ops(struct host_vm_change *hvc, int end, |
| 61 | int finished) |
| 62 | { |
| 63 | struct host_vm_op *op; |
| 64 | int i, ret = 0; |
| 65 | |
| 66 | for (i = 0; i < end && !ret; i++) { |
| 67 | op = &hvc->ops[i]; |
Jeff Dike | c5d4bb1 | 2008-02-04 22:31:14 -0800 | [diff] [blame] | 68 | switch (op->type) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 69 | case MMAP: |
| 70 | ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len, |
| 71 | op->u.mmap.prot, op->u.mmap.fd, |
| 72 | op->u.mmap.offset, finished, &hvc->data); |
| 73 | break; |
| 74 | case MUNMAP: |
| 75 | ret = unmap(hvc->id, op->u.munmap.addr, |
| 76 | op->u.munmap.len, finished, &hvc->data); |
| 77 | break; |
| 78 | case MPROTECT: |
| 79 | ret = protect(hvc->id, op->u.mprotect.addr, |
| 80 | op->u.mprotect.len, op->u.mprotect.prot, |
| 81 | finished, &hvc->data); |
| 82 | break; |
| 83 | default: |
| 84 | printk(KERN_ERR "Unknown op type %d in do_ops\n", |
| 85 | op->type); |
Richard Weinberger | 62179d47 | 2012-04-13 15:54:01 +0200 | [diff] [blame] | 86 | BUG(); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 87 | break; |
| 88 | } |
| 89 | } |
| 90 | |
Richard Weinberger | 70c8205 | 2015-10-25 22:26:09 +0100 | [diff] [blame] | 91 | if (ret == -ENOMEM) |
| 92 | report_enomem(); |
| 93 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 94 | return ret; |
| 95 | } |
| 96 | |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 97 | static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 98 | unsigned int prot, struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 99 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 100 | __u64 offset; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 101 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 102 | int fd, ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 103 | |
| 104 | fd = phys_mapping(phys, &offset); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 105 | if (hvc->index != 0) { |
| 106 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 107 | if ((last->type == MMAP) && |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 108 | (last->u.mmap.addr + last->u.mmap.len == virt) && |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 109 | (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) && |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 110 | (last->u.mmap.offset + last->u.mmap.len == offset)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 111 | last->u.mmap.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 112 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 113 | } |
| 114 | } |
| 115 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 116 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 117 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 118 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 119 | } |
| 120 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 121 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 122 | { .type = MMAP, |
| 123 | .u = { .mmap = { .addr = virt, |
| 124 | .len = len, |
| 125 | .prot = prot, |
| 126 | .fd = fd, |
| 127 | .offset = offset } |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 128 | } }); |
| 129 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 130 | } |
| 131 | |
| 132 | static int add_munmap(unsigned long addr, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 133 | struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 134 | { |
| 135 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 136 | int ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 137 | |
Richard Weinberger | 284e6d3 | 2014-07-20 13:09:15 +0200 | [diff] [blame] | 138 | if ((addr >= STUB_START) && (addr < STUB_END)) |
| 139 | return -EINVAL; |
| 140 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 141 | if (hvc->index != 0) { |
| 142 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 143 | if ((last->type == MUNMAP) && |
| 144 | (last->u.munmap.addr + last->u.mmap.len == addr)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 145 | last->u.munmap.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 146 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 147 | } |
| 148 | } |
| 149 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 150 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 151 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 152 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 153 | } |
| 154 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 155 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 156 | { .type = MUNMAP, |
| 157 | .u = { .munmap = { .addr = addr, |
| 158 | .len = len } } }); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 159 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 160 | } |
| 161 | |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 162 | static int add_mprotect(unsigned long addr, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 163 | unsigned int prot, struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 164 | { |
| 165 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 166 | int ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 167 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 168 | if (hvc->index != 0) { |
| 169 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 170 | if ((last->type == MPROTECT) && |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 171 | (last->u.mprotect.addr + last->u.mprotect.len == addr) && |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 172 | (last->u.mprotect.prot == prot)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 173 | last->u.mprotect.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 174 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 175 | } |
| 176 | } |
| 177 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 178 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 179 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 180 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 181 | } |
| 182 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 183 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 184 | { .type = MPROTECT, |
| 185 | .u = { .mprotect = { .addr = addr, |
| 186 | .len = len, |
| 187 | .prot = prot } } }); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 188 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 189 | } |
| 190 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1)) |
| 192 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 193 | static inline int update_pte_range(pmd_t *pmd, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 194 | unsigned long end, |
| 195 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 196 | { |
| 197 | pte_t *pte; |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 198 | int r, w, x, prot, ret = 0; |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 199 | |
| 200 | pte = pte_offset_kernel(pmd, addr); |
| 201 | do { |
Jeff Dike | 3963333 | 2008-02-04 22:31:01 -0800 | [diff] [blame] | 202 | if ((addr >= STUB_START) && (addr < STUB_END)) |
| 203 | continue; |
| 204 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 205 | r = pte_read(*pte); |
| 206 | w = pte_write(*pte); |
| 207 | x = pte_exec(*pte); |
| 208 | if (!pte_young(*pte)) { |
| 209 | r = 0; |
| 210 | w = 0; |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 211 | } else if (!pte_dirty(*pte)) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 212 | w = 0; |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 213 | |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 214 | prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) | |
| 215 | (x ? UM_PROT_EXEC : 0)); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 216 | if (hvc->force || pte_newpage(*pte)) { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 217 | if (pte_present(*pte)) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 218 | ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 219 | PAGE_SIZE, prot, hvc); |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 220 | else |
| 221 | ret = add_munmap(addr, PAGE_SIZE, hvc); |
| 222 | } else if (pte_newprot(*pte)) |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 223 | ret = add_mprotect(addr, PAGE_SIZE, prot, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 224 | *pte = pte_mkuptodate(*pte); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 225 | } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 226 | return ret; |
| 227 | } |
| 228 | |
| 229 | static inline int update_pmd_range(pud_t *pud, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 230 | unsigned long end, |
| 231 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 232 | { |
| 233 | pmd_t *pmd; |
| 234 | unsigned long next; |
| 235 | int ret = 0; |
| 236 | |
| 237 | pmd = pmd_offset(pud, addr); |
| 238 | do { |
| 239 | next = pmd_addr_end(addr, end); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 240 | if (!pmd_present(*pmd)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 241 | if (hvc->force || pmd_newpage(*pmd)) { |
| 242 | ret = add_munmap(addr, next - addr, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 243 | pmd_mkuptodate(*pmd); |
| 244 | } |
| 245 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 246 | else ret = update_pte_range(pmd, addr, next, hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 247 | } while (pmd++, addr = next, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 248 | return ret; |
| 249 | } |
| 250 | |
| 251 | static inline int update_pud_range(pgd_t *pgd, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 252 | unsigned long end, |
| 253 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 254 | { |
| 255 | pud_t *pud; |
| 256 | unsigned long next; |
| 257 | int ret = 0; |
| 258 | |
| 259 | pud = pud_offset(pgd, addr); |
| 260 | do { |
| 261 | next = pud_addr_end(addr, end); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 262 | if (!pud_present(*pud)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 263 | if (hvc->force || pud_newpage(*pud)) { |
| 264 | ret = add_munmap(addr, next - addr, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 265 | pud_mkuptodate(*pud); |
| 266 | } |
| 267 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 268 | else ret = update_pmd_range(pud, addr, next, hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 269 | } while (pud++, addr = next, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 270 | return ret; |
| 271 | } |
| 272 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 273 | void fix_range_common(struct mm_struct *mm, unsigned long start_addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 274 | unsigned long end_addr, int force) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | { |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 276 | pgd_t *pgd; |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 277 | struct host_vm_change hvc; |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 278 | unsigned long addr = start_addr, next; |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 279 | int ret = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 281 | hvc = INIT_HVC(mm, force); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 282 | pgd = pgd_offset(mm, addr); |
| 283 | do { |
| 284 | next = pgd_addr_end(addr, end_addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 285 | if (!pgd_present(*pgd)) { |
| 286 | if (force || pgd_newpage(*pgd)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 287 | ret = add_munmap(addr, next - addr, &hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 288 | pgd_mkuptodate(*pgd); |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 289 | } |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 290 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 291 | else ret = update_pud_range(pgd, addr, next, &hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 292 | } while (pgd++, addr = next, ((addr < end_addr) && !ret)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 293 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 294 | if (!ret) |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 295 | ret = do_ops(&hvc, hvc.index, 1); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 296 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 297 | /* This is not an else because ret is modified above */ |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 298 | if (ret) { |
| 299 | printk(KERN_ERR "fix_range_common: failed, killing current " |
Richard Weinberger | 468f659 | 2014-07-20 13:16:20 +0200 | [diff] [blame] | 300 | "process: %d\n", task_tgid_vnr(current)); |
| 301 | /* We are under mmap_sem, release it such that current can terminate */ |
| 302 | up_write(¤t->mm->mmap_sem); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 303 | force_sig(SIGKILL, current); |
Ingo Molnar | ccaee5f | 2015-07-03 12:44:20 -0700 | [diff] [blame] | 304 | do_signal(¤t->thread.regs); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 305 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 306 | } |
| 307 | |
Al Viro | c75d053 | 2011-08-18 20:07:49 +0100 | [diff] [blame] | 308 | static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 309 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 310 | struct mm_struct *mm; |
| 311 | pgd_t *pgd; |
| 312 | pud_t *pud; |
| 313 | pmd_t *pmd; |
| 314 | pte_t *pte; |
| 315 | unsigned long addr, last; |
| 316 | int updated = 0, err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 318 | mm = &init_mm; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 319 | for (addr = start; addr < end;) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 320 | pgd = pgd_offset(mm, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 321 | if (!pgd_present(*pgd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 322 | last = ADD_ROUND(addr, PGDIR_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 323 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 324 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 325 | if (pgd_newpage(*pgd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 326 | updated = 1; |
| 327 | err = os_unmap_memory((void *) addr, |
| 328 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 329 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 330 | panic("munmap failed, errno = %d\n", |
| 331 | -err); |
| 332 | } |
| 333 | addr = last; |
| 334 | continue; |
| 335 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 337 | pud = pud_offset(pgd, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 338 | if (!pud_present(*pud)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 339 | last = ADD_ROUND(addr, PUD_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 340 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 341 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 342 | if (pud_newpage(*pud)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 343 | updated = 1; |
| 344 | err = os_unmap_memory((void *) addr, |
| 345 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 346 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 347 | panic("munmap failed, errno = %d\n", |
| 348 | -err); |
| 349 | } |
| 350 | addr = last; |
| 351 | continue; |
| 352 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 353 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 354 | pmd = pmd_offset(pud, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 355 | if (!pmd_present(*pmd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 356 | last = ADD_ROUND(addr, PMD_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 357 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 358 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 359 | if (pmd_newpage(*pmd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 360 | updated = 1; |
| 361 | err = os_unmap_memory((void *) addr, |
| 362 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 363 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 364 | panic("munmap failed, errno = %d\n", |
| 365 | -err); |
| 366 | } |
| 367 | addr = last; |
| 368 | continue; |
| 369 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 371 | pte = pte_offset_kernel(pmd, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 372 | if (!pte_present(*pte) || pte_newpage(*pte)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 373 | updated = 1; |
| 374 | err = os_unmap_memory((void *) addr, |
| 375 | PAGE_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 376 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 377 | panic("munmap failed, errno = %d\n", |
| 378 | -err); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 379 | if (pte_present(*pte)) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 380 | map_memory(addr, |
| 381 | pte_val(*pte) & PAGE_MASK, |
| 382 | PAGE_SIZE, 1, 1, 1); |
| 383 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 384 | else if (pte_newprot(*pte)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 385 | updated = 1; |
| 386 | os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1); |
| 387 | } |
| 388 | addr += PAGE_SIZE; |
| 389 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 390 | return updated; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | } |
| 392 | |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 393 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) |
| 394 | { |
| 395 | pgd_t *pgd; |
| 396 | pud_t *pud; |
| 397 | pmd_t *pmd; |
| 398 | pte_t *pte; |
| 399 | struct mm_struct *mm = vma->vm_mm; |
| 400 | void *flush = NULL; |
| 401 | int r, w, x, prot, err = 0; |
| 402 | struct mm_id *mm_id; |
| 403 | |
| 404 | address &= PAGE_MASK; |
| 405 | pgd = pgd_offset(mm, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 406 | if (!pgd_present(*pgd)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 407 | goto kill; |
| 408 | |
| 409 | pud = pud_offset(pgd, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 410 | if (!pud_present(*pud)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 411 | goto kill; |
| 412 | |
| 413 | pmd = pmd_offset(pud, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 414 | if (!pmd_present(*pmd)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 415 | goto kill; |
| 416 | |
| 417 | pte = pte_offset_kernel(pmd, address); |
| 418 | |
| 419 | r = pte_read(*pte); |
| 420 | w = pte_write(*pte); |
| 421 | x = pte_exec(*pte); |
| 422 | if (!pte_young(*pte)) { |
| 423 | r = 0; |
| 424 | w = 0; |
| 425 | } else if (!pte_dirty(*pte)) { |
| 426 | w = 0; |
| 427 | } |
| 428 | |
Jeff Dike | 6c738ff | 2007-10-16 01:27:06 -0700 | [diff] [blame] | 429 | mm_id = &mm->context.id; |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 430 | prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) | |
| 431 | (x ? UM_PROT_EXEC : 0)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 432 | if (pte_newpage(*pte)) { |
| 433 | if (pte_present(*pte)) { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 434 | unsigned long long offset; |
| 435 | int fd; |
| 436 | |
| 437 | fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset); |
| 438 | err = map(mm_id, address, PAGE_SIZE, prot, fd, offset, |
| 439 | 1, &flush); |
| 440 | } |
| 441 | else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush); |
| 442 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 443 | else if (pte_newprot(*pte)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 444 | err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush); |
| 445 | |
Richard Weinberger | 70c8205 | 2015-10-25 22:26:09 +0100 | [diff] [blame] | 446 | if (err) { |
| 447 | if (err == -ENOMEM) |
| 448 | report_enomem(); |
| 449 | |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 450 | goto kill; |
Richard Weinberger | 70c8205 | 2015-10-25 22:26:09 +0100 | [diff] [blame] | 451 | } |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 452 | |
| 453 | *pte = pte_mkuptodate(*pte); |
| 454 | |
| 455 | return; |
| 456 | |
| 457 | kill: |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 458 | printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address); |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 459 | force_sig(SIGKILL, current); |
| 460 | } |
| 461 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 462 | pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address) |
| 463 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 464 | return pgd_offset(mm, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | } |
| 466 | |
| 467 | pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address) |
| 468 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 469 | return pud_offset(pgd, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 470 | } |
| 471 | |
| 472 | pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address) |
| 473 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 474 | return pmd_offset(pud, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | } |
| 476 | |
| 477 | pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address) |
| 478 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 479 | return pte_offset_kernel(pmd, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 480 | } |
| 481 | |
| 482 | pte_t *addr_pte(struct task_struct *task, unsigned long addr) |
| 483 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 484 | pgd_t *pgd = pgd_offset(task->mm, addr); |
| 485 | pud_t *pud = pud_offset(pgd, addr); |
| 486 | pmd_t *pmd = pmd_offset(pud, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 487 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 488 | return pte_offset_map(pmd, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 489 | } |
| 490 | |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 491 | void flush_tlb_all(void) |
| 492 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 493 | flush_tlb_mm(current->mm); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 494 | } |
| 495 | |
| 496 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) |
| 497 | { |
Jeff Dike | 6aa802c | 2007-10-16 01:26:56 -0700 | [diff] [blame] | 498 | flush_tlb_kernel_range_common(start, end); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 499 | } |
| 500 | |
| 501 | void flush_tlb_kernel_vm(void) |
| 502 | { |
Jeff Dike | 6aa802c | 2007-10-16 01:26:56 -0700 | [diff] [blame] | 503 | flush_tlb_kernel_range_common(start_vm, end_vm); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 504 | } |
| 505 | |
| 506 | void __flush_tlb_one(unsigned long addr) |
| 507 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 508 | flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE); |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 509 | } |
| 510 | |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 511 | static void fix_range(struct mm_struct *mm, unsigned long start_addr, |
| 512 | unsigned long end_addr, int force) |
| 513 | { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 514 | fix_range_common(mm, start_addr, end_addr, force); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 515 | } |
| 516 | |
| 517 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 518 | unsigned long end) |
| 519 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 520 | if (vma->vm_mm == NULL) |
| 521 | flush_tlb_kernel_range_common(start, end); |
| 522 | else fix_range(vma->vm_mm, start, end, 0); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 523 | } |
Al Viro | 73395a0 | 2011-08-18 20:14:10 +0100 | [diff] [blame] | 524 | EXPORT_SYMBOL(flush_tlb_range); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 525 | |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 526 | void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, |
| 527 | unsigned long end) |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 528 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 529 | /* |
| 530 | * Don't bother flushing if this address space is about to be |
| 531 | * destroyed. |
| 532 | */ |
| 533 | if (atomic_read(&mm->mm_users) == 0) |
| 534 | return; |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 535 | |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 536 | fix_range(mm, start, end, 0); |
| 537 | } |
| 538 | |
| 539 | void flush_tlb_mm(struct mm_struct *mm) |
| 540 | { |
| 541 | struct vm_area_struct *vma = mm->mmap; |
| 542 | |
| 543 | while (vma != NULL) { |
| 544 | fix_range(mm, vma->vm_start, vma->vm_end, 0); |
| 545 | vma = vma->vm_next; |
| 546 | } |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 547 | } |
| 548 | |
| 549 | void force_flush_all(void) |
| 550 | { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 551 | struct mm_struct *mm = current->mm; |
| 552 | struct vm_area_struct *vma = mm->mmap; |
| 553 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 554 | while (vma != NULL) { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 555 | fix_range(mm, vma->vm_start, vma->vm_end, 1); |
| 556 | vma = vma->vm_next; |
| 557 | } |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 558 | } |