| /* |
| * mpx.c - Memory Protection eXtensions |
| * |
| * Copyright (c) 2014, Intel Corporation. |
| * Qiaowei Ren <qiaowei.ren@intel.com> |
| * Dave Hansen <dave.hansen@intel.com> |
| */ |
| #include <linux/kernel.h> |
| #include <linux/slab.h> |
| #include <linux/syscalls.h> |
| #include <linux/sched/sysctl.h> |
| |
| #include <asm/insn.h> |
| #include <asm/mman.h> |
| #include <asm/mmu_context.h> |
| #include <asm/mpx.h> |
| #include <asm/processor.h> |
| #include <asm/fpu/internal.h> |
| |
| #define CREATE_TRACE_POINTS |
| #include <asm/trace/mpx.h> |
| |
| static inline unsigned long mpx_bd_size_bytes(struct mm_struct *mm) |
| { |
| if (is_64bit_mm(mm)) |
| return MPX_BD_SIZE_BYTES_64; |
| else |
| return MPX_BD_SIZE_BYTES_32; |
| } |
| |
| static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm) |
| { |
| if (is_64bit_mm(mm)) |
| return MPX_BT_SIZE_BYTES_64; |
| else |
| return MPX_BT_SIZE_BYTES_32; |
| } |
| |
| /* |
| * This is really a simplified "vm_mmap". it only handles MPX |
| * bounds tables (the bounds directory is user-allocated). |
| */ |
| static unsigned long mpx_mmap(unsigned long len) |
| { |
| unsigned long ret; |
| unsigned long addr, pgoff; |
| struct mm_struct *mm = current->mm; |
| vm_flags_t vm_flags; |
| struct vm_area_struct *vma; |
| |
| /* Only bounds table can be allocated here */ |
| if (len != mpx_bt_size_bytes(mm)) |
| return -EINVAL; |
| |
| down_write(&mm->mmap_sem); |
| |
| /* Too many mappings? */ |
| if (mm->map_count > sysctl_max_map_count) { |
| ret = -ENOMEM; |
| goto out; |
| } |
| |
| /* Obtain the address to map to. we verify (or select) it and ensure |
| * that it represents a valid section of the address space. |
| */ |
| addr = get_unmapped_area(NULL, 0, len, 0, MAP_ANONYMOUS | MAP_PRIVATE); |
| if (addr & ~PAGE_MASK) { |
| ret = addr; |
| goto out; |
| } |
| |
| vm_flags = VM_READ | VM_WRITE | VM_MPX | |
| mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; |
| |
| /* Set pgoff according to addr for anon_vma */ |
| pgoff = addr >> PAGE_SHIFT; |
| |
| ret = mmap_region(NULL, addr, len, vm_flags, pgoff); |
| if (IS_ERR_VALUE(ret)) |
| goto out; |
| |
| vma = find_vma(mm, ret); |
| if (!vma) { |
| ret = -ENOMEM; |
| goto out; |
| } |
| |
| if (vm_flags & VM_LOCKED) { |
| up_write(&mm->mmap_sem); |
| mm_populate(ret, len); |
| return ret; |
| } |
| |
| out: |
| up_write(&mm->mmap_sem); |
| return ret; |
| } |
| |
| enum reg_type { |
| REG_TYPE_RM = 0, |
| REG_TYPE_INDEX, |
| REG_TYPE_BASE, |
| }; |
| |
| static int get_reg_offset(struct insn *insn, struct pt_regs *regs, |
| enum reg_type type) |
| { |
| int regno = 0; |
| |
| static const int regoff[] = { |
| offsetof(struct pt_regs, ax), |
| offsetof(struct pt_regs, cx), |
| offsetof(struct pt_regs, dx), |
| offsetof(struct pt_regs, bx), |
| offsetof(struct pt_regs, sp), |
| offsetof(struct pt_regs, bp), |
| offsetof(struct pt_regs, si), |
| offsetof(struct pt_regs, di), |
| #ifdef CONFIG_X86_64 |
| offsetof(struct pt_regs, r8), |
| offsetof(struct pt_regs, r9), |
| offsetof(struct pt_regs, r10), |
| offsetof(struct pt_regs, r11), |
| offsetof(struct pt_regs, r12), |
| offsetof(struct pt_regs, r13), |
| offsetof(struct pt_regs, r14), |
| offsetof(struct pt_regs, r15), |
| #endif |
| }; |
| int nr_registers = ARRAY_SIZE(regoff); |
| /* |
| * Don't possibly decode a 32-bit instructions as |
| * reading a 64-bit-only register. |
| */ |
| if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64) |
| nr_registers -= 8; |
| |
| switch (type) { |
| case REG_TYPE_RM: |
| regno = X86_MODRM_RM(insn->modrm.value); |
| if (X86_REX_B(insn->rex_prefix.value) == 1) |
| regno += 8; |
| break; |
| |
| case REG_TYPE_INDEX: |
| regno = X86_SIB_INDEX(insn->sib.value); |
| if (X86_REX_X(insn->rex_prefix.value) == 1) |
| regno += 8; |
| break; |
| |
| case REG_TYPE_BASE: |
| regno = X86_SIB_BASE(insn->sib.value); |
| if (X86_REX_B(insn->rex_prefix.value) == 1) |
| regno += 8; |
| break; |
| |
| default: |
| pr_err("invalid register type"); |
| BUG(); |
| break; |
| } |
| |
| if (regno > nr_registers) { |
| WARN_ONCE(1, "decoded an instruction with an invalid register"); |
| return -EINVAL; |
| } |
| return regoff[regno]; |
| } |
| |
| /* |
| * return the address being referenced be instruction |
| * for rm=3 returning the content of the rm reg |
| * for rm!=3 calculates the address using SIB and Disp |
| */ |
| static void __user *mpx_get_addr_ref(struct insn *insn, struct pt_regs *regs) |
| { |
| unsigned long addr, base, indx; |
| int addr_offset, base_offset, indx_offset; |
| insn_byte_t sib; |
| |
| insn_get_modrm(insn); |
| insn_get_sib(insn); |
| sib = insn->sib.value; |
| |
| if (X86_MODRM_MOD(insn->modrm.value) == 3) { |
| addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM); |
| if (addr_offset < 0) |
| goto out_err; |
| addr = regs_get_register(regs, addr_offset); |
| } else { |
| if (insn->sib.nbytes) { |
| base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE); |
| if (base_offset < 0) |
| goto out_err; |
| |
| indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX); |
| if (indx_offset < 0) |
| goto out_err; |
| |
| base = regs_get_register(regs, base_offset); |
| indx = regs_get_register(regs, indx_offset); |
| addr = base + indx * (1 << X86_SIB_SCALE(sib)); |
| } else { |
| addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM); |
| if (addr_offset < 0) |
| goto out_err; |
| addr = regs_get_register(regs, addr_offset); |
| } |
| addr += insn->displacement.value; |
| } |
| return (void __user *)addr; |
| out_err: |
| return (void __user *)-1; |
| } |
| |
| static int mpx_insn_decode(struct insn *insn, |
| struct pt_regs *regs) |
| { |
| unsigned char buf[MAX_INSN_SIZE]; |
| int x86_64 = !test_thread_flag(TIF_IA32); |
| int not_copied; |
| int nr_copied; |
| |
| not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf)); |
| nr_copied = sizeof(buf) - not_copied; |
| /* |
| * The decoder _should_ fail nicely if we pass it a short buffer. |
| * But, let's not depend on that implementation detail. If we |
| * did not get anything, just error out now. |
| */ |
| if (!nr_copied) |
| return -EFAULT; |
| insn_init(insn, buf, nr_copied, x86_64); |
| insn_get_length(insn); |
| /* |
| * copy_from_user() tries to get as many bytes as we could see in |
| * the largest possible instruction. If the instruction we are |
| * after is shorter than that _and_ we attempt to copy from |
| * something unreadable, we might get a short read. This is OK |
| * as long as the read did not stop in the middle of the |
| * instruction. Check to see if we got a partial instruction. |
| */ |
| if (nr_copied < insn->length) |
| return -EFAULT; |
| |
| insn_get_opcode(insn); |
| /* |
| * We only _really_ need to decode bndcl/bndcn/bndcu |
| * Error out on anything else. |
| */ |
| if (insn->opcode.bytes[0] != 0x0f) |
| goto bad_opcode; |
| if ((insn->opcode.bytes[1] != 0x1a) && |
| (insn->opcode.bytes[1] != 0x1b)) |
| goto bad_opcode; |
| |
| return 0; |
| bad_opcode: |
| return -EINVAL; |
| } |
| |
| /* |
| * If a bounds overflow occurs then a #BR is generated. This |
| * function decodes MPX instructions to get violation address |
| * and set this address into extended struct siginfo. |
| * |
| * Note that this is not a super precise way of doing this. |
| * Userspace could have, by the time we get here, written |
| * anything it wants in to the instructions. We can not |
| * trust anything about it. They might not be valid |
| * instructions or might encode invalid registers, etc... |
| * |
| * The caller is expected to kfree() the returned siginfo_t. |
| */ |
| siginfo_t *mpx_generate_siginfo(struct pt_regs *regs) |
| { |
| const struct bndreg *bndregs, *bndreg; |
| siginfo_t *info = NULL; |
| struct insn insn; |
| uint8_t bndregno; |
| int err; |
| |
| err = mpx_insn_decode(&insn, regs); |
| if (err) |
| goto err_out; |
| |
| /* |
| * We know at this point that we are only dealing with |
| * MPX instructions. |
| */ |
| insn_get_modrm(&insn); |
| bndregno = X86_MODRM_REG(insn.modrm.value); |
| if (bndregno > 3) { |
| err = -EINVAL; |
| goto err_out; |
| } |
| /* get bndregs field from current task's xsave area */ |
| bndregs = get_xsave_field_ptr(XSTATE_BNDREGS); |
| if (!bndregs) { |
| err = -EINVAL; |
| goto err_out; |
| } |
| /* now go select the individual register in the set of 4 */ |
| bndreg = &bndregs[bndregno]; |
| |
| info = kzalloc(sizeof(*info), GFP_KERNEL); |
| if (!info) { |
| err = -ENOMEM; |
| goto err_out; |
| } |
| /* |
| * The registers are always 64-bit, but the upper 32 |
| * bits are ignored in 32-bit mode. Also, note that the |
| * upper bounds are architecturally represented in 1's |
| * complement form. |
| * |
| * The 'unsigned long' cast is because the compiler |
| * complains when casting from integers to different-size |
| * pointers. |
| */ |
| info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound; |
| info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound; |
| info->si_addr_lsb = 0; |
| info->si_signo = SIGSEGV; |
| info->si_errno = 0; |
| info->si_code = SEGV_BNDERR; |
| info->si_addr = mpx_get_addr_ref(&insn, regs); |
| /* |
| * We were not able to extract an address from the instruction, |
| * probably because there was something invalid in it. |
| */ |
| if (info->si_addr == (void *)-1) { |
| err = -EINVAL; |
| goto err_out; |
| } |
| trace_mpx_bounds_register_exception(info->si_addr, bndreg); |
| return info; |
| err_out: |
| /* info might be NULL, but kfree() handles that */ |
| kfree(info); |
| return ERR_PTR(err); |
| } |
| |
| static __user void *mpx_get_bounds_dir(void) |
| { |
| const struct bndcsr *bndcsr; |
| |
| if (!cpu_feature_enabled(X86_FEATURE_MPX)) |
| return MPX_INVALID_BOUNDS_DIR; |
| |
| /* |
| * The bounds directory pointer is stored in a register |
| * only accessible if we first do an xsave. |
| */ |
| bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR); |
| if (!bndcsr) |
| return MPX_INVALID_BOUNDS_DIR; |
| |
| /* |
| * Make sure the register looks valid by checking the |
| * enable bit. |
| */ |
| if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG)) |
| return MPX_INVALID_BOUNDS_DIR; |
| |
| /* |
| * Lastly, mask off the low bits used for configuration |
| * flags, and return the address of the bounds table. |
| */ |
| return (void __user *)(unsigned long) |
| (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK); |
| } |
| |
| int mpx_enable_management(void) |
| { |
| void __user *bd_base = MPX_INVALID_BOUNDS_DIR; |
| struct mm_struct *mm = current->mm; |
| int ret = 0; |
| |
| /* |
| * runtime in the userspace will be responsible for allocation of |
| * the bounds directory. Then, it will save the base of the bounds |
| * directory into XSAVE/XRSTOR Save Area and enable MPX through |
| * XRSTOR instruction. |
| * |
| * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is |
| * expected to be relatively expensive. Storing the bounds |
| * directory here means that we do not have to do xsave in the |
| * unmap path; we can just use mm->bd_addr instead. |
| */ |
| bd_base = mpx_get_bounds_dir(); |
| down_write(&mm->mmap_sem); |
| mm->bd_addr = bd_base; |
| if (mm->bd_addr == MPX_INVALID_BOUNDS_DIR) |
| ret = -ENXIO; |
| |
| up_write(&mm->mmap_sem); |
| return ret; |
| } |
| |
| int mpx_disable_management(void) |
| { |
| struct mm_struct *mm = current->mm; |
| |
| if (!cpu_feature_enabled(X86_FEATURE_MPX)) |
| return -ENXIO; |
| |
| down_write(&mm->mmap_sem); |
| mm->bd_addr = MPX_INVALID_BOUNDS_DIR; |
| up_write(&mm->mmap_sem); |
| return 0; |
| } |
| |
| static int mpx_cmpxchg_bd_entry(struct mm_struct *mm, |
| unsigned long *curval, |
| unsigned long __user *addr, |
| unsigned long old_val, unsigned long new_val) |
| { |
| int ret; |
| /* |
| * user_atomic_cmpxchg_inatomic() actually uses sizeof() |
| * the pointer that we pass to it to figure out how much |
| * data to cmpxchg. We have to be careful here not to |
| * pass a pointer to a 64-bit data type when we only want |
| * a 32-bit copy. |
| */ |
| if (is_64bit_mm(mm)) { |
| ret = user_atomic_cmpxchg_inatomic(curval, |
| addr, old_val, new_val); |
| } else { |
| u32 uninitialized_var(curval_32); |
| u32 old_val_32 = old_val; |
| u32 new_val_32 = new_val; |
| u32 __user *addr_32 = (u32 __user *)addr; |
| |
| ret = user_atomic_cmpxchg_inatomic(&curval_32, |
| addr_32, old_val_32, new_val_32); |
| *curval = curval_32; |
| } |
| return ret; |
| } |
| |
| /* |
| * With 32-bit mode, a bounds directory is 4MB, and the size of each |
| * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB, |
| * and the size of each bounds table is 4MB. |
| */ |
| static int allocate_bt(struct mm_struct *mm, long __user *bd_entry) |
| { |
| unsigned long expected_old_val = 0; |
| unsigned long actual_old_val = 0; |
| unsigned long bt_addr; |
| unsigned long bd_new_entry; |
| int ret = 0; |
| |
| /* |
| * Carve the virtual space out of userspace for the new |
| * bounds table: |
| */ |
| bt_addr = mpx_mmap(mpx_bt_size_bytes(mm)); |
| if (IS_ERR((void *)bt_addr)) |
| return PTR_ERR((void *)bt_addr); |
| /* |
| * Set the valid flag (kinda like _PAGE_PRESENT in a pte) |
| */ |
| bd_new_entry = bt_addr | MPX_BD_ENTRY_VALID_FLAG; |
| |
| /* |
| * Go poke the address of the new bounds table in to the |
| * bounds directory entry out in userspace memory. Note: |
| * we may race with another CPU instantiating the same table. |
| * In that case the cmpxchg will see an unexpected |
| * 'actual_old_val'. |
| * |
| * This can fault, but that's OK because we do not hold |
| * mmap_sem at this point, unlike some of the other part |
| * of the MPX code that have to pagefault_disable(). |
| */ |
| ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, bd_entry, |
| expected_old_val, bd_new_entry); |
| if (ret) |
| goto out_unmap; |
| |
| /* |
| * The user_atomic_cmpxchg_inatomic() will only return nonzero |
| * for faults, *not* if the cmpxchg itself fails. Now we must |
| * verify that the cmpxchg itself completed successfully. |
| */ |
| /* |
| * We expected an empty 'expected_old_val', but instead found |
| * an apparently valid entry. Assume we raced with another |
| * thread to instantiate this table and desclare succecss. |
| */ |
| if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) { |
| ret = 0; |
| goto out_unmap; |
| } |
| /* |
| * We found a non-empty bd_entry but it did not have the |
| * VALID_FLAG set. Return an error which will result in |
| * a SEGV since this probably means that somebody scribbled |
| * some invalid data in to a bounds table. |
| */ |
| if (expected_old_val != actual_old_val) { |
| ret = -EINVAL; |
| goto out_unmap; |
| } |
| trace_mpx_new_bounds_table(bt_addr); |
| return 0; |
| out_unmap: |
| vm_munmap(bt_addr, mpx_bt_size_bytes(mm)); |
| return ret; |
| } |
| |
| /* |
| * When a BNDSTX instruction attempts to save bounds to a bounds |
| * table, it will first attempt to look up the table in the |
| * first-level bounds directory. If it does not find a table in |
| * the directory, a #BR is generated and we get here in order to |
| * allocate a new table. |
| * |
| * With 32-bit mode, the size of BD is 4MB, and the size of each |
| * bound table is 16KB. With 64-bit mode, the size of BD is 2GB, |
| * and the size of each bound table is 4MB. |
| */ |
| static int do_mpx_bt_fault(void) |
| { |
| unsigned long bd_entry, bd_base; |
| const struct bndcsr *bndcsr; |
| struct mm_struct *mm = current->mm; |
| |
| bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR); |
| if (!bndcsr) |
| return -EINVAL; |
| /* |
| * Mask off the preserve and enable bits |
| */ |
| bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK; |
| /* |
| * The hardware provides the address of the missing or invalid |
| * entry via BNDSTATUS, so we don't have to go look it up. |
| */ |
| bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK; |
| /* |
| * Make sure the directory entry is within where we think |
| * the directory is. |
| */ |
| if ((bd_entry < bd_base) || |
| (bd_entry >= bd_base + mpx_bd_size_bytes(mm))) |
| return -EINVAL; |
| |
| return allocate_bt(mm, (long __user *)bd_entry); |
| } |
| |
| int mpx_handle_bd_fault(void) |
| { |
| /* |
| * Userspace never asked us to manage the bounds tables, |
| * so refuse to help. |
| */ |
| if (!kernel_managing_mpx_tables(current->mm)) |
| return -EINVAL; |
| |
| if (do_mpx_bt_fault()) { |
| force_sig(SIGSEGV, current); |
| /* |
| * The force_sig() is essentially "handling" this |
| * exception, so we do not pass up the error |
| * from do_mpx_bt_fault(). |
| */ |
| } |
| return 0; |
| } |
| |
| /* |
| * A thin wrapper around get_user_pages(). Returns 0 if the |
| * fault was resolved or -errno if not. |
| */ |
| static int mpx_resolve_fault(long __user *addr, int write) |
| { |
| long gup_ret; |
| int nr_pages = 1; |
| int force = 0; |
| |
| gup_ret = get_user_pages(current, current->mm, (unsigned long)addr, |
| nr_pages, write, force, NULL, NULL); |
| /* |
| * get_user_pages() returns number of pages gotten. |
| * 0 means we failed to fault in and get anything, |
| * probably because 'addr' is bad. |
| */ |
| if (!gup_ret) |
| return -EFAULT; |
| /* Other error, return it */ |
| if (gup_ret < 0) |
| return gup_ret; |
| /* must have gup'd a page and gup_ret>0, success */ |
| return 0; |
| } |
| |
| static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct *mm, |
| unsigned long bd_entry) |
| { |
| unsigned long bt_addr = bd_entry; |
| int align_to_bytes; |
| /* |
| * Bit 0 in a bt_entry is always the valid bit. |
| */ |
| bt_addr &= ~MPX_BD_ENTRY_VALID_FLAG; |
| /* |
| * Tables are naturally aligned at 8-byte boundaries |
| * on 64-bit and 4-byte boundaries on 32-bit. The |
| * documentation makes it appear that the low bits |
| * are ignored by the hardware, so we do the same. |
| */ |
| if (is_64bit_mm(mm)) |
| align_to_bytes = 8; |
| else |
| align_to_bytes = 4; |
| bt_addr &= ~(align_to_bytes-1); |
| return bt_addr; |
| } |
| |
| /* |
| * Get the base of bounds tables pointed by specific bounds |
| * directory entry. |
| */ |
| static int get_bt_addr(struct mm_struct *mm, |
| long __user *bd_entry_ptr, |
| unsigned long *bt_addr_result) |
| { |
| int ret; |
| int valid_bit; |
| unsigned long bd_entry; |
| unsigned long bt_addr; |
| |
| if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr))) |
| return -EFAULT; |
| |
| while (1) { |
| int need_write = 0; |
| |
| pagefault_disable(); |
| ret = get_user(bd_entry, bd_entry_ptr); |
| pagefault_enable(); |
| if (!ret) |
| break; |
| if (ret == -EFAULT) |
| ret = mpx_resolve_fault(bd_entry_ptr, need_write); |
| /* |
| * If we could not resolve the fault, consider it |
| * userspace's fault and error out. |
| */ |
| if (ret) |
| return ret; |
| } |
| |
| valid_bit = bd_entry & MPX_BD_ENTRY_VALID_FLAG; |
| bt_addr = mpx_bd_entry_to_bt_addr(mm, bd_entry); |
| |
| /* |
| * When the kernel is managing bounds tables, a bounds directory |
| * entry will either have a valid address (plus the valid bit) |
| * *OR* be completely empty. If we see a !valid entry *and* some |
| * data in the address field, we know something is wrong. This |
| * -EINVAL return will cause a SIGSEGV. |
| */ |
| if (!valid_bit && bt_addr) |
| return -EINVAL; |
| /* |
| * Do we have an completely zeroed bt entry? That is OK. It |
| * just means there was no bounds table for this memory. Make |
| * sure to distinguish this from -EINVAL, which will cause |
| * a SEGV. |
| */ |
| if (!valid_bit) |
| return -ENOENT; |
| |
| *bt_addr_result = bt_addr; |
| return 0; |
| } |
| |
| static inline int bt_entry_size_bytes(struct mm_struct *mm) |
| { |
| if (is_64bit_mm(mm)) |
| return MPX_BT_ENTRY_BYTES_64; |
| else |
| return MPX_BT_ENTRY_BYTES_32; |
| } |
| |
| /* |
| * Take a virtual address and turns it in to the offset in bytes |
| * inside of the bounds table where the bounds table entry |
| * controlling 'addr' can be found. |
| */ |
| static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct *mm, |
| unsigned long addr) |
| { |
| unsigned long bt_table_nr_entries; |
| unsigned long offset = addr; |
| |
| if (is_64bit_mm(mm)) { |
| /* Bottom 3 bits are ignored on 64-bit */ |
| offset >>= 3; |
| bt_table_nr_entries = MPX_BT_NR_ENTRIES_64; |
| } else { |
| /* Bottom 2 bits are ignored on 32-bit */ |
| offset >>= 2; |
| bt_table_nr_entries = MPX_BT_NR_ENTRIES_32; |
| } |
| /* |
| * We know the size of the table in to which we are |
| * indexing, and we have eliminated all the low bits |
| * which are ignored for indexing. |
| * |
| * Mask out all the high bits which we do not need |
| * to index in to the table. Note that the tables |
| * are always powers of two so this gives us a proper |
| * mask. |
| */ |
| offset &= (bt_table_nr_entries-1); |
| /* |
| * We now have an entry offset in terms of *entries* in |
| * the table. We need to scale it back up to bytes. |
| */ |
| offset *= bt_entry_size_bytes(mm); |
| return offset; |
| } |
| |
| /* |
| * How much virtual address space does a single bounds |
| * directory entry cover? |
| * |
| * Note, we need a long long because 4GB doesn't fit in |
| * to a long on 32-bit. |
| */ |
| static inline unsigned long bd_entry_virt_space(struct mm_struct *mm) |
| { |
| unsigned long long virt_space = (1ULL << boot_cpu_data.x86_virt_bits); |
| if (is_64bit_mm(mm)) |
| return virt_space / MPX_BD_NR_ENTRIES_64; |
| else |
| return virt_space / MPX_BD_NR_ENTRIES_32; |
| } |
| |
| /* |
| * Free the backing physical pages of bounds table 'bt_addr'. |
| * Assume start...end is within that bounds table. |
| */ |
| static noinline int zap_bt_entries_mapping(struct mm_struct *mm, |
| unsigned long bt_addr, |
| unsigned long start_mapping, unsigned long end_mapping) |
| { |
| struct vm_area_struct *vma; |
| unsigned long addr, len; |
| unsigned long start; |
| unsigned long end; |
| |
| /* |
| * if we 'end' on a boundary, the offset will be 0 which |
| * is not what we want. Back it up a byte to get the |
| * last bt entry. Then once we have the entry itself, |
| * move 'end' back up by the table entry size. |
| */ |
| start = bt_addr + mpx_get_bt_entry_offset_bytes(mm, start_mapping); |
| end = bt_addr + mpx_get_bt_entry_offset_bytes(mm, end_mapping - 1); |
| /* |
| * Move end back up by one entry. Among other things |
| * this ensures that it remains page-aligned and does |
| * not screw up zap_page_range() |
| */ |
| end += bt_entry_size_bytes(mm); |
| |
| /* |
| * Find the first overlapping vma. If vma->vm_start > start, there |
| * will be a hole in the bounds table. This -EINVAL return will |
| * cause a SIGSEGV. |
| */ |
| vma = find_vma(mm, start); |
| if (!vma || vma->vm_start > start) |
| return -EINVAL; |
| |
| /* |
| * A NUMA policy on a VM_MPX VMA could cause this bounds table to |
| * be split. So we need to look across the entire 'start -> end' |
| * range of this bounds table, find all of the VM_MPX VMAs, and |
| * zap only those. |
| */ |
| addr = start; |
| while (vma && vma->vm_start < end) { |
| /* |
| * We followed a bounds directory entry down |
| * here. If we find a non-MPX VMA, that's bad, |
| * so stop immediately and return an error. This |
| * probably results in a SIGSEGV. |
| */ |
| if (!(vma->vm_flags & VM_MPX)) |
| return -EINVAL; |
| |
| len = min(vma->vm_end, end) - addr; |
| zap_page_range(vma, addr, len, NULL); |
| trace_mpx_unmap_zap(addr, addr+len); |
| |
| vma = vma->vm_next; |
| addr = vma->vm_start; |
| } |
| return 0; |
| } |
| |
| static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm, |
| unsigned long addr) |
| { |
| /* |
| * There are several ways to derive the bd offsets. We |
| * use the following approach here: |
| * 1. We know the size of the virtual address space |
| * 2. We know the number of entries in a bounds table |
| * 3. We know that each entry covers a fixed amount of |
| * virtual address space. |
| * So, we can just divide the virtual address by the |
| * virtual space used by one entry to determine which |
| * entry "controls" the given virtual address. |
| */ |
| if (is_64bit_mm(mm)) { |
| int bd_entry_size = 8; /* 64-bit pointer */ |
| /* |
| * Take the 64-bit addressing hole in to account. |
| */ |
| addr &= ((1UL << boot_cpu_data.x86_virt_bits) - 1); |
| return (addr / bd_entry_virt_space(mm)) * bd_entry_size; |
| } else { |
| int bd_entry_size = 4; /* 32-bit pointer */ |
| /* |
| * 32-bit has no hole so this case needs no mask |
| */ |
| return (addr / bd_entry_virt_space(mm)) * bd_entry_size; |
| } |
| /* |
| * The two return calls above are exact copies. If we |
| * pull out a single copy and put it in here, gcc won't |
| * realize that we're doing a power-of-2 divide and use |
| * shifts. It uses a real divide. If we put them up |
| * there, it manages to figure it out (gcc 4.8.3). |
| */ |
| } |
| |
| static int unmap_entire_bt(struct mm_struct *mm, |
| long __user *bd_entry, unsigned long bt_addr) |
| { |
| unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG; |
| unsigned long uninitialized_var(actual_old_val); |
| int ret; |
| |
| while (1) { |
| int need_write = 1; |
| unsigned long cleared_bd_entry = 0; |
| |
| pagefault_disable(); |
| ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, |
| bd_entry, expected_old_val, cleared_bd_entry); |
| pagefault_enable(); |
| if (!ret) |
| break; |
| if (ret == -EFAULT) |
| ret = mpx_resolve_fault(bd_entry, need_write); |
| /* |
| * If we could not resolve the fault, consider it |
| * userspace's fault and error out. |
| */ |
| if (ret) |
| return ret; |
| } |
| /* |
| * The cmpxchg was performed, check the results. |
| */ |
| if (actual_old_val != expected_old_val) { |
| /* |
| * Someone else raced with us to unmap the table. |
| * That is OK, since we were both trying to do |
| * the same thing. Declare success. |
| */ |
| if (!actual_old_val) |
| return 0; |
| /* |
| * Something messed with the bounds directory |
| * entry. We hold mmap_sem for read or write |
| * here, so it could not be a _new_ bounds table |
| * that someone just allocated. Something is |
| * wrong, so pass up the error and SIGSEGV. |
| */ |
| return -EINVAL; |
| } |
| /* |
| * Note, we are likely being called under do_munmap() already. To |
| * avoid recursion, do_munmap() will check whether it comes |
| * from one bounds table through VM_MPX flag. |
| */ |
| return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm)); |
| } |
| |
| static int try_unmap_single_bt(struct mm_struct *mm, |
| unsigned long start, unsigned long end) |
| { |
| struct vm_area_struct *next; |
| struct vm_area_struct *prev; |
| /* |
| * "bta" == Bounds Table Area: the area controlled by the |
| * bounds table that we are unmapping. |
| */ |
| unsigned long bta_start_vaddr = start & ~(bd_entry_virt_space(mm)-1); |
| unsigned long bta_end_vaddr = bta_start_vaddr + bd_entry_virt_space(mm); |
| unsigned long uninitialized_var(bt_addr); |
| void __user *bde_vaddr; |
| int ret; |
| /* |
| * We already unlinked the VMAs from the mm's rbtree so 'start' |
| * is guaranteed to be in a hole. This gets us the first VMA |
| * before the hole in to 'prev' and the next VMA after the hole |
| * in to 'next'. |
| */ |
| next = find_vma_prev(mm, start, &prev); |
| /* |
| * Do not count other MPX bounds table VMAs as neighbors. |
| * Although theoretically possible, we do not allow bounds |
| * tables for bounds tables so our heads do not explode. |
| * If we count them as neighbors here, we may end up with |
| * lots of tables even though we have no actual table |
| * entries in use. |
| */ |
| while (next && (next->vm_flags & VM_MPX)) |
| next = next->vm_next; |
| while (prev && (prev->vm_flags & VM_MPX)) |
| prev = prev->vm_prev; |
| /* |
| * We know 'start' and 'end' lie within an area controlled |
| * by a single bounds table. See if there are any other |
| * VMAs controlled by that bounds table. If there are not |
| * then we can "expand" the are we are unmapping to possibly |
| * cover the entire table. |
| */ |
| next = find_vma_prev(mm, start, &prev); |
| if ((!prev || prev->vm_end <= bta_start_vaddr) && |
| (!next || next->vm_start >= bta_end_vaddr)) { |
| /* |
| * No neighbor VMAs controlled by same bounds |
| * table. Try to unmap the whole thing |
| */ |
| start = bta_start_vaddr; |
| end = bta_end_vaddr; |
| } |
| |
| bde_vaddr = mm->bd_addr + mpx_get_bd_entry_offset(mm, start); |
| ret = get_bt_addr(mm, bde_vaddr, &bt_addr); |
| /* |
| * No bounds table there, so nothing to unmap. |
| */ |
| if (ret == -ENOENT) { |
| ret = 0; |
| return 0; |
| } |
| if (ret) |
| return ret; |
| /* |
| * We are unmapping an entire table. Either because the |
| * unmap that started this whole process was large enough |
| * to cover an entire table, or that the unmap was small |
| * but was the area covered by a bounds table. |
| */ |
| if ((start == bta_start_vaddr) && |
| (end == bta_end_vaddr)) |
| return unmap_entire_bt(mm, bde_vaddr, bt_addr); |
| return zap_bt_entries_mapping(mm, bt_addr, start, end); |
| } |
| |
| static int mpx_unmap_tables(struct mm_struct *mm, |
| unsigned long start, unsigned long end) |
| { |
| unsigned long one_unmap_start; |
| trace_mpx_unmap_search(start, end); |
| |
| one_unmap_start = start; |
| while (one_unmap_start < end) { |
| int ret; |
| unsigned long next_unmap_start = ALIGN(one_unmap_start+1, |
| bd_entry_virt_space(mm)); |
| unsigned long one_unmap_end = end; |
| /* |
| * if the end is beyond the current bounds table, |
| * move it back so we only deal with a single one |
| * at a time |
| */ |
| if (one_unmap_end > next_unmap_start) |
| one_unmap_end = next_unmap_start; |
| ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end); |
| if (ret) |
| return ret; |
| |
| one_unmap_start = next_unmap_start; |
| } |
| return 0; |
| } |
| |
| /* |
| * Free unused bounds tables covered in a virtual address region being |
| * munmap()ed. Assume end > start. |
| * |
| * This function will be called by do_munmap(), and the VMAs covering |
| * the virtual address region start...end have already been split if |
| * necessary, and the 'vma' is the first vma in this range (start -> end). |
| */ |
| void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma, |
| unsigned long start, unsigned long end) |
| { |
| int ret; |
| |
| /* |
| * Refuse to do anything unless userspace has asked |
| * the kernel to help manage the bounds tables, |
| */ |
| if (!kernel_managing_mpx_tables(current->mm)) |
| return; |
| /* |
| * This will look across the entire 'start -> end' range, |
| * and find all of the non-VM_MPX VMAs. |
| * |
| * To avoid recursion, if a VM_MPX vma is found in the range |
| * (start->end), we will not continue follow-up work. This |
| * recursion represents having bounds tables for bounds tables, |
| * which should not occur normally. Being strict about it here |
| * helps ensure that we do not have an exploitable stack overflow. |
| */ |
| do { |
| if (vma->vm_flags & VM_MPX) |
| return; |
| vma = vma->vm_next; |
| } while (vma && vma->vm_start < end); |
| |
| ret = mpx_unmap_tables(mm, start, end); |
| if (ret) |
| force_sig(SIGSEGV, current); |
| } |