Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Based on arch/arm/mm/fault.c |
| 3 | * |
| 4 | * Copyright (C) 1995 Linus Torvalds |
| 5 | * Copyright (C) 1995-2004 Russell King |
| 6 | * Copyright (C) 2012 ARM Ltd. |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License version 2 as |
| 10 | * published by the Free Software Foundation. |
| 11 | * |
| 12 | * This program is distributed in the hope that it will be useful, |
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | * GNU General Public License for more details. |
| 16 | * |
| 17 | * You should have received a copy of the GNU General Public License |
| 18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | */ |
| 20 | |
Paul Gortmaker | 0edfa83 | 2016-09-19 17:38:55 -0400 | [diff] [blame] | 21 | #include <linux/extable.h> |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 22 | #include <linux/signal.h> |
| 23 | #include <linux/mm.h> |
| 24 | #include <linux/hardirq.h> |
| 25 | #include <linux/init.h> |
| 26 | #include <linux/kprobes.h> |
| 27 | #include <linux/uaccess.h> |
| 28 | #include <linux/page-flags.h> |
| 29 | #include <linux/sched.h> |
| 30 | #include <linux/highmem.h> |
| 31 | #include <linux/perf_event.h> |
James Morse | 7209c86 | 2016-10-18 11:27:47 +0100 | [diff] [blame] | 32 | #include <linux/preempt.h> |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 33 | |
James Morse | 7209c86 | 2016-10-18 11:27:47 +0100 | [diff] [blame] | 34 | #include <asm/bug.h> |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 35 | #include <asm/cpufeature.h> |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 36 | #include <asm/exception.h> |
| 37 | #include <asm/debug-monitors.h> |
Catalin Marinas | 9141300 | 2014-04-06 23:04:12 +0100 | [diff] [blame] | 38 | #include <asm/esr.h> |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 39 | #include <asm/sysreg.h> |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 40 | #include <asm/system_misc.h> |
| 41 | #include <asm/pgtable.h> |
| 42 | #include <asm/tlbflush.h> |
| 43 | |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 44 | struct fault_info { |
| 45 | int (*fn)(unsigned long addr, unsigned int esr, |
| 46 | struct pt_regs *regs); |
| 47 | int sig; |
| 48 | int code; |
| 49 | const char *name; |
| 50 | }; |
| 51 | |
| 52 | static const struct fault_info fault_info[]; |
| 53 | |
| 54 | static inline const struct fault_info *esr_to_fault_info(unsigned int esr) |
| 55 | { |
| 56 | return fault_info + (esr & 63); |
| 57 | } |
Catalin Marinas | 3495386b | 2012-10-24 16:34:02 +0100 | [diff] [blame] | 58 | |
Sandeepa Prabhu | 2dd0e8d | 2016-07-08 12:35:48 -0400 | [diff] [blame] | 59 | #ifdef CONFIG_KPROBES |
| 60 | static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) |
| 61 | { |
| 62 | int ret = 0; |
| 63 | |
| 64 | /* kprobe_running() needs smp_processor_id() */ |
| 65 | if (!user_mode(regs)) { |
| 66 | preempt_disable(); |
| 67 | if (kprobe_running() && kprobe_fault_handler(regs, esr)) |
| 68 | ret = 1; |
| 69 | preempt_enable(); |
| 70 | } |
| 71 | |
| 72 | return ret; |
| 73 | } |
| 74 | #else |
| 75 | static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) |
| 76 | { |
| 77 | return 0; |
| 78 | } |
| 79 | #endif |
| 80 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 81 | /* |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 82 | * Dump out the page tables associated with 'addr' in the currently active mm. |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 83 | */ |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 84 | void show_pte(unsigned long addr) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 85 | { |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 86 | struct mm_struct *mm; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 87 | pgd_t *pgd; |
| 88 | |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 89 | if (addr < TASK_SIZE) { |
| 90 | /* TTBR0 */ |
| 91 | mm = current->active_mm; |
| 92 | if (mm == &init_mm) { |
| 93 | pr_alert("[%016lx] user address but active_mm is swapper\n", |
| 94 | addr); |
| 95 | return; |
| 96 | } |
| 97 | } else if (addr >= VA_START) { |
| 98 | /* TTBR1 */ |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 99 | mm = &init_mm; |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 100 | } else { |
| 101 | pr_alert("[%016lx] address between user and kernel address ranges\n", |
| 102 | addr); |
| 103 | return; |
| 104 | } |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 105 | |
| 106 | pr_alert("pgd = %p\n", mm->pgd); |
| 107 | pgd = pgd_offset(mm, addr); |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 108 | pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd)); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 109 | |
| 110 | do { |
| 111 | pud_t *pud; |
| 112 | pmd_t *pmd; |
| 113 | pte_t *pte; |
| 114 | |
Steve Capper | 4339e3f3 | 2013-04-19 15:49:31 +0100 | [diff] [blame] | 115 | if (pgd_none(*pgd) || pgd_bad(*pgd)) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 116 | break; |
| 117 | |
| 118 | pud = pud_offset(pgd, addr); |
Mark Rutland | e95ec35 | 2017-01-03 14:27:26 +0000 | [diff] [blame] | 119 | pr_cont(", *pud=%016llx", pud_val(*pud)); |
Steve Capper | 4339e3f3 | 2013-04-19 15:49:31 +0100 | [diff] [blame] | 120 | if (pud_none(*pud) || pud_bad(*pud)) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 121 | break; |
| 122 | |
| 123 | pmd = pmd_offset(pud, addr); |
Mark Rutland | e95ec35 | 2017-01-03 14:27:26 +0000 | [diff] [blame] | 124 | pr_cont(", *pmd=%016llx", pmd_val(*pmd)); |
Steve Capper | 4339e3f3 | 2013-04-19 15:49:31 +0100 | [diff] [blame] | 125 | if (pmd_none(*pmd) || pmd_bad(*pmd)) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 126 | break; |
| 127 | |
| 128 | pte = pte_offset_map(pmd, addr); |
Mark Rutland | e95ec35 | 2017-01-03 14:27:26 +0000 | [diff] [blame] | 129 | pr_cont(", *pte=%016llx", pte_val(*pte)); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 130 | pte_unmap(pte); |
| 131 | } while(0); |
| 132 | |
Mark Rutland | e95ec35 | 2017-01-03 14:27:26 +0000 | [diff] [blame] | 133 | pr_cont("\n"); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 134 | } |
| 135 | |
Catalin Marinas | 66dbd6e | 2016-04-13 16:01:22 +0100 | [diff] [blame] | 136 | #ifdef CONFIG_ARM64_HW_AFDBM |
| 137 | /* |
| 138 | * This function sets the access flags (dirty, accessed), as well as write |
| 139 | * permission, and only to a more permissive setting. |
| 140 | * |
| 141 | * It needs to cope with hardware update of the accessed/dirty state by other |
| 142 | * agents in the system and can safely skip the __sync_icache_dcache() call as, |
| 143 | * like set_pte_at(), the PTE is never changed from no-exec to exec here. |
| 144 | * |
| 145 | * Returns whether or not the PTE actually changed. |
| 146 | */ |
| 147 | int ptep_set_access_flags(struct vm_area_struct *vma, |
| 148 | unsigned long address, pte_t *ptep, |
| 149 | pte_t entry, int dirty) |
| 150 | { |
| 151 | pteval_t old_pteval; |
| 152 | unsigned int tmp; |
| 153 | |
| 154 | if (pte_same(*ptep, entry)) |
| 155 | return 0; |
| 156 | |
| 157 | /* only preserve the access flags and write permission */ |
| 158 | pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; |
| 159 | |
| 160 | /* |
| 161 | * PTE_RDONLY is cleared by default in the asm below, so set it in |
| 162 | * back if necessary (read-only or clean PTE). |
| 163 | */ |
Will Deacon | 0106d45 | 2016-06-07 17:55:15 +0100 | [diff] [blame] | 164 | if (!pte_write(entry) || !pte_sw_dirty(entry)) |
Catalin Marinas | 66dbd6e | 2016-04-13 16:01:22 +0100 | [diff] [blame] | 165 | pte_val(entry) |= PTE_RDONLY; |
| 166 | |
| 167 | /* |
| 168 | * Setting the flags must be done atomically to avoid racing with the |
| 169 | * hardware update of the access/dirty state. |
| 170 | */ |
| 171 | asm volatile("// ptep_set_access_flags\n" |
| 172 | " prfm pstl1strm, %2\n" |
| 173 | "1: ldxr %0, %2\n" |
| 174 | " and %0, %0, %3 // clear PTE_RDONLY\n" |
| 175 | " orr %0, %0, %4 // set flags\n" |
| 176 | " stxr %w1, %0, %2\n" |
| 177 | " cbnz %w1, 1b\n" |
| 178 | : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) |
| 179 | : "L" (~PTE_RDONLY), "r" (pte_val(entry))); |
| 180 | |
| 181 | flush_tlb_fix_spurious_fault(vma, address); |
| 182 | return 1; |
| 183 | } |
| 184 | #endif |
| 185 | |
Laura Abbott | 9adeb8e | 2016-08-09 18:25:26 -0700 | [diff] [blame] | 186 | static bool is_el1_instruction_abort(unsigned int esr) |
| 187 | { |
| 188 | return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; |
| 189 | } |
| 190 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 191 | /* |
| 192 | * The kernel tried to access some page that wasn't present. |
| 193 | */ |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 194 | static void __do_kernel_fault(unsigned long addr, unsigned int esr, |
| 195 | struct pt_regs *regs) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 196 | { |
| 197 | /* |
| 198 | * Are we prepared to handle this kernel fault? |
Laura Abbott | 9adeb8e | 2016-08-09 18:25:26 -0700 | [diff] [blame] | 199 | * We are almost certainly not prepared to handle instruction faults. |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 200 | */ |
Laura Abbott | 9adeb8e | 2016-08-09 18:25:26 -0700 | [diff] [blame] | 201 | if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 202 | return; |
| 203 | |
| 204 | /* |
| 205 | * No handler, we'll have to terminate things with extreme prejudice. |
| 206 | */ |
| 207 | bust_spinlocks(1); |
| 208 | pr_alert("Unable to handle kernel %s at virtual address %08lx\n", |
| 209 | (addr < PAGE_SIZE) ? "NULL pointer dereference" : |
| 210 | "paging request", addr); |
| 211 | |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 212 | show_pte(addr); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 213 | die("Oops", regs, esr); |
| 214 | bust_spinlocks(0); |
| 215 | do_exit(SIGKILL); |
| 216 | } |
| 217 | |
| 218 | /* |
| 219 | * Something tried to access memory that isn't in our memory map. User mode |
| 220 | * accesses just cause a SIGSEGV |
| 221 | */ |
| 222 | static void __do_user_fault(struct task_struct *tsk, unsigned long addr, |
| 223 | unsigned int esr, unsigned int sig, int code, |
| 224 | struct pt_regs *regs) |
| 225 | { |
| 226 | struct siginfo si; |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 227 | const struct fault_info *inf; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 228 | |
Suzuki K. Poulose | f871d26 | 2015-07-03 15:08:08 +0100 | [diff] [blame] | 229 | if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 230 | inf = esr_to_fault_info(esr); |
Catalin Marinas | 3495386b | 2012-10-24 16:34:02 +0100 | [diff] [blame] | 231 | pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n", |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 232 | tsk->comm, task_pid_nr(tsk), inf->name, sig, |
Catalin Marinas | 3495386b | 2012-10-24 16:34:02 +0100 | [diff] [blame] | 233 | addr, esr); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 234 | show_regs(regs); |
| 235 | } |
| 236 | |
| 237 | tsk->thread.fault_address = addr; |
Catalin Marinas | 9141300 | 2014-04-06 23:04:12 +0100 | [diff] [blame] | 238 | tsk->thread.fault_code = esr; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 239 | si.si_signo = sig; |
| 240 | si.si_errno = 0; |
| 241 | si.si_code = code; |
| 242 | si.si_addr = (void __user *)addr; |
| 243 | force_sig_info(sig, &si, tsk); |
| 244 | } |
| 245 | |
Catalin Marinas | 59f67e1 | 2013-09-16 15:18:28 +0100 | [diff] [blame] | 246 | static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 247 | { |
| 248 | struct task_struct *tsk = current; |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 249 | const struct fault_info *inf; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 250 | |
| 251 | /* |
| 252 | * If we are in kernel mode at this point, we have no context to |
| 253 | * handle this fault with. |
| 254 | */ |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 255 | if (user_mode(regs)) { |
| 256 | inf = esr_to_fault_info(esr); |
| 257 | __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs); |
| 258 | } else |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 259 | __do_kernel_fault(addr, esr, regs); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 260 | } |
| 261 | |
| 262 | #define VM_FAULT_BADMAP 0x010000 |
| 263 | #define VM_FAULT_BADACCESS 0x020000 |
| 264 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 265 | static int __do_page_fault(struct mm_struct *mm, unsigned long addr, |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 266 | unsigned int mm_flags, unsigned long vm_flags, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 267 | struct task_struct *tsk) |
| 268 | { |
| 269 | struct vm_area_struct *vma; |
| 270 | int fault; |
| 271 | |
| 272 | vma = find_vma(mm, addr); |
| 273 | fault = VM_FAULT_BADMAP; |
| 274 | if (unlikely(!vma)) |
| 275 | goto out; |
| 276 | if (unlikely(vma->vm_start > addr)) |
| 277 | goto check_stack; |
| 278 | |
| 279 | /* |
| 280 | * Ok, we have a good vm_area for this memory access, so we can handle |
| 281 | * it. |
| 282 | */ |
| 283 | good_area: |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 284 | /* |
| 285 | * Check that the permissions on the VMA allow for the fault which |
Catalin Marinas | cab15ce | 2016-08-11 18:44:50 +0100 | [diff] [blame] | 286 | * occurred. |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 287 | */ |
| 288 | if (!(vma->vm_flags & vm_flags)) { |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 289 | fault = VM_FAULT_BADACCESS; |
| 290 | goto out; |
| 291 | } |
| 292 | |
Kirill A. Shutemov | dcddffd | 2016-07-26 15:25:18 -0700 | [diff] [blame] | 293 | return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 294 | |
| 295 | check_stack: |
| 296 | if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) |
| 297 | goto good_area; |
| 298 | out: |
| 299 | return fault; |
| 300 | } |
| 301 | |
Catalin Marinas | 27b729b | 2016-07-01 18:22:39 +0100 | [diff] [blame] | 302 | static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs) |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 303 | { |
Mark Rutland | 275f344 | 2016-05-31 12:33:01 +0100 | [diff] [blame] | 304 | unsigned int ec = ESR_ELx_EC(esr); |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 305 | unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; |
| 306 | |
Catalin Marinas | 27b729b | 2016-07-01 18:22:39 +0100 | [diff] [blame] | 307 | if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) |
| 308 | return false; |
| 309 | |
| 310 | if (system_uses_ttbr0_pan()) |
| 311 | return fsc_type == ESR_ELx_FSC_FAULT && |
| 312 | (regs->pstate & PSR_PAN_BIT); |
| 313 | else |
| 314 | return fsc_type == ESR_ELx_FSC_PERM; |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 315 | } |
| 316 | |
Mark Rutland | 541ec87 | 2016-05-31 12:33:03 +0100 | [diff] [blame] | 317 | static bool is_el0_instruction_abort(unsigned int esr) |
| 318 | { |
| 319 | return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; |
| 320 | } |
| 321 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 322 | static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, |
| 323 | struct pt_regs *regs) |
| 324 | { |
| 325 | struct task_struct *tsk; |
| 326 | struct mm_struct *mm; |
| 327 | int fault, sig, code; |
Catalin Marinas | 68f9032 | 2020-01-06 14:35:39 +0000 | [diff] [blame] | 328 | unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 329 | unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
| 330 | |
Sandeepa Prabhu | 2dd0e8d | 2016-07-08 12:35:48 -0400 | [diff] [blame] | 331 | if (notify_page_fault(regs, esr)) |
| 332 | return 0; |
| 333 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 334 | tsk = current; |
| 335 | mm = tsk->mm; |
| 336 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 337 | /* |
| 338 | * If we're in an interrupt or have no user context, we must not take |
| 339 | * the fault. |
| 340 | */ |
David Hildenbrand | 70ffdb9 | 2015-05-11 17:52:11 +0200 | [diff] [blame] | 341 | if (faulthandler_disabled() || !mm) |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 342 | goto no_context; |
| 343 | |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 344 | if (user_mode(regs)) |
| 345 | mm_flags |= FAULT_FLAG_USER; |
| 346 | |
Mark Rutland | 541ec87 | 2016-05-31 12:33:03 +0100 | [diff] [blame] | 347 | if (is_el0_instruction_abort(esr)) { |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 348 | vm_flags = VM_EXEC; |
Mark Rutland | aed40e0 | 2014-11-24 12:31:40 +0000 | [diff] [blame] | 349 | } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 350 | vm_flags = VM_WRITE; |
| 351 | mm_flags |= FAULT_FLAG_WRITE; |
| 352 | } |
| 353 | |
Greg Hackmann | b4c14c2 | 2018-04-20 10:06:49 -0700 | [diff] [blame] | 354 | if (addr < TASK_SIZE && is_permission_fault(esr, regs)) { |
James Morse | e19a6ee | 2016-06-20 18:28:01 +0100 | [diff] [blame] | 355 | /* regs->orig_addr_limit may be 0 if we entered from EL0 */ |
| 356 | if (regs->orig_addr_limit == KERNEL_DS) |
Catalin Marinas | 70c8abc | 2016-02-19 14:28:58 +0000 | [diff] [blame] | 357 | die("Accessing user space memory with fs=KERNEL_DS", regs, esr); |
James Morse | 7054419 | 2016-02-05 14:58:50 +0000 | [diff] [blame] | 358 | |
Laura Abbott | 9adeb8e | 2016-08-09 18:25:26 -0700 | [diff] [blame] | 359 | if (is_el1_instruction_abort(esr)) |
| 360 | die("Attempting to execute userspace memory", regs, esr); |
| 361 | |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 362 | if (!search_exception_tables(regs->pc)) |
Catalin Marinas | 70c8abc | 2016-02-19 14:28:58 +0000 | [diff] [blame] | 363 | die("Accessing user space memory outside uaccess.h routines", regs, esr); |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 364 | } |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 365 | |
| 366 | /* |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 367 | * As per x86, we may deadlock here. However, since the kernel only |
| 368 | * validly references user space from well defined areas of the code, |
| 369 | * we can bug out early if this is from code which shouldn't. |
| 370 | */ |
| 371 | if (!down_read_trylock(&mm->mmap_sem)) { |
| 372 | if (!user_mode(regs) && !search_exception_tables(regs->pc)) |
| 373 | goto no_context; |
| 374 | retry: |
| 375 | down_read(&mm->mmap_sem); |
| 376 | } else { |
| 377 | /* |
| 378 | * The above down_read_trylock() might have succeeded in which |
| 379 | * case, we'll have missed the might_sleep() from down_read(). |
| 380 | */ |
| 381 | might_sleep(); |
| 382 | #ifdef CONFIG_DEBUG_VM |
| 383 | if (!user_mode(regs) && !search_exception_tables(regs->pc)) |
| 384 | goto no_context; |
| 385 | #endif |
| 386 | } |
| 387 | |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 388 | fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 389 | |
| 390 | /* |
| 391 | * If we need to retry but a fatal signal is pending, handle the |
| 392 | * signal first. We do not need to release the mmap_sem because it |
| 393 | * would already be released in __lock_page_or_retry in mm/filemap.c. |
| 394 | */ |
Mark Rutland | 509d8b5 | 2017-07-11 15:19:22 +0100 | [diff] [blame] | 395 | if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { |
| 396 | if (!user_mode(regs)) |
| 397 | goto no_context; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 398 | return 0; |
Mark Rutland | 509d8b5 | 2017-07-11 15:19:22 +0100 | [diff] [blame] | 399 | } |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 400 | |
| 401 | /* |
| 402 | * Major/minor page fault accounting is only done on the initial |
| 403 | * attempt. If we go through a retry, it is extremely likely that the |
| 404 | * page will be found in page cache at that point. |
| 405 | */ |
| 406 | |
| 407 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 408 | if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 409 | if (fault & VM_FAULT_MAJOR) { |
| 410 | tsk->maj_flt++; |
| 411 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, |
| 412 | addr); |
| 413 | } else { |
| 414 | tsk->min_flt++; |
| 415 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, |
| 416 | addr); |
| 417 | } |
| 418 | if (fault & VM_FAULT_RETRY) { |
| 419 | /* |
| 420 | * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of |
| 421 | * starvation. |
| 422 | */ |
Will Deacon | db6f410 | 2013-07-19 15:37:12 +0100 | [diff] [blame] | 423 | mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; |
Mark Salyzyn | 569ba74 | 2015-09-21 21:39:50 +0100 | [diff] [blame] | 424 | mm_flags |= FAULT_FLAG_TRIED; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 425 | goto retry; |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | up_read(&mm->mmap_sem); |
| 430 | |
| 431 | /* |
Jan Kara | 0e8fb93 | 2016-03-17 14:19:55 -0700 | [diff] [blame] | 432 | * Handle the "normal" case first - VM_FAULT_MAJOR |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 433 | */ |
| 434 | if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | |
| 435 | VM_FAULT_BADACCESS)))) |
| 436 | return 0; |
| 437 | |
Johannes Weiner | 8713410 | 2013-09-12 15:13:38 -0700 | [diff] [blame] | 438 | /* |
| 439 | * If we are in kernel mode at this point, we have no context to |
| 440 | * handle this fault with. |
| 441 | */ |
| 442 | if (!user_mode(regs)) |
| 443 | goto no_context; |
| 444 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 445 | if (fault & VM_FAULT_OOM) { |
| 446 | /* |
| 447 | * We ran out of memory, call the OOM killer, and return to |
| 448 | * userspace (which will retry the fault, or kill us if we got |
| 449 | * oom-killed). |
| 450 | */ |
| 451 | pagefault_out_of_memory(); |
| 452 | return 0; |
| 453 | } |
| 454 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 455 | if (fault & VM_FAULT_SIGBUS) { |
| 456 | /* |
| 457 | * We had some memory, but were unable to successfully fix up |
| 458 | * this page fault. |
| 459 | */ |
| 460 | sig = SIGBUS; |
| 461 | code = BUS_ADRERR; |
| 462 | } else { |
| 463 | /* |
| 464 | * Something tried to access memory that isn't in our memory |
| 465 | * map. |
| 466 | */ |
| 467 | sig = SIGSEGV; |
| 468 | code = fault == VM_FAULT_BADACCESS ? |
| 469 | SEGV_ACCERR : SEGV_MAPERR; |
| 470 | } |
| 471 | |
| 472 | __do_user_fault(tsk, addr, esr, sig, code, regs); |
| 473 | return 0; |
| 474 | |
| 475 | no_context: |
Kristina Martsenko | 9cec5be | 2017-06-09 16:35:52 +0100 | [diff] [blame] | 476 | __do_kernel_fault(addr, esr, regs); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 477 | return 0; |
| 478 | } |
| 479 | |
| 480 | /* |
| 481 | * First Level Translation Fault Handler |
| 482 | * |
| 483 | * We enter here because the first level page table doesn't contain a valid |
| 484 | * entry for the address. |
| 485 | * |
| 486 | * If the address is in kernel space (>= TASK_SIZE), then we are probably |
| 487 | * faulting in the vmalloc() area. |
| 488 | * |
| 489 | * If the init_task's first level page tables contains the relevant entry, we |
| 490 | * copy the it to this task. If not, we send the process a signal, fixup the |
| 491 | * exception, or oops the kernel. |
| 492 | * |
| 493 | * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt |
| 494 | * or a critical region, and should only copy the information from the master |
| 495 | * page table, nothing more. |
| 496 | */ |
| 497 | static int __kprobes do_translation_fault(unsigned long addr, |
| 498 | unsigned int esr, |
| 499 | struct pt_regs *regs) |
| 500 | { |
| 501 | if (addr < TASK_SIZE) |
| 502 | return do_page_fault(addr, esr, regs); |
| 503 | |
| 504 | do_bad_area(addr, esr, regs); |
| 505 | return 0; |
| 506 | } |
| 507 | |
EunTaik Lee | 52d7523 | 2016-02-16 04:44:35 +0000 | [diff] [blame] | 508 | static int do_alignment_fault(unsigned long addr, unsigned int esr, |
| 509 | struct pt_regs *regs) |
| 510 | { |
| 511 | do_bad_area(addr, esr, regs); |
| 512 | return 0; |
| 513 | } |
| 514 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 515 | /* |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 516 | * This abort handler always returns "fault". |
| 517 | */ |
| 518 | static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) |
| 519 | { |
| 520 | return 1; |
| 521 | } |
| 522 | |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 523 | static const struct fault_info fault_info[] = { |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 524 | { do_bad, SIGBUS, 0, "ttbr address size fault" }, |
| 525 | { do_bad, SIGBUS, 0, "level 1 address size fault" }, |
| 526 | { do_bad, SIGBUS, 0, "level 2 address size fault" }, |
| 527 | { do_bad, SIGBUS, 0, "level 3 address size fault" }, |
Will Deacon | 7f73f7a | 2014-11-21 14:22:22 +0000 | [diff] [blame] | 528 | { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 529 | { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, |
| 530 | { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, |
Will Deacon | d49527e | 2017-09-29 12:27:41 +0100 | [diff] [blame] | 531 | { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 532 | { do_bad, SIGBUS, 0, "unknown 8" }, |
Steve Capper | 084bd29 | 2013-04-10 13:48:00 +0100 | [diff] [blame] | 533 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, |
| 534 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 535 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 536 | { do_bad, SIGBUS, 0, "unknown 12" }, |
Steve Capper | 084bd29 | 2013-04-10 13:48:00 +0100 | [diff] [blame] | 537 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, |
| 538 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 539 | { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, |
| 540 | { do_bad, SIGBUS, 0, "synchronous external abort" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 541 | { do_bad, SIGBUS, 0, "unknown 17" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 542 | { do_bad, SIGBUS, 0, "unknown 18" }, |
| 543 | { do_bad, SIGBUS, 0, "unknown 19" }, |
| 544 | { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, |
| 545 | { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, |
| 546 | { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, |
| 547 | { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, |
| 548 | { do_bad, SIGBUS, 0, "synchronous parity error" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 549 | { do_bad, SIGBUS, 0, "unknown 25" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 550 | { do_bad, SIGBUS, 0, "unknown 26" }, |
| 551 | { do_bad, SIGBUS, 0, "unknown 27" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 552 | { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, |
| 553 | { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, |
| 554 | { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, |
| 555 | { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 556 | { do_bad, SIGBUS, 0, "unknown 32" }, |
EunTaik Lee | 52d7523 | 2016-02-16 04:44:35 +0000 | [diff] [blame] | 557 | { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 558 | { do_bad, SIGBUS, 0, "unknown 34" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 559 | { do_bad, SIGBUS, 0, "unknown 35" }, |
| 560 | { do_bad, SIGBUS, 0, "unknown 36" }, |
| 561 | { do_bad, SIGBUS, 0, "unknown 37" }, |
| 562 | { do_bad, SIGBUS, 0, "unknown 38" }, |
| 563 | { do_bad, SIGBUS, 0, "unknown 39" }, |
| 564 | { do_bad, SIGBUS, 0, "unknown 40" }, |
| 565 | { do_bad, SIGBUS, 0, "unknown 41" }, |
| 566 | { do_bad, SIGBUS, 0, "unknown 42" }, |
| 567 | { do_bad, SIGBUS, 0, "unknown 43" }, |
| 568 | { do_bad, SIGBUS, 0, "unknown 44" }, |
| 569 | { do_bad, SIGBUS, 0, "unknown 45" }, |
| 570 | { do_bad, SIGBUS, 0, "unknown 46" }, |
| 571 | { do_bad, SIGBUS, 0, "unknown 47" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 572 | { do_bad, SIGBUS, 0, "TLB conflict abort" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 573 | { do_bad, SIGBUS, 0, "unknown 49" }, |
| 574 | { do_bad, SIGBUS, 0, "unknown 50" }, |
| 575 | { do_bad, SIGBUS, 0, "unknown 51" }, |
| 576 | { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 577 | { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 578 | { do_bad, SIGBUS, 0, "unknown 54" }, |
| 579 | { do_bad, SIGBUS, 0, "unknown 55" }, |
| 580 | { do_bad, SIGBUS, 0, "unknown 56" }, |
| 581 | { do_bad, SIGBUS, 0, "unknown 57" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 582 | { do_bad, SIGBUS, 0, "unknown 58" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 583 | { do_bad, SIGBUS, 0, "unknown 59" }, |
| 584 | { do_bad, SIGBUS, 0, "unknown 60" }, |
Mark Rutland | c03784ee8 | 2015-11-23 15:09:36 +0000 | [diff] [blame] | 585 | { do_bad, SIGBUS, 0, "section domain fault" }, |
| 586 | { do_bad, SIGBUS, 0, "page domain fault" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 587 | { do_bad, SIGBUS, 0, "unknown 63" }, |
| 588 | }; |
| 589 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 590 | /* |
| 591 | * Dispatch a data abort to the relevant handler. |
| 592 | */ |
| 593 | asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, |
| 594 | struct pt_regs *regs) |
| 595 | { |
Victor Kamensky | 3715dbf | 2017-04-03 22:51:01 -0700 | [diff] [blame] | 596 | const struct fault_info *inf = esr_to_fault_info(esr); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 597 | struct siginfo info; |
| 598 | |
| 599 | if (!inf->fn(addr, esr, regs)) |
| 600 | return; |
| 601 | |
| 602 | pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", |
| 603 | inf->name, esr, addr); |
| 604 | |
| 605 | info.si_signo = inf->sig; |
| 606 | info.si_errno = 0; |
| 607 | info.si_code = inf->code; |
| 608 | info.si_addr = (void __user *)addr; |
| 609 | arm64_notify_die("", regs, &info, esr); |
| 610 | } |
| 611 | |
Mark Rutland | 34dc20b | 2018-04-12 12:11:18 +0100 | [diff] [blame] | 612 | asmlinkage void __exception do_el0_irq_bp_hardening(void) |
| 613 | { |
| 614 | /* PC has already been checked in entry.S */ |
| 615 | arm64_apply_bp_hardening(); |
| 616 | } |
| 617 | |
Mark Rutland | 4732001 | 2018-04-12 12:11:13 +0100 | [diff] [blame] | 618 | asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, |
| 619 | unsigned int esr, |
| 620 | struct pt_regs *regs) |
| 621 | { |
| 622 | /* |
| 623 | * We've taken an instruction abort from userspace and not yet |
| 624 | * re-enabled IRQs. If the address is a kernel address, apply |
| 625 | * BP hardening prior to enabling IRQs and pre-emption. |
| 626 | */ |
| 627 | if (addr > TASK_SIZE) |
| 628 | arm64_apply_bp_hardening(); |
| 629 | |
| 630 | local_irq_enable(); |
| 631 | do_mem_abort(addr, esr, regs); |
| 632 | } |
| 633 | |
| 634 | |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 635 | /* |
| 636 | * Handle stack alignment exceptions. |
| 637 | */ |
| 638 | asmlinkage void __exception do_sp_pc_abort(unsigned long addr, |
| 639 | unsigned int esr, |
| 640 | struct pt_regs *regs) |
| 641 | { |
| 642 | struct siginfo info; |
Vladimir Murzin | 9e793ab | 2015-06-19 15:28:16 +0100 | [diff] [blame] | 643 | struct task_struct *tsk = current; |
| 644 | |
Mark Rutland | e7c3b24 | 2018-04-12 12:11:17 +0100 | [diff] [blame] | 645 | if (user_mode(regs)) { |
| 646 | if (instruction_pointer(regs) > TASK_SIZE) |
| 647 | arm64_apply_bp_hardening(); |
| 648 | local_irq_enable(); |
| 649 | } |
| 650 | |
Vladimir Murzin | 9e793ab | 2015-06-19 15:28:16 +0100 | [diff] [blame] | 651 | if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) |
| 652 | pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", |
| 653 | tsk->comm, task_pid_nr(tsk), |
| 654 | esr_get_class_string(esr), (void *)regs->pc, |
| 655 | (void *)regs->sp); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 656 | |
| 657 | info.si_signo = SIGBUS; |
| 658 | info.si_errno = 0; |
| 659 | info.si_code = BUS_ADRALN; |
| 660 | info.si_addr = (void __user *)addr; |
Vladimir Murzin | 9e793ab | 2015-06-19 15:28:16 +0100 | [diff] [blame] | 661 | arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 662 | } |
| 663 | |
Dave P Martin | 9fb7410 | 2015-07-24 16:37:48 +0100 | [diff] [blame] | 664 | int __init early_brk64(unsigned long addr, unsigned int esr, |
| 665 | struct pt_regs *regs); |
| 666 | |
| 667 | /* |
| 668 | * __refdata because early_brk64 is __init, but the reference to it is |
| 669 | * clobbered at arch_initcall time. |
| 670 | * See traps.c and debug-monitors.c:debug_traps_init(). |
| 671 | */ |
| 672 | static struct fault_info __refdata debug_fault_info[] = { |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 673 | { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, |
| 674 | { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, |
| 675 | { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, |
| 676 | { do_bad, SIGBUS, 0, "unknown 3" }, |
| 677 | { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, |
| 678 | { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, |
Dave P Martin | 9fb7410 | 2015-07-24 16:37:48 +0100 | [diff] [blame] | 679 | { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 680 | { do_bad, SIGBUS, 0, "unknown 7" }, |
| 681 | }; |
| 682 | |
| 683 | void __init hook_debug_fault_code(int nr, |
| 684 | int (*fn)(unsigned long, unsigned int, struct pt_regs *), |
| 685 | int sig, int code, const char *name) |
| 686 | { |
| 687 | BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); |
| 688 | |
| 689 | debug_fault_info[nr].fn = fn; |
| 690 | debug_fault_info[nr].sig = sig; |
| 691 | debug_fault_info[nr].code = code; |
| 692 | debug_fault_info[nr].name = name; |
| 693 | } |
| 694 | |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 695 | asmlinkage int __exception do_debug_exception(unsigned long addr_if_watchpoint, |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 696 | unsigned int esr, |
| 697 | struct pt_regs *regs) |
| 698 | { |
| 699 | const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 700 | unsigned long pc = instruction_pointer(regs); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 701 | struct siginfo info; |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 702 | int rv; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 703 | |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 704 | /* |
| 705 | * Tell lockdep we disabled irqs in entry.S. Do nothing if they were |
| 706 | * already disabled to preserve the last enabled/disabled addresses. |
| 707 | */ |
| 708 | if (interrupts_enabled(regs)) |
| 709 | trace_hardirqs_off(); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 710 | |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 711 | if (user_mode(regs) && pc > TASK_SIZE) |
Mark Rutland | e7c3b24 | 2018-04-12 12:11:17 +0100 | [diff] [blame] | 712 | arm64_apply_bp_hardening(); |
| 713 | |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 714 | if (!inf->fn(addr_if_watchpoint, esr, regs)) { |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 715 | rv = 1; |
| 716 | } else { |
| 717 | pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 718 | inf->name, esr, pc); |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 719 | |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 720 | info.si_signo = inf->sig; |
| 721 | info.si_errno = 0; |
| 722 | info.si_code = inf->code; |
Will Deacon | cc785dc | 2019-03-01 13:28:00 +0000 | [diff] [blame] | 723 | info.si_addr = (void __user *)pc; |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 724 | arm64_notify_die("", regs, &info, 0); |
| 725 | rv = 0; |
| 726 | } |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 727 | |
James Morse | 6afedcd | 2016-04-13 13:40:00 +0100 | [diff] [blame] | 728 | if (interrupts_enabled(regs)) |
| 729 | trace_hardirqs_on(); |
| 730 | |
| 731 | return rv; |
Catalin Marinas | 1d18c47 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 732 | } |
Sandeepa Prabhu | 2dd0e8d | 2016-07-08 12:35:48 -0400 | [diff] [blame] | 733 | NOKPROBE_SYMBOL(do_debug_exception); |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 734 | |
| 735 | #ifdef CONFIG_ARM64_PAN |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame] | 736 | int cpu_enable_pan(void *__unused) |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 737 | { |
James Morse | 7209c86 | 2016-10-18 11:27:47 +0100 | [diff] [blame] | 738 | /* |
| 739 | * We modify PSTATE. This won't work from irq context as the PSTATE |
| 740 | * is discarded once we return from the exception. |
| 741 | */ |
| 742 | WARN_ON_ONCE(in_interrupt()); |
| 743 | |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 744 | config_sctlr_el1(SCTLR_EL1_SPAN, 0); |
James Morse | 7209c86 | 2016-10-18 11:27:47 +0100 | [diff] [blame] | 745 | asm(SET_PSTATE_PAN(1)); |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame] | 746 | return 0; |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 747 | } |
| 748 | #endif /* CONFIG_ARM64_PAN */ |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 749 | |
| 750 | #ifdef CONFIG_ARM64_UAO |
| 751 | /* |
| 752 | * Kernel threads have fs=KERNEL_DS by default, and don't need to call |
| 753 | * set_fs(), devtmpfs in particular relies on this behaviour. |
| 754 | * We need to enable the feature at runtime (instead of adding it to |
| 755 | * PSR_MODE_EL1h) as the feature may not be implemented by the cpu. |
| 756 | */ |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame] | 757 | int cpu_enable_uao(void *__unused) |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 758 | { |
| 759 | asm(SET_PSTATE_UAO(1)); |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame] | 760 | return 0; |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 761 | } |
| 762 | #endif /* CONFIG_ARM64_UAO */ |