Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/i386/mm/fault.c |
| 3 | * |
| 4 | * Copyright (C) 1995 Linus Torvalds |
| 5 | */ |
| 6 | |
| 7 | #include <linux/signal.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/kernel.h> |
| 10 | #include <linux/errno.h> |
| 11 | #include <linux/string.h> |
| 12 | #include <linux/types.h> |
| 13 | #include <linux/ptrace.h> |
| 14 | #include <linux/mman.h> |
| 15 | #include <linux/mm.h> |
| 16 | #include <linux/smp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <linux/interrupt.h> |
| 18 | #include <linux/init.h> |
| 19 | #include <linux/tty.h> |
| 20 | #include <linux/vt_kern.h> /* For unblank_screen() */ |
| 21 | #include <linux/highmem.h> |
Jan Beulich | 28609f6 | 2007-05-02 19:27:04 +0200 | [diff] [blame] | 22 | #include <linux/bootmem.h> /* for max_low_pfn */ |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 23 | #include <linux/vmalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | #include <linux/module.h> |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 25 | #include <linux/kprobes.h> |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 26 | #include <linux/uaccess.h> |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 27 | #include <linux/kdebug.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | |
| 29 | #include <asm/system.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | #include <asm/desc.h> |
Rusty Russell | 78be370 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 31 | #include <asm/segment.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | |
| 33 | extern void die(const char *,struct pt_regs *,long); |
| 34 | |
Andi Kleen | 474c256 | 2006-09-26 10:52:35 +0200 | [diff] [blame] | 35 | static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain); |
| 36 | |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 37 | int register_page_fault_notifier(struct notifier_block *nb) |
| 38 | { |
| 39 | vmalloc_sync_all(); |
| 40 | return atomic_notifier_chain_register(¬ify_page_fault_chain, nb); |
| 41 | } |
Andi Kleen | 474c256 | 2006-09-26 10:52:35 +0200 | [diff] [blame] | 42 | EXPORT_SYMBOL_GPL(register_page_fault_notifier); |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 43 | |
| 44 | int unregister_page_fault_notifier(struct notifier_block *nb) |
| 45 | { |
| 46 | return atomic_notifier_chain_unregister(¬ify_page_fault_chain, nb); |
| 47 | } |
Andi Kleen | 474c256 | 2006-09-26 10:52:35 +0200 | [diff] [blame] | 48 | EXPORT_SYMBOL_GPL(unregister_page_fault_notifier); |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 49 | |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 50 | static inline int notify_page_fault(struct pt_regs *regs, long err) |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 51 | { |
| 52 | struct die_args args = { |
| 53 | .regs = regs, |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 54 | .str = "page fault", |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 55 | .err = err, |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 56 | .trapnr = 14, |
| 57 | .signr = SIGSEGV |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 58 | }; |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 59 | return atomic_notifier_call_chain(¬ify_page_fault_chain, |
| 60 | DIE_PAGE_FAULT, &args); |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 61 | } |
Anil S Keshavamurthy | b71b5b6 | 2006-06-26 00:25:25 -0700 | [diff] [blame] | 62 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | * Return EIP plus the CS segment base. The segment limit is also |
| 65 | * adjusted, clamped to the kernel/user address space (whichever is |
| 66 | * appropriate), and returned in *eip_limit. |
| 67 | * |
| 68 | * The segment is checked, because it might have been changed by another |
| 69 | * task between the original faulting instruction and here. |
| 70 | * |
| 71 | * If CS is no longer a valid code segment, or if EIP is beyond the |
| 72 | * limit, or if it is a kernel address when CS is not a kernel segment, |
| 73 | * then the returned value will be greater than *eip_limit. |
| 74 | * |
| 75 | * This is slow, but is very rarely executed. |
| 76 | */ |
| 77 | static inline unsigned long get_segment_eip(struct pt_regs *regs, |
| 78 | unsigned long *eip_limit) |
| 79 | { |
| 80 | unsigned long eip = regs->eip; |
| 81 | unsigned seg = regs->xcs & 0xffff; |
| 82 | u32 seg_ar, seg_limit, base, *desc; |
| 83 | |
Chuck Ebbert | 19964fe | 2006-06-23 02:04:29 -0700 | [diff] [blame] | 84 | /* Unlikely, but must come before segment checks. */ |
| 85 | if (unlikely(regs->eflags & VM_MASK)) { |
| 86 | base = seg << 4; |
| 87 | *eip_limit = base + 0xffff; |
| 88 | return base + (eip & 0xffff); |
| 89 | } |
| 90 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | /* The standard kernel/user address space limit. */ |
Rusty Russell | 78be370 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 92 | *eip_limit = user_mode(regs) ? USER_DS.seg : KERNEL_DS.seg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | |
| 94 | /* By far the most common cases. */ |
Rusty Russell | 78be370 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 95 | if (likely(SEGMENT_IS_FLAT_CODE(seg))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | return eip; |
| 97 | |
| 98 | /* Check the segment exists, is within the current LDT/GDT size, |
| 99 | that kernel/user (ring 0..3) has the appropriate privilege, |
| 100 | that it's a code segment, and get the limit. */ |
| 101 | __asm__ ("larl %3,%0; lsll %3,%1" |
| 102 | : "=&r" (seg_ar), "=r" (seg_limit) : "0" (0), "rm" (seg)); |
| 103 | if ((~seg_ar & 0x9800) || eip > seg_limit) { |
| 104 | *eip_limit = 0; |
| 105 | return 1; /* So that returned eip > *eip_limit. */ |
| 106 | } |
| 107 | |
| 108 | /* Get the GDT/LDT descriptor base. |
| 109 | When you look for races in this code remember that |
| 110 | LDT and other horrors are only used in user space. */ |
| 111 | if (seg & (1<<2)) { |
| 112 | /* Must lock the LDT while reading it. */ |
| 113 | down(¤t->mm->context.sem); |
| 114 | desc = current->mm->context.ldt; |
| 115 | desc = (void *)desc + (seg & ~7); |
| 116 | } else { |
| 117 | /* Must disable preemption while reading the GDT. */ |
Zachary Amsden | 251e691 | 2005-10-30 14:59:34 -0800 | [diff] [blame] | 118 | desc = (u32 *)get_cpu_gdt_table(get_cpu()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 119 | desc = (void *)desc + (seg & ~7); |
| 120 | } |
| 121 | |
| 122 | /* Decode the code segment base from the descriptor */ |
| 123 | base = get_desc_base((unsigned long *)desc); |
| 124 | |
| 125 | if (seg & (1<<2)) { |
| 126 | up(¤t->mm->context.sem); |
| 127 | } else |
| 128 | put_cpu(); |
| 129 | |
| 130 | /* Adjust EIP and segment limit, and clamp at the kernel limit. |
| 131 | It's legitimate for segments to wrap at 0xffffffff. */ |
| 132 | seg_limit += base; |
| 133 | if (seg_limit < *eip_limit && seg_limit >= base) |
| 134 | *eip_limit = seg_limit; |
| 135 | return eip + base; |
| 136 | } |
| 137 | |
| 138 | /* |
| 139 | * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch. |
| 140 | * Check that here and ignore it. |
| 141 | */ |
| 142 | static int __is_prefetch(struct pt_regs *regs, unsigned long addr) |
| 143 | { |
| 144 | unsigned long limit; |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 145 | unsigned char *instr = (unsigned char *)get_segment_eip (regs, &limit); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | int scan_more = 1; |
| 147 | int prefetch = 0; |
| 148 | int i; |
| 149 | |
| 150 | for (i = 0; scan_more && i < 15; i++) { |
| 151 | unsigned char opcode; |
| 152 | unsigned char instr_hi; |
| 153 | unsigned char instr_lo; |
| 154 | |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 155 | if (instr > (unsigned char *)limit) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | break; |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 157 | if (probe_kernel_address(instr, opcode)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 158 | break; |
| 159 | |
| 160 | instr_hi = opcode & 0xf0; |
| 161 | instr_lo = opcode & 0x0f; |
| 162 | instr++; |
| 163 | |
| 164 | switch (instr_hi) { |
| 165 | case 0x20: |
| 166 | case 0x30: |
| 167 | /* Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes. */ |
| 168 | scan_more = ((instr_lo & 7) == 0x6); |
| 169 | break; |
| 170 | |
| 171 | case 0x60: |
| 172 | /* 0x64 thru 0x67 are valid prefixes in all modes. */ |
| 173 | scan_more = (instr_lo & 0xC) == 0x4; |
| 174 | break; |
| 175 | case 0xF0: |
| 176 | /* 0xF0, 0xF2, and 0xF3 are valid prefixes */ |
| 177 | scan_more = !instr_lo || (instr_lo>>1) == 1; |
| 178 | break; |
| 179 | case 0x00: |
| 180 | /* Prefetch instruction is 0x0F0D or 0x0F18 */ |
| 181 | scan_more = 0; |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 182 | if (instr > (unsigned char *)limit) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 183 | break; |
Andi Kleen | 11a4180 | 2006-12-07 02:14:06 +0100 | [diff] [blame] | 184 | if (probe_kernel_address(instr, opcode)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 185 | break; |
| 186 | prefetch = (instr_lo == 0xF) && |
| 187 | (opcode == 0x0D || opcode == 0x18); |
| 188 | break; |
| 189 | default: |
| 190 | scan_more = 0; |
| 191 | break; |
| 192 | } |
| 193 | } |
| 194 | return prefetch; |
| 195 | } |
| 196 | |
| 197 | static inline int is_prefetch(struct pt_regs *regs, unsigned long addr, |
| 198 | unsigned long error_code) |
| 199 | { |
| 200 | if (unlikely(boot_cpu_data.x86_vendor == X86_VENDOR_AMD && |
| 201 | boot_cpu_data.x86 >= 6)) { |
| 202 | /* Catch an obscure case of prefetch inside an NX page. */ |
| 203 | if (nx_enabled && (error_code & 16)) |
| 204 | return 0; |
| 205 | return __is_prefetch(regs, addr); |
| 206 | } |
| 207 | return 0; |
| 208 | } |
| 209 | |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 210 | static noinline void force_sig_info_fault(int si_signo, int si_code, |
| 211 | unsigned long address, struct task_struct *tsk) |
| 212 | { |
| 213 | siginfo_t info; |
| 214 | |
| 215 | info.si_signo = si_signo; |
| 216 | info.si_errno = 0; |
| 217 | info.si_code = si_code; |
| 218 | info.si_addr = (void __user *)address; |
| 219 | force_sig_info(si_signo, &info, tsk); |
| 220 | } |
| 221 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | fastcall void do_invalid_op(struct pt_regs *, unsigned long); |
| 223 | |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 224 | static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) |
| 225 | { |
| 226 | unsigned index = pgd_index(address); |
| 227 | pgd_t *pgd_k; |
| 228 | pud_t *pud, *pud_k; |
| 229 | pmd_t *pmd, *pmd_k; |
| 230 | |
| 231 | pgd += index; |
| 232 | pgd_k = init_mm.pgd + index; |
| 233 | |
| 234 | if (!pgd_present(*pgd_k)) |
| 235 | return NULL; |
| 236 | |
| 237 | /* |
| 238 | * set_pgd(pgd, *pgd_k); here would be useless on PAE |
| 239 | * and redundant with the set_pmd() on non-PAE. As would |
| 240 | * set_pud. |
| 241 | */ |
| 242 | |
| 243 | pud = pud_offset(pgd, address); |
| 244 | pud_k = pud_offset(pgd_k, address); |
| 245 | if (!pud_present(*pud_k)) |
| 246 | return NULL; |
| 247 | |
| 248 | pmd = pmd_offset(pud, address); |
| 249 | pmd_k = pmd_offset(pud_k, address); |
| 250 | if (!pmd_present(*pmd_k)) |
| 251 | return NULL; |
Zachary Amsden | 8b14cb9 | 2007-08-21 18:30:36 -0700 | [diff] [blame^] | 252 | if (!pmd_present(*pmd)) { |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 253 | set_pmd(pmd, *pmd_k); |
Zachary Amsden | 8b14cb9 | 2007-08-21 18:30:36 -0700 | [diff] [blame^] | 254 | arch_flush_lazy_mmu_mode(); |
| 255 | } else |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 256 | BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k)); |
| 257 | return pmd_k; |
| 258 | } |
| 259 | |
| 260 | /* |
| 261 | * Handle a fault on the vmalloc or module mapping area |
| 262 | * |
| 263 | * This assumes no large pages in there. |
| 264 | */ |
| 265 | static inline int vmalloc_fault(unsigned long address) |
| 266 | { |
| 267 | unsigned long pgd_paddr; |
| 268 | pmd_t *pmd_k; |
| 269 | pte_t *pte_k; |
| 270 | /* |
| 271 | * Synchronize this task's top level page-table |
| 272 | * with the 'reference' page table. |
| 273 | * |
| 274 | * Do _not_ use "current" here. We might be inside |
| 275 | * an interrupt in the middle of a task switch.. |
| 276 | */ |
| 277 | pgd_paddr = read_cr3(); |
| 278 | pmd_k = vmalloc_sync_one(__va(pgd_paddr), address); |
| 279 | if (!pmd_k) |
| 280 | return -1; |
| 281 | pte_k = pte_offset_kernel(pmd_k, address); |
| 282 | if (!pte_present(*pte_k)) |
| 283 | return -1; |
| 284 | return 0; |
| 285 | } |
| 286 | |
Masoud Asgharifard Sharbiani | abd4f75 | 2007-07-22 11:12:28 +0200 | [diff] [blame] | 287 | int show_unhandled_signals = 1; |
| 288 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 289 | /* |
| 290 | * This routine handles page faults. It determines the address, |
| 291 | * and the problem, and then passes it off to one of the appropriate |
| 292 | * routines. |
| 293 | * |
| 294 | * error_code: |
| 295 | * bit 0 == 0 means no page found, 1 means protection fault |
| 296 | * bit 1 == 0 means read, 1 means write |
| 297 | * bit 2 == 0 means kernel, 1 means user-mode |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 298 | * bit 3 == 1 means use of reserved bit detected |
| 299 | * bit 4 == 1 means fault was an instruction fetch |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | */ |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 301 | fastcall void __kprobes do_page_fault(struct pt_regs *regs, |
| 302 | unsigned long error_code) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 303 | { |
| 304 | struct task_struct *tsk; |
| 305 | struct mm_struct *mm; |
| 306 | struct vm_area_struct * vma; |
| 307 | unsigned long address; |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 308 | int write, si_code; |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 309 | int fault; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | |
| 311 | /* get the address */ |
Zachary Amsden | 4bb0d3e | 2005-09-03 15:56:36 -0700 | [diff] [blame] | 312 | address = read_cr2(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | tsk = current; |
| 315 | |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 316 | si_code = SEGV_MAPERR; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | |
| 318 | /* |
| 319 | * We fault-in kernel-space virtual memory on-demand. The |
| 320 | * 'reference' page table is init_mm.pgd. |
| 321 | * |
| 322 | * NOTE! We MUST NOT take any locks for this case. We may |
| 323 | * be in an interrupt or a critical region, and should |
| 324 | * only copy the information from the master page table, |
| 325 | * nothing more. |
| 326 | * |
| 327 | * This verifies that the fault happens in kernel space |
| 328 | * (error_code & 4) == 0, and that the fault was not a |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 329 | * protection error (error_code & 9) == 0. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | */ |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 331 | if (unlikely(address >= TASK_SIZE)) { |
| 332 | if (!(error_code & 0x0000000d) && vmalloc_fault(address) >= 0) |
| 333 | return; |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 334 | if (notify_page_fault(regs, error_code) == NOTIFY_STOP) |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 335 | return; |
| 336 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | * Don't take the mm semaphore here. If we fixup a prefetch |
| 338 | * fault we could otherwise deadlock. |
| 339 | */ |
| 340 | goto bad_area_nosemaphore; |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 341 | } |
| 342 | |
Jan Beulich | 9b35589 | 2007-02-13 13:26:23 +0100 | [diff] [blame] | 343 | if (notify_page_fault(regs, error_code) == NOTIFY_STOP) |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 344 | return; |
| 345 | |
| 346 | /* It's safe to allow irq's after cr2 has been saved and the vmalloc |
| 347 | fault has been handled. */ |
| 348 | if (regs->eflags & (X86_EFLAGS_IF|VM_MASK)) |
| 349 | local_irq_enable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 350 | |
| 351 | mm = tsk->mm; |
| 352 | |
| 353 | /* |
| 354 | * If we're in an interrupt, have no user context or are running in an |
| 355 | * atomic region then we must not take the fault.. |
| 356 | */ |
| 357 | if (in_atomic() || !mm) |
| 358 | goto bad_area_nosemaphore; |
| 359 | |
| 360 | /* When running in the kernel we expect faults to occur only to |
| 361 | * addresses in user space. All other faults represent errors in the |
| 362 | * kernel and should generate an OOPS. Unfortunatly, in the case of an |
Adrian Bunk | 80f7228 | 2006-06-30 18:27:16 +0200 | [diff] [blame] | 363 | * erroneous fault occurring in a code path which already holds mmap_sem |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | * we will deadlock attempting to validate the fault against the |
| 365 | * address space. Luckily the kernel only validly references user |
| 366 | * space from well defined areas of code, which are listed in the |
| 367 | * exceptions table. |
| 368 | * |
| 369 | * As the vast majority of faults will be valid we will only perform |
| 370 | * the source reference check when there is a possibilty of a deadlock. |
| 371 | * Attempt to lock the address space, if we cannot we then validate the |
| 372 | * source. If this is invalid we can skip the address space check, |
| 373 | * thus avoiding the deadlock. |
| 374 | */ |
| 375 | if (!down_read_trylock(&mm->mmap_sem)) { |
| 376 | if ((error_code & 4) == 0 && |
| 377 | !search_exception_tables(regs->eip)) |
| 378 | goto bad_area_nosemaphore; |
| 379 | down_read(&mm->mmap_sem); |
| 380 | } |
| 381 | |
| 382 | vma = find_vma(mm, address); |
| 383 | if (!vma) |
| 384 | goto bad_area; |
| 385 | if (vma->vm_start <= address) |
| 386 | goto good_area; |
| 387 | if (!(vma->vm_flags & VM_GROWSDOWN)) |
| 388 | goto bad_area; |
| 389 | if (error_code & 4) { |
| 390 | /* |
Chuck Ebbert | 2152845 | 2006-06-23 02:04:23 -0700 | [diff] [blame] | 391 | * Accessing the stack below %esp is always a bug. |
| 392 | * The large cushion allows instructions like enter |
| 393 | * and pusha to work. ("enter $65535,$31" pushes |
| 394 | * 32 pointers and then decrements %esp by 65535.) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 395 | */ |
Chuck Ebbert | 2152845 | 2006-06-23 02:04:23 -0700 | [diff] [blame] | 396 | if (address + 65536 + 32 * sizeof(unsigned long) < regs->esp) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | goto bad_area; |
| 398 | } |
| 399 | if (expand_stack(vma, address)) |
| 400 | goto bad_area; |
| 401 | /* |
| 402 | * Ok, we have a good vm_area for this memory access, so |
| 403 | * we can handle it.. |
| 404 | */ |
| 405 | good_area: |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 406 | si_code = SEGV_ACCERR; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | write = 0; |
| 408 | switch (error_code & 3) { |
| 409 | default: /* 3: write, present */ |
Rusty Russell | 78be370 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 410 | /* fall through */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 411 | case 2: /* write, not present */ |
| 412 | if (!(vma->vm_flags & VM_WRITE)) |
| 413 | goto bad_area; |
| 414 | write++; |
| 415 | break; |
| 416 | case 1: /* read, present */ |
| 417 | goto bad_area; |
| 418 | case 0: /* read, not present */ |
Jason Baron | df67b3d | 2006-09-29 01:58:58 -0700 | [diff] [blame] | 419 | if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 420 | goto bad_area; |
| 421 | } |
| 422 | |
| 423 | survive: |
| 424 | /* |
| 425 | * If for any reason at all we couldn't handle the fault, |
| 426 | * make sure we exit gracefully rather than endlessly redo |
| 427 | * the fault. |
| 428 | */ |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 429 | fault = handle_mm_fault(mm, vma, address, write); |
| 430 | if (unlikely(fault & VM_FAULT_ERROR)) { |
| 431 | if (fault & VM_FAULT_OOM) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 432 | goto out_of_memory; |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 433 | else if (fault & VM_FAULT_SIGBUS) |
| 434 | goto do_sigbus; |
| 435 | BUG(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 436 | } |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 437 | if (fault & VM_FAULT_MAJOR) |
| 438 | tsk->maj_flt++; |
| 439 | else |
| 440 | tsk->min_flt++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 441 | |
| 442 | /* |
| 443 | * Did it hit the DOS screen memory VA from vm86 mode? |
| 444 | */ |
| 445 | if (regs->eflags & VM_MASK) { |
| 446 | unsigned long bit = (address - 0xA0000) >> PAGE_SHIFT; |
| 447 | if (bit < 32) |
| 448 | tsk->thread.screen_bitmap |= 1 << bit; |
| 449 | } |
| 450 | up_read(&mm->mmap_sem); |
| 451 | return; |
| 452 | |
| 453 | /* |
| 454 | * Something tried to access memory that isn't in our memory map.. |
| 455 | * Fix it, but check if it's kernel or user first.. |
| 456 | */ |
| 457 | bad_area: |
| 458 | up_read(&mm->mmap_sem); |
| 459 | |
| 460 | bad_area_nosemaphore: |
| 461 | /* User mode accesses just cause a SIGSEGV */ |
| 462 | if (error_code & 4) { |
Steven Rostedt | e5e3c84 | 2007-06-06 23:34:04 -0400 | [diff] [blame] | 463 | /* |
| 464 | * It's possible to have interrupts off here. |
| 465 | */ |
| 466 | local_irq_enable(); |
| 467 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | /* |
| 469 | * Valid to do another page fault here because this one came |
| 470 | * from user space. |
| 471 | */ |
| 472 | if (is_prefetch(regs, address, error_code)) |
| 473 | return; |
| 474 | |
Masoud Asgharifard Sharbiani | abd4f75 | 2007-07-22 11:12:28 +0200 | [diff] [blame] | 475 | if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) && |
| 476 | printk_ratelimit()) { |
| 477 | printk("%s%s[%d]: segfault at %08lx eip %08lx " |
| 478 | "esp %08lx error %lx\n", |
| 479 | tsk->pid > 1 ? KERN_INFO : KERN_EMERG, |
| 480 | tsk->comm, tsk->pid, address, regs->eip, |
| 481 | regs->esp, error_code); |
| 482 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | tsk->thread.cr2 = address; |
| 484 | /* Kernel addresses are always protection faults */ |
| 485 | tsk->thread.error_code = error_code | (address >= TASK_SIZE); |
| 486 | tsk->thread.trap_no = 14; |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 487 | force_sig_info_fault(SIGSEGV, si_code, address, tsk); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 488 | return; |
| 489 | } |
| 490 | |
| 491 | #ifdef CONFIG_X86_F00F_BUG |
| 492 | /* |
| 493 | * Pentium F0 0F C7 C8 bug workaround. |
| 494 | */ |
| 495 | if (boot_cpu_data.f00f_bug) { |
| 496 | unsigned long nr; |
| 497 | |
| 498 | nr = (address - idt_descr.address) >> 3; |
| 499 | |
| 500 | if (nr == 6) { |
| 501 | do_invalid_op(regs, 0); |
| 502 | return; |
| 503 | } |
| 504 | } |
| 505 | #endif |
| 506 | |
| 507 | no_context: |
| 508 | /* Are we prepared to handle this kernel fault? */ |
| 509 | if (fixup_exception(regs)) |
| 510 | return; |
| 511 | |
| 512 | /* |
| 513 | * Valid to do another page fault here, because if this fault |
| 514 | * had been triggered by is_prefetch fixup_exception would have |
| 515 | * handled it. |
| 516 | */ |
| 517 | if (is_prefetch(regs, address, error_code)) |
| 518 | return; |
| 519 | |
| 520 | /* |
| 521 | * Oops. The kernel tried to access some bad page. We'll have to |
| 522 | * terminate things with extreme prejudice. |
| 523 | */ |
| 524 | |
| 525 | bust_spinlocks(1); |
| 526 | |
Andrew Morton | dd28779 | 2006-03-23 03:00:57 -0800 | [diff] [blame] | 527 | if (oops_may_print()) { |
Jan Beulich | 28609f6 | 2007-05-02 19:27:04 +0200 | [diff] [blame] | 528 | __typeof__(pte_val(__pte(0))) page; |
| 529 | |
| 530 | #ifdef CONFIG_X86_PAE |
Andrew Morton | dd28779 | 2006-03-23 03:00:57 -0800 | [diff] [blame] | 531 | if (error_code & 16) { |
| 532 | pte_t *pte = lookup_address(address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 533 | |
Andrew Morton | dd28779 | 2006-03-23 03:00:57 -0800 | [diff] [blame] | 534 | if (pte && pte_present(*pte) && !pte_exec_kernel(*pte)) |
| 535 | printk(KERN_CRIT "kernel tried to execute " |
| 536 | "NX-protected page - exploit attempt? " |
| 537 | "(uid: %d)\n", current->uid); |
| 538 | } |
Jan Beulich | 28609f6 | 2007-05-02 19:27:04 +0200 | [diff] [blame] | 539 | #endif |
Andrew Morton | dd28779 | 2006-03-23 03:00:57 -0800 | [diff] [blame] | 540 | if (address < PAGE_SIZE) |
| 541 | printk(KERN_ALERT "BUG: unable to handle kernel NULL " |
| 542 | "pointer dereference"); |
| 543 | else |
| 544 | printk(KERN_ALERT "BUG: unable to handle kernel paging" |
| 545 | " request"); |
| 546 | printk(" at virtual address %08lx\n",address); |
| 547 | printk(KERN_ALERT " printing eip:\n"); |
| 548 | printk("%08lx\n", regs->eip); |
Jan Beulich | 28609f6 | 2007-05-02 19:27:04 +0200 | [diff] [blame] | 549 | |
| 550 | page = read_cr3(); |
| 551 | page = ((__typeof__(page) *) __va(page))[address >> PGDIR_SHIFT]; |
| 552 | #ifdef CONFIG_X86_PAE |
| 553 | printk(KERN_ALERT "*pdpt = %016Lx\n", page); |
| 554 | if ((page >> PAGE_SHIFT) < max_low_pfn |
| 555 | && page & _PAGE_PRESENT) { |
| 556 | page &= PAGE_MASK; |
| 557 | page = ((__typeof__(page) *) __va(page))[(address >> PMD_SHIFT) |
| 558 | & (PTRS_PER_PMD - 1)]; |
| 559 | printk(KERN_ALERT "*pde = %016Lx\n", page); |
| 560 | page &= ~_PAGE_NX; |
| 561 | } |
| 562 | #else |
Andrew Morton | dd28779 | 2006-03-23 03:00:57 -0800 | [diff] [blame] | 563 | printk(KERN_ALERT "*pde = %08lx\n", page); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 564 | #endif |
Jan Beulich | 28609f6 | 2007-05-02 19:27:04 +0200 | [diff] [blame] | 565 | |
| 566 | /* |
| 567 | * We must not directly access the pte in the highpte |
| 568 | * case if the page table is located in highmem. |
| 569 | * And let's rather not kmap-atomic the pte, just in case |
| 570 | * it's allocated already. |
| 571 | */ |
| 572 | if ((page >> PAGE_SHIFT) < max_low_pfn |
| 573 | && (page & _PAGE_PRESENT)) { |
| 574 | page &= PAGE_MASK; |
| 575 | page = ((__typeof__(page) *) __va(page))[(address >> PAGE_SHIFT) |
| 576 | & (PTRS_PER_PTE - 1)]; |
| 577 | printk(KERN_ALERT "*pte = %0*Lx\n", sizeof(page)*2, (u64)page); |
| 578 | } |
| 579 | } |
| 580 | |
Alexander Nyberg | 4f339ec | 2005-06-25 14:58:27 -0700 | [diff] [blame] | 581 | tsk->thread.cr2 = address; |
| 582 | tsk->thread.trap_no = 14; |
| 583 | tsk->thread.error_code = error_code; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 584 | die("Oops", regs, error_code); |
| 585 | bust_spinlocks(0); |
| 586 | do_exit(SIGKILL); |
| 587 | |
| 588 | /* |
| 589 | * We ran out of memory, or some other thing happened to us that made |
| 590 | * us unable to handle the page fault gracefully. |
| 591 | */ |
| 592 | out_of_memory: |
| 593 | up_read(&mm->mmap_sem); |
Sukadev Bhattiprolu | f400e19 | 2006-09-29 02:00:07 -0700 | [diff] [blame] | 594 | if (is_init(tsk)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 595 | yield(); |
| 596 | down_read(&mm->mmap_sem); |
| 597 | goto survive; |
| 598 | } |
| 599 | printk("VM: killing process %s\n", tsk->comm); |
| 600 | if (error_code & 4) |
| 601 | do_exit(SIGKILL); |
| 602 | goto no_context; |
| 603 | |
| 604 | do_sigbus: |
| 605 | up_read(&mm->mmap_sem); |
| 606 | |
| 607 | /* Kernel mode? Handle exceptions or die */ |
| 608 | if (!(error_code & 4)) |
| 609 | goto no_context; |
| 610 | |
| 611 | /* User space => ok to do another page fault */ |
| 612 | if (is_prefetch(regs, address, error_code)) |
| 613 | return; |
| 614 | |
| 615 | tsk->thread.cr2 = address; |
| 616 | tsk->thread.error_code = error_code; |
| 617 | tsk->thread.trap_no = 14; |
Ingo Molnar | 869f96a | 2005-09-03 15:56:26 -0700 | [diff] [blame] | 618 | force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk); |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 619 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 620 | |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 621 | void vmalloc_sync_all(void) |
| 622 | { |
| 623 | /* |
| 624 | * Note that races in the updates of insync and start aren't |
| 625 | * problematic: insync can only get set bits added, and updates to |
| 626 | * start are only improving performance (without affecting correctness |
| 627 | * if undone). |
| 628 | */ |
| 629 | static DECLARE_BITMAP(insync, PTRS_PER_PGD); |
| 630 | static unsigned long start = TASK_SIZE; |
| 631 | unsigned long address; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 632 | |
Jeremy Fitzhardinge | 5311ab6 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 633 | if (SHARED_KERNEL_PMD) |
| 634 | return; |
| 635 | |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 636 | BUILD_BUG_ON(TASK_SIZE & ~PGDIR_MASK); |
| 637 | for (address = start; address >= TASK_SIZE; address += PGDIR_SIZE) { |
| 638 | if (!test_bit(pgd_index(address), insync)) { |
| 639 | unsigned long flags; |
| 640 | struct page *page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 641 | |
Jan Beulich | 101f12a | 2006-03-23 02:59:45 -0800 | [diff] [blame] | 642 | spin_lock_irqsave(&pgd_lock, flags); |
| 643 | for (page = pgd_list; page; page = |
| 644 | (struct page *)page->index) |
| 645 | if (!vmalloc_sync_one(page_address(page), |
| 646 | address)) { |
| 647 | BUG_ON(page != pgd_list); |
| 648 | break; |
| 649 | } |
| 650 | spin_unlock_irqrestore(&pgd_lock, flags); |
| 651 | if (!page) |
| 652 | set_bit(pgd_index(address), insync); |
| 653 | } |
| 654 | if (address == start && test_bit(pgd_index(address), insync)) |
| 655 | start = address + PGDIR_SIZE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 656 | } |
| 657 | } |