blob: 55ae802dc0ad6ffd6c06b9bd12e860f9a777267b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* $Id: fault.c,v 1.59 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc.
3 *
4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
6 */
7
8#include <asm/head.h>
9
10#include <linux/string.h>
11#include <linux/types.h>
12#include <linux/sched.h>
13#include <linux/ptrace.h>
14#include <linux/mman.h>
15#include <linux/signal.h>
16#include <linux/mm.h>
17#include <linux/module.h>
18#include <linux/smp_lock.h>
19#include <linux/init.h>
20#include <linux/interrupt.h>
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -070021#include <linux/kprobes.h>
David S. Millereb398d12006-07-22 01:12:09 -070022#include <linux/kallsyms.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24#include <asm/page.h>
25#include <asm/pgtable.h>
26#include <asm/openprom.h>
27#include <asm/oplib.h>
28#include <asm/uaccess.h>
29#include <asm/asi.h>
30#include <asm/lsu.h>
31#include <asm/sections.h>
32#include <asm/kdebug.h>
David S. Miller7a1ac522006-03-16 02:02:32 -080033#include <asm/mmu_context.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -070035#ifdef CONFIG_KPROBES
36ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain);
37
38/* Hook to register for page fault notifications */
39int register_page_fault_notifier(struct notifier_block *nb)
40{
41 return atomic_notifier_chain_register(&notify_page_fault_chain, nb);
42}
43
44int unregister_page_fault_notifier(struct notifier_block *nb)
45{
46 return atomic_notifier_chain_unregister(&notify_page_fault_chain, nb);
47}
48
49static inline int notify_page_fault(enum die_val val, const char *str,
50 struct pt_regs *regs, long err, int trap, int sig)
51{
52 struct die_args args = {
53 .regs = regs,
54 .str = str,
55 .err = err,
56 .trapnr = trap,
57 .signr = sig
58 };
59 return atomic_notifier_call_chain(&notify_page_fault_chain, val, &args);
60}
61#else
62static inline int notify_page_fault(enum die_val val, const char *str,
63 struct pt_regs *regs, long err, int trap, int sig)
64{
65 return NOTIFY_DONE;
66}
67#endif
68
Linus Torvalds1da177e2005-04-16 15:20:36 -070069/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 * To debug kernel to catch accesses to certain virtual/physical addresses.
71 * Mode = 0 selects physical watchpoints, mode = 1 selects virtual watchpoints.
72 * flags = VM_READ watches memread accesses, flags = VM_WRITE watches memwrite accesses.
73 * Caller passes in a 64bit aligned addr, with mask set to the bytes that need to be
74 * watched. This is only useful on a single cpu machine for now. After the watchpoint
75 * is detected, the process causing it will be killed, thus preventing an infinite loop.
76 */
77void set_brkpt(unsigned long addr, unsigned char mask, int flags, int mode)
78{
79 unsigned long lsubits;
80
81 __asm__ __volatile__("ldxa [%%g0] %1, %0"
82 : "=r" (lsubits)
83 : "i" (ASI_LSU_CONTROL));
84 lsubits &= ~(LSU_CONTROL_PM | LSU_CONTROL_VM |
85 LSU_CONTROL_PR | LSU_CONTROL_VR |
86 LSU_CONTROL_PW | LSU_CONTROL_VW);
87
88 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
89 "membar #Sync"
90 : /* no outputs */
91 : "r" (addr), "r" (mode ? VIRT_WATCHPOINT : PHYS_WATCHPOINT),
92 "i" (ASI_DMMU));
93
94 lsubits |= ((unsigned long)mask << (mode ? 25 : 33));
95 if (flags & VM_READ)
96 lsubits |= (mode ? LSU_CONTROL_VR : LSU_CONTROL_PR);
97 if (flags & VM_WRITE)
98 lsubits |= (mode ? LSU_CONTROL_VW : LSU_CONTROL_PW);
99 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
100 "membar #Sync"
101 : /* no outputs */
102 : "r" (lsubits), "i" (ASI_LSU_CONTROL)
103 : "memory");
104}
105
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -0700106static void __kprobes unhandled_fault(unsigned long address,
107 struct task_struct *tsk,
108 struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109{
110 if ((unsigned long) address < PAGE_SIZE) {
111 printk(KERN_ALERT "Unable to handle kernel NULL "
112 "pointer dereference\n");
113 } else {
114 printk(KERN_ALERT "Unable to handle kernel paging request "
115 "at virtual address %016lx\n", (unsigned long)address);
116 }
117 printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n",
118 (tsk->mm ?
119 CTX_HWBITS(tsk->mm->context) :
120 CTX_HWBITS(tsk->active_mm->context)));
121 printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n",
122 (tsk->mm ? (unsigned long) tsk->mm->pgd :
123 (unsigned long) tsk->active_mm->pgd));
124 if (notify_die(DIE_GPF, "general protection fault", regs,
125 0, 0, SIGSEGV) == NOTIFY_STOP)
126 return;
127 die_if_kernel("Oops", regs);
128}
129
David S. Millerbf941d62006-02-13 18:07:45 -0800130static void bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131{
132 unsigned long *ksp;
133
134 printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n",
135 regs->tpc);
David S. Millereb398d12006-07-22 01:12:09 -0700136 printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]);
137 print_symbol("RPC: <%s>\n", regs->u_regs[15]);
David S. Millerbf941d62006-02-13 18:07:45 -0800138 printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 __asm__("mov %%sp, %0" : "=r" (ksp));
140 show_stack(current, ksp);
141 unhandled_fault(regs->tpc, current, regs);
142}
143
144/*
145 * We now make sure that mmap_sem is held in all paths that call
146 * this. Additionally, to prevent kswapd from ripping ptes from
147 * under us, raise interrupts around the time that we look at the
148 * pte, kswapd will have to wait to get his smp ipi response from
Hugh Dickinsda160542005-11-08 10:00:55 -0800149 * us. vmtruncate likewise. This saves us having to get pte lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 */
151static unsigned int get_user_insn(unsigned long tpc)
152{
153 pgd_t *pgdp = pgd_offset(current->mm, tpc);
154 pud_t *pudp;
155 pmd_t *pmdp;
156 pte_t *ptep, pte;
157 unsigned long pa;
158 u32 insn = 0;
159 unsigned long pstate;
160
161 if (pgd_none(*pgdp))
162 goto outret;
163 pudp = pud_offset(pgdp, tpc);
164 if (pud_none(*pudp))
165 goto outret;
166 pmdp = pmd_offset(pudp, tpc);
167 if (pmd_none(*pmdp))
168 goto outret;
169
170 /* This disables preemption for us as well. */
171 __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
172 __asm__ __volatile__("wrpr %0, %1, %%pstate"
173 : : "r" (pstate), "i" (PSTATE_IE));
174 ptep = pte_offset_map(pmdp, tpc);
175 pte = *ptep;
176 if (!pte_present(pte))
177 goto out;
178
David S. Millerc4bce902006-02-11 21:57:54 -0800179 pa = (pte_pfn(pte) << PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 pa += (tpc & ~PAGE_MASK);
181
182 /* Use phys bypass so we don't pollute dtlb/dcache. */
183 __asm__ __volatile__("lduwa [%1] %2, %0"
184 : "=r" (insn)
185 : "r" (pa), "i" (ASI_PHYS_USE_EC));
186
187out:
188 pte_unmap(ptep);
189 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
190outret:
191 return insn;
192}
193
194extern unsigned long compute_effective_address(struct pt_regs *, unsigned int, unsigned int);
195
196static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
197 unsigned int insn, int fault_code)
198{
199 siginfo_t info;
200
201 info.si_code = code;
202 info.si_signo = sig;
203 info.si_errno = 0;
204 if (fault_code & FAULT_CODE_ITLB)
205 info.si_addr = (void __user *) regs->tpc;
206 else
207 info.si_addr = (void __user *)
208 compute_effective_address(regs, insn, 0);
209 info.si_trapno = 0;
210 force_sig_info(sig, &info, current);
211}
212
213extern int handle_ldf_stq(u32, struct pt_regs *);
214extern int handle_ld_nf(u32, struct pt_regs *);
215
216static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
217{
218 if (!insn) {
219 if (!regs->tpc || (regs->tpc & 0x3))
220 return 0;
221 if (regs->tstate & TSTATE_PRIV) {
222 insn = *(unsigned int *) regs->tpc;
223 } else {
224 insn = get_user_insn(regs->tpc);
225 }
226 }
227 return insn;
228}
229
230static void do_kernel_fault(struct pt_regs *regs, int si_code, int fault_code,
231 unsigned int insn, unsigned long address)
232{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 unsigned char asi = ASI_P;
234
235 if ((!insn) && (regs->tstate & TSTATE_PRIV))
236 goto cannot_handle;
237
238 /* If user insn could be read (thus insn is zero), that
239 * is fine. We will just gun down the process with a signal
240 * in that case.
241 */
242
243 if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) &&
244 (insn & 0xc0800000) == 0xc0800000) {
245 if (insn & 0x2000)
246 asi = (regs->tstate >> 24);
247 else
248 asi = (insn >> 5);
249 if ((asi & 0xf2) == 0x82) {
250 if (insn & 0x1000000) {
251 handle_ldf_stq(insn, regs);
252 } else {
253 /* This was a non-faulting load. Just clear the
254 * destination register(s) and continue with the next
255 * instruction. -jj
256 */
257 handle_ld_nf(insn, regs);
258 }
259 return;
260 }
261 }
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 /* Is this in ex_table? */
264 if (regs->tstate & TSTATE_PRIV) {
David S. Miller8cf14af2005-09-28 20:21:11 -0700265 const struct exception_table_entry *entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267 if (asi == ASI_P && (insn & 0xc0800000) == 0xc0800000) {
268 if (insn & 0x2000)
269 asi = (regs->tstate >> 24);
270 else
271 asi = (insn >> 5);
272 }
273
274 /* Look in asi.h: All _S asis have LS bit set */
275 if ((asi & 0x1) &&
David S. Miller8cf14af2005-09-28 20:21:11 -0700276 (entry = search_exception_tables(regs->tpc))) {
277 regs->tpc = entry->fixup;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 regs->tnpc = regs->tpc + 4;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return;
280 }
281 } else {
282 /* The si_code was set to make clear whether
283 * this was a SEGV_MAPERR or SEGV_ACCERR fault.
284 */
285 do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code);
286 return;
287 }
288
289cannot_handle:
290 unhandled_fault (address, current, regs);
291}
292
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -0700293asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294{
295 struct mm_struct *mm = current->mm;
296 struct vm_area_struct *vma;
297 unsigned int insn = 0;
298 int si_code, fault_code;
David S. Miller7a1ac522006-03-16 02:02:32 -0800299 unsigned long address, mm_rss;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
301 fault_code = get_thread_fault_code();
302
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -0700303 if (notify_page_fault(DIE_PAGE_FAULT, "page_fault", regs,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 fault_code, 0, SIGSEGV) == NOTIFY_STOP)
305 return;
306
307 si_code = SEGV_MAPERR;
308 address = current_thread_info()->fault_address;
309
310 if ((fault_code & FAULT_CODE_ITLB) &&
311 (fault_code & FAULT_CODE_DTLB))
312 BUG();
313
314 if (regs->tstate & TSTATE_PRIV) {
315 unsigned long tpc = regs->tpc;
316
317 /* Sanity check the PC. */
318 if ((tpc >= KERNBASE && tpc < (unsigned long) _etext) ||
319 (tpc >= MODULES_VADDR && tpc < MODULES_END)) {
320 /* Valid, no problems... */
321 } else {
David S. Millerbf941d62006-02-13 18:07:45 -0800322 bad_kernel_pc(regs, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 return;
324 }
325 }
326
327 /*
328 * If we're in an interrupt or have no user
329 * context, we must not take the fault..
330 */
331 if (in_atomic() || !mm)
332 goto intr_or_no_mm;
333
334 if (test_thread_flag(TIF_32BIT)) {
335 if (!(regs->tstate & TSTATE_PRIV))
336 regs->tpc &= 0xffffffff;
337 address &= 0xffffffff;
338 }
339
340 if (!down_read_trylock(&mm->mmap_sem)) {
341 if ((regs->tstate & TSTATE_PRIV) &&
342 !search_exception_tables(regs->tpc)) {
343 insn = get_fault_insn(regs, insn);
344 goto handle_kernel_fault;
345 }
346 down_read(&mm->mmap_sem);
347 }
348
349 vma = find_vma(mm, address);
350 if (!vma)
351 goto bad_area;
352
353 /* Pure DTLB misses do not tell us whether the fault causing
354 * load/store/atomic was a write or not, it only says that there
355 * was no match. So in such a case we (carefully) read the
356 * instruction to try and figure this out. It's an optimization
357 * so it's ok if we can't do this.
358 *
359 * Special hack, window spill/fill knows the exact fault type.
360 */
361 if (((fault_code &
362 (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) &&
363 (vma->vm_flags & VM_WRITE) != 0) {
364 insn = get_fault_insn(regs, 0);
365 if (!insn)
366 goto continue_fault;
David S. Miller73c50a22006-03-28 13:32:24 -0800367 /* All loads, stores and atomics have bits 30 and 31 both set
368 * in the instruction. Bit 21 is set in all stores, but we
369 * have to avoid prefetches which also have bit 21 set.
370 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 if ((insn & 0xc0200000) == 0xc0200000 &&
David S. Miller73c50a22006-03-28 13:32:24 -0800372 (insn & 0x01780000) != 0x01680000) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 /* Don't bother updating thread struct value,
374 * because update_mmu_cache only cares which tlb
375 * the access came from.
376 */
377 fault_code |= FAULT_CODE_WRITE;
378 }
379 }
380continue_fault:
381
382 if (vma->vm_start <= address)
383 goto good_area;
384 if (!(vma->vm_flags & VM_GROWSDOWN))
385 goto bad_area;
386 if (!(fault_code & FAULT_CODE_WRITE)) {
387 /* Non-faulting loads shouldn't expand stack. */
388 insn = get_fault_insn(regs, insn);
389 if ((insn & 0xc0800000) == 0xc0800000) {
390 unsigned char asi;
391
392 if (insn & 0x2000)
393 asi = (regs->tstate >> 24);
394 else
395 asi = (insn >> 5);
396 if ((asi & 0xf2) == 0x82)
397 goto bad_area;
398 }
399 }
400 if (expand_stack(vma, address))
401 goto bad_area;
402 /*
403 * Ok, we have a good vm_area for this memory access, so
404 * we can handle it..
405 */
406good_area:
407 si_code = SEGV_ACCERR;
408
409 /* If we took a ITLB miss on a non-executable page, catch
410 * that here.
411 */
412 if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) {
413 BUG_ON(address != regs->tpc);
414 BUG_ON(regs->tstate & TSTATE_PRIV);
415 goto bad_area;
416 }
417
418 if (fault_code & FAULT_CODE_WRITE) {
419 if (!(vma->vm_flags & VM_WRITE))
420 goto bad_area;
421
422 /* Spitfire has an icache which does not snoop
423 * processor stores. Later processors do...
424 */
425 if (tlb_type == spitfire &&
426 (vma->vm_flags & VM_EXEC) != 0 &&
427 vma->vm_file != NULL)
428 set_thread_fault_code(fault_code |
429 FAULT_CODE_BLKCOMMIT);
430 } else {
431 /* Allow reads even for write-only mappings */
432 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
433 goto bad_area;
434 }
435
436 switch (handle_mm_fault(mm, vma, address, (fault_code & FAULT_CODE_WRITE))) {
437 case VM_FAULT_MINOR:
438 current->min_flt++;
439 break;
440 case VM_FAULT_MAJOR:
441 current->maj_flt++;
442 break;
443 case VM_FAULT_SIGBUS:
444 goto do_sigbus;
445 case VM_FAULT_OOM:
446 goto out_of_memory;
447 default:
448 BUG();
449 }
450
451 up_read(&mm->mmap_sem);
David S. Miller7a1ac522006-03-16 02:02:32 -0800452
453 mm_rss = get_mm_rss(mm);
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800454#ifdef CONFIG_HUGETLB_PAGE
455 mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
456#endif
David S. Miller7bebd832006-03-27 01:07:55 -0800457 if (unlikely(mm_rss >
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800458 mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
459 tsb_grow(mm, MM_TSB_BASE, mm_rss);
460#ifdef CONFIG_HUGETLB_PAGE
461 mm_rss = mm->context.huge_pte_count;
David S. Miller7bebd832006-03-27 01:07:55 -0800462 if (unlikely(mm_rss >
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800463 mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit))
464 tsb_grow(mm, MM_TSB_HUGE, mm_rss);
465#endif
David S. Millerefdc1e22005-09-28 21:06:47 -0700466 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
468 /*
469 * Something tried to access memory that isn't in our memory map..
470 * Fix it, but check if it's kernel or user first..
471 */
472bad_area:
473 insn = get_fault_insn(regs, insn);
474 up_read(&mm->mmap_sem);
475
476handle_kernel_fault:
477 do_kernel_fault(regs, si_code, fault_code, insn, address);
David S. Millerefdc1e22005-09-28 21:06:47 -0700478 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479
480/*
481 * We ran out of memory, or some other thing happened to us that made
482 * us unable to handle the page fault gracefully.
483 */
484out_of_memory:
485 insn = get_fault_insn(regs, insn);
486 up_read(&mm->mmap_sem);
487 printk("VM: killing process %s\n", current->comm);
488 if (!(regs->tstate & TSTATE_PRIV))
489 do_exit(SIGKILL);
490 goto handle_kernel_fault;
491
492intr_or_no_mm:
493 insn = get_fault_insn(regs, 0);
494 goto handle_kernel_fault;
495
496do_sigbus:
497 insn = get_fault_insn(regs, insn);
498 up_read(&mm->mmap_sem);
499
500 /*
501 * Send a sigbus, regardless of whether we were in kernel
502 * or user mode.
503 */
504 do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code);
505
506 /* Kernel mode? Handle exceptions or die */
507 if (regs->tstate & TSTATE_PRIV)
508 goto handle_kernel_fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509}