blob: fa636fc6b7b90403a2a8771ef3c5f023a1a82cd4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
8#include <linux/signal.h>
9#include <linux/sched.h>
10#include <linux/interrupt.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/vt_kern.h> /* For unblank_screen() */
20#include <linux/module.h>
21
22#include <asm/branch.h>
23#include <asm/mmu_context.h>
24#include <asm/system.h>
25#include <asm/uaccess.h>
26#include <asm/ptrace.h>
Thiemo Seufer16033d62005-02-19 13:56:04 +000027#include <asm/highmem.h> /* For VMALLOC_END */
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
29/*
30 * This routine handles page faults. It determines the address,
31 * and the problem, and then passes it off to one of the appropriate
32 * routines.
33 */
34asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
35 unsigned long address)
36{
37 struct vm_area_struct * vma = NULL;
38 struct task_struct *tsk = current;
39 struct mm_struct *mm = tsk->mm;
40 const int field = sizeof(unsigned long) * 2;
41 siginfo_t info;
Nick Piggin83c54072007-07-19 01:47:05 -070042 int fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
44#if 0
Ralf Baechled6f70362007-03-29 22:30:01 +010045 printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", raw_smp_processor_id(),
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 current->comm, current->pid, field, address, write,
47 field, regs->cp0_epc);
48#endif
49
50 info.si_code = SEGV_MAPERR;
51
52 /*
53 * We fault-in kernel-space virtual memory on-demand. The
54 * 'reference' page table is init_mm.pgd.
55 *
56 * NOTE! We MUST NOT take any locks for this case. We may
57 * be in an interrupt or a critical region, and should
58 * only copy the information from the master page table,
59 * nothing more.
60 */
Thiemo Seufer16033d62005-02-19 13:56:04 +000061 if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 goto vmalloc_fault;
Atsushi Nemoto656be922006-10-26 00:08:31 +090063#ifdef MODULE_START
64 if (unlikely(address >= MODULE_START && address < MODULE_END))
65 goto vmalloc_fault;
66#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070067
68 /*
69 * If we're in an interrupt or have no user
70 * context, we must not take the fault..
71 */
72 if (in_atomic() || !mm)
73 goto bad_area_nosemaphore;
74
75 down_read(&mm->mmap_sem);
76 vma = find_vma(mm, address);
77 if (!vma)
78 goto bad_area;
79 if (vma->vm_start <= address)
80 goto good_area;
81 if (!(vma->vm_flags & VM_GROWSDOWN))
82 goto bad_area;
83 if (expand_stack(vma, address))
84 goto bad_area;
85/*
86 * Ok, we have a good vm_area for this memory access, so
87 * we can handle it..
88 */
89good_area:
90 info.si_code = SEGV_ACCERR;
91
92 if (write) {
93 if (!(vma->vm_flags & VM_WRITE))
94 goto bad_area;
95 } else {
Ralf Baechle00932ba2006-09-16 01:29:37 +010096 if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 goto bad_area;
98 }
99
100survive:
101 /*
102 * If for any reason at all we couldn't handle the fault,
103 * make sure we exit gracefully rather than endlessly redo
104 * the fault.
105 */
Nick Piggin83c54072007-07-19 01:47:05 -0700106 fault = handle_mm_fault(mm, vma, address, write);
107 if (unlikely(fault & VM_FAULT_ERROR)) {
108 if (fault & VM_FAULT_OOM)
109 goto out_of_memory;
110 else if (fault & VM_FAULT_SIGBUS)
111 goto do_sigbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 BUG();
113 }
Nick Piggin83c54072007-07-19 01:47:05 -0700114 if (fault & VM_FAULT_MAJOR)
115 tsk->maj_flt++;
116 else
117 tsk->min_flt++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
119 up_read(&mm->mmap_sem);
120 return;
121
122/*
123 * Something tried to access memory that isn't in our memory map..
124 * Fix it, but check if it's kernel or user first..
125 */
126bad_area:
127 up_read(&mm->mmap_sem);
128
129bad_area_nosemaphore:
130 /* User mode accesses just cause a SIGSEGV */
131 if (user_mode(regs)) {
132 tsk->thread.cp0_badvaddr = address;
133 tsk->thread.error_code = write;
134#if 0
135 printk("do_page_fault() #2: sending SIGSEGV to %s for "
136 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
137 tsk->comm,
138 write ? "write access to" : "read access from",
139 field, address,
140 field, (unsigned long) regs->cp0_epc,
141 field, (unsigned long) regs->regs[31]);
142#endif
143 info.si_signo = SIGSEGV;
144 info.si_errno = 0;
145 /* info.si_code has been set above */
Ralf Baechlefe00f942005-03-01 19:22:29 +0000146 info.si_addr = (void __user *) address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 force_sig_info(SIGSEGV, &info, tsk);
148 return;
149 }
150
151no_context:
152 /* Are we prepared to handle this kernel fault? */
153 if (fixup_exception(regs)) {
154 current->thread.cp0_baduaddr = address;
155 return;
156 }
157
158 /*
159 * Oops. The kernel tried to access some bad page. We'll have to
160 * terminate things with extreme prejudice.
161 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 bust_spinlocks(1);
163
164 printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at "
165 "virtual address %0*lx, epc == %0*lx, ra == %0*lx\n",
Ralf Baechled6f70362007-03-29 22:30:01 +0100166 raw_smp_processor_id(), field, address, field, regs->cp0_epc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 field, regs->regs[31]);
168 die("Oops", regs);
169
170/*
171 * We ran out of memory, or some other thing happened to us that made
172 * us unable to handle the page fault gracefully.
173 */
174out_of_memory:
175 up_read(&mm->mmap_sem);
Serge E. Hallynb460cbc2007-10-18 23:39:52 -0700176 if (is_global_init(tsk)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 yield();
178 down_read(&mm->mmap_sem);
179 goto survive;
180 }
181 printk("VM: killing process %s\n", tsk->comm);
182 if (user_mode(regs))
Will Schmidtdcca2bd2007-10-16 01:24:18 -0700183 do_group_exit(SIGKILL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 goto no_context;
185
186do_sigbus:
187 up_read(&mm->mmap_sem);
188
189 /* Kernel mode? Handle exceptions or die */
190 if (!user_mode(regs))
191 goto no_context;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100192 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 /*
194 * Send a sigbus, regardless of whether we were in kernel
195 * or user mode.
196 */
Ralf Baechle41c594a2006-04-05 09:45:45 +0100197#if 0
198 printk("do_page_fault() #3: sending SIGBUS to %s for "
199 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
200 tsk->comm,
201 write ? "write access to" : "read access from",
202 field, address,
203 field, (unsigned long) regs->cp0_epc,
204 field, (unsigned long) regs->regs[31]);
205#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 tsk->thread.cp0_badvaddr = address;
207 info.si_signo = SIGBUS;
208 info.si_errno = 0;
209 info.si_code = BUS_ADRERR;
Ralf Baechlefe00f942005-03-01 19:22:29 +0000210 info.si_addr = (void __user *) address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 force_sig_info(SIGBUS, &info, tsk);
212
213 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214vmalloc_fault:
215 {
216 /*
217 * Synchronize this task's top level page-table
218 * with the 'reference' page table.
219 *
220 * Do _not_ use "tsk" here. We might be inside
221 * an interrupt in the middle of a task switch..
222 */
223 int offset = __pgd_offset(address);
224 pgd_t *pgd, *pgd_k;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000225 pud_t *pud, *pud_k;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 pmd_t *pmd, *pmd_k;
227 pte_t *pte_k;
228
Ralf Baechled6f70362007-03-29 22:30:01 +0100229 pgd = (pgd_t *) pgd_current[raw_smp_processor_id()] + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 pgd_k = init_mm.pgd + offset;
231
232 if (!pgd_present(*pgd_k))
233 goto no_context;
234 set_pgd(pgd, *pgd_k);
235
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000236 pud = pud_offset(pgd, address);
237 pud_k = pud_offset(pgd_k, address);
238 if (!pud_present(*pud_k))
239 goto no_context;
240
241 pmd = pmd_offset(pud, address);
242 pmd_k = pmd_offset(pud_k, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 if (!pmd_present(*pmd_k))
244 goto no_context;
245 set_pmd(pmd, *pmd_k);
246
247 pte_k = pte_offset_kernel(pmd_k, address);
248 if (!pte_present(*pte_k))
249 goto no_context;
250 return;
251 }
252}