blob: e3430e093d436d300bb1928ea4320ce230b1d632 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Paul Mundt3eeffb32007-11-19 18:57:03 +09002 * arch/sh/mm/tlb-flush_64.c
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
Paul Mundt163b2f02009-06-25 02:49:03 +09006 * Copyright (C) 2003 - 2009 Paul Mundt
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Paul Mundt3eeffb32007-11-19 18:57:03 +09008 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/signal.h>
13#include <linux/rwsem.h>
14#include <linux/sched.h>
15#include <linux/kernel.h>
16#include <linux/errno.h>
17#include <linux/string.h>
18#include <linux/types.h>
19#include <linux/ptrace.h>
20#include <linux/mman.h>
21#include <linux/mm.h>
22#include <linux/smp.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020023#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/interrupt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <asm/system.h>
26#include <asm/io.h>
27#include <asm/tlb.h>
28#include <asm/uaccess.h>
29#include <asm/pgalloc.h>
30#include <asm/mmu_context.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
32extern void die(const char *,struct pt_regs *,long);
33
34#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
35#define PPROT(flag) PFLAG(pgprot_val(prot),flag)
36
37static inline void print_prots(pgprot_t prot)
38{
Matt Fleming24ef7fc2009-11-19 21:11:05 +000039 printk("prot is 0x%016llx\n",pgprot_val(prot));
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
42 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
43}
44
45static inline void print_vma(struct vm_area_struct *vma)
46{
47 printk("vma start 0x%08lx\n", vma->vm_start);
48 printk("vma end 0x%08lx\n", vma->vm_end);
49
50 print_prots(vma->vm_page_prot);
51 printk("vm_flags 0x%08lx\n", vma->vm_flags);
52}
53
54static inline void print_task(struct task_struct *tsk)
55{
Alexey Dobriyan19c58702007-10-18 23:40:41 -070056 printk("Task pid %d\n", task_pid_nr(tsk));
Linus Torvalds1da177e2005-04-16 15:20:36 -070057}
58
59static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
60{
61 pgd_t *dir;
Paul Mundt3eeffb32007-11-19 18:57:03 +090062 pud_t *pud;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063 pmd_t *pmd;
64 pte_t *pte;
65 pte_t entry;
66
67 dir = pgd_offset(mm, address);
Paul Mundt3eeffb32007-11-19 18:57:03 +090068 if (pgd_none(*dir))
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Paul Mundt3eeffb32007-11-19 18:57:03 +090071 pud = pud_offset(dir, address);
72 if (pud_none(*pud))
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 return NULL;
Paul Mundt3eeffb32007-11-19 18:57:03 +090074
75 pmd = pmd_offset(pud, address);
76 if (pmd_none(*pmd))
77 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79 pte = pte_offset_kernel(pmd, address);
80 entry = *pte;
Paul Mundt3eeffb32007-11-19 18:57:03 +090081 if (pte_none(entry) || !pte_present(entry))
Linus Torvalds1da177e2005-04-16 15:20:36 -070082 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
84 return pte;
85}
86
87/*
88 * This routine handles page faults. It determines the address,
89 * and the problem, and then passes it off to one of the appropriate
90 * routines.
91 */
92asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
93 unsigned long textaccess, unsigned long address)
94{
95 struct task_struct *tsk;
96 struct mm_struct *mm;
97 struct vm_area_struct * vma;
98 const struct exception_table_entry *fixup;
99 pte_t *pte;
Nick Piggin83c54072007-07-19 01:47:05 -0700100 int fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 /* SIM
103 * Note this is now called with interrupts still disabled
104 * This is to cope with being called for a missing IO port
Simon Arlott0a354772007-05-14 08:25:48 +0900105 * address with interrupts disabled. This should be fixed as
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 * soon as we have a better 'fast path' miss handler.
107 *
108 * Plus take care how you try and debug this stuff.
109 * For example, writing debug data to a port which you
110 * have just faulted on is not going to work.
111 */
112
113 tsk = current;
114 mm = tsk->mm;
115
116 /* Not an IO address, so reenable interrupts */
117 local_irq_enable();
118
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200119 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
Paul Mundt163b2f02009-06-25 02:49:03 +0900120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 /*
122 * If we're in an interrupt or have no user
123 * context, we must not take the fault..
124 */
Peter Zijlstra6edaf682006-12-06 20:32:18 -0800125 if (in_atomic() || !mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 goto no_context;
127
128 /* TLB misses upon some cache flushes get done under cli() */
129 down_read(&mm->mmap_sem);
130
131 vma = find_vma(mm, address);
132
133 if (!vma) {
134#ifdef DEBUG_FAULT
135 print_task(tsk);
136 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
Harvey Harrison866e6b92008-03-04 15:23:47 -0800137 __func__, __LINE__,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 address,regs->pc,textaccess,writeaccess);
139 show_regs(regs);
140#endif
141 goto bad_area;
142 }
143 if (vma->vm_start <= address) {
144 goto good_area;
145 }
146
147 if (!(vma->vm_flags & VM_GROWSDOWN)) {
148#ifdef DEBUG_FAULT
149 print_task(tsk);
150 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
Harvey Harrison866e6b92008-03-04 15:23:47 -0800151 __func__, __LINE__,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 address,regs->pc,textaccess,writeaccess);
153 show_regs(regs);
154
155 print_vma(vma);
156#endif
157 goto bad_area;
158 }
159 if (expand_stack(vma, address)) {
160#ifdef DEBUG_FAULT
161 print_task(tsk);
162 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
Harvey Harrison866e6b92008-03-04 15:23:47 -0800163 __func__, __LINE__,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 address,regs->pc,textaccess,writeaccess);
165 show_regs(regs);
166#endif
167 goto bad_area;
168 }
169/*
170 * Ok, we have a good vm_area for this memory access, so
171 * we can handle it..
172 */
173good_area:
174 if (textaccess) {
175 if (!(vma->vm_flags & VM_EXEC))
176 goto bad_area;
177 } else {
178 if (writeaccess) {
179 if (!(vma->vm_flags & VM_WRITE))
180 goto bad_area;
181 } else {
182 if (!(vma->vm_flags & VM_READ))
183 goto bad_area;
184 }
185 }
186
187 /*
188 * If for any reason at all we couldn't handle the fault,
189 * make sure we exit gracefully rather than endlessly redo
190 * the fault.
191 */
Linus Torvaldsd06063c2009-04-10 09:01:23 -0700192 fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
Nick Piggin83c54072007-07-19 01:47:05 -0700193 if (unlikely(fault & VM_FAULT_ERROR)) {
194 if (fault & VM_FAULT_OOM)
195 goto out_of_memory;
196 else if (fault & VM_FAULT_SIGBUS)
197 goto do_sigbus;
198 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 }
Paul Mundt163b2f02009-06-25 02:49:03 +0900200
201 if (fault & VM_FAULT_MAJOR) {
Nick Piggin83c54072007-07-19 01:47:05 -0700202 tsk->maj_flt++;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200203 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
Paul Mundt163b2f02009-06-25 02:49:03 +0900204 regs, address);
205 } else {
Nick Piggin83c54072007-07-19 01:47:05 -0700206 tsk->min_flt++;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200207 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
Paul Mundt163b2f02009-06-25 02:49:03 +0900208 regs, address);
209 }
Nick Piggin83c54072007-07-19 01:47:05 -0700210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 /* If we get here, the page fault has been handled. Do the TLB refill
212 now from the newly-setup PTE, to avoid having to fault again right
213 away on the same instruction. */
214 pte = lookup_pte (mm, address);
215 if (!pte) {
216 /* From empirical evidence, we can get here, due to
217 !pte_present(pte). (e.g. if a swap-in occurs, and the page
218 is swapped back out again before the process that wanted it
219 gets rescheduled?) */
220 goto no_pte;
221 }
222
223 __do_tlb_refill(address, textaccess, pte);
224
225no_pte:
226
227 up_read(&mm->mmap_sem);
228 return;
229
230/*
231 * Something tried to access memory that isn't in our memory map..
232 * Fix it, but check if it's kernel or user first..
233 */
234bad_area:
235#ifdef DEBUG_FAULT
236 printk("fault:bad area\n");
237#endif
238 up_read(&mm->mmap_sem);
239
240 if (user_mode(regs)) {
241 static int count=0;
242 siginfo_t info;
243 if (count < 4) {
244 /* This is really to help debug faults when starting
245 * usermode, so only need a few */
246 count++;
247 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
Alexey Dobriyan19c58702007-10-18 23:40:41 -0700248 address, task_pid_nr(current), current->comm,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 (unsigned long) regs->pc);
250#if 0
251 show_regs(regs);
252#endif
253 }
Serge E. Hallynb460cbc2007-10-18 23:39:52 -0700254 if (is_global_init(tsk)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 panic("INIT had user mode bad_area\n");
256 }
257 tsk->thread.address = address;
258 tsk->thread.error_code = writeaccess;
259 info.si_signo = SIGSEGV;
260 info.si_errno = 0;
261 info.si_addr = (void *) address;
262 force_sig_info(SIGSEGV, &info, tsk);
263 return;
264 }
265
266no_context:
267#ifdef DEBUG_FAULT
268 printk("fault:No context\n");
269#endif
270 /* Are we prepared to handle this kernel fault? */
271 fixup = search_exception_tables(regs->pc);
272 if (fixup) {
273 regs->pc = fixup->fixup;
274 return;
275 }
276
277/*
278 * Oops. The kernel tried to access some bad page. We'll have to
279 * terminate things with extreme prejudice.
280 *
281 */
282 if (address < PAGE_SIZE)
283 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
284 else
285 printk(KERN_ALERT "Unable to handle kernel paging request");
286 printk(" at virtual address %08lx\n", address);
287 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
288 die("Oops", regs, writeaccess);
289 do_exit(SIGKILL);
290
291/*
292 * We ran out of memory, or some other thing happened to us that made
293 * us unable to handle the page fault gracefully.
294 */
295out_of_memory:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 up_read(&mm->mmap_sem);
Nick Piggin6b6b18e2010-04-22 16:06:26 +0000297 if (!user_mode(regs))
298 goto no_context;
299 pagefault_out_of_memory();
300 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301
302do_sigbus:
303 printk("fault:Do sigbus\n");
304 up_read(&mm->mmap_sem);
305
306 /*
307 * Send a sigbus, regardless of whether we were in kernel
308 * or user mode.
309 */
310 tsk->thread.address = address;
311 tsk->thread.error_code = writeaccess;
312 tsk->thread.trap_no = 14;
313 force_sig(SIGBUS, tsk);
314
315 /* Kernel mode? Handle exceptions or die */
316 if (!user_mode(regs))
317 goto no_context;
318}
319
Paul Mundt3eeffb32007-11-19 18:57:03 +0900320void local_flush_tlb_one(unsigned long asid, unsigned long page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
322 unsigned long long match, pteh=0, lpage;
323 unsigned long tlb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325 /*
326 * Sign-extend based on neff.
327 */
Paul Mundtc7914832009-08-04 17:14:39 +0900328 lpage = neff_sign_extend(page);
Paul Mundt3eeffb32007-11-19 18:57:03 +0900329 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330 match |= lpage;
331
Paul Mundt3eeffb32007-11-19 18:57:03 +0900332 for_each_itlb_entry(tlb) {
333 asm volatile ("getcfg %1, 0, %0"
334 : "=r" (pteh)
335 : "r" (tlb) );
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336
Paul Mundt3eeffb32007-11-19 18:57:03 +0900337 if (pteh == match) {
338 __flush_tlb_slot(tlb);
339 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 }
341 }
342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 for_each_dtlb_entry(tlb) {
344 asm volatile ("getcfg %1, 0, %0"
345 : "=r" (pteh)
346 : "r" (tlb) );
347
348 if (pteh == match) {
349 __flush_tlb_slot(tlb);
350 break;
351 }
352
353 }
354}
355
Paul Mundt3eeffb32007-11-19 18:57:03 +0900356void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357{
358 unsigned long flags;
359
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 if (vma->vm_mm) {
361 page &= PAGE_MASK;
362 local_irq_save(flags);
Paul Mundt3eeffb32007-11-19 18:57:03 +0900363 local_flush_tlb_one(get_asid(), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 local_irq_restore(flags);
365 }
366}
367
Paul Mundt3eeffb32007-11-19 18:57:03 +0900368void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
369 unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370{
371 unsigned long flags;
372 unsigned long long match, pteh=0, pteh_epn, pteh_low;
373 unsigned long tlb;
Paul Mundt3eeffb32007-11-19 18:57:03 +0900374 unsigned int cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 struct mm_struct *mm;
376
377 mm = vma->vm_mm;
Paul Mundt3eeffb32007-11-19 18:57:03 +0900378 if (cpu_context(cpu, mm) == NO_CONTEXT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 return;
380
381 local_irq_save(flags);
382
383 start &= PAGE_MASK;
384 end &= PAGE_MASK;
385
Paul Mundt3eeffb32007-11-19 18:57:03 +0900386 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
388 /* Flush ITLB */
389 for_each_itlb_entry(tlb) {
390 asm volatile ("getcfg %1, 0, %0"
391 : "=r" (pteh)
392 : "r" (tlb) );
393
394 pteh_epn = pteh & PAGE_MASK;
395 pteh_low = pteh & ~PAGE_MASK;
396
397 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
398 __flush_tlb_slot(tlb);
399 }
400
401 /* Flush DTLB */
402 for_each_dtlb_entry(tlb) {
403 asm volatile ("getcfg %1, 0, %0"
404 : "=r" (pteh)
405 : "r" (tlb) );
406
407 pteh_epn = pteh & PAGE_MASK;
408 pteh_low = pteh & ~PAGE_MASK;
409
410 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
411 __flush_tlb_slot(tlb);
412 }
413
414 local_irq_restore(flags);
415}
416
Paul Mundt3eeffb32007-11-19 18:57:03 +0900417void local_flush_tlb_mm(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418{
419 unsigned long flags;
Paul Mundt3eeffb32007-11-19 18:57:03 +0900420 unsigned int cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421
Paul Mundt3eeffb32007-11-19 18:57:03 +0900422 if (cpu_context(cpu, mm) == NO_CONTEXT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 return;
424
425 local_irq_save(flags);
426
Paul Mundt3eeffb32007-11-19 18:57:03 +0900427 cpu_context(cpu, mm) = NO_CONTEXT;
428 if (mm == current->mm)
429 activate_context(mm, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430
431 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432}
433
Paul Mundt3eeffb32007-11-19 18:57:03 +0900434void local_flush_tlb_all(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435{
436 /* Invalidate all, including shared pages, excluding fixed TLBs */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 unsigned long flags, tlb;
438
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 local_irq_save(flags);
440
441 /* Flush each ITLB entry */
Paul Mundt3eeffb32007-11-19 18:57:03 +0900442 for_each_itlb_entry(tlb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 __flush_tlb_slot(tlb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444
445 /* Flush each DTLB entry */
Paul Mundt3eeffb32007-11-19 18:57:03 +0900446 for_each_dtlb_entry(tlb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 __flush_tlb_slot(tlb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
449 local_irq_restore(flags);
450}
451
Paul Mundt3eeffb32007-11-19 18:57:03 +0900452void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453{
454 /* FIXME: Optimize this later.. */
455 flush_tlb_all();
456}
Paul Mundt9cef7492009-07-29 00:12:17 +0900457
Paul Mundt59615ec2010-07-02 15:44:09 +0900458void __flush_tlb_global(void)
459{
460 flush_tlb_all();
461}
462
Paul Mundt9cef7492009-07-29 00:12:17 +0900463void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
464{
465}