Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 2 | * arch/sh/mm/tlb-flush_64.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2000, 2001 Paolo Alberelli |
| 5 | * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 6 | * Copyright (C) 2003 - 2009 Paul Mundt |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 8 | * This file is subject to the terms and conditions of the GNU General Public |
| 9 | * License. See the file "COPYING" in the main directory of this archive |
| 10 | * for more details. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/signal.h> |
| 13 | #include <linux/rwsem.h> |
| 14 | #include <linux/sched.h> |
| 15 | #include <linux/kernel.h> |
| 16 | #include <linux/errno.h> |
| 17 | #include <linux/string.h> |
| 18 | #include <linux/types.h> |
| 19 | #include <linux/ptrace.h> |
| 20 | #include <linux/mman.h> |
| 21 | #include <linux/mm.h> |
| 22 | #include <linux/smp.h> |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 23 | #include <linux/perf_event.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | #include <linux/interrupt.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | #include <asm/system.h> |
| 26 | #include <asm/io.h> |
| 27 | #include <asm/tlb.h> |
| 28 | #include <asm/uaccess.h> |
| 29 | #include <asm/pgalloc.h> |
| 30 | #include <asm/mmu_context.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | |
| 32 | extern void die(const char *,struct pt_regs *,long); |
| 33 | |
| 34 | #define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) |
| 35 | #define PPROT(flag) PFLAG(pgprot_val(prot),flag) |
| 36 | |
| 37 | static inline void print_prots(pgprot_t prot) |
| 38 | { |
Matt Fleming | 24ef7fc | 2009-11-19 21:11:05 +0000 | [diff] [blame] | 39 | printk("prot is 0x%016llx\n",pgprot_val(prot)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | |
| 41 | printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), |
| 42 | PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); |
| 43 | } |
| 44 | |
| 45 | static inline void print_vma(struct vm_area_struct *vma) |
| 46 | { |
| 47 | printk("vma start 0x%08lx\n", vma->vm_start); |
| 48 | printk("vma end 0x%08lx\n", vma->vm_end); |
| 49 | |
| 50 | print_prots(vma->vm_page_prot); |
| 51 | printk("vm_flags 0x%08lx\n", vma->vm_flags); |
| 52 | } |
| 53 | |
| 54 | static inline void print_task(struct task_struct *tsk) |
| 55 | { |
Alexey Dobriyan | 19c5870 | 2007-10-18 23:40:41 -0700 | [diff] [blame] | 56 | printk("Task pid %d\n", task_pid_nr(tsk)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | } |
| 58 | |
| 59 | static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) |
| 60 | { |
| 61 | pgd_t *dir; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 62 | pud_t *pud; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | pmd_t *pmd; |
| 64 | pte_t *pte; |
| 65 | pte_t entry; |
| 66 | |
| 67 | dir = pgd_offset(mm, address); |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 68 | if (pgd_none(*dir)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 71 | pud = pud_offset(dir, address); |
| 72 | if (pud_none(*pud)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | return NULL; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 74 | |
| 75 | pmd = pmd_offset(pud, address); |
| 76 | if (pmd_none(*pmd)) |
| 77 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
| 79 | pte = pte_offset_kernel(pmd, address); |
| 80 | entry = *pte; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 81 | if (pte_none(entry) || !pte_present(entry)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | |
| 84 | return pte; |
| 85 | } |
| 86 | |
| 87 | /* |
| 88 | * This routine handles page faults. It determines the address, |
| 89 | * and the problem, and then passes it off to one of the appropriate |
| 90 | * routines. |
| 91 | */ |
| 92 | asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, |
| 93 | unsigned long textaccess, unsigned long address) |
| 94 | { |
| 95 | struct task_struct *tsk; |
| 96 | struct mm_struct *mm; |
| 97 | struct vm_area_struct * vma; |
| 98 | const struct exception_table_entry *fixup; |
| 99 | pte_t *pte; |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 100 | int fault; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | /* SIM |
| 103 | * Note this is now called with interrupts still disabled |
| 104 | * This is to cope with being called for a missing IO port |
Simon Arlott | 0a35477 | 2007-05-14 08:25:48 +0900 | [diff] [blame] | 105 | * address with interrupts disabled. This should be fixed as |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | * soon as we have a better 'fast path' miss handler. |
| 107 | * |
| 108 | * Plus take care how you try and debug this stuff. |
| 109 | * For example, writing debug data to a port which you |
| 110 | * have just faulted on is not going to work. |
| 111 | */ |
| 112 | |
| 113 | tsk = current; |
| 114 | mm = tsk->mm; |
| 115 | |
| 116 | /* Not an IO address, so reenable interrupts */ |
| 117 | local_irq_enable(); |
| 118 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 119 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 120 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | /* |
| 122 | * If we're in an interrupt or have no user |
| 123 | * context, we must not take the fault.. |
| 124 | */ |
Peter Zijlstra | 6edaf68 | 2006-12-06 20:32:18 -0800 | [diff] [blame] | 125 | if (in_atomic() || !mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | goto no_context; |
| 127 | |
| 128 | /* TLB misses upon some cache flushes get done under cli() */ |
| 129 | down_read(&mm->mmap_sem); |
| 130 | |
| 131 | vma = find_vma(mm, address); |
| 132 | |
| 133 | if (!vma) { |
| 134 | #ifdef DEBUG_FAULT |
| 135 | print_task(tsk); |
| 136 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 137 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | address,regs->pc,textaccess,writeaccess); |
| 139 | show_regs(regs); |
| 140 | #endif |
| 141 | goto bad_area; |
| 142 | } |
| 143 | if (vma->vm_start <= address) { |
| 144 | goto good_area; |
| 145 | } |
| 146 | |
| 147 | if (!(vma->vm_flags & VM_GROWSDOWN)) { |
| 148 | #ifdef DEBUG_FAULT |
| 149 | print_task(tsk); |
| 150 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 151 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | address,regs->pc,textaccess,writeaccess); |
| 153 | show_regs(regs); |
| 154 | |
| 155 | print_vma(vma); |
| 156 | #endif |
| 157 | goto bad_area; |
| 158 | } |
| 159 | if (expand_stack(vma, address)) { |
| 160 | #ifdef DEBUG_FAULT |
| 161 | print_task(tsk); |
| 162 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 163 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 164 | address,regs->pc,textaccess,writeaccess); |
| 165 | show_regs(regs); |
| 166 | #endif |
| 167 | goto bad_area; |
| 168 | } |
| 169 | /* |
| 170 | * Ok, we have a good vm_area for this memory access, so |
| 171 | * we can handle it.. |
| 172 | */ |
| 173 | good_area: |
| 174 | if (textaccess) { |
| 175 | if (!(vma->vm_flags & VM_EXEC)) |
| 176 | goto bad_area; |
| 177 | } else { |
| 178 | if (writeaccess) { |
| 179 | if (!(vma->vm_flags & VM_WRITE)) |
| 180 | goto bad_area; |
| 181 | } else { |
| 182 | if (!(vma->vm_flags & VM_READ)) |
| 183 | goto bad_area; |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | /* |
| 188 | * If for any reason at all we couldn't handle the fault, |
| 189 | * make sure we exit gracefully rather than endlessly redo |
| 190 | * the fault. |
| 191 | */ |
Linus Torvalds | d06063c | 2009-04-10 09:01:23 -0700 | [diff] [blame] | 192 | fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0); |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 193 | if (unlikely(fault & VM_FAULT_ERROR)) { |
| 194 | if (fault & VM_FAULT_OOM) |
| 195 | goto out_of_memory; |
| 196 | else if (fault & VM_FAULT_SIGBUS) |
| 197 | goto do_sigbus; |
| 198 | BUG(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | } |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 200 | |
| 201 | if (fault & VM_FAULT_MAJOR) { |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 202 | tsk->maj_flt++; |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 203 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 204 | regs, address); |
| 205 | } else { |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 206 | tsk->min_flt++; |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 207 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 208 | regs, address); |
| 209 | } |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 210 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | /* If we get here, the page fault has been handled. Do the TLB refill |
| 212 | now from the newly-setup PTE, to avoid having to fault again right |
| 213 | away on the same instruction. */ |
| 214 | pte = lookup_pte (mm, address); |
| 215 | if (!pte) { |
| 216 | /* From empirical evidence, we can get here, due to |
| 217 | !pte_present(pte). (e.g. if a swap-in occurs, and the page |
| 218 | is swapped back out again before the process that wanted it |
| 219 | gets rescheduled?) */ |
| 220 | goto no_pte; |
| 221 | } |
| 222 | |
| 223 | __do_tlb_refill(address, textaccess, pte); |
| 224 | |
| 225 | no_pte: |
| 226 | |
| 227 | up_read(&mm->mmap_sem); |
| 228 | return; |
| 229 | |
| 230 | /* |
| 231 | * Something tried to access memory that isn't in our memory map.. |
| 232 | * Fix it, but check if it's kernel or user first.. |
| 233 | */ |
| 234 | bad_area: |
| 235 | #ifdef DEBUG_FAULT |
| 236 | printk("fault:bad area\n"); |
| 237 | #endif |
| 238 | up_read(&mm->mmap_sem); |
| 239 | |
| 240 | if (user_mode(regs)) { |
| 241 | static int count=0; |
| 242 | siginfo_t info; |
| 243 | if (count < 4) { |
| 244 | /* This is really to help debug faults when starting |
| 245 | * usermode, so only need a few */ |
| 246 | count++; |
| 247 | printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", |
Alexey Dobriyan | 19c5870 | 2007-10-18 23:40:41 -0700 | [diff] [blame] | 248 | address, task_pid_nr(current), current->comm, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 249 | (unsigned long) regs->pc); |
| 250 | #if 0 |
| 251 | show_regs(regs); |
| 252 | #endif |
| 253 | } |
Serge E. Hallyn | b460cbc | 2007-10-18 23:39:52 -0700 | [diff] [blame] | 254 | if (is_global_init(tsk)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | panic("INIT had user mode bad_area\n"); |
| 256 | } |
| 257 | tsk->thread.address = address; |
| 258 | tsk->thread.error_code = writeaccess; |
| 259 | info.si_signo = SIGSEGV; |
| 260 | info.si_errno = 0; |
| 261 | info.si_addr = (void *) address; |
| 262 | force_sig_info(SIGSEGV, &info, tsk); |
| 263 | return; |
| 264 | } |
| 265 | |
| 266 | no_context: |
| 267 | #ifdef DEBUG_FAULT |
| 268 | printk("fault:No context\n"); |
| 269 | #endif |
| 270 | /* Are we prepared to handle this kernel fault? */ |
| 271 | fixup = search_exception_tables(regs->pc); |
| 272 | if (fixup) { |
| 273 | regs->pc = fixup->fixup; |
| 274 | return; |
| 275 | } |
| 276 | |
| 277 | /* |
| 278 | * Oops. The kernel tried to access some bad page. We'll have to |
| 279 | * terminate things with extreme prejudice. |
| 280 | * |
| 281 | */ |
| 282 | if (address < PAGE_SIZE) |
| 283 | printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); |
| 284 | else |
| 285 | printk(KERN_ALERT "Unable to handle kernel paging request"); |
| 286 | printk(" at virtual address %08lx\n", address); |
| 287 | printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); |
| 288 | die("Oops", regs, writeaccess); |
| 289 | do_exit(SIGKILL); |
| 290 | |
| 291 | /* |
| 292 | * We ran out of memory, or some other thing happened to us that made |
| 293 | * us unable to handle the page fault gracefully. |
| 294 | */ |
| 295 | out_of_memory: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | up_read(&mm->mmap_sem); |
Nick Piggin | 6b6b18e | 2010-04-22 16:06:26 +0000 | [diff] [blame] | 297 | if (!user_mode(regs)) |
| 298 | goto no_context; |
| 299 | pagefault_out_of_memory(); |
| 300 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 301 | |
| 302 | do_sigbus: |
| 303 | printk("fault:Do sigbus\n"); |
| 304 | up_read(&mm->mmap_sem); |
| 305 | |
| 306 | /* |
| 307 | * Send a sigbus, regardless of whether we were in kernel |
| 308 | * or user mode. |
| 309 | */ |
| 310 | tsk->thread.address = address; |
| 311 | tsk->thread.error_code = writeaccess; |
| 312 | tsk->thread.trap_no = 14; |
| 313 | force_sig(SIGBUS, tsk); |
| 314 | |
| 315 | /* Kernel mode? Handle exceptions or die */ |
| 316 | if (!user_mode(regs)) |
| 317 | goto no_context; |
| 318 | } |
| 319 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 320 | void local_flush_tlb_one(unsigned long asid, unsigned long page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 321 | { |
| 322 | unsigned long long match, pteh=0, lpage; |
| 323 | unsigned long tlb; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 324 | |
| 325 | /* |
| 326 | * Sign-extend based on neff. |
| 327 | */ |
Paul Mundt | c791483 | 2009-08-04 17:14:39 +0900 | [diff] [blame] | 328 | lpage = neff_sign_extend(page); |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 329 | match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | match |= lpage; |
| 331 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 332 | for_each_itlb_entry(tlb) { |
| 333 | asm volatile ("getcfg %1, 0, %0" |
| 334 | : "=r" (pteh) |
| 335 | : "r" (tlb) ); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 337 | if (pteh == match) { |
| 338 | __flush_tlb_slot(tlb); |
| 339 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | } |
| 341 | } |
| 342 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 343 | for_each_dtlb_entry(tlb) { |
| 344 | asm volatile ("getcfg %1, 0, %0" |
| 345 | : "=r" (pteh) |
| 346 | : "r" (tlb) ); |
| 347 | |
| 348 | if (pteh == match) { |
| 349 | __flush_tlb_slot(tlb); |
| 350 | break; |
| 351 | } |
| 352 | |
| 353 | } |
| 354 | } |
| 355 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 356 | void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 357 | { |
| 358 | unsigned long flags; |
| 359 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | if (vma->vm_mm) { |
| 361 | page &= PAGE_MASK; |
| 362 | local_irq_save(flags); |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 363 | local_flush_tlb_one(get_asid(), page); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | local_irq_restore(flags); |
| 365 | } |
| 366 | } |
| 367 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 368 | void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 369 | unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | { |
| 371 | unsigned long flags; |
| 372 | unsigned long long match, pteh=0, pteh_epn, pteh_low; |
| 373 | unsigned long tlb; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 374 | unsigned int cpu = smp_processor_id(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 375 | struct mm_struct *mm; |
| 376 | |
| 377 | mm = vma->vm_mm; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 378 | if (cpu_context(cpu, mm) == NO_CONTEXT) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 379 | return; |
| 380 | |
| 381 | local_irq_save(flags); |
| 382 | |
| 383 | start &= PAGE_MASK; |
| 384 | end &= PAGE_MASK; |
| 385 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 386 | match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 387 | |
| 388 | /* Flush ITLB */ |
| 389 | for_each_itlb_entry(tlb) { |
| 390 | asm volatile ("getcfg %1, 0, %0" |
| 391 | : "=r" (pteh) |
| 392 | : "r" (tlb) ); |
| 393 | |
| 394 | pteh_epn = pteh & PAGE_MASK; |
| 395 | pteh_low = pteh & ~PAGE_MASK; |
| 396 | |
| 397 | if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) |
| 398 | __flush_tlb_slot(tlb); |
| 399 | } |
| 400 | |
| 401 | /* Flush DTLB */ |
| 402 | for_each_dtlb_entry(tlb) { |
| 403 | asm volatile ("getcfg %1, 0, %0" |
| 404 | : "=r" (pteh) |
| 405 | : "r" (tlb) ); |
| 406 | |
| 407 | pteh_epn = pteh & PAGE_MASK; |
| 408 | pteh_low = pteh & ~PAGE_MASK; |
| 409 | |
| 410 | if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) |
| 411 | __flush_tlb_slot(tlb); |
| 412 | } |
| 413 | |
| 414 | local_irq_restore(flags); |
| 415 | } |
| 416 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 417 | void local_flush_tlb_mm(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | { |
| 419 | unsigned long flags; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 420 | unsigned int cpu = smp_processor_id(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 421 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 422 | if (cpu_context(cpu, mm) == NO_CONTEXT) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 423 | return; |
| 424 | |
| 425 | local_irq_save(flags); |
| 426 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 427 | cpu_context(cpu, mm) = NO_CONTEXT; |
| 428 | if (mm == current->mm) |
| 429 | activate_context(mm, cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 430 | |
| 431 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 432 | } |
| 433 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 434 | void local_flush_tlb_all(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 435 | { |
| 436 | /* Invalidate all, including shared pages, excluding fixed TLBs */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | unsigned long flags, tlb; |
| 438 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 439 | local_irq_save(flags); |
| 440 | |
| 441 | /* Flush each ITLB entry */ |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 442 | for_each_itlb_entry(tlb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 443 | __flush_tlb_slot(tlb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | |
| 445 | /* Flush each DTLB entry */ |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 446 | for_each_dtlb_entry(tlb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | __flush_tlb_slot(tlb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 448 | |
| 449 | local_irq_restore(flags); |
| 450 | } |
| 451 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 452 | void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 453 | { |
| 454 | /* FIXME: Optimize this later.. */ |
| 455 | flush_tlb_all(); |
| 456 | } |
Paul Mundt | 9cef749 | 2009-07-29 00:12:17 +0900 | [diff] [blame] | 457 | |
Paul Mundt | 59615ec | 2010-07-02 15:44:09 +0900 | [diff] [blame] | 458 | void __flush_tlb_global(void) |
| 459 | { |
| 460 | flush_tlb_all(); |
| 461 | } |
| 462 | |
Paul Mundt | 9cef749 | 2009-07-29 00:12:17 +0900 | [diff] [blame] | 463 | void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte) |
| 464 | { |
| 465 | } |