Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 2 | * arch/sh/mm/tlb-flush_64.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2000, 2001 Paolo Alberelli |
| 5 | * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 6 | * Copyright (C) 2003 - 2009 Paul Mundt |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 8 | * This file is subject to the terms and conditions of the GNU General Public |
| 9 | * License. See the file "COPYING" in the main directory of this archive |
| 10 | * for more details. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/signal.h> |
| 13 | #include <linux/rwsem.h> |
| 14 | #include <linux/sched.h> |
| 15 | #include <linux/kernel.h> |
| 16 | #include <linux/errno.h> |
| 17 | #include <linux/string.h> |
| 18 | #include <linux/types.h> |
| 19 | #include <linux/ptrace.h> |
| 20 | #include <linux/mman.h> |
| 21 | #include <linux/mm.h> |
| 22 | #include <linux/smp.h> |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 23 | #include <linux/perf_counter.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | #include <linux/interrupt.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | #include <asm/system.h> |
| 26 | #include <asm/io.h> |
| 27 | #include <asm/tlb.h> |
| 28 | #include <asm/uaccess.h> |
| 29 | #include <asm/pgalloc.h> |
| 30 | #include <asm/mmu_context.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | |
| 32 | extern void die(const char *,struct pt_regs *,long); |
| 33 | |
| 34 | #define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) |
| 35 | #define PPROT(flag) PFLAG(pgprot_val(prot),flag) |
| 36 | |
| 37 | static inline void print_prots(pgprot_t prot) |
| 38 | { |
| 39 | printk("prot is 0x%08lx\n",pgprot_val(prot)); |
| 40 | |
| 41 | printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), |
| 42 | PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); |
| 43 | } |
| 44 | |
| 45 | static inline void print_vma(struct vm_area_struct *vma) |
| 46 | { |
| 47 | printk("vma start 0x%08lx\n", vma->vm_start); |
| 48 | printk("vma end 0x%08lx\n", vma->vm_end); |
| 49 | |
| 50 | print_prots(vma->vm_page_prot); |
| 51 | printk("vm_flags 0x%08lx\n", vma->vm_flags); |
| 52 | } |
| 53 | |
| 54 | static inline void print_task(struct task_struct *tsk) |
| 55 | { |
Alexey Dobriyan | 19c5870 | 2007-10-18 23:40:41 -0700 | [diff] [blame] | 56 | printk("Task pid %d\n", task_pid_nr(tsk)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | } |
| 58 | |
| 59 | static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) |
| 60 | { |
| 61 | pgd_t *dir; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 62 | pud_t *pud; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | pmd_t *pmd; |
| 64 | pte_t *pte; |
| 65 | pte_t entry; |
| 66 | |
| 67 | dir = pgd_offset(mm, address); |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 68 | if (pgd_none(*dir)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 71 | pud = pud_offset(dir, address); |
| 72 | if (pud_none(*pud)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | return NULL; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 74 | |
| 75 | pmd = pmd_offset(pud, address); |
| 76 | if (pmd_none(*pmd)) |
| 77 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
| 79 | pte = pte_offset_kernel(pmd, address); |
| 80 | entry = *pte; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 81 | if (pte_none(entry) || !pte_present(entry)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | |
| 84 | return pte; |
| 85 | } |
| 86 | |
| 87 | /* |
| 88 | * This routine handles page faults. It determines the address, |
| 89 | * and the problem, and then passes it off to one of the appropriate |
| 90 | * routines. |
| 91 | */ |
| 92 | asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, |
| 93 | unsigned long textaccess, unsigned long address) |
| 94 | { |
| 95 | struct task_struct *tsk; |
| 96 | struct mm_struct *mm; |
| 97 | struct vm_area_struct * vma; |
| 98 | const struct exception_table_entry *fixup; |
| 99 | pte_t *pte; |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 100 | int fault; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | /* SIM |
| 103 | * Note this is now called with interrupts still disabled |
| 104 | * This is to cope with being called for a missing IO port |
Simon Arlott | 0a35477 | 2007-05-14 08:25:48 +0900 | [diff] [blame] | 105 | * address with interrupts disabled. This should be fixed as |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | * soon as we have a better 'fast path' miss handler. |
| 107 | * |
| 108 | * Plus take care how you try and debug this stuff. |
| 109 | * For example, writing debug data to a port which you |
| 110 | * have just faulted on is not going to work. |
| 111 | */ |
| 112 | |
| 113 | tsk = current; |
| 114 | mm = tsk->mm; |
| 115 | |
| 116 | /* Not an IO address, so reenable interrupts */ |
| 117 | local_irq_enable(); |
| 118 | |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 119 | perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); |
| 120 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | /* |
| 122 | * If we're in an interrupt or have no user |
| 123 | * context, we must not take the fault.. |
| 124 | */ |
Peter Zijlstra | 6edaf68 | 2006-12-06 20:32:18 -0800 | [diff] [blame] | 125 | if (in_atomic() || !mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | goto no_context; |
| 127 | |
| 128 | /* TLB misses upon some cache flushes get done under cli() */ |
| 129 | down_read(&mm->mmap_sem); |
| 130 | |
| 131 | vma = find_vma(mm, address); |
| 132 | |
| 133 | if (!vma) { |
| 134 | #ifdef DEBUG_FAULT |
| 135 | print_task(tsk); |
| 136 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 137 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | address,regs->pc,textaccess,writeaccess); |
| 139 | show_regs(regs); |
| 140 | #endif |
| 141 | goto bad_area; |
| 142 | } |
| 143 | if (vma->vm_start <= address) { |
| 144 | goto good_area; |
| 145 | } |
| 146 | |
| 147 | if (!(vma->vm_flags & VM_GROWSDOWN)) { |
| 148 | #ifdef DEBUG_FAULT |
| 149 | print_task(tsk); |
| 150 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 151 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | address,regs->pc,textaccess,writeaccess); |
| 153 | show_regs(regs); |
| 154 | |
| 155 | print_vma(vma); |
| 156 | #endif |
| 157 | goto bad_area; |
| 158 | } |
| 159 | if (expand_stack(vma, address)) { |
| 160 | #ifdef DEBUG_FAULT |
| 161 | print_task(tsk); |
| 162 | printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", |
Harvey Harrison | 866e6b9 | 2008-03-04 15:23:47 -0800 | [diff] [blame] | 163 | __func__, __LINE__, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 164 | address,regs->pc,textaccess,writeaccess); |
| 165 | show_regs(regs); |
| 166 | #endif |
| 167 | goto bad_area; |
| 168 | } |
| 169 | /* |
| 170 | * Ok, we have a good vm_area for this memory access, so |
| 171 | * we can handle it.. |
| 172 | */ |
| 173 | good_area: |
| 174 | if (textaccess) { |
| 175 | if (!(vma->vm_flags & VM_EXEC)) |
| 176 | goto bad_area; |
| 177 | } else { |
| 178 | if (writeaccess) { |
| 179 | if (!(vma->vm_flags & VM_WRITE)) |
| 180 | goto bad_area; |
| 181 | } else { |
| 182 | if (!(vma->vm_flags & VM_READ)) |
| 183 | goto bad_area; |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | /* |
| 188 | * If for any reason at all we couldn't handle the fault, |
| 189 | * make sure we exit gracefully rather than endlessly redo |
| 190 | * the fault. |
| 191 | */ |
| 192 | survive: |
Linus Torvalds | d06063c | 2009-04-10 09:01:23 -0700 | [diff] [blame] | 193 | fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0); |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 194 | if (unlikely(fault & VM_FAULT_ERROR)) { |
| 195 | if (fault & VM_FAULT_OOM) |
| 196 | goto out_of_memory; |
| 197 | else if (fault & VM_FAULT_SIGBUS) |
| 198 | goto do_sigbus; |
| 199 | BUG(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | } |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 201 | |
| 202 | if (fault & VM_FAULT_MAJOR) { |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 203 | tsk->maj_flt++; |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 204 | perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, |
| 205 | regs, address); |
| 206 | } else { |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 207 | tsk->min_flt++; |
Paul Mundt | 163b2f0 | 2009-06-25 02:49:03 +0900 | [diff] [blame] | 208 | perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, |
| 209 | regs, address); |
| 210 | } |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 211 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 212 | /* If we get here, the page fault has been handled. Do the TLB refill |
| 213 | now from the newly-setup PTE, to avoid having to fault again right |
| 214 | away on the same instruction. */ |
| 215 | pte = lookup_pte (mm, address); |
| 216 | if (!pte) { |
| 217 | /* From empirical evidence, we can get here, due to |
| 218 | !pte_present(pte). (e.g. if a swap-in occurs, and the page |
| 219 | is swapped back out again before the process that wanted it |
| 220 | gets rescheduled?) */ |
| 221 | goto no_pte; |
| 222 | } |
| 223 | |
| 224 | __do_tlb_refill(address, textaccess, pte); |
| 225 | |
| 226 | no_pte: |
| 227 | |
| 228 | up_read(&mm->mmap_sem); |
| 229 | return; |
| 230 | |
| 231 | /* |
| 232 | * Something tried to access memory that isn't in our memory map.. |
| 233 | * Fix it, but check if it's kernel or user first.. |
| 234 | */ |
| 235 | bad_area: |
| 236 | #ifdef DEBUG_FAULT |
| 237 | printk("fault:bad area\n"); |
| 238 | #endif |
| 239 | up_read(&mm->mmap_sem); |
| 240 | |
| 241 | if (user_mode(regs)) { |
| 242 | static int count=0; |
| 243 | siginfo_t info; |
| 244 | if (count < 4) { |
| 245 | /* This is really to help debug faults when starting |
| 246 | * usermode, so only need a few */ |
| 247 | count++; |
| 248 | printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", |
Alexey Dobriyan | 19c5870 | 2007-10-18 23:40:41 -0700 | [diff] [blame] | 249 | address, task_pid_nr(current), current->comm, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | (unsigned long) regs->pc); |
| 251 | #if 0 |
| 252 | show_regs(regs); |
| 253 | #endif |
| 254 | } |
Serge E. Hallyn | b460cbc | 2007-10-18 23:39:52 -0700 | [diff] [blame] | 255 | if (is_global_init(tsk)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | panic("INIT had user mode bad_area\n"); |
| 257 | } |
| 258 | tsk->thread.address = address; |
| 259 | tsk->thread.error_code = writeaccess; |
| 260 | info.si_signo = SIGSEGV; |
| 261 | info.si_errno = 0; |
| 262 | info.si_addr = (void *) address; |
| 263 | force_sig_info(SIGSEGV, &info, tsk); |
| 264 | return; |
| 265 | } |
| 266 | |
| 267 | no_context: |
| 268 | #ifdef DEBUG_FAULT |
| 269 | printk("fault:No context\n"); |
| 270 | #endif |
| 271 | /* Are we prepared to handle this kernel fault? */ |
| 272 | fixup = search_exception_tables(regs->pc); |
| 273 | if (fixup) { |
| 274 | regs->pc = fixup->fixup; |
| 275 | return; |
| 276 | } |
| 277 | |
| 278 | /* |
| 279 | * Oops. The kernel tried to access some bad page. We'll have to |
| 280 | * terminate things with extreme prejudice. |
| 281 | * |
| 282 | */ |
| 283 | if (address < PAGE_SIZE) |
| 284 | printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); |
| 285 | else |
| 286 | printk(KERN_ALERT "Unable to handle kernel paging request"); |
| 287 | printk(" at virtual address %08lx\n", address); |
| 288 | printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); |
| 289 | die("Oops", regs, writeaccess); |
| 290 | do_exit(SIGKILL); |
| 291 | |
| 292 | /* |
| 293 | * We ran out of memory, or some other thing happened to us that made |
| 294 | * us unable to handle the page fault gracefully. |
| 295 | */ |
| 296 | out_of_memory: |
Serge E. Hallyn | b460cbc | 2007-10-18 23:39:52 -0700 | [diff] [blame] | 297 | if (is_global_init(current)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 298 | panic("INIT out of memory\n"); |
| 299 | yield(); |
| 300 | goto survive; |
| 301 | } |
| 302 | printk("fault:Out of memory\n"); |
| 303 | up_read(&mm->mmap_sem); |
Serge E. Hallyn | b460cbc | 2007-10-18 23:39:52 -0700 | [diff] [blame] | 304 | if (is_global_init(current)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 305 | yield(); |
| 306 | down_read(&mm->mmap_sem); |
| 307 | goto survive; |
| 308 | } |
| 309 | printk("VM: killing process %s\n", tsk->comm); |
| 310 | if (user_mode(regs)) |
Will Schmidt | dcca2bd | 2007-10-16 01:24:18 -0700 | [diff] [blame] | 311 | do_group_exit(SIGKILL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 312 | goto no_context; |
| 313 | |
| 314 | do_sigbus: |
| 315 | printk("fault:Do sigbus\n"); |
| 316 | up_read(&mm->mmap_sem); |
| 317 | |
| 318 | /* |
| 319 | * Send a sigbus, regardless of whether we were in kernel |
| 320 | * or user mode. |
| 321 | */ |
| 322 | tsk->thread.address = address; |
| 323 | tsk->thread.error_code = writeaccess; |
| 324 | tsk->thread.trap_no = 14; |
| 325 | force_sig(SIGBUS, tsk); |
| 326 | |
| 327 | /* Kernel mode? Handle exceptions or die */ |
| 328 | if (!user_mode(regs)) |
| 329 | goto no_context; |
| 330 | } |
| 331 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 332 | void update_mmu_cache(struct vm_area_struct * vma, |
| 333 | unsigned long address, pte_t pte) |
| 334 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 335 | /* |
| 336 | * This appears to get called once for every pte entry that gets |
| 337 | * established => I don't think it's efficient to try refilling the |
| 338 | * TLBs with the pages - some may not get accessed even. Also, for |
| 339 | * executable pages, it is impossible to determine reliably here which |
| 340 | * TLB they should be mapped into (or both even). |
| 341 | * |
| 342 | * So, just do nothing here and handle faults on demand. In the |
| 343 | * TLBMISS handling case, the refill is now done anyway after the pte |
| 344 | * has been fixed up, so that deals with most useful cases. |
| 345 | */ |
| 346 | } |
| 347 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 348 | void local_flush_tlb_one(unsigned long asid, unsigned long page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 349 | { |
| 350 | unsigned long long match, pteh=0, lpage; |
| 351 | unsigned long tlb; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | |
| 353 | /* |
| 354 | * Sign-extend based on neff. |
| 355 | */ |
| 356 | lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 357 | match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 358 | match |= lpage; |
| 359 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 360 | for_each_itlb_entry(tlb) { |
| 361 | asm volatile ("getcfg %1, 0, %0" |
| 362 | : "=r" (pteh) |
| 363 | : "r" (tlb) ); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 365 | if (pteh == match) { |
| 366 | __flush_tlb_slot(tlb); |
| 367 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | } |
| 369 | } |
| 370 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 371 | for_each_dtlb_entry(tlb) { |
| 372 | asm volatile ("getcfg %1, 0, %0" |
| 373 | : "=r" (pteh) |
| 374 | : "r" (tlb) ); |
| 375 | |
| 376 | if (pteh == match) { |
| 377 | __flush_tlb_slot(tlb); |
| 378 | break; |
| 379 | } |
| 380 | |
| 381 | } |
| 382 | } |
| 383 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 384 | void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 385 | { |
| 386 | unsigned long flags; |
| 387 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | if (vma->vm_mm) { |
| 389 | page &= PAGE_MASK; |
| 390 | local_irq_save(flags); |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 391 | local_flush_tlb_one(get_asid(), page); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 392 | local_irq_restore(flags); |
| 393 | } |
| 394 | } |
| 395 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 396 | void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 397 | unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 398 | { |
| 399 | unsigned long flags; |
| 400 | unsigned long long match, pteh=0, pteh_epn, pteh_low; |
| 401 | unsigned long tlb; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 402 | unsigned int cpu = smp_processor_id(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 403 | struct mm_struct *mm; |
| 404 | |
| 405 | mm = vma->vm_mm; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 406 | if (cpu_context(cpu, mm) == NO_CONTEXT) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | return; |
| 408 | |
| 409 | local_irq_save(flags); |
| 410 | |
| 411 | start &= PAGE_MASK; |
| 412 | end &= PAGE_MASK; |
| 413 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 414 | match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 415 | |
| 416 | /* Flush ITLB */ |
| 417 | for_each_itlb_entry(tlb) { |
| 418 | asm volatile ("getcfg %1, 0, %0" |
| 419 | : "=r" (pteh) |
| 420 | : "r" (tlb) ); |
| 421 | |
| 422 | pteh_epn = pteh & PAGE_MASK; |
| 423 | pteh_low = pteh & ~PAGE_MASK; |
| 424 | |
| 425 | if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) |
| 426 | __flush_tlb_slot(tlb); |
| 427 | } |
| 428 | |
| 429 | /* Flush DTLB */ |
| 430 | for_each_dtlb_entry(tlb) { |
| 431 | asm volatile ("getcfg %1, 0, %0" |
| 432 | : "=r" (pteh) |
| 433 | : "r" (tlb) ); |
| 434 | |
| 435 | pteh_epn = pteh & PAGE_MASK; |
| 436 | pteh_low = pteh & ~PAGE_MASK; |
| 437 | |
| 438 | if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) |
| 439 | __flush_tlb_slot(tlb); |
| 440 | } |
| 441 | |
| 442 | local_irq_restore(flags); |
| 443 | } |
| 444 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 445 | void local_flush_tlb_mm(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 446 | { |
| 447 | unsigned long flags; |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 448 | unsigned int cpu = smp_processor_id(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 450 | if (cpu_context(cpu, mm) == NO_CONTEXT) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 451 | return; |
| 452 | |
| 453 | local_irq_save(flags); |
| 454 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 455 | cpu_context(cpu, mm) = NO_CONTEXT; |
| 456 | if (mm == current->mm) |
| 457 | activate_context(mm, cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 458 | |
| 459 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 460 | } |
| 461 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 462 | void local_flush_tlb_all(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 463 | { |
| 464 | /* Invalidate all, including shared pages, excluding fixed TLBs */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | unsigned long flags, tlb; |
| 466 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | local_irq_save(flags); |
| 468 | |
| 469 | /* Flush each ITLB entry */ |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 470 | for_each_itlb_entry(tlb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | __flush_tlb_slot(tlb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 472 | |
| 473 | /* Flush each DTLB entry */ |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 474 | for_each_dtlb_entry(tlb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | __flush_tlb_slot(tlb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 476 | |
| 477 | local_irq_restore(flags); |
| 478 | } |
| 479 | |
Paul Mundt | 3eeffb3 | 2007-11-19 18:57:03 +0900 | [diff] [blame] | 480 | void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 481 | { |
| 482 | /* FIXME: Optimize this later.. */ |
| 483 | flush_tlb_all(); |
| 484 | } |