Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Russell King | 4baa992 | 2008-08-02 10:55:55 +0100 | [diff] [blame] | 2 | * arch/arm/include/asm/tlbflush.h |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 1999-2003 Russell King |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | #ifndef _ASMARM_TLBFLUSH_H |
| 11 | #define _ASMARM_TLBFLUSH_H |
| 12 | |
Hyok S. Choi | 0157903 | 2006-02-24 21:41:25 +0000 | [diff] [blame] | 13 | |
| 14 | #ifndef CONFIG_MMU |
| 15 | |
| 16 | #define tlb_flush(tlb) ((void) tlb) |
| 17 | |
Russell King | fb1c776 | 2006-02-24 21:44:56 +0000 | [diff] [blame] | 18 | #else /* CONFIG_MMU */ |
Hyok S. Choi | 0157903 | 2006-02-24 21:41:25 +0000 | [diff] [blame] | 19 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <asm/glue.h> |
| 21 | |
| 22 | #define TLB_V3_PAGE (1 << 0) |
| 23 | #define TLB_V4_U_PAGE (1 << 1) |
| 24 | #define TLB_V4_D_PAGE (1 << 2) |
| 25 | #define TLB_V4_I_PAGE (1 << 3) |
| 26 | #define TLB_V6_U_PAGE (1 << 4) |
| 27 | #define TLB_V6_D_PAGE (1 << 5) |
| 28 | #define TLB_V6_I_PAGE (1 << 6) |
| 29 | |
| 30 | #define TLB_V3_FULL (1 << 8) |
| 31 | #define TLB_V4_U_FULL (1 << 9) |
| 32 | #define TLB_V4_D_FULL (1 << 10) |
| 33 | #define TLB_V4_I_FULL (1 << 11) |
| 34 | #define TLB_V6_U_FULL (1 << 12) |
| 35 | #define TLB_V6_D_FULL (1 << 13) |
| 36 | #define TLB_V6_I_FULL (1 << 14) |
| 37 | |
| 38 | #define TLB_V6_U_ASID (1 << 16) |
| 39 | #define TLB_V6_D_ASID (1 << 17) |
| 40 | #define TLB_V6_I_ASID (1 << 18) |
| 41 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 42 | #define TLB_BTB (1 << 28) |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 43 | |
| 44 | /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ |
| 45 | #define TLB_V7_UIS_PAGE (1 << 19) |
| 46 | #define TLB_V7_UIS_FULL (1 << 20) |
| 47 | #define TLB_V7_UIS_ASID (1 << 21) |
| 48 | |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 49 | /* Inner Shareable BTB operation (ARMv7 MP extensions) */ |
| 50 | #define TLB_V7_IS_BTB (1 << 22) |
| 51 | |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 52 | #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | #define TLB_DCLEAN (1 << 30) |
| 54 | #define TLB_WB (1 << 31) |
| 55 | |
| 56 | /* |
| 57 | * MMU TLB Model |
| 58 | * ============= |
| 59 | * |
| 60 | * We have the following to choose from: |
| 61 | * v3 - ARMv3 |
| 62 | * v4 - ARMv4 without write buffer |
| 63 | * v4wb - ARMv4 with write buffer without I TLB flush entry instruction |
| 64 | * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 65 | * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 66 | * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 68 | * v7wbi - identical to v6wbi |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | */ |
| 70 | #undef _TLB |
| 71 | #undef MULTI_TLB |
| 72 | |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame^] | 73 | #ifdef CONFIG_SMP_ON_UP |
| 74 | #define MULTI_TLB 1 |
| 75 | #endif |
| 76 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 77 | #define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE) |
| 78 | |
| 79 | #ifdef CONFIG_CPU_TLB_V3 |
| 80 | # define v3_possible_flags v3_tlb_flags |
| 81 | # define v3_always_flags v3_tlb_flags |
| 82 | # ifdef _TLB |
| 83 | # define MULTI_TLB 1 |
| 84 | # else |
| 85 | # define _TLB v3 |
| 86 | # endif |
| 87 | #else |
| 88 | # define v3_possible_flags 0 |
| 89 | # define v3_always_flags (-1UL) |
| 90 | #endif |
| 91 | |
| 92 | #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE) |
| 93 | |
| 94 | #ifdef CONFIG_CPU_TLB_V4WT |
| 95 | # define v4_possible_flags v4_tlb_flags |
| 96 | # define v4_always_flags v4_tlb_flags |
| 97 | # ifdef _TLB |
| 98 | # define MULTI_TLB 1 |
| 99 | # else |
| 100 | # define _TLB v4 |
| 101 | # endif |
| 102 | #else |
| 103 | # define v4_possible_flags 0 |
| 104 | # define v4_always_flags (-1UL) |
| 105 | #endif |
| 106 | |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 107 | #define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \ |
| 108 | TLB_V4_U_FULL | TLB_V4_U_PAGE) |
| 109 | |
| 110 | #ifdef CONFIG_CPU_TLB_FA |
| 111 | # define fa_possible_flags fa_tlb_flags |
| 112 | # define fa_always_flags fa_tlb_flags |
| 113 | # ifdef _TLB |
| 114 | # define MULTI_TLB 1 |
| 115 | # else |
| 116 | # define _TLB fa |
| 117 | # endif |
| 118 | #else |
| 119 | # define fa_possible_flags 0 |
| 120 | # define fa_always_flags (-1UL) |
| 121 | #endif |
| 122 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
| 124 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 125 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) |
| 126 | |
| 127 | #ifdef CONFIG_CPU_TLB_V4WBI |
| 128 | # define v4wbi_possible_flags v4wbi_tlb_flags |
| 129 | # define v4wbi_always_flags v4wbi_tlb_flags |
| 130 | # ifdef _TLB |
| 131 | # define MULTI_TLB 1 |
| 132 | # else |
| 133 | # define _TLB v4wbi |
| 134 | # endif |
| 135 | #else |
| 136 | # define v4wbi_possible_flags 0 |
| 137 | # define v4wbi_always_flags (-1UL) |
| 138 | #endif |
| 139 | |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 140 | #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \ |
| 141 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 142 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) |
| 143 | |
| 144 | #ifdef CONFIG_CPU_TLB_FEROCEON |
| 145 | # define fr_possible_flags fr_tlb_flags |
| 146 | # define fr_always_flags fr_tlb_flags |
| 147 | # ifdef _TLB |
| 148 | # define MULTI_TLB 1 |
| 149 | # else |
| 150 | # define _TLB v4wbi |
| 151 | # endif |
| 152 | #else |
| 153 | # define fr_possible_flags 0 |
| 154 | # define fr_always_flags (-1UL) |
| 155 | #endif |
| 156 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
| 158 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 159 | TLB_V4_D_PAGE) |
| 160 | |
| 161 | #ifdef CONFIG_CPU_TLB_V4WB |
| 162 | # define v4wb_possible_flags v4wb_tlb_flags |
| 163 | # define v4wb_always_flags v4wb_tlb_flags |
| 164 | # ifdef _TLB |
| 165 | # define MULTI_TLB 1 |
| 166 | # else |
| 167 | # define _TLB v4wb |
| 168 | # endif |
| 169 | #else |
| 170 | # define v4wb_possible_flags 0 |
| 171 | # define v4wb_always_flags (-1UL) |
| 172 | #endif |
| 173 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 174 | #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | TLB_V6_I_FULL | TLB_V6_D_FULL | \ |
| 176 | TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ |
| 177 | TLB_V6_I_ASID | TLB_V6_D_ASID) |
| 178 | |
| 179 | #ifdef CONFIG_CPU_TLB_V6 |
| 180 | # define v6wbi_possible_flags v6wbi_tlb_flags |
| 181 | # define v6wbi_always_flags v6wbi_tlb_flags |
| 182 | # ifdef _TLB |
| 183 | # define MULTI_TLB 1 |
| 184 | # else |
| 185 | # define _TLB v6wbi |
| 186 | # endif |
| 187 | #else |
| 188 | # define v6wbi_possible_flags 0 |
| 189 | # define v6wbi_always_flags (-1UL) |
| 190 | #endif |
| 191 | |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame^] | 192 | #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \ |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 193 | TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame^] | 194 | #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \ |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 195 | TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 196 | |
Catalin Marinas | 2ccdd1e | 2007-05-18 11:25:31 +0100 | [diff] [blame] | 197 | #ifdef CONFIG_CPU_TLB_V7 |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame^] | 198 | |
| 199 | # ifdef CONFIG_SMP_ON_UP |
| 200 | # define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up) |
| 201 | # define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up) |
| 202 | # elif defined(CONFIG_SMP) |
| 203 | # define v7wbi_possible_flags v7wbi_tlb_flags_smp |
| 204 | # define v7wbi_always_flags v7wbi_tlb_flags_smp |
| 205 | # else |
| 206 | # define v7wbi_possible_flags v7wbi_tlb_flags_up |
| 207 | # define v7wbi_always_flags v7wbi_tlb_flags_up |
| 208 | # endif |
Catalin Marinas | 2ccdd1e | 2007-05-18 11:25:31 +0100 | [diff] [blame] | 209 | # ifdef _TLB |
| 210 | # define MULTI_TLB 1 |
| 211 | # else |
| 212 | # define _TLB v7wbi |
| 213 | # endif |
| 214 | #else |
| 215 | # define v7wbi_possible_flags 0 |
| 216 | # define v7wbi_always_flags (-1UL) |
| 217 | #endif |
| 218 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 219 | #ifndef _TLB |
| 220 | #error Unknown TLB model |
| 221 | #endif |
| 222 | |
| 223 | #ifndef __ASSEMBLY__ |
| 224 | |
Alexey Dobriyan | e8edc6e | 2007-05-21 01:22:52 +0400 | [diff] [blame] | 225 | #include <linux/sched.h> |
| 226 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 227 | struct cpu_tlb_fns { |
| 228 | void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); |
| 229 | void (*flush_kern_range)(unsigned long, unsigned long); |
| 230 | unsigned long tlb_flags; |
| 231 | }; |
| 232 | |
| 233 | /* |
| 234 | * Select the calling method |
| 235 | */ |
| 236 | #ifdef MULTI_TLB |
| 237 | |
| 238 | #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range |
| 239 | #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range |
| 240 | |
| 241 | #else |
| 242 | |
| 243 | #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range) |
| 244 | #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range) |
| 245 | |
| 246 | extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); |
| 247 | extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long); |
| 248 | |
| 249 | #endif |
| 250 | |
| 251 | extern struct cpu_tlb_fns cpu_tlb; |
| 252 | |
| 253 | #define __cpu_tlb_flags cpu_tlb.tlb_flags |
| 254 | |
| 255 | /* |
| 256 | * TLB Management |
| 257 | * ============== |
| 258 | * |
| 259 | * The arch/arm/mm/tlb-*.S files implement these methods. |
| 260 | * |
| 261 | * The TLB specific code is expected to perform whatever tests it |
| 262 | * needs to determine if it should invalidate the TLB for each |
| 263 | * call. Start addresses are inclusive and end addresses are |
| 264 | * exclusive; it is safe to round these addresses down. |
| 265 | * |
| 266 | * flush_tlb_all() |
| 267 | * |
| 268 | * Invalidate the entire TLB. |
| 269 | * |
| 270 | * flush_tlb_mm(mm) |
| 271 | * |
| 272 | * Invalidate all TLB entries in a particular address |
| 273 | * space. |
| 274 | * - mm - mm_struct describing address space |
| 275 | * |
| 276 | * flush_tlb_range(mm,start,end) |
| 277 | * |
| 278 | * Invalidate a range of TLB entries in the specified |
| 279 | * address space. |
| 280 | * - mm - mm_struct describing address space |
| 281 | * - start - start address (may not be aligned) |
| 282 | * - end - end address (exclusive, may not be aligned) |
| 283 | * |
| 284 | * flush_tlb_page(vaddr,vma) |
| 285 | * |
| 286 | * Invalidate the specified page in the specified address range. |
| 287 | * - vaddr - virtual address (may not be aligned) |
| 288 | * - vma - vma_struct describing address range |
| 289 | * |
| 290 | * flush_kern_tlb_page(kaddr) |
| 291 | * |
| 292 | * Invalidate the TLB entry for the specified page. The address |
| 293 | * will be in the kernels virtual memory space. Current uses |
| 294 | * only require the D-TLB to be invalidated. |
| 295 | * - kaddr - Kernel virtual memory address |
| 296 | */ |
| 297 | |
| 298 | /* |
| 299 | * We optimise the code below by: |
| 300 | * - building a set of TLB flags that might be set in __cpu_tlb_flags |
| 301 | * - building a set of TLB flags that will always be set in __cpu_tlb_flags |
| 302 | * - if we're going to need __cpu_tlb_flags, access it once and only once |
| 303 | * |
| 304 | * This allows us to build optimal assembly for the single-CPU type case, |
| 305 | * and as close to optimal given the compiler constrants for multi-CPU |
| 306 | * case. We could do better for the multi-CPU case if the compiler |
| 307 | * implemented the "%?" method, but this has been discontinued due to too |
| 308 | * many people getting it wrong. |
| 309 | */ |
| 310 | #define possible_tlb_flags (v3_possible_flags | \ |
| 311 | v4_possible_flags | \ |
| 312 | v4wbi_possible_flags | \ |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 313 | fr_possible_flags | \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | v4wb_possible_flags | \ |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 315 | fa_possible_flags | \ |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 316 | v6wbi_possible_flags | \ |
| 317 | v7wbi_possible_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 318 | |
| 319 | #define always_tlb_flags (v3_always_flags & \ |
| 320 | v4_always_flags & \ |
| 321 | v4wbi_always_flags & \ |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 322 | fr_always_flags & \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | v4wb_always_flags & \ |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 324 | fa_always_flags & \ |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 325 | v6wbi_always_flags & \ |
| 326 | v7wbi_always_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | |
| 328 | #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) |
| 329 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 330 | static inline void local_flush_tlb_all(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | { |
| 332 | const int zero = 0; |
| 333 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 334 | |
| 335 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 336 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | |
| 338 | if (tlb_flag(TLB_V3_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 339 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | if (tlb_flag(TLB_V4_U_FULL | TLB_V6_U_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 341 | asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 342 | if (tlb_flag(TLB_V4_D_FULL | TLB_V6_D_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 343 | asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 344 | if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 345 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 346 | if (tlb_flag(TLB_V7_UIS_FULL)) |
| 347 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 348 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 349 | if (tlb_flag(TLB_BTB)) { |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 350 | /* flush the branch target cache */ |
| 351 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); |
| 352 | dsb(); |
| 353 | isb(); |
| 354 | } |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 355 | if (tlb_flag(TLB_V7_IS_BTB)) { |
| 356 | /* flush the branch target cache */ |
| 357 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); |
| 358 | dsb(); |
| 359 | isb(); |
| 360 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 361 | } |
| 362 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 363 | static inline void local_flush_tlb_mm(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | { |
| 365 | const int zero = 0; |
| 366 | const int asid = ASID(mm); |
| 367 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 368 | |
| 369 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 370 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 371 | |
Santosh Shilimkar | daaeb6c | 2009-10-15 15:06:47 +0100 | [diff] [blame] | 372 | if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 373 | if (tlb_flag(TLB_V3_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 374 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 375 | if (tlb_flag(TLB_V4_U_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 376 | asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 377 | if (tlb_flag(TLB_V4_D_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 378 | asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 379 | if (tlb_flag(TLB_V4_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 380 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 381 | } |
Santosh Shilimkar | daaeb6c | 2009-10-15 15:06:47 +0100 | [diff] [blame] | 382 | put_cpu(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | |
| 384 | if (tlb_flag(TLB_V6_U_ASID)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 385 | asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | if (tlb_flag(TLB_V6_D_ASID)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 387 | asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | if (tlb_flag(TLB_V6_I_ASID)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 389 | asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 390 | if (tlb_flag(TLB_V7_UIS_ASID)) |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 391 | #ifdef CONFIG_ARM_ERRATA_720789 |
| 392 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); |
| 393 | #else |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 394 | asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 395 | #endif |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 396 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 397 | if (tlb_flag(TLB_BTB)) { |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 398 | /* flush the branch target cache */ |
| 399 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); |
| 400 | dsb(); |
| 401 | } |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 402 | if (tlb_flag(TLB_V7_IS_BTB)) { |
| 403 | /* flush the branch target cache */ |
| 404 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); |
| 405 | dsb(); |
| 406 | isb(); |
| 407 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 408 | } |
| 409 | |
| 410 | static inline void |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 411 | local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 412 | { |
| 413 | const int zero = 0; |
| 414 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 415 | |
| 416 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); |
| 417 | |
| 418 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 419 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 420 | |
Rusty Russell | 56f8ba8 | 2009-09-24 09:34:49 -0600 | [diff] [blame] | 421 | if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 422 | if (tlb_flag(TLB_V3_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 423 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 424 | if (tlb_flag(TLB_V4_U_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 425 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 426 | if (tlb_flag(TLB_V4_D_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 427 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 428 | if (tlb_flag(TLB_V4_I_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 429 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 430 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 431 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 432 | } |
| 433 | |
| 434 | if (tlb_flag(TLB_V6_U_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 435 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 436 | if (tlb_flag(TLB_V6_D_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 437 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 438 | if (tlb_flag(TLB_V6_I_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 439 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 440 | if (tlb_flag(TLB_V7_UIS_PAGE)) |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 441 | #ifdef CONFIG_ARM_ERRATA_720789 |
| 442 | asm("mcr p15, 0, %0, c8, c3, 3" : : "r" (uaddr & PAGE_MASK) : "cc"); |
| 443 | #else |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 444 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 445 | #endif |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 446 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 447 | if (tlb_flag(TLB_BTB)) { |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 448 | /* flush the branch target cache */ |
| 449 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); |
| 450 | dsb(); |
| 451 | } |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 452 | if (tlb_flag(TLB_V7_IS_BTB)) { |
| 453 | /* flush the branch target cache */ |
| 454 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); |
| 455 | dsb(); |
| 456 | isb(); |
| 457 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 458 | } |
| 459 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 460 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 461 | { |
| 462 | const int zero = 0; |
| 463 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 464 | |
| 465 | kaddr &= PAGE_MASK; |
| 466 | |
| 467 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 468 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | |
| 470 | if (tlb_flag(TLB_V3_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 471 | asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 472 | if (tlb_flag(TLB_V4_U_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 473 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | if (tlb_flag(TLB_V4_D_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 475 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 476 | if (tlb_flag(TLB_V4_I_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 477 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 478 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 479 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 480 | |
| 481 | if (tlb_flag(TLB_V6_U_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 482 | asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | if (tlb_flag(TLB_V6_D_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 484 | asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 485 | if (tlb_flag(TLB_V6_I_PAGE)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 486 | asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 487 | if (tlb_flag(TLB_V7_UIS_PAGE)) |
| 488 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); |
Catalin Marinas | 6a0e243 | 2006-03-07 14:42:27 +0000 | [diff] [blame] | 489 | |
Paulius Zaleckas | bba7d0b | 2009-03-25 10:58:47 +0200 | [diff] [blame] | 490 | if (tlb_flag(TLB_BTB)) { |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 491 | /* flush the branch target cache */ |
| 492 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); |
| 493 | dsb(); |
| 494 | isb(); |
| 495 | } |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 496 | if (tlb_flag(TLB_V7_IS_BTB)) { |
| 497 | /* flush the branch target cache */ |
| 498 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); |
| 499 | dsb(); |
| 500 | isb(); |
| 501 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | } |
| 503 | |
| 504 | /* |
| 505 | * flush_pmd_entry |
| 506 | * |
| 507 | * Flush a PMD entry (word aligned, or double-word aligned) to |
| 508 | * RAM if the TLB for the CPU we are running on requires this. |
| 509 | * This is typically used when we are creating PMD entries. |
| 510 | * |
| 511 | * clean_pmd_entry |
| 512 | * |
| 513 | * Clean (but don't drain the write buffer) if the CPU requires |
| 514 | * these operations. This is typically used when we are removing |
| 515 | * PMD entries. |
| 516 | */ |
| 517 | static inline void flush_pmd_entry(pmd_t *pmd) |
| 518 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 519 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 520 | |
| 521 | if (tlb_flag(TLB_DCLEAN)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 522 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" |
| 523 | : : "r" (pmd) : "cc"); |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 524 | |
| 525 | if (tlb_flag(TLB_L2CLEAN_FR)) |
| 526 | asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" |
| 527 | : : "r" (pmd) : "cc"); |
| 528 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 529 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 530 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | static inline void clean_pmd_entry(pmd_t *pmd) |
| 534 | { |
| 535 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 536 | |
| 537 | if (tlb_flag(TLB_DCLEAN)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 538 | asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" |
| 539 | : : "r" (pmd) : "cc"); |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 540 | |
| 541 | if (tlb_flag(TLB_L2CLEAN_FR)) |
| 542 | asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" |
| 543 | : : "r" (pmd) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | } |
| 545 | |
| 546 | #undef tlb_flag |
| 547 | #undef always_tlb_flags |
| 548 | #undef possible_tlb_flags |
| 549 | |
| 550 | /* |
| 551 | * Convert calls to our calling convention. |
| 552 | */ |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 553 | #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) |
| 554 | #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) |
| 555 | |
| 556 | #ifndef CONFIG_SMP |
| 557 | #define flush_tlb_all local_flush_tlb_all |
| 558 | #define flush_tlb_mm local_flush_tlb_mm |
| 559 | #define flush_tlb_page local_flush_tlb_page |
| 560 | #define flush_tlb_kernel_page local_flush_tlb_kernel_page |
| 561 | #define flush_tlb_range local_flush_tlb_range |
| 562 | #define flush_tlb_kernel_range local_flush_tlb_kernel_range |
| 563 | #else |
| 564 | extern void flush_tlb_all(void); |
| 565 | extern void flush_tlb_mm(struct mm_struct *mm); |
| 566 | extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); |
| 567 | extern void flush_tlb_kernel_page(unsigned long kaddr); |
| 568 | extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); |
| 569 | extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); |
| 570 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 571 | |
| 572 | /* |
| 573 | * if PG_dcache_dirty is set for the page, we need to ensure that any |
| 574 | * cache entries for the kernels virtual memory range are written |
| 575 | * back to the page. |
| 576 | */ |
Russell King | 4b3073e | 2009-12-18 16:40:18 +0000 | [diff] [blame] | 577 | extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, |
| 578 | pte_t *ptep); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 579 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 580 | #endif |
| 581 | |
Hyok S. Choi | 0157903 | 2006-02-24 21:41:25 +0000 | [diff] [blame] | 582 | #endif /* CONFIG_MMU */ |
| 583 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 584 | #endif |