Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Russell King | 4baa992 | 2008-08-02 10:55:55 +0100 | [diff] [blame] | 2 | * arch/arm/include/asm/tlbflush.h |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 1999-2003 Russell King |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | #ifndef _ASMARM_TLBFLUSH_H |
| 11 | #define _ASMARM_TLBFLUSH_H |
| 12 | |
Russell King | 58e9c47 | 2011-02-20 12:27:49 +0000 | [diff] [blame] | 13 | #ifdef CONFIG_MMU |
Hyok S. Choi | 0157903 | 2006-02-24 21:41:25 +0000 | [diff] [blame] | 14 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <asm/glue.h> |
| 16 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #define TLB_V4_U_PAGE (1 << 1) |
| 18 | #define TLB_V4_D_PAGE (1 << 2) |
| 19 | #define TLB_V4_I_PAGE (1 << 3) |
| 20 | #define TLB_V6_U_PAGE (1 << 4) |
| 21 | #define TLB_V6_D_PAGE (1 << 5) |
| 22 | #define TLB_V6_I_PAGE (1 << 6) |
| 23 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | #define TLB_V4_U_FULL (1 << 9) |
| 25 | #define TLB_V4_D_FULL (1 << 10) |
| 26 | #define TLB_V4_I_FULL (1 << 11) |
| 27 | #define TLB_V6_U_FULL (1 << 12) |
| 28 | #define TLB_V6_D_FULL (1 << 13) |
| 29 | #define TLB_V6_I_FULL (1 << 14) |
| 30 | |
| 31 | #define TLB_V6_U_ASID (1 << 16) |
| 32 | #define TLB_V6_D_ASID (1 << 17) |
| 33 | #define TLB_V6_I_ASID (1 << 18) |
| 34 | |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 35 | #define TLB_V6_BP (1 << 19) |
| 36 | |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 37 | /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 38 | #define TLB_V7_UIS_PAGE (1 << 20) |
| 39 | #define TLB_V7_UIS_FULL (1 << 21) |
| 40 | #define TLB_V7_UIS_ASID (1 << 22) |
| 41 | #define TLB_V7_UIS_BP (1 << 23) |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 42 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 43 | #define TLB_BARRIER (1 << 28) |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 44 | #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | #define TLB_DCLEAN (1 << 30) |
| 46 | #define TLB_WB (1 << 31) |
| 47 | |
| 48 | /* |
| 49 | * MMU TLB Model |
| 50 | * ============= |
| 51 | * |
| 52 | * We have the following to choose from: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | * v4 - ARMv4 without write buffer |
| 54 | * v4wb - ARMv4 with write buffer without I TLB flush entry instruction |
| 55 | * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 56 | * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 57 | * fa - Faraday (v4 with write buffer with UTLB) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 59 | * v7wbi - identical to v6wbi |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | */ |
| 61 | #undef _TLB |
| 62 | #undef MULTI_TLB |
| 63 | |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame] | 64 | #ifdef CONFIG_SMP_ON_UP |
| 65 | #define MULTI_TLB 1 |
| 66 | #endif |
| 67 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE) |
| 69 | |
| 70 | #ifdef CONFIG_CPU_TLB_V4WT |
| 71 | # define v4_possible_flags v4_tlb_flags |
| 72 | # define v4_always_flags v4_tlb_flags |
| 73 | # ifdef _TLB |
| 74 | # define MULTI_TLB 1 |
| 75 | # else |
| 76 | # define _TLB v4 |
| 77 | # endif |
| 78 | #else |
| 79 | # define v4_possible_flags 0 |
| 80 | # define v4_always_flags (-1UL) |
| 81 | #endif |
| 82 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 83 | #define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 84 | TLB_V4_U_FULL | TLB_V4_U_PAGE) |
| 85 | |
| 86 | #ifdef CONFIG_CPU_TLB_FA |
| 87 | # define fa_possible_flags fa_tlb_flags |
| 88 | # define fa_always_flags fa_tlb_flags |
| 89 | # ifdef _TLB |
| 90 | # define MULTI_TLB 1 |
| 91 | # else |
| 92 | # define _TLB fa |
| 93 | # endif |
| 94 | #else |
| 95 | # define fa_possible_flags 0 |
| 96 | # define fa_always_flags (-1UL) |
| 97 | #endif |
| 98 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
| 100 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 101 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) |
| 102 | |
| 103 | #ifdef CONFIG_CPU_TLB_V4WBI |
| 104 | # define v4wbi_possible_flags v4wbi_tlb_flags |
| 105 | # define v4wbi_always_flags v4wbi_tlb_flags |
| 106 | # ifdef _TLB |
| 107 | # define MULTI_TLB 1 |
| 108 | # else |
| 109 | # define _TLB v4wbi |
| 110 | # endif |
| 111 | #else |
| 112 | # define v4wbi_possible_flags 0 |
| 113 | # define v4wbi_always_flags (-1UL) |
| 114 | #endif |
| 115 | |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 116 | #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \ |
| 117 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 118 | TLB_V4_I_PAGE | TLB_V4_D_PAGE) |
| 119 | |
| 120 | #ifdef CONFIG_CPU_TLB_FEROCEON |
| 121 | # define fr_possible_flags fr_tlb_flags |
| 122 | # define fr_always_flags fr_tlb_flags |
| 123 | # ifdef _TLB |
| 124 | # define MULTI_TLB 1 |
| 125 | # else |
| 126 | # define _TLB v4wbi |
| 127 | # endif |
| 128 | #else |
| 129 | # define fr_possible_flags 0 |
| 130 | # define fr_always_flags (-1UL) |
| 131 | #endif |
| 132 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \ |
| 134 | TLB_V4_I_FULL | TLB_V4_D_FULL | \ |
| 135 | TLB_V4_D_PAGE) |
| 136 | |
| 137 | #ifdef CONFIG_CPU_TLB_V4WB |
| 138 | # define v4wb_possible_flags v4wb_tlb_flags |
| 139 | # define v4wb_always_flags v4wb_tlb_flags |
| 140 | # ifdef _TLB |
| 141 | # define MULTI_TLB 1 |
| 142 | # else |
| 143 | # define _TLB v4wb |
| 144 | # endif |
| 145 | #else |
| 146 | # define v4wb_possible_flags 0 |
| 147 | # define v4wb_always_flags (-1UL) |
| 148 | #endif |
| 149 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 150 | #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | TLB_V6_I_FULL | TLB_V6_D_FULL | \ |
| 152 | TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 153 | TLB_V6_I_ASID | TLB_V6_D_ASID | \ |
| 154 | TLB_V6_BP) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | |
| 156 | #ifdef CONFIG_CPU_TLB_V6 |
| 157 | # define v6wbi_possible_flags v6wbi_tlb_flags |
| 158 | # define v6wbi_always_flags v6wbi_tlb_flags |
| 159 | # ifdef _TLB |
| 160 | # define MULTI_TLB 1 |
| 161 | # else |
| 162 | # define _TLB v6wbi |
| 163 | # endif |
| 164 | #else |
| 165 | # define v6wbi_possible_flags 0 |
| 166 | # define v6wbi_always_flags (-1UL) |
| 167 | #endif |
| 168 | |
Will Deacon | ae8a8b9 | 2013-04-03 17:16:57 +0100 | [diff] [blame] | 169 | #define v7wbi_tlb_flags_smp (TLB_WB | TLB_BARRIER | \ |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 170 | TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \ |
| 171 | TLB_V7_UIS_ASID | TLB_V7_UIS_BP) |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 172 | #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 173 | TLB_V6_U_FULL | TLB_V6_U_PAGE | \ |
| 174 | TLB_V6_U_ASID | TLB_V6_BP) |
Catalin Marinas | faa7bc5 | 2009-05-30 14:00:14 +0100 | [diff] [blame] | 175 | |
Catalin Marinas | 2ccdd1e | 2007-05-18 11:25:31 +0100 | [diff] [blame] | 176 | #ifdef CONFIG_CPU_TLB_V7 |
Russell King | f00ec48 | 2010-09-04 10:47:48 +0100 | [diff] [blame] | 177 | |
| 178 | # ifdef CONFIG_SMP_ON_UP |
| 179 | # define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up) |
| 180 | # define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up) |
| 181 | # elif defined(CONFIG_SMP) |
| 182 | # define v7wbi_possible_flags v7wbi_tlb_flags_smp |
| 183 | # define v7wbi_always_flags v7wbi_tlb_flags_smp |
| 184 | # else |
| 185 | # define v7wbi_possible_flags v7wbi_tlb_flags_up |
| 186 | # define v7wbi_always_flags v7wbi_tlb_flags_up |
| 187 | # endif |
Catalin Marinas | 2ccdd1e | 2007-05-18 11:25:31 +0100 | [diff] [blame] | 188 | # ifdef _TLB |
| 189 | # define MULTI_TLB 1 |
| 190 | # else |
| 191 | # define _TLB v7wbi |
| 192 | # endif |
| 193 | #else |
| 194 | # define v7wbi_possible_flags 0 |
| 195 | # define v7wbi_always_flags (-1UL) |
| 196 | #endif |
| 197 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | #ifndef _TLB |
| 199 | #error Unknown TLB model |
| 200 | #endif |
| 201 | |
| 202 | #ifndef __ASSEMBLY__ |
| 203 | |
Alexey Dobriyan | e8edc6e | 2007-05-21 01:22:52 +0400 | [diff] [blame] | 204 | #include <linux/sched.h> |
| 205 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | struct cpu_tlb_fns { |
| 207 | void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); |
| 208 | void (*flush_kern_range)(unsigned long, unsigned long); |
| 209 | unsigned long tlb_flags; |
| 210 | }; |
| 211 | |
| 212 | /* |
| 213 | * Select the calling method |
| 214 | */ |
| 215 | #ifdef MULTI_TLB |
| 216 | |
| 217 | #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range |
| 218 | #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range |
| 219 | |
| 220 | #else |
| 221 | |
| 222 | #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range) |
| 223 | #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range) |
| 224 | |
| 225 | extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); |
| 226 | extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long); |
| 227 | |
| 228 | #endif |
| 229 | |
| 230 | extern struct cpu_tlb_fns cpu_tlb; |
| 231 | |
| 232 | #define __cpu_tlb_flags cpu_tlb.tlb_flags |
| 233 | |
| 234 | /* |
| 235 | * TLB Management |
| 236 | * ============== |
| 237 | * |
| 238 | * The arch/arm/mm/tlb-*.S files implement these methods. |
| 239 | * |
| 240 | * The TLB specific code is expected to perform whatever tests it |
| 241 | * needs to determine if it should invalidate the TLB for each |
| 242 | * call. Start addresses are inclusive and end addresses are |
| 243 | * exclusive; it is safe to round these addresses down. |
| 244 | * |
| 245 | * flush_tlb_all() |
| 246 | * |
| 247 | * Invalidate the entire TLB. |
| 248 | * |
| 249 | * flush_tlb_mm(mm) |
| 250 | * |
| 251 | * Invalidate all TLB entries in a particular address |
| 252 | * space. |
| 253 | * - mm - mm_struct describing address space |
| 254 | * |
| 255 | * flush_tlb_range(mm,start,end) |
| 256 | * |
| 257 | * Invalidate a range of TLB entries in the specified |
| 258 | * address space. |
| 259 | * - mm - mm_struct describing address space |
| 260 | * - start - start address (may not be aligned) |
| 261 | * - end - end address (exclusive, may not be aligned) |
| 262 | * |
| 263 | * flush_tlb_page(vaddr,vma) |
| 264 | * |
| 265 | * Invalidate the specified page in the specified address range. |
| 266 | * - vaddr - virtual address (may not be aligned) |
| 267 | * - vma - vma_struct describing address range |
| 268 | * |
| 269 | * flush_kern_tlb_page(kaddr) |
| 270 | * |
| 271 | * Invalidate the TLB entry for the specified page. The address |
| 272 | * will be in the kernels virtual memory space. Current uses |
| 273 | * only require the D-TLB to be invalidated. |
| 274 | * - kaddr - Kernel virtual memory address |
| 275 | */ |
| 276 | |
| 277 | /* |
| 278 | * We optimise the code below by: |
| 279 | * - building a set of TLB flags that might be set in __cpu_tlb_flags |
| 280 | * - building a set of TLB flags that will always be set in __cpu_tlb_flags |
| 281 | * - if we're going to need __cpu_tlb_flags, access it once and only once |
| 282 | * |
| 283 | * This allows us to build optimal assembly for the single-CPU type case, |
| 284 | * and as close to optimal given the compiler constrants for multi-CPU |
| 285 | * case. We could do better for the multi-CPU case if the compiler |
| 286 | * implemented the "%?" method, but this has been discontinued due to too |
| 287 | * many people getting it wrong. |
| 288 | */ |
Russell King | 357c9c1 | 2012-05-04 12:04:26 +0100 | [diff] [blame] | 289 | #define possible_tlb_flags (v4_possible_flags | \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 290 | v4wbi_possible_flags | \ |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 291 | fr_possible_flags | \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 292 | v4wb_possible_flags | \ |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 293 | fa_possible_flags | \ |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 294 | v6wbi_possible_flags | \ |
| 295 | v7wbi_possible_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | |
Russell King | 357c9c1 | 2012-05-04 12:04:26 +0100 | [diff] [blame] | 297 | #define always_tlb_flags (v4_always_flags & \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 298 | v4wbi_always_flags & \ |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 299 | fr_always_flags & \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | v4wb_always_flags & \ |
Paulius Zaleckas | 28853ac | 2009-03-25 13:10:01 +0200 | [diff] [blame] | 301 | fa_always_flags & \ |
Paul Walmsley | 61db7fb | 2008-08-12 00:04:15 +0100 | [diff] [blame] | 302 | v6wbi_always_flags & \ |
| 303 | v7wbi_always_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 304 | |
| 305 | #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) |
| 306 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 307 | #define __tlb_op(f, insnarg, arg) \ |
| 308 | do { \ |
| 309 | if (always_tlb_flags & (f)) \ |
| 310 | asm("mcr " insnarg \ |
| 311 | : : "r" (arg) : "cc"); \ |
| 312 | else if (possible_tlb_flags & (f)) \ |
| 313 | asm("tst %1, %2\n\t" \ |
| 314 | "mcrne " insnarg \ |
| 315 | : : "r" (arg), "r" (__tlb_flag), "Ir" (f) \ |
| 316 | : "cc"); \ |
| 317 | } while (0) |
| 318 | |
| 319 | #define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg) |
| 320 | #define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg) |
| 321 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 322 | static inline void local_flush_tlb_all(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | { |
| 324 | const int zero = 0; |
| 325 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 326 | |
| 327 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 328 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 329 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 330 | tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero); |
| 331 | tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero); |
| 332 | tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero); |
| 333 | tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero); |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 334 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 335 | if (tlb_flag(TLB_BARRIER)) { |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 336 | dsb(); |
| 337 | isb(); |
| 338 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 339 | } |
| 340 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 341 | static inline void local_flush_tlb_mm(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 342 | { |
| 343 | const int zero = 0; |
| 344 | const int asid = ASID(mm); |
| 345 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 346 | |
| 347 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 348 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 349 | |
Will Deacon | 4cc3daaf | 2013-01-14 20:48:55 +0000 | [diff] [blame] | 350 | if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) { |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 351 | if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) { |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 352 | tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero); |
| 353 | tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero); |
| 354 | tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero); |
| 355 | } |
| 356 | put_cpu(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 357 | } |
| 358 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 359 | tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid); |
| 360 | tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid); |
| 361 | tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 362 | #ifdef CONFIG_ARM_ERRATA_720789 |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 363 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", zero); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 364 | #else |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 365 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", asid); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 366 | #endif |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 367 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 368 | if (tlb_flag(TLB_BARRIER)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 369 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | } |
| 371 | |
| 372 | static inline void |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 373 | local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 374 | { |
| 375 | const int zero = 0; |
| 376 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 377 | |
| 378 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); |
| 379 | |
| 380 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 381 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 382 | |
Will Deacon | 4cc3daaf | 2013-01-14 20:48:55 +0000 | [diff] [blame] | 383 | if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) && |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 384 | cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 385 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr); |
| 386 | tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr); |
| 387 | tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 389 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 390 | } |
| 391 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 392 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr); |
| 393 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr); |
| 394 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 395 | #ifdef CONFIG_ARM_ERRATA_720789 |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 396 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 397 | #else |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 398 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr); |
Will Deacon | cdf357f | 2010-08-05 11:20:51 +0100 | [diff] [blame] | 399 | #endif |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 400 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 401 | if (tlb_flag(TLB_BARRIER)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 402 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 403 | } |
| 404 | |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 405 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 406 | { |
| 407 | const int zero = 0; |
| 408 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 409 | |
| 410 | kaddr &= PAGE_MASK; |
| 411 | |
| 412 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 413 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 415 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr); |
| 416 | tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr); |
| 417 | tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) |
Daniel Jacobowitz | 6a39dd6 | 2006-08-30 15:02:08 +0100 | [diff] [blame] | 419 | asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 420 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 421 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr); |
| 422 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr); |
| 423 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr); |
| 424 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr); |
Catalin Marinas | 6a0e243 | 2006-03-07 14:42:27 +0000 | [diff] [blame] | 425 | |
Russell King | 4348810 | 2011-07-05 09:01:13 +0100 | [diff] [blame] | 426 | if (tlb_flag(TLB_BARRIER)) { |
Catalin Marinas | b8349b5 | 2010-05-07 18:03:05 +0100 | [diff] [blame] | 427 | dsb(); |
| 428 | isb(); |
| 429 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 430 | } |
| 431 | |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 432 | static inline void local_flush_bp_all(void) |
| 433 | { |
| 434 | const int zero = 0; |
| 435 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 436 | |
| 437 | if (tlb_flag(TLB_V7_UIS_BP)) |
| 438 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero)); |
| 439 | else if (tlb_flag(TLB_V6_BP)) |
| 440 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); |
| 441 | |
| 442 | if (tlb_flag(TLB_BARRIER)) |
| 443 | isb(); |
| 444 | } |
| 445 | |
Fabio Estevam | 1f49856 | 2013-07-23 15:13:06 +0100 | [diff] [blame] | 446 | #include <asm/cputype.h> |
Catalin Marinas | 93dc688 | 2013-03-26 23:35:04 +0100 | [diff] [blame] | 447 | #ifdef CONFIG_ARM_ERRATA_798181 |
Fabio Estevam | 1f49856 | 2013-07-23 15:13:06 +0100 | [diff] [blame] | 448 | static inline int erratum_a15_798181(void) |
| 449 | { |
| 450 | unsigned int midr = read_cpuid_id(); |
| 451 | |
| 452 | /* Cortex-A15 r0p0..r3p2 affected */ |
| 453 | if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2) |
| 454 | return 0; |
| 455 | return 1; |
| 456 | } |
| 457 | |
Catalin Marinas | 93dc688 | 2013-03-26 23:35:04 +0100 | [diff] [blame] | 458 | static inline void dummy_flush_tlb_a15_erratum(void) |
| 459 | { |
| 460 | /* |
| 461 | * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0. |
| 462 | */ |
| 463 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0)); |
| 464 | dsb(); |
| 465 | } |
| 466 | #else |
Fabio Estevam | 1f49856 | 2013-07-23 15:13:06 +0100 | [diff] [blame] | 467 | static inline int erratum_a15_798181(void) |
| 468 | { |
| 469 | return 0; |
| 470 | } |
| 471 | |
Catalin Marinas | 93dc688 | 2013-03-26 23:35:04 +0100 | [diff] [blame] | 472 | static inline void dummy_flush_tlb_a15_erratum(void) |
| 473 | { |
| 474 | } |
| 475 | #endif |
| 476 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 477 | /* |
| 478 | * flush_pmd_entry |
| 479 | * |
| 480 | * Flush a PMD entry (word aligned, or double-word aligned) to |
| 481 | * RAM if the TLB for the CPU we are running on requires this. |
| 482 | * This is typically used when we are creating PMD entries. |
| 483 | * |
| 484 | * clean_pmd_entry |
| 485 | * |
| 486 | * Clean (but don't drain the write buffer) if the CPU requires |
| 487 | * these operations. This is typically used when we are removing |
| 488 | * PMD entries. |
| 489 | */ |
Catalin Marinas | 442e70c | 2011-09-05 17:51:56 +0100 | [diff] [blame] | 490 | static inline void flush_pmd_entry(void *pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 491 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 493 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 494 | tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd); |
| 495 | tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); |
Lennert Buytenhek | 99c6dc1 | 2008-06-22 22:45:04 +0200 | [diff] [blame] | 496 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 497 | if (tlb_flag(TLB_WB)) |
Catalin Marinas | e6a5d66 | 2007-02-05 14:47:51 +0100 | [diff] [blame] | 498 | dsb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 499 | } |
| 500 | |
Catalin Marinas | 442e70c | 2011-09-05 17:51:56 +0100 | [diff] [blame] | 501 | static inline void clean_pmd_entry(void *pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | { |
| 503 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
| 504 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 505 | tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd); |
| 506 | tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 507 | } |
| 508 | |
Russell King | 87067a9 | 2012-02-04 10:55:38 +0000 | [diff] [blame] | 509 | #undef tlb_op |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 510 | #undef tlb_flag |
| 511 | #undef always_tlb_flags |
| 512 | #undef possible_tlb_flags |
| 513 | |
| 514 | /* |
| 515 | * Convert calls to our calling convention. |
| 516 | */ |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 517 | #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) |
| 518 | #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) |
| 519 | |
| 520 | #ifndef CONFIG_SMP |
| 521 | #define flush_tlb_all local_flush_tlb_all |
| 522 | #define flush_tlb_mm local_flush_tlb_mm |
| 523 | #define flush_tlb_page local_flush_tlb_page |
| 524 | #define flush_tlb_kernel_page local_flush_tlb_kernel_page |
| 525 | #define flush_tlb_range local_flush_tlb_range |
| 526 | #define flush_tlb_kernel_range local_flush_tlb_kernel_range |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 527 | #define flush_bp_all local_flush_bp_all |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 528 | #else |
| 529 | extern void flush_tlb_all(void); |
| 530 | extern void flush_tlb_mm(struct mm_struct *mm); |
| 531 | extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); |
| 532 | extern void flush_tlb_kernel_page(unsigned long kaddr); |
| 533 | extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); |
| 534 | extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); |
Will Deacon | 862c588 | 2013-02-28 17:48:11 +0100 | [diff] [blame] | 535 | extern void flush_bp_all(void); |
Russell King | 603fff5 | 2005-06-28 13:40:39 +0100 | [diff] [blame] | 536 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | |
| 538 | /* |
Catalin Marinas | c017780 | 2010-09-13 15:57:36 +0100 | [diff] [blame] | 539 | * If PG_dcache_clean is not set for the page, we need to ensure that any |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 540 | * cache entries for the kernels virtual memory range are written |
Catalin Marinas | 6012191 | 2010-09-13 15:58:06 +0100 | [diff] [blame] | 541 | * back to the page. On ARMv6 and later, the cache coherency is handled via |
| 542 | * the set_pte_at() function. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 543 | */ |
Catalin Marinas | 6012191 | 2010-09-13 15:58:06 +0100 | [diff] [blame] | 544 | #if __LINUX_ARM_ARCH__ < 6 |
Russell King | 4b3073e | 2009-12-18 16:40:18 +0000 | [diff] [blame] | 545 | extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, |
| 546 | pte_t *ptep); |
Catalin Marinas | 6012191 | 2010-09-13 15:58:06 +0100 | [diff] [blame] | 547 | #else |
| 548 | static inline void update_mmu_cache(struct vm_area_struct *vma, |
| 549 | unsigned long addr, pte_t *ptep) |
| 550 | { |
| 551 | } |
| 552 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 553 | |
Catalin Marinas | 8d96250 | 2012-07-25 14:39:26 +0100 | [diff] [blame] | 554 | #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) |
| 555 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 556 | #endif |
| 557 | |
Will Deacon | 5c709e6 | 2012-02-28 12:56:06 +0000 | [diff] [blame] | 558 | #elif defined(CONFIG_SMP) /* !CONFIG_MMU */ |
| 559 | |
| 560 | #ifndef __ASSEMBLY__ |
| 561 | |
| 562 | #include <linux/mm_types.h> |
| 563 | |
| 564 | static inline void local_flush_tlb_all(void) { } |
| 565 | static inline void local_flush_tlb_mm(struct mm_struct *mm) { } |
| 566 | static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { } |
| 567 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) { } |
| 568 | static inline void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { } |
| 569 | static inline void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) { } |
Jonathan Austin | 8d655d8 | 2013-04-24 11:56:09 +0100 | [diff] [blame] | 570 | static inline void local_flush_bp_all(void) { } |
Will Deacon | 5c709e6 | 2012-02-28 12:56:06 +0000 | [diff] [blame] | 571 | |
| 572 | extern void flush_tlb_all(void); |
| 573 | extern void flush_tlb_mm(struct mm_struct *mm); |
| 574 | extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); |
| 575 | extern void flush_tlb_kernel_page(unsigned long kaddr); |
| 576 | extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); |
| 577 | extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); |
Jonathan Austin | 8d655d8 | 2013-04-24 11:56:09 +0100 | [diff] [blame] | 578 | extern void flush_bp_all(void); |
Will Deacon | 5c709e6 | 2012-02-28 12:56:06 +0000 | [diff] [blame] | 579 | #endif /* __ASSEMBLY__ */ |
| 580 | |
| 581 | #endif |
Hyok S. Choi | 0157903 | 2006-02-24 21:41:25 +0000 | [diff] [blame] | 582 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 583 | #endif |