Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Inline assembly cache operations. |
| 7 | * |
| 8 | * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) |
| 9 | * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org) |
| 10 | * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org) |
| 11 | */ |
| 12 | #ifndef _ASM_R4KCACHE_H |
| 13 | #define _ASM_R4KCACHE_H |
| 14 | |
| 15 | #include <asm/asm.h> |
| 16 | #include <asm/cacheops.h> |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 17 | #include <asm/cpu-features.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
| 19 | /* |
| 20 | * This macro return a properly sign-extended address suitable as base address |
| 21 | * for indexed cache operations. Two issues here: |
| 22 | * |
| 23 | * - The MIPS32 and MIPS64 specs permit an implementation to directly derive |
| 24 | * the index bits from the virtual address. This breaks with tradition |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 25 | * set by the R4000. To keep unpleasant surprises from happening we pick |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | * an address in KSEG0 / CKSEG0. |
| 27 | * - We need a properly sign extended address for 64-bit code. To get away |
| 28 | * without ifdefs we let the compiler do it by a type cast. |
| 29 | */ |
| 30 | #define INDEX_BASE CKSEG0 |
| 31 | |
| 32 | #define cache_op(op,addr) \ |
| 33 | __asm__ __volatile__( \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 34 | " .set push \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | " .set noreorder \n" \ |
| 36 | " .set mips3\n\t \n" \ |
| 37 | " cache %0, %1 \n" \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 38 | " .set pop \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | : \ |
| 40 | : "i" (op), "m" (*(unsigned char *)(addr))) |
| 41 | |
| 42 | static inline void flush_icache_line_indexed(unsigned long addr) |
| 43 | { |
| 44 | cache_op(Index_Invalidate_I, addr); |
| 45 | } |
| 46 | |
| 47 | static inline void flush_dcache_line_indexed(unsigned long addr) |
| 48 | { |
| 49 | cache_op(Index_Writeback_Inv_D, addr); |
| 50 | } |
| 51 | |
| 52 | static inline void flush_scache_line_indexed(unsigned long addr) |
| 53 | { |
| 54 | cache_op(Index_Writeback_Inv_SD, addr); |
| 55 | } |
| 56 | |
| 57 | static inline void flush_icache_line(unsigned long addr) |
| 58 | { |
| 59 | cache_op(Hit_Invalidate_I, addr); |
| 60 | } |
| 61 | |
| 62 | static inline void flush_dcache_line(unsigned long addr) |
| 63 | { |
| 64 | cache_op(Hit_Writeback_Inv_D, addr); |
| 65 | } |
| 66 | |
| 67 | static inline void invalidate_dcache_line(unsigned long addr) |
| 68 | { |
| 69 | cache_op(Hit_Invalidate_D, addr); |
| 70 | } |
| 71 | |
| 72 | static inline void invalidate_scache_line(unsigned long addr) |
| 73 | { |
| 74 | cache_op(Hit_Invalidate_SD, addr); |
| 75 | } |
| 76 | |
| 77 | static inline void flush_scache_line(unsigned long addr) |
| 78 | { |
| 79 | cache_op(Hit_Writeback_Inv_SD, addr); |
| 80 | } |
| 81 | |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 82 | #define protected_cache_op(op,addr) \ |
| 83 | __asm__ __volatile__( \ |
| 84 | " .set push \n" \ |
| 85 | " .set noreorder \n" \ |
| 86 | " .set mips3 \n" \ |
| 87 | "1: cache %0, (%1) \n" \ |
| 88 | "2: .set pop \n" \ |
| 89 | " .section __ex_table,\"a\" \n" \ |
| 90 | " "STR(PTR)" 1b, 2b \n" \ |
| 91 | " .previous" \ |
| 92 | : \ |
| 93 | : "i" (op), "r" (addr)) |
| 94 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 95 | /* |
| 96 | * The next two are for badland addresses like signal trampolines. |
| 97 | */ |
| 98 | static inline void protected_flush_icache_line(unsigned long addr) |
| 99 | { |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 100 | protected_cache_op(Hit_Invalidate_I, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | } |
| 102 | |
| 103 | /* |
| 104 | * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D |
| 105 | * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style |
| 106 | * caches. We're talking about one cacheline unnecessarily getting invalidated |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 107 | * here so the penalty isn't overly hard. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | */ |
| 109 | static inline void protected_writeback_dcache_line(unsigned long addr) |
| 110 | { |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 111 | protected_cache_op(Hit_Writeback_Inv_D, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | } |
| 113 | |
| 114 | static inline void protected_writeback_scache_line(unsigned long addr) |
| 115 | { |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 116 | protected_cache_op(Hit_Writeback_Inv_SD, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | } |
| 118 | |
| 119 | /* |
| 120 | * This one is RM7000-specific |
| 121 | */ |
| 122 | static inline void invalidate_tcache_page(unsigned long addr) |
| 123 | { |
| 124 | cache_op(Page_Invalidate_T, addr); |
| 125 | } |
| 126 | |
| 127 | #define cache16_unroll32(base,op) \ |
| 128 | __asm__ __volatile__( \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 129 | " .set push \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | " .set noreorder \n" \ |
| 131 | " .set mips3 \n" \ |
| 132 | " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \ |
| 133 | " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \ |
| 134 | " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \ |
| 135 | " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \ |
| 136 | " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \ |
| 137 | " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \ |
| 138 | " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \ |
| 139 | " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \ |
| 140 | " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \ |
| 141 | " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \ |
| 142 | " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \ |
| 143 | " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \ |
| 144 | " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \ |
| 145 | " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \ |
| 146 | " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \ |
| 147 | " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 148 | " .set pop \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 149 | : \ |
| 150 | : "r" (base), \ |
| 151 | "i" (op)); |
| 152 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | #define cache32_unroll32(base,op) \ |
| 154 | __asm__ __volatile__( \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 155 | " .set push \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | " .set noreorder \n" \ |
| 157 | " .set mips3 \n" \ |
| 158 | " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \ |
| 159 | " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \ |
| 160 | " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \ |
| 161 | " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \ |
| 162 | " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \ |
| 163 | " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \ |
| 164 | " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \ |
| 165 | " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \ |
| 166 | " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \ |
| 167 | " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \ |
| 168 | " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \ |
| 169 | " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \ |
| 170 | " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \ |
| 171 | " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \ |
| 172 | " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \ |
| 173 | " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 174 | " .set pop \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | : \ |
| 176 | : "r" (base), \ |
| 177 | "i" (op)); |
| 178 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | #define cache64_unroll32(base,op) \ |
| 180 | __asm__ __volatile__( \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 181 | " .set push \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | " .set noreorder \n" \ |
| 183 | " .set mips3 \n" \ |
| 184 | " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \ |
| 185 | " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \ |
| 186 | " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \ |
| 187 | " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \ |
| 188 | " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \ |
| 189 | " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \ |
| 190 | " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \ |
| 191 | " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \ |
| 192 | " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \ |
| 193 | " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \ |
| 194 | " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \ |
| 195 | " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \ |
| 196 | " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \ |
| 197 | " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \ |
| 198 | " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \ |
| 199 | " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 200 | " .set pop \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 201 | : \ |
| 202 | : "r" (base), \ |
| 203 | "i" (op)); |
| 204 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | #define cache128_unroll32(base,op) \ |
| 206 | __asm__ __volatile__( \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 207 | " .set push \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | " .set noreorder \n" \ |
| 209 | " .set mips3 \n" \ |
| 210 | " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \ |
| 211 | " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \ |
| 212 | " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \ |
| 213 | " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \ |
| 214 | " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \ |
| 215 | " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \ |
| 216 | " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \ |
| 217 | " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \ |
| 218 | " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \ |
| 219 | " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \ |
| 220 | " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \ |
| 221 | " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \ |
| 222 | " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \ |
| 223 | " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \ |
| 224 | " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \ |
| 225 | " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \ |
Thiemo Seufer | 2fe25f6 | 2005-09-01 08:59:55 +0000 | [diff] [blame] | 226 | " .set pop \n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 227 | : \ |
| 228 | : "r" (base), \ |
| 229 | "i" (op)); |
| 230 | |
Atsushi Nemoto | 76f072a | 2006-01-29 02:30:55 +0900 | [diff] [blame] | 231 | /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */ |
| 232 | #define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize) \ |
| 233 | static inline void blast_##pfx##cache##lsize(void) \ |
| 234 | { \ |
| 235 | unsigned long start = INDEX_BASE; \ |
| 236 | unsigned long end = start + current_cpu_data.desc.waysize; \ |
| 237 | unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ |
| 238 | unsigned long ws_end = current_cpu_data.desc.ways << \ |
| 239 | current_cpu_data.desc.waybit; \ |
| 240 | unsigned long ws, addr; \ |
| 241 | \ |
| 242 | for (ws = 0; ws < ws_end; ws += ws_inc) \ |
| 243 | for (addr = start; addr < end; addr += lsize * 32) \ |
| 244 | cache##lsize##_unroll32(addr|ws,indexop); \ |
| 245 | } \ |
| 246 | \ |
| 247 | static inline void blast_##pfx##cache##lsize##_page(unsigned long page) \ |
| 248 | { \ |
| 249 | unsigned long start = page; \ |
| 250 | unsigned long end = page + PAGE_SIZE; \ |
| 251 | \ |
| 252 | do { \ |
| 253 | cache##lsize##_unroll32(start,hitop); \ |
| 254 | start += lsize * 32; \ |
| 255 | } while (start < end); \ |
| 256 | } \ |
| 257 | \ |
| 258 | static inline void blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \ |
| 259 | { \ |
Atsushi Nemoto | de62893 | 2006-03-13 18:23:03 +0900 | [diff] [blame] | 260 | unsigned long indexmask = current_cpu_data.desc.waysize - 1; \ |
| 261 | unsigned long start = INDEX_BASE + (page & indexmask); \ |
Atsushi Nemoto | 76f072a | 2006-01-29 02:30:55 +0900 | [diff] [blame] | 262 | unsigned long end = start + PAGE_SIZE; \ |
| 263 | unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ |
| 264 | unsigned long ws_end = current_cpu_data.desc.ways << \ |
| 265 | current_cpu_data.desc.waybit; \ |
| 266 | unsigned long ws, addr; \ |
| 267 | \ |
| 268 | for (ws = 0; ws < ws_end; ws += ws_inc) \ |
| 269 | for (addr = start; addr < end; addr += lsize * 32) \ |
| 270 | cache##lsize##_unroll32(addr|ws,indexop); \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 271 | } |
| 272 | |
Atsushi Nemoto | 76f072a | 2006-01-29 02:30:55 +0900 | [diff] [blame] | 273 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16) |
| 274 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16) |
| 275 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16) |
| 276 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32) |
| 277 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32) |
| 278 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32) |
| 279 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64) |
| 280 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64) |
| 281 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | |
Atsushi Nemoto | 41700e7 | 2006-02-10 00:39:06 +0900 | [diff] [blame] | 283 | /* build blast_xxx_range, protected_blast_xxx_range */ |
| 284 | #define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot) \ |
| 285 | static inline void prot##blast_##pfx##cache##_range(unsigned long start, \ |
| 286 | unsigned long end) \ |
| 287 | { \ |
| 288 | unsigned long lsize = cpu_##desc##_line_size(); \ |
| 289 | unsigned long addr = start & ~(lsize - 1); \ |
| 290 | unsigned long aend = (end - 1) & ~(lsize - 1); \ |
| 291 | while (1) { \ |
| 292 | prot##cache_op(hitop, addr); \ |
| 293 | if (addr == aend) \ |
| 294 | break; \ |
| 295 | addr += lsize; \ |
| 296 | } \ |
| 297 | } |
| 298 | |
| 299 | __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_) |
| 300 | __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_) |
| 301 | __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_) |
| 302 | __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, ) |
| 303 | __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, ) |
| 304 | /* blast_inv_dcache_range */ |
| 305 | __BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, ) |
| 306 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 307 | #endif /* _ASM_R4KCACHE_H */ |