Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * r2300.c: R2000 and R3000 specific mmu/cache code. |
| 3 | * |
Justin P. Mattock | 79add62 | 2011-04-04 14:15:29 -0700 | [diff] [blame] | 4 | * Copyright (C) 1996 David S. Miller (davem@davemloft.net) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * |
| 6 | * with a lot of changes to make this thing work for R3000s |
| 7 | * Tx39XX R4k style caches added. HK |
| 8 | * Copyright (C) 1998, 1999, 2000 Harald Koerfgen |
| 9 | * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov |
Maciej W. Rozycki | 21b2aec | 2007-10-17 11:51:39 +0100 | [diff] [blame] | 10 | * Copyright (C) 2001, 2004, 2007 Maciej W. Rozycki |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/kernel.h> |
| 13 | #include <linux/sched.h> |
Ralf Baechle | 631330f | 2009-06-19 14:05:26 +0100 | [diff] [blame] | 14 | #include <linux/smp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/mm.h> |
| 16 | |
| 17 | #include <asm/page.h> |
| 18 | #include <asm/pgtable.h> |
| 19 | #include <asm/mmu_context.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <asm/isadep.h> |
| 21 | #include <asm/io.h> |
| 22 | #include <asm/bootinfo.h> |
| 23 | #include <asm/cpu.h> |
| 24 | |
| 25 | static unsigned long icache_size, dcache_size; /* Size in bytes */ |
| 26 | static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ |
| 27 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 28 | unsigned long r3k_cache_size(unsigned long ca_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | { |
| 30 | unsigned long flags, status, dummy, size; |
| 31 | volatile unsigned long *p; |
| 32 | |
| 33 | p = (volatile unsigned long *) KSEG0; |
| 34 | |
| 35 | flags = read_c0_status(); |
| 36 | |
| 37 | /* isolate cache space */ |
| 38 | write_c0_status((ca_flags|flags)&~ST0_IEC); |
| 39 | |
| 40 | *p = 0xa5a55a5a; |
| 41 | dummy = *p; |
| 42 | status = read_c0_status(); |
| 43 | |
| 44 | if (dummy != 0xa5a55a5a || (status & ST0_CM)) { |
| 45 | size = 0; |
| 46 | } else { |
| 47 | for (size = 128; size <= 0x40000; size <<= 1) |
| 48 | *(p + size) = 0; |
| 49 | *p = -1; |
| 50 | for (size = 128; |
| 51 | (size <= 0x40000) && (*(p + size) == 0); |
| 52 | size <<= 1) |
| 53 | ; |
| 54 | if (size > 0x40000) |
| 55 | size = 0; |
| 56 | } |
| 57 | |
| 58 | write_c0_status(flags); |
| 59 | |
| 60 | return size * sizeof(*p); |
| 61 | } |
| 62 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 63 | unsigned long r3k_cache_lsize(unsigned long ca_flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | { |
| 65 | unsigned long flags, status, lsize, i; |
| 66 | volatile unsigned long *p; |
| 67 | |
| 68 | p = (volatile unsigned long *) KSEG0; |
| 69 | |
| 70 | flags = read_c0_status(); |
| 71 | |
| 72 | /* isolate cache space */ |
| 73 | write_c0_status((ca_flags|flags)&~ST0_IEC); |
| 74 | |
| 75 | for (i = 0; i < 128; i++) |
| 76 | *(p + i) = 0; |
| 77 | *(volatile unsigned char *)p = 0; |
| 78 | for (lsize = 1; lsize < 128; lsize <<= 1) { |
| 79 | *(p + lsize); |
| 80 | status = read_c0_status(); |
| 81 | if (!(status & ST0_CM)) |
| 82 | break; |
| 83 | } |
| 84 | for (i = 0; i < 128; i += lsize) |
| 85 | *(volatile unsigned char *)(p + i) = 0; |
| 86 | |
| 87 | write_c0_status(flags); |
| 88 | |
| 89 | return lsize * sizeof(*p); |
| 90 | } |
| 91 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 92 | static void r3k_probe_cache(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | { |
| 94 | dcache_size = r3k_cache_size(ST0_ISC); |
| 95 | if (dcache_size) |
| 96 | dcache_lsize = r3k_cache_lsize(ST0_ISC); |
| 97 | |
| 98 | icache_size = r3k_cache_size(ST0_ISC|ST0_SWC); |
| 99 | if (icache_size) |
| 100 | icache_lsize = r3k_cache_lsize(ST0_ISC|ST0_SWC); |
| 101 | } |
| 102 | |
| 103 | static void r3k_flush_icache_range(unsigned long start, unsigned long end) |
| 104 | { |
| 105 | unsigned long size, i, flags; |
| 106 | volatile unsigned char *p; |
| 107 | |
| 108 | size = end - start; |
| 109 | if (size > icache_size || KSEGX(start) != KSEG0) { |
| 110 | start = KSEG0; |
| 111 | size = icache_size; |
| 112 | } |
| 113 | p = (char *)start; |
| 114 | |
| 115 | flags = read_c0_status(); |
| 116 | |
| 117 | /* isolate cache space */ |
| 118 | write_c0_status((ST0_ISC|ST0_SWC|flags)&~ST0_IEC); |
| 119 | |
| 120 | for (i = 0; i < size; i += 0x080) { |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 121 | asm( "sb\t$0, 0x000(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | "sb\t$0, 0x004(%0)\n\t" |
| 123 | "sb\t$0, 0x008(%0)\n\t" |
| 124 | "sb\t$0, 0x00c(%0)\n\t" |
| 125 | "sb\t$0, 0x010(%0)\n\t" |
| 126 | "sb\t$0, 0x014(%0)\n\t" |
| 127 | "sb\t$0, 0x018(%0)\n\t" |
| 128 | "sb\t$0, 0x01c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 129 | "sb\t$0, 0x020(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | "sb\t$0, 0x024(%0)\n\t" |
| 131 | "sb\t$0, 0x028(%0)\n\t" |
| 132 | "sb\t$0, 0x02c(%0)\n\t" |
| 133 | "sb\t$0, 0x030(%0)\n\t" |
| 134 | "sb\t$0, 0x034(%0)\n\t" |
| 135 | "sb\t$0, 0x038(%0)\n\t" |
| 136 | "sb\t$0, 0x03c(%0)\n\t" |
| 137 | "sb\t$0, 0x040(%0)\n\t" |
| 138 | "sb\t$0, 0x044(%0)\n\t" |
| 139 | "sb\t$0, 0x048(%0)\n\t" |
| 140 | "sb\t$0, 0x04c(%0)\n\t" |
| 141 | "sb\t$0, 0x050(%0)\n\t" |
| 142 | "sb\t$0, 0x054(%0)\n\t" |
| 143 | "sb\t$0, 0x058(%0)\n\t" |
| 144 | "sb\t$0, 0x05c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 145 | "sb\t$0, 0x060(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | "sb\t$0, 0x064(%0)\n\t" |
| 147 | "sb\t$0, 0x068(%0)\n\t" |
| 148 | "sb\t$0, 0x06c(%0)\n\t" |
| 149 | "sb\t$0, 0x070(%0)\n\t" |
| 150 | "sb\t$0, 0x074(%0)\n\t" |
| 151 | "sb\t$0, 0x078(%0)\n\t" |
| 152 | "sb\t$0, 0x07c(%0)\n\t" |
| 153 | : : "r" (p) ); |
| 154 | p += 0x080; |
| 155 | } |
| 156 | |
| 157 | write_c0_status(flags); |
| 158 | } |
| 159 | |
| 160 | static void r3k_flush_dcache_range(unsigned long start, unsigned long end) |
| 161 | { |
| 162 | unsigned long size, i, flags; |
| 163 | volatile unsigned char *p; |
| 164 | |
| 165 | size = end - start; |
| 166 | if (size > dcache_size || KSEGX(start) != KSEG0) { |
| 167 | start = KSEG0; |
| 168 | size = dcache_size; |
| 169 | } |
| 170 | p = (char *)start; |
| 171 | |
| 172 | flags = read_c0_status(); |
| 173 | |
| 174 | /* isolate cache space */ |
| 175 | write_c0_status((ST0_ISC|flags)&~ST0_IEC); |
| 176 | |
| 177 | for (i = 0; i < size; i += 0x080) { |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 178 | asm( "sb\t$0, 0x000(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | "sb\t$0, 0x004(%0)\n\t" |
| 180 | "sb\t$0, 0x008(%0)\n\t" |
| 181 | "sb\t$0, 0x00c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 182 | "sb\t$0, 0x010(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 183 | "sb\t$0, 0x014(%0)\n\t" |
| 184 | "sb\t$0, 0x018(%0)\n\t" |
| 185 | "sb\t$0, 0x01c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 186 | "sb\t$0, 0x020(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 187 | "sb\t$0, 0x024(%0)\n\t" |
| 188 | "sb\t$0, 0x028(%0)\n\t" |
| 189 | "sb\t$0, 0x02c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 190 | "sb\t$0, 0x030(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | "sb\t$0, 0x034(%0)\n\t" |
| 192 | "sb\t$0, 0x038(%0)\n\t" |
| 193 | "sb\t$0, 0x03c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 194 | "sb\t$0, 0x040(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | "sb\t$0, 0x044(%0)\n\t" |
| 196 | "sb\t$0, 0x048(%0)\n\t" |
| 197 | "sb\t$0, 0x04c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 198 | "sb\t$0, 0x050(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | "sb\t$0, 0x054(%0)\n\t" |
| 200 | "sb\t$0, 0x058(%0)\n\t" |
| 201 | "sb\t$0, 0x05c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 202 | "sb\t$0, 0x060(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | "sb\t$0, 0x064(%0)\n\t" |
| 204 | "sb\t$0, 0x068(%0)\n\t" |
| 205 | "sb\t$0, 0x06c(%0)\n\t" |
Ralf Baechle | a3dddd5 | 2006-03-11 08:18:41 +0000 | [diff] [blame] | 206 | "sb\t$0, 0x070(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | "sb\t$0, 0x074(%0)\n\t" |
| 208 | "sb\t$0, 0x078(%0)\n\t" |
| 209 | "sb\t$0, 0x07c(%0)\n\t" |
| 210 | : : "r" (p) ); |
| 211 | p += 0x080; |
| 212 | } |
| 213 | |
| 214 | write_c0_status(flags); |
| 215 | } |
| 216 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 217 | static inline void r3k_flush_cache_all(void) |
| 218 | { |
| 219 | } |
| 220 | |
| 221 | static inline void r3k___flush_cache_all(void) |
| 222 | { |
| 223 | r3k_flush_dcache_range(KSEG0, KSEG0 + dcache_size); |
| 224 | r3k_flush_icache_range(KSEG0, KSEG0 + icache_size); |
| 225 | } |
| 226 | |
| 227 | static void r3k_flush_cache_mm(struct mm_struct *mm) |
| 228 | { |
| 229 | } |
| 230 | |
| 231 | static void r3k_flush_cache_range(struct vm_area_struct *vma, |
Maciej W. Rozycki | 21b2aec | 2007-10-17 11:51:39 +0100 | [diff] [blame] | 232 | unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 233 | { |
| 234 | } |
| 235 | |
Maciej W. Rozycki | 21b2aec | 2007-10-17 11:51:39 +0100 | [diff] [blame] | 236 | static void r3k_flush_cache_page(struct vm_area_struct *vma, |
| 237 | unsigned long addr, unsigned long pfn) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 238 | { |
Maciej W. Rozycki | 21b2aec | 2007-10-17 11:51:39 +0100 | [diff] [blame] | 239 | unsigned long kaddr = KSEG0ADDR(pfn << PAGE_SHIFT); |
| 240 | int exec = vma->vm_flags & VM_EXEC; |
| 241 | struct mm_struct *mm = vma->vm_mm; |
| 242 | pgd_t *pgdp; |
| 243 | pud_t *pudp; |
| 244 | pmd_t *pmdp; |
| 245 | pte_t *ptep; |
| 246 | |
| 247 | pr_debug("cpage[%08lx,%08lx]\n", |
| 248 | cpu_context(smp_processor_id(), mm), addr); |
| 249 | |
| 250 | /* No ASID => no such page in the cache. */ |
| 251 | if (cpu_context(smp_processor_id(), mm) == 0) |
| 252 | return; |
| 253 | |
| 254 | pgdp = pgd_offset(mm, addr); |
| 255 | pudp = pud_offset(pgdp, addr); |
| 256 | pmdp = pmd_offset(pudp, addr); |
| 257 | ptep = pte_offset(pmdp, addr); |
| 258 | |
| 259 | /* Invalid => no such page in the cache. */ |
| 260 | if (!(pte_val(*ptep) & _PAGE_PRESENT)) |
| 261 | return; |
| 262 | |
| 263 | r3k_flush_dcache_range(kaddr, kaddr + PAGE_SIZE); |
| 264 | if (exec) |
| 265 | r3k_flush_icache_range(kaddr, kaddr + PAGE_SIZE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | } |
| 267 | |
Ralf Baechle | e007566 | 2007-03-24 13:20:15 +0000 | [diff] [blame] | 268 | static void local_r3k_flush_data_cache_page(void *addr) |
Ralf Baechle | 7e3bfc7 | 2006-04-05 20:42:04 +0100 | [diff] [blame] | 269 | { |
| 270 | } |
| 271 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 272 | static void r3k_flush_data_cache_page(unsigned long addr) |
| 273 | { |
| 274 | } |
| 275 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | static void r3k_flush_cache_sigtramp(unsigned long addr) |
| 277 | { |
| 278 | unsigned long flags; |
| 279 | |
Maciej W. Rozycki | 21b2aec | 2007-10-17 11:51:39 +0100 | [diff] [blame] | 280 | pr_debug("csigtramp[%08lx]\n", addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 281 | |
| 282 | flags = read_c0_status(); |
| 283 | |
| 284 | write_c0_status(flags&~ST0_IEC); |
| 285 | |
| 286 | /* Fill the TLB to avoid an exception with caches isolated. */ |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 287 | asm( "lw\t$0, 0x000(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | "lw\t$0, 0x004(%0)\n\t" |
| 289 | : : "r" (addr) ); |
| 290 | |
| 291 | write_c0_status((ST0_ISC|ST0_SWC|flags)&~ST0_IEC); |
| 292 | |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 293 | asm( "sb\t$0, 0x000(%0)\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | "sb\t$0, 0x004(%0)\n\t" |
| 295 | : : "r" (addr) ); |
| 296 | |
| 297 | write_c0_status(flags); |
| 298 | } |
| 299 | |
Ralf Baechle | d9cdc901 | 2011-06-17 16:20:28 +0100 | [diff] [blame] | 300 | static void r3k_flush_kernel_vmap_range(unsigned long vaddr, int size) |
| 301 | { |
| 302 | BUG(); |
| 303 | } |
| 304 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 305 | static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size) |
| 306 | { |
| 307 | /* Catch bad driver code */ |
| 308 | BUG_ON(size == 0); |
| 309 | |
| 310 | iob(); |
| 311 | r3k_flush_dcache_range(start, start + size); |
| 312 | } |
| 313 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 314 | void r3k_cache_init(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 315 | { |
| 316 | extern void build_clear_page(void); |
| 317 | extern void build_copy_page(void); |
| 318 | |
| 319 | r3k_probe_cache(); |
| 320 | |
| 321 | flush_cache_all = r3k_flush_cache_all; |
| 322 | __flush_cache_all = r3k___flush_cache_all; |
| 323 | flush_cache_mm = r3k_flush_cache_mm; |
| 324 | flush_cache_range = r3k_flush_cache_range; |
| 325 | flush_cache_page = r3k_flush_cache_page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 326 | flush_icache_range = r3k_flush_icache_range; |
Thomas Bogendoerfer | e0cee3e | 2008-08-04 20:53:57 +0200 | [diff] [blame] | 327 | local_flush_icache_range = r3k_flush_icache_range; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 328 | |
Ralf Baechle | d9cdc901 | 2011-06-17 16:20:28 +0100 | [diff] [blame] | 329 | __flush_kernel_vmap_range = r3k_flush_kernel_vmap_range; |
| 330 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | flush_cache_sigtramp = r3k_flush_cache_sigtramp; |
Ralf Baechle | 7e3bfc7 | 2006-04-05 20:42:04 +0100 | [diff] [blame] | 332 | local_flush_data_cache_page = local_r3k_flush_data_cache_page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 333 | flush_data_cache_page = r3k_flush_data_cache_page; |
| 334 | |
| 335 | _dma_cache_wback_inv = r3k_dma_cache_wback_inv; |
| 336 | _dma_cache_wback = r3k_dma_cache_wback_inv; |
| 337 | _dma_cache_inv = r3k_dma_cache_wback_inv; |
| 338 | |
| 339 | printk("Primary instruction cache %ldkB, linesize %ld bytes.\n", |
| 340 | icache_size >> 10, icache_lsize); |
| 341 | printk("Primary data cache %ldkB, linesize %ld bytes.\n", |
| 342 | dcache_size >> 10, dcache_lsize); |
| 343 | |
| 344 | build_clear_page(); |
| 345 | build_copy_page(); |
| 346 | } |