blob: 0f7d788e88106fafa98e824d8b6d22f0bdcd2067 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Justin P. Mattock79add622011-04-04 14:15:29 -07006 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
Ralf Baechlea754f702007-11-03 01:01:37 +000010#include <linux/hardirq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Ralf Baechledb813fe2007-09-27 18:26:43 +010012#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/kernel.h>
Ralf Baechle641e97f2007-10-11 23:46:05 +010014#include <linux/linkage.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010016#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/mm.h>
Chris Dearman35133692007-09-19 00:58:24 +010018#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/bitops.h>
20
21#include <asm/bcache.h>
22#include <asm/bootinfo.h>
Ralf Baechleec74e362005-07-13 11:48:45 +000023#include <asm/cache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <asm/cacheops.h>
25#include <asm/cpu.h>
26#include <asm/cpu-features.h>
27#include <asm/io.h>
28#include <asm/page.h>
29#include <asm/pgtable.h>
30#include <asm/r4kcache.h>
Ralf Baechlee001e522007-07-28 12:45:47 +010031#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <asm/mmu_context.h>
33#include <asm/war.h>
Thiemo Seuferba5187d2005-04-25 16:36:23 +000034#include <asm/cacheflush.h> /* for run_uncached() */
David Daney9cd9669b2012-05-15 00:04:49 -070035#include <asm/traps.h>
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010036
37/*
38 * Special Variant of smp_call_function for use by cache functions:
39 *
40 * o No return value
41 * o collapses to normal function call on UP kernels
42 * o collapses to normal function call on systems with a single shared
43 * primary cache.
Ralf Baechlec8c5f3f2010-10-29 19:08:25 +010044 * o doesn't disable interrupts on the local CPU
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010045 */
Ralf Baechle48a26e62010-10-29 19:08:25 +010046static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010047{
48 preempt_disable();
49
50#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
Ralf Baechle48a26e62010-10-29 19:08:25 +010051 smp_call_function(func, info, 1);
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010052#endif
53 func(info);
54 preempt_enable();
55}
56
Ralf Baechle39b8d522008-04-28 17:14:26 +010057#if defined(CONFIG_MIPS_CMP)
58#define cpu_has_safe_index_cacheops 0
59#else
60#define cpu_has_safe_index_cacheops 1
61#endif
62
Ralf Baechleec74e362005-07-13 11:48:45 +000063/*
64 * Must die.
65 */
66static unsigned long icache_size __read_mostly;
67static unsigned long dcache_size __read_mostly;
68static unsigned long scache_size __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
70/*
71 * Dummy cache handling routines for machines without boardcaches
72 */
Chris Dearman73f40352006-06-20 18:06:52 +010073static void cache_noop(void) {}
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75static struct bcache_ops no_sc_ops = {
Chris Dearman73f40352006-06-20 18:06:52 +010076 .bc_enable = (void *)cache_noop,
77 .bc_disable = (void *)cache_noop,
78 .bc_wback_inv = (void *)cache_noop,
79 .bc_inv = (void *)cache_noop
Linus Torvalds1da177e2005-04-16 15:20:36 -070080};
81
82struct bcache_ops *bcops = &no_sc_ops;
83
Thiemo Seufer330cfe02005-09-01 18:33:58 +000084#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
85#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
87#define R4600_HIT_CACHEOP_WAR_IMPL \
88do { \
89 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
90 *(volatile unsigned long *)CKSEG1; \
91 if (R4600_V1_HIT_CACHEOP_WAR) \
92 __asm__ __volatile__("nop;nop;nop;nop"); \
93} while (0)
94
95static void (*r4k_blast_dcache_page)(unsigned long addr);
96
97static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
98{
99 R4600_HIT_CACHEOP_WAR_IMPL;
100 blast_dcache32_page(addr);
101}
102
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700103static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
104{
105 R4600_HIT_CACHEOP_WAR_IMPL;
106 blast_dcache64_page(addr);
107}
108
Ralf Baechle234fcd12008-03-08 09:56:28 +0000109static void __cpuinit r4k_blast_dcache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110{
111 unsigned long dc_lsize = cpu_dcache_line_size();
112
Chris Dearman73f40352006-06-20 18:06:52 +0100113 if (dc_lsize == 0)
114 r4k_blast_dcache_page = (void *)cache_noop;
115 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 r4k_blast_dcache_page = blast_dcache16_page;
117 else if (dc_lsize == 32)
118 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700119 else if (dc_lsize == 64)
120 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121}
122
123static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
124
Ralf Baechle234fcd12008-03-08 09:56:28 +0000125static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
127 unsigned long dc_lsize = cpu_dcache_line_size();
128
Chris Dearman73f40352006-06-20 18:06:52 +0100129 if (dc_lsize == 0)
130 r4k_blast_dcache_page_indexed = (void *)cache_noop;
131 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
133 else if (dc_lsize == 32)
134 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700135 else if (dc_lsize == 64)
136 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137}
138
139static void (* r4k_blast_dcache)(void);
140
Ralf Baechle234fcd12008-03-08 09:56:28 +0000141static void __cpuinit r4k_blast_dcache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142{
143 unsigned long dc_lsize = cpu_dcache_line_size();
144
Chris Dearman73f40352006-06-20 18:06:52 +0100145 if (dc_lsize == 0)
146 r4k_blast_dcache = (void *)cache_noop;
147 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 r4k_blast_dcache = blast_dcache16;
149 else if (dc_lsize == 32)
150 r4k_blast_dcache = blast_dcache32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700151 else if (dc_lsize == 64)
152 r4k_blast_dcache = blast_dcache64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153}
154
155/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
156#define JUMP_TO_ALIGN(order) \
157 __asm__ __volatile__( \
158 "b\t1f\n\t" \
159 ".align\t" #order "\n\t" \
160 "1:\n\t" \
161 )
162#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
163#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
164
165static inline void blast_r4600_v1_icache32(void)
166{
167 unsigned long flags;
168
169 local_irq_save(flags);
170 blast_icache32();
171 local_irq_restore(flags);
172}
173
174static inline void tx49_blast_icache32(void)
175{
176 unsigned long start = INDEX_BASE;
177 unsigned long end = start + current_cpu_data.icache.waysize;
178 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
179 unsigned long ws_end = current_cpu_data.icache.ways <<
180 current_cpu_data.icache.waybit;
181 unsigned long ws, addr;
182
183 CACHE32_UNROLL32_ALIGN2;
184 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700185 for (ws = 0; ws < ws_end; ws += ws_inc)
186 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100187 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 CACHE32_UNROLL32_ALIGN;
189 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700190 for (ws = 0; ws < ws_end; ws += ws_inc)
191 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100192 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193}
194
195static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
196{
197 unsigned long flags;
198
199 local_irq_save(flags);
200 blast_icache32_page_indexed(page);
201 local_irq_restore(flags);
202}
203
204static inline void tx49_blast_icache32_page_indexed(unsigned long page)
205{
Atsushi Nemoto67a3f6de2006-04-04 17:34:14 +0900206 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
207 unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 unsigned long end = start + PAGE_SIZE;
209 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
210 unsigned long ws_end = current_cpu_data.icache.ways <<
211 current_cpu_data.icache.waybit;
212 unsigned long ws, addr;
213
214 CACHE32_UNROLL32_ALIGN2;
215 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700216 for (ws = 0; ws < ws_end; ws += ws_inc)
217 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100218 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 CACHE32_UNROLL32_ALIGN;
220 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700221 for (ws = 0; ws < ws_end; ws += ws_inc)
222 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100223 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224}
225
226static void (* r4k_blast_icache_page)(unsigned long addr);
227
Ralf Baechle234fcd12008-03-08 09:56:28 +0000228static void __cpuinit r4k_blast_icache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229{
230 unsigned long ic_lsize = cpu_icache_line_size();
231
Chris Dearman73f40352006-06-20 18:06:52 +0100232 if (ic_lsize == 0)
233 r4k_blast_icache_page = (void *)cache_noop;
234 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 r4k_blast_icache_page = blast_icache16_page;
236 else if (ic_lsize == 32)
237 r4k_blast_icache_page = blast_icache32_page;
238 else if (ic_lsize == 64)
239 r4k_blast_icache_page = blast_icache64_page;
240}
241
242
243static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
244
Ralf Baechle234fcd12008-03-08 09:56:28 +0000245static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246{
247 unsigned long ic_lsize = cpu_icache_line_size();
248
Chris Dearman73f40352006-06-20 18:06:52 +0100249 if (ic_lsize == 0)
250 r4k_blast_icache_page_indexed = (void *)cache_noop;
251 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
253 else if (ic_lsize == 32) {
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000254 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 r4k_blast_icache_page_indexed =
256 blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000257 else if (TX49XX_ICACHE_INDEX_INV_WAR)
258 r4k_blast_icache_page_indexed =
259 tx49_blast_icache32_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 else
261 r4k_blast_icache_page_indexed =
262 blast_icache32_page_indexed;
263 } else if (ic_lsize == 64)
264 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
265}
266
267static void (* r4k_blast_icache)(void);
268
Ralf Baechle234fcd12008-03-08 09:56:28 +0000269static void __cpuinit r4k_blast_icache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270{
271 unsigned long ic_lsize = cpu_icache_line_size();
272
Chris Dearman73f40352006-06-20 18:06:52 +0100273 if (ic_lsize == 0)
274 r4k_blast_icache = (void *)cache_noop;
275 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 r4k_blast_icache = blast_icache16;
277 else if (ic_lsize == 32) {
278 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
279 r4k_blast_icache = blast_r4600_v1_icache32;
280 else if (TX49XX_ICACHE_INDEX_INV_WAR)
281 r4k_blast_icache = tx49_blast_icache32;
282 else
283 r4k_blast_icache = blast_icache32;
284 } else if (ic_lsize == 64)
285 r4k_blast_icache = blast_icache64;
286}
287
288static void (* r4k_blast_scache_page)(unsigned long addr);
289
Ralf Baechle234fcd12008-03-08 09:56:28 +0000290static void __cpuinit r4k_blast_scache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291{
292 unsigned long sc_lsize = cpu_scache_line_size();
293
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000294 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100295 r4k_blast_scache_page = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000296 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 r4k_blast_scache_page = blast_scache16_page;
298 else if (sc_lsize == 32)
299 r4k_blast_scache_page = blast_scache32_page;
300 else if (sc_lsize == 64)
301 r4k_blast_scache_page = blast_scache64_page;
302 else if (sc_lsize == 128)
303 r4k_blast_scache_page = blast_scache128_page;
304}
305
306static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
307
Ralf Baechle234fcd12008-03-08 09:56:28 +0000308static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309{
310 unsigned long sc_lsize = cpu_scache_line_size();
311
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000312 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100313 r4k_blast_scache_page_indexed = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000314 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
316 else if (sc_lsize == 32)
317 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
318 else if (sc_lsize == 64)
319 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
320 else if (sc_lsize == 128)
321 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
322}
323
324static void (* r4k_blast_scache)(void);
325
Ralf Baechle234fcd12008-03-08 09:56:28 +0000326static void __cpuinit r4k_blast_scache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327{
328 unsigned long sc_lsize = cpu_scache_line_size();
329
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000330 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100331 r4k_blast_scache = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000332 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 r4k_blast_scache = blast_scache16;
334 else if (sc_lsize == 32)
335 r4k_blast_scache = blast_scache32;
336 else if (sc_lsize == 64)
337 r4k_blast_scache = blast_scache64;
338 else if (sc_lsize == 128)
339 r4k_blast_scache = blast_scache128;
340}
341
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342static inline void local_r4k___flush_cache_all(void * args)
343{
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800344#if defined(CONFIG_CPU_LOONGSON2)
345 r4k_blast_scache();
346 return;
347#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 r4k_blast_dcache();
349 r4k_blast_icache();
350
Ralf Baechle10cc3522007-10-11 23:46:15 +0100351 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 case CPU_R4000SC:
353 case CPU_R4000MC:
354 case CPU_R4400SC:
355 case CPU_R4400MC:
356 case CPU_R10000:
357 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400358 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 r4k_blast_scache();
360 }
361}
362
363static void r4k___flush_cache_all(void)
364{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100365 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366}
367
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100368static inline int has_valid_asid(const struct mm_struct *mm)
369{
370#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
371 int i;
372
373 for_each_online_cpu(i)
374 if (cpu_context(i, mm))
375 return 1;
376
377 return 0;
378#else
379 return cpu_context(smp_processor_id(), mm);
380#endif
381}
382
Ralf Baechle9c5a3d72008-04-05 15:13:23 +0100383static void r4k__flush_cache_vmap(void)
384{
385 r4k_blast_dcache();
386}
387
388static void r4k__flush_cache_vunmap(void)
389{
390 r4k_blast_dcache();
391}
392
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393static inline void local_r4k_flush_cache_range(void * args)
394{
395 struct vm_area_struct *vma = args;
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000396 int exec = vma->vm_flags & VM_EXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100398 if (!(has_valid_asid(vma->vm_mm)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 return;
400
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900401 r4k_blast_dcache();
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000402 if (exec)
403 r4k_blast_icache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404}
405
406static void r4k_flush_cache_range(struct vm_area_struct *vma,
407 unsigned long start, unsigned long end)
408{
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000409 int exec = vma->vm_flags & VM_EXEC;
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900410
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000411 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
Ralf Baechle48a26e62010-10-29 19:08:25 +0100412 r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413}
414
415static inline void local_r4k_flush_cache_mm(void * args)
416{
417 struct mm_struct *mm = args;
418
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100419 if (!has_valid_asid(mm))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 return;
421
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 /*
423 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
424 * only flush the primary caches but R10000 and R12000 behave sane ...
Ralf Baechle617667b2006-11-30 01:14:48 +0000425 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
426 * caches, so we can bail out early.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 */
Ralf Baechle10cc3522007-10-11 23:46:15 +0100428 if (current_cpu_type() == CPU_R4000SC ||
429 current_cpu_type() == CPU_R4000MC ||
430 current_cpu_type() == CPU_R4400SC ||
431 current_cpu_type() == CPU_R4400MC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 r4k_blast_scache();
Ralf Baechle617667b2006-11-30 01:14:48 +0000433 return;
434 }
435
436 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
438
439static void r4k_flush_cache_mm(struct mm_struct *mm)
440{
441 if (!cpu_has_dc_aliases)
442 return;
443
Ralf Baechle48a26e62010-10-29 19:08:25 +0100444 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445}
446
447struct flush_cache_page_args {
448 struct vm_area_struct *vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100449 unsigned long addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900450 unsigned long pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451};
452
453static inline void local_r4k_flush_cache_page(void *args)
454{
455 struct flush_cache_page_args *fcp_args = args;
456 struct vm_area_struct *vma = fcp_args->vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100457 unsigned long addr = fcp_args->addr;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100458 struct page *page = pfn_to_page(fcp_args->pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 int exec = vma->vm_flags & VM_EXEC;
460 struct mm_struct *mm = vma->vm_mm;
Ralf Baechlec9c50232008-06-14 22:22:08 +0100461 int map_coherent = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 pgd_t *pgdp;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000463 pud_t *pudp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 pmd_t *pmdp;
465 pte_t *ptep;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100466 void *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
Ralf Baechle79acf832005-02-10 13:54:37 +0000468 /*
469 * If ownes no valid ASID yet, cannot possibly have gotten
470 * this page into the cache.
471 */
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100472 if (!has_valid_asid(mm))
Ralf Baechle79acf832005-02-10 13:54:37 +0000473 return;
474
Ralf Baechle6ec25802005-10-12 00:02:34 +0100475 addr &= PAGE_MASK;
476 pgdp = pgd_offset(mm, addr);
477 pudp = pud_offset(pgdp, addr);
478 pmdp = pmd_offset(pudp, addr);
479 ptep = pte_offset(pmdp, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480
481 /*
482 * If the page isn't marked valid, the page cannot possibly be
483 * in the cache.
484 */
Ralf Baechle526af352008-01-29 10:14:55 +0000485 if (!(pte_present(*ptep)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 return;
487
Ralf Baechledb813fe2007-09-27 18:26:43 +0100488 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
489 vaddr = NULL;
490 else {
491 /*
492 * Use kmap_coherent or kmap_atomic to do flushes for
493 * another ASID than the current one.
494 */
Ralf Baechlec9c50232008-06-14 22:22:08 +0100495 map_coherent = (cpu_has_dc_aliases &&
496 page_mapped(page) && !Page_dcache_dirty(page));
497 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100498 vaddr = kmap_coherent(page, addr);
499 else
Cong Wang9c020482011-11-25 23:14:15 +0800500 vaddr = kmap_atomic(page);
Ralf Baechledb813fe2007-09-27 18:26:43 +0100501 addr = (unsigned long)vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 }
503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100505 r4k_blast_dcache_page(addr);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100506 if (exec && !cpu_icache_snoops_remote_store)
507 r4k_blast_scache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 }
509 if (exec) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100510 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 int cpu = smp_processor_id();
512
Thiemo Seufer26a51b22005-02-19 13:32:02 +0000513 if (cpu_context(cpu, mm) != 0)
514 drop_mmu_context(mm, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 } else
Ralf Baechledb813fe2007-09-27 18:26:43 +0100516 r4k_blast_icache_page(addr);
517 }
518
519 if (vaddr) {
Ralf Baechlec9c50232008-06-14 22:22:08 +0100520 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100521 kunmap_coherent();
522 else
Cong Wang9c020482011-11-25 23:14:15 +0800523 kunmap_atomic(vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 }
525}
526
Ralf Baechle6ec25802005-10-12 00:02:34 +0100527static void r4k_flush_cache_page(struct vm_area_struct *vma,
528 unsigned long addr, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529{
530 struct flush_cache_page_args args;
531
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 args.vma = vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100533 args.addr = addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900534 args.pfn = pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
Ralf Baechle48a26e62010-10-29 19:08:25 +0100536 r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537}
538
539static inline void local_r4k_flush_data_cache_page(void * addr)
540{
541 r4k_blast_dcache_page((unsigned long) addr);
542}
543
544static void r4k_flush_data_cache_page(unsigned long addr)
545{
Ralf Baechlea754f702007-11-03 01:01:37 +0000546 if (in_atomic())
547 local_r4k_flush_data_cache_page((void *)addr);
548 else
Ralf Baechle48a26e62010-10-29 19:08:25 +0100549 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550}
551
552struct flush_icache_range_args {
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900553 unsigned long start;
554 unsigned long end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555};
556
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200557static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 if (!cpu_has_ic_fills_f_dc) {
Chris Dearman73f40352006-06-20 18:06:52 +0100560 if (end - start >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 r4k_blast_dcache();
562 } else {
Thiemo Seufer10a3dab2005-09-09 20:26:54 +0000563 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900564 protected_blast_dcache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 }
567
568 if (end - start > icache_size)
569 r4k_blast_icache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900570 else
571 protected_blast_icache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572}
573
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200574static inline void local_r4k_flush_icache_range_ipi(void *args)
575{
576 struct flush_icache_range_args *fir_args = args;
577 unsigned long start = fir_args->start;
578 unsigned long end = fir_args->end;
579
580 local_r4k_flush_icache_range(start, end);
581}
582
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900583static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584{
585 struct flush_icache_range_args args;
586
587 args.start = start;
588 args.end = end;
589
Ralf Baechle48a26e62010-10-29 19:08:25 +0100590 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
Ralf Baechlecc61c1f2005-07-12 18:35:38 +0000591 instruction_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592}
593
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594#ifdef CONFIG_DMA_NONCOHERENT
595
596static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
597{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 /* Catch bad driver code */
599 BUG_ON(size == 0);
600
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100601 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900602 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900604 else
605 blast_scache_range(addr, addr + size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700606 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 return;
608 }
609
610 /*
611 * Either no secondary cache or the available caches don't have the
612 * subset property so we have to flush the primary caches
613 * explicitly
614 */
Ralf Baechle39b8d522008-04-28 17:14:26 +0100615 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 r4k_blast_dcache();
617 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900619 blast_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 }
621
622 bc_wback_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700623 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624}
625
626static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
627{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 /* Catch bad driver code */
629 BUG_ON(size == 0);
630
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100631 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900632 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 r4k_blast_scache();
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000634 else {
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000635 /*
636 * There is no clearly documented alignment requirement
637 * for the cache instruction on MIPS processors and
638 * some processors, among them the RM5200 and RM7000
639 * QED processors will throw an address error for cache
640 * hit ops with insufficient alignment. Solved by
641 * aligning the address to cache line size.
642 */
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100643 blast_inv_scache_range(addr, addr + size);
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000644 }
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700645 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 return;
647 }
648
Ralf Baechle39b8d522008-04-28 17:14:26 +0100649 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 r4k_blast_dcache();
651 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 R4600_HIT_CACHEOP_WAR_IMPL;
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100653 blast_inv_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 }
655
656 bc_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700657 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658}
659#endif /* CONFIG_DMA_NONCOHERENT */
660
661/*
662 * While we're protected against bad userland addresses we don't care
663 * very much about what happens in that case. Usually a segmentation
664 * fault will dump the process later on anyway ...
665 */
666static void local_r4k_flush_cache_sigtramp(void * arg)
667{
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000668 unsigned long ic_lsize = cpu_icache_line_size();
669 unsigned long dc_lsize = cpu_dcache_line_size();
670 unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 unsigned long addr = (unsigned long) arg;
672
673 R4600_HIT_CACHEOP_WAR_IMPL;
Chris Dearman73f40352006-06-20 18:06:52 +0100674 if (dc_lsize)
675 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000676 if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
Chris Dearman73f40352006-06-20 18:06:52 +0100678 if (ic_lsize)
679 protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 if (MIPS4K_ICACHE_REFILL_WAR) {
681 __asm__ __volatile__ (
682 ".set push\n\t"
683 ".set noat\n\t"
684 ".set mips3\n\t"
Ralf Baechle875d43e2005-09-03 15:56:16 -0700685#ifdef CONFIG_32BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 "la $at,1f\n\t"
687#endif
Ralf Baechle875d43e2005-09-03 15:56:16 -0700688#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 "dla $at,1f\n\t"
690#endif
691 "cache %0,($at)\n\t"
692 "nop; nop; nop\n"
693 "1:\n\t"
694 ".set pop"
695 :
696 : "i" (Hit_Invalidate_I));
697 }
698 if (MIPS_CACHE_SYNC_WAR)
699 __asm__ __volatile__ ("sync");
700}
701
702static void r4k_flush_cache_sigtramp(unsigned long addr)
703{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100704 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705}
706
707static void r4k_flush_icache_all(void)
708{
709 if (cpu_has_vtag_icache)
710 r4k_blast_icache();
711}
712
Ralf Baechled9cdc9012011-06-17 16:20:28 +0100713struct flush_kernel_vmap_range_args {
714 unsigned long vaddr;
715 int size;
716};
717
718static inline void local_r4k_flush_kernel_vmap_range(void *args)
719{
720 struct flush_kernel_vmap_range_args *vmra = args;
721 unsigned long vaddr = vmra->vaddr;
722 int size = vmra->size;
723
724 /*
725 * Aliases only affect the primary caches so don't bother with
726 * S-caches or T-caches.
727 */
728 if (cpu_has_safe_index_cacheops && size >= dcache_size)
729 r4k_blast_dcache();
730 else {
731 R4600_HIT_CACHEOP_WAR_IMPL;
732 blast_dcache_range(vaddr, vaddr + size);
733 }
734}
735
736static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
737{
738 struct flush_kernel_vmap_range_args args;
739
740 args.vaddr = (unsigned long) vaddr;
741 args.size = size;
742
743 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
744}
745
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746static inline void rm7k_erratum31(void)
747{
748 const unsigned long ic_lsize = 32;
749 unsigned long addr;
750
751 /* RM7000 erratum #31. The icache is screwed at startup. */
752 write_c0_taglo(0);
753 write_c0_taghi(0);
754
755 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
756 __asm__ __volatile__ (
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000757 ".set push\n\t"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 ".set noreorder\n\t"
759 ".set mips3\n\t"
760 "cache\t%1, 0(%0)\n\t"
761 "cache\t%1, 0x1000(%0)\n\t"
762 "cache\t%1, 0x2000(%0)\n\t"
763 "cache\t%1, 0x3000(%0)\n\t"
764 "cache\t%2, 0(%0)\n\t"
765 "cache\t%2, 0x1000(%0)\n\t"
766 "cache\t%2, 0x2000(%0)\n\t"
767 "cache\t%2, 0x3000(%0)\n\t"
768 "cache\t%1, 0(%0)\n\t"
769 "cache\t%1, 0x1000(%0)\n\t"
770 "cache\t%1, 0x2000(%0)\n\t"
771 "cache\t%1, 0x3000(%0)\n\t"
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000772 ".set pop\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 :
774 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
775 }
776}
777
Steven J. Hill006a8512012-06-26 04:11:03 +0000778static inline void alias_74k_erratum(struct cpuinfo_mips *c)
779{
780 /*
781 * Early versions of the 74K do not update the cache tags on a
782 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
783 * aliases. In this case it is better to treat the cache as always
784 * having aliases.
785 */
786 if ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(2, 4, 0))
787 c->dcache.flags |= MIPS_CACHE_VTAG;
788 if ((c->processor_id & 0xff) == PRID_REV_ENCODE_332(2, 4, 0))
789 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
790 if (((c->processor_id & 0xff00) == PRID_IMP_1074K) &&
791 ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(1, 1, 0))) {
792 c->dcache.flags |= MIPS_CACHE_VTAG;
793 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
794 }
795}
796
Ralf Baechle234fcd12008-03-08 09:56:28 +0000797static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
799};
800
Ralf Baechle234fcd12008-03-08 09:56:28 +0000801static void __cpuinit probe_pcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802{
803 struct cpuinfo_mips *c = &current_cpu_data;
804 unsigned int config = read_c0_config();
805 unsigned int prid = read_c0_prid();
806 unsigned long config1;
807 unsigned int lsize;
808
809 switch (c->cputype) {
810 case CPU_R4600: /* QED style two way caches? */
811 case CPU_R4700:
812 case CPU_R5000:
813 case CPU_NEVADA:
814 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
815 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
816 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900817 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818
819 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
820 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
821 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900822 c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
824 c->options |= MIPS_CPU_CACHE_CDEX_P;
825 break;
826
827 case CPU_R5432:
828 case CPU_R5500:
829 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
830 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
831 c->icache.ways = 2;
832 c->icache.waybit= 0;
833
834 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
835 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
836 c->dcache.ways = 2;
837 c->dcache.waybit = 0;
838
Shinya Kuribayashi58648102009-03-18 09:04:01 +0900839 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 break;
841
842 case CPU_TX49XX:
843 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
844 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
845 c->icache.ways = 4;
846 c->icache.waybit= 0;
847
848 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
849 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
850 c->dcache.ways = 4;
851 c->dcache.waybit = 0;
852
853 c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemotode862b42006-03-17 12:59:22 +0900854 c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 break;
856
857 case CPU_R4000PC:
858 case CPU_R4000SC:
859 case CPU_R4000MC:
860 case CPU_R4400PC:
861 case CPU_R4400SC:
862 case CPU_R4400MC:
863 case CPU_R4300:
864 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
865 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
866 c->icache.ways = 1;
867 c->icache.waybit = 0; /* doesn't matter */
868
869 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
870 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
871 c->dcache.ways = 1;
872 c->dcache.waybit = 0; /* does not matter */
873
874 c->options |= MIPS_CPU_CACHE_CDEX_P;
875 break;
876
877 case CPU_R10000:
878 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400879 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
881 c->icache.linesz = 64;
882 c->icache.ways = 2;
883 c->icache.waybit = 0;
884
885 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
886 c->dcache.linesz = 32;
887 c->dcache.ways = 2;
888 c->dcache.waybit = 0;
889
890 c->options |= MIPS_CPU_PREFETCH;
891 break;
892
893 case CPU_VR4133:
Yoichi Yuasa2874fe52006-07-08 00:42:12 +0900894 write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 case CPU_VR4131:
896 /* Workaround for cache instruction bug of VR4131 */
897 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
898 c->processor_id == 0x0c82U) {
Yoichi Yuasa4e8ab362006-07-04 22:59:41 +0900899 config |= 0x00400000U;
900 if (c->processor_id == 0x0c80U)
901 config |= VR41_CONF_BP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 write_c0_config(config);
Yoichi Yuasa1058ecd2006-07-08 00:42:01 +0900903 } else
904 c->options |= MIPS_CPU_CACHE_CDEX_P;
905
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
907 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
908 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900909 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
911 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
912 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
913 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900914 c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 break;
916
917 case CPU_VR41XX:
918 case CPU_VR4111:
919 case CPU_VR4121:
920 case CPU_VR4122:
921 case CPU_VR4181:
922 case CPU_VR4181A:
923 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
924 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
925 c->icache.ways = 1;
926 c->icache.waybit = 0; /* doesn't matter */
927
928 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
929 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
930 c->dcache.ways = 1;
931 c->dcache.waybit = 0; /* does not matter */
932
933 c->options |= MIPS_CPU_CACHE_CDEX_P;
934 break;
935
936 case CPU_RM7000:
937 rm7k_erratum31();
938
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
940 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
941 c->icache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900942 c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
944 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
945 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
946 c->dcache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900947 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 c->options |= MIPS_CPU_CACHE_CDEX_P;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 c->options |= MIPS_CPU_PREFETCH;
951 break;
952
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800953 case CPU_LOONGSON2:
954 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
955 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
956 if (prid & 0x3)
957 c->icache.ways = 4;
958 else
959 c->icache.ways = 2;
960 c->icache.waybit = 0;
961
962 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
963 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
964 if (prid & 0x3)
965 c->dcache.ways = 4;
966 else
967 c->dcache.ways = 2;
968 c->dcache.waybit = 0;
969 break;
970
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 default:
972 if (!(config & MIPS_CONF_M))
973 panic("Don't know how to probe P-caches on this cpu.");
974
975 /*
976 * So we seem to be a MIPS32 or MIPS64 CPU
977 * So let's probe the I-cache ...
978 */
979 config1 = read_c0_config1();
980
981 if ((lsize = ((config1 >> 19) & 7)))
982 c->icache.linesz = 2 << lsize;
983 else
984 c->icache.linesz = lsize;
Douglas Leungdc34b052012-07-19 09:11:13 +0200985 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 c->icache.ways = 1 + ((config1 >> 16) & 7);
987
988 icache_size = c->icache.sets *
989 c->icache.ways *
990 c->icache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900991 c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992
993 if (config & 0x8) /* VI bit */
994 c->icache.flags |= MIPS_CACHE_VTAG;
995
996 /*
997 * Now probe the MIPS32 / MIPS64 data cache.
998 */
999 c->dcache.flags = 0;
1000
1001 if ((lsize = ((config1 >> 10) & 7)))
1002 c->dcache.linesz = 2 << lsize;
1003 else
1004 c->dcache.linesz= lsize;
Douglas Leungdc34b052012-07-19 09:11:13 +02001005 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1007
1008 dcache_size = c->dcache.sets *
1009 c->dcache.ways *
1010 c->dcache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +09001011 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
1013 c->options |= MIPS_CPU_PREFETCH;
1014 break;
1015 }
1016
1017 /*
1018 * Processor configuration sanity check for the R4000SC erratum
1019 * #5. With page sizes larger than 32kB there is no possibility
1020 * to get a VCE exception anymore so we don't care about this
1021 * misconfiguration. The case is rather theoretical anyway;
1022 * presumably no vendor is shipping his hardware in the "bad"
1023 * configuration.
1024 */
1025 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
1026 !(config & CONF_SC) && c->icache.linesz != 16 &&
1027 PAGE_SIZE <= 0x8000)
1028 panic("Improper R4000SC processor configuration detected");
1029
1030 /* compute a couple of other cache variables */
1031 c->icache.waysize = icache_size / c->icache.ways;
1032 c->dcache.waysize = dcache_size / c->dcache.ways;
1033
Chris Dearman73f40352006-06-20 18:06:52 +01001034 c->icache.sets = c->icache.linesz ?
1035 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1036 c->dcache.sets = c->dcache.linesz ?
1037 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
1039 /*
1040 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB
1041 * 2-way virtually indexed so normally would suffer from aliases. So
1042 * normally they'd suffer from aliases but magic in the hardware deals
1043 * with that for us so we don't need to take care ourselves.
1044 */
Ralf Baechled1e344e2005-02-04 15:51:26 +00001045 switch (c->cputype) {
Ralf Baechlea95970f2005-02-07 21:41:32 +00001046 case CPU_20KC:
Ralf Baechle505403b2005-02-07 21:53:39 +00001047 case CPU_25KF:
Ralf Baechle641e97f2007-10-11 23:46:05 +01001048 case CPU_SB1:
1049 case CPU_SB1A:
Jayachandran Cefa0f812011-05-07 01:36:21 +05301050 case CPU_XLR:
Atsushi Nemotode628932006-03-13 18:23:03 +09001051 c->dcache.flags |= MIPS_CACHE_PINDEX;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001052 break;
1053
Ralf Baechled1e344e2005-02-04 15:51:26 +00001054 case CPU_R10000:
1055 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001056 case CPU_R14000:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001057 break;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001058
Steven J. Hill113c62d2012-07-06 23:56:00 +02001059 case CPU_M14KC:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001060 case CPU_24K:
Nigel Stephens98a41de2006-04-27 15:50:32 +01001061 case CPU_34K:
Ralf Baechle2e78ae32006-06-23 18:48:21 +01001062 case CPU_74K:
Ralf Baechle39b8d522008-04-28 17:14:26 +01001063 case CPU_1004K:
Steven J. Hill006a8512012-06-26 04:11:03 +00001064 if (c->cputype == CPU_74K)
1065 alias_74k_erratum(c);
Ralf Baechlebeab3752006-06-19 21:56:25 +01001066 if ((read_c0_config7() & (1 << 16))) {
1067 /* effectively physically indexed dcache,
1068 thus no virtual aliases. */
1069 c->dcache.flags |= MIPS_CACHE_PINDEX;
1070 break;
1071 }
Ralf Baechled1e344e2005-02-04 15:51:26 +00001072 default:
Ralf Baechlebeab3752006-06-19 21:56:25 +01001073 if (c->dcache.waysize > PAGE_SIZE)
1074 c->dcache.flags |= MIPS_CACHE_ALIASES;
Ralf Baechled1e344e2005-02-04 15:51:26 +00001075 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
1077 switch (c->cputype) {
1078 case CPU_20KC:
1079 /*
1080 * Some older 20Kc chips doesn't have the 'VI' bit in
1081 * the config register.
1082 */
1083 c->icache.flags |= MIPS_CACHE_VTAG;
1084 break;
1085
Manuel Lauss270717a2009-03-25 17:49:28 +01001086 case CPU_ALCHEMY:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1088 break;
1089 }
1090
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001091#ifdef CONFIG_CPU_LOONGSON2
1092 /*
1093 * LOONGSON2 has 4 way icache, but when using indexed cache op,
1094 * one op will act on all 4 ways
1095 */
1096 c->icache.ways = 1;
1097#endif
1098
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1100 icache_size >> 10,
Ralf Baechle7fc73162009-04-01 16:11:53 +02001101 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 way_string[c->icache.ways], c->icache.linesz);
1103
Ralf Baechle64bfca52007-10-15 16:35:45 +01001104 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1105 dcache_size >> 10, way_string[c->dcache.ways],
1106 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1107 (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1108 "cache aliases" : "no aliases",
1109 c->dcache.linesz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110}
1111
1112/*
1113 * If you even _breathe_ on this function, look at the gcc output and make sure
1114 * it does not pop things on and off the stack for the cache sizing loop that
1115 * executes in KSEG1 space or else you will crash and burn badly. You have
1116 * been warned.
1117 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001118static int __cpuinit probe_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 unsigned long flags, addr, begin, end, pow2;
1121 unsigned int config = read_c0_config();
1122 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
1124 if (config & CONF_SC)
1125 return 0;
1126
Ralf Baechlee001e522007-07-28 12:45:47 +01001127 begin = (unsigned long) &_stext;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 begin &= ~((4 * 1024 * 1024) - 1);
1129 end = begin + (4 * 1024 * 1024);
1130
1131 /*
1132 * This is such a bitch, you'd think they would make it easy to do
1133 * this. Away you daemons of stupidity!
1134 */
1135 local_irq_save(flags);
1136
1137 /* Fill each size-multiple cache line with a valid tag. */
1138 pow2 = (64 * 1024);
1139 for (addr = begin; addr < end; addr = (begin + pow2)) {
1140 unsigned long *p = (unsigned long *) addr;
1141 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1142 pow2 <<= 1;
1143 }
1144
1145 /* Load first line with zero (therefore invalid) tag. */
1146 write_c0_taglo(0);
1147 write_c0_taghi(0);
1148 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1149 cache_op(Index_Store_Tag_I, begin);
1150 cache_op(Index_Store_Tag_D, begin);
1151 cache_op(Index_Store_Tag_SD, begin);
1152
1153 /* Now search for the wrap around point. */
1154 pow2 = (128 * 1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1156 cache_op(Index_Load_Tag_SD, addr);
1157 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1158 if (!read_c0_taglo())
1159 break;
1160 pow2 <<= 1;
1161 }
1162 local_irq_restore(flags);
1163 addr -= begin;
1164
1165 scache_size = addr;
1166 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1167 c->scache.ways = 1;
1168 c->dcache.waybit = 0; /* does not matter */
1169
1170 return 1;
1171}
1172
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001173#if defined(CONFIG_CPU_LOONGSON2)
1174static void __init loongson2_sc_init(void)
1175{
1176 struct cpuinfo_mips *c = &current_cpu_data;
1177
1178 scache_size = 512*1024;
1179 c->scache.linesz = 32;
1180 c->scache.ways = 4;
1181 c->scache.waybit = 0;
1182 c->scache.waysize = scache_size / (c->scache.ways);
1183 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1184 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1185 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1186
1187 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1188}
1189#endif
1190
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191extern int r5k_sc_init(void);
1192extern int rm7k_sc_init(void);
Chris Dearman9318c512006-06-20 17:15:20 +01001193extern int mips_sc_init(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194
Ralf Baechle234fcd12008-03-08 09:56:28 +00001195static void __cpuinit setup_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196{
1197 struct cpuinfo_mips *c = &current_cpu_data;
1198 unsigned int config = read_c0_config();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 int sc_present = 0;
1200
1201 /*
1202 * Do the probing thing on R4000SC and R4400SC processors. Other
1203 * processors don't have a S-cache that would be relevant to the
Joe Perches603e82e2008-02-03 16:54:53 +02001204 * Linux memory management.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 */
1206 switch (c->cputype) {
1207 case CPU_R4000SC:
1208 case CPU_R4000MC:
1209 case CPU_R4400SC:
1210 case CPU_R4400MC:
Thiemo Seuferba5187d2005-04-25 16:36:23 +00001211 sc_present = run_uncached(probe_scache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 if (sc_present)
1213 c->options |= MIPS_CPU_CACHE_CDEX_S;
1214 break;
1215
1216 case CPU_R10000:
1217 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001218 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1220 c->scache.linesz = 64 << ((config >> 13) & 1);
1221 c->scache.ways = 2;
1222 c->scache.waybit= 0;
1223 sc_present = 1;
1224 break;
1225
1226 case CPU_R5000:
1227 case CPU_NEVADA:
1228#ifdef CONFIG_R5000_CPU_SCACHE
1229 r5k_sc_init();
1230#endif
1231 return;
1232
1233 case CPU_RM7000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234#ifdef CONFIG_RM7000_CPU_SCACHE
1235 rm7k_sc_init();
1236#endif
1237 return;
1238
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001239#if defined(CONFIG_CPU_LOONGSON2)
1240 case CPU_LOONGSON2:
1241 loongson2_sc_init();
1242 return;
1243#endif
Jayachandran Ca3d4fb22011-11-16 00:21:20 +00001244 case CPU_XLP:
1245 /* don't need to worry about L2, fully coherent */
1246 return;
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001247
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 default:
Chris Dearman9318c512006-06-20 17:15:20 +01001249 if (c->isa_level == MIPS_CPU_ISA_M32R1 ||
1250 c->isa_level == MIPS_CPU_ISA_M32R2 ||
1251 c->isa_level == MIPS_CPU_ISA_M64R1 ||
1252 c->isa_level == MIPS_CPU_ISA_M64R2) {
1253#ifdef CONFIG_MIPS_CPU_SCACHE
1254 if (mips_sc_init ()) {
1255 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1256 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1257 scache_size >> 10,
1258 way_string[c->scache.ways], c->scache.linesz);
1259 }
1260#else
1261 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1262 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1263#endif
1264 return;
1265 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 sc_present = 0;
1267 }
1268
1269 if (!sc_present)
1270 return;
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272 /* compute a couple of other cache variables */
1273 c->scache.waysize = scache_size / c->scache.ways;
1274
1275 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1276
1277 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1278 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1279
Ralf Baechlefc5d2d22006-07-06 13:04:01 +01001280 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281}
1282
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001283void au1x00_fixup_config_od(void)
1284{
1285 /*
1286 * c0_config.od (bit 19) was write only (and read as 0)
1287 * on the early revisions of Alchemy SOCs. It disables the bus
1288 * transaction overlapping and needs to be set to fix various errata.
1289 */
1290 switch (read_c0_prid()) {
1291 case 0x00030100: /* Au1000 DA */
1292 case 0x00030201: /* Au1000 HA */
1293 case 0x00030202: /* Au1000 HB */
1294 case 0x01030200: /* Au1500 AB */
1295 /*
1296 * Au1100 errata actually keeps silence about this bit, so we set it
1297 * just in case for those revisions that require it to be set according
Manuel Lauss270717a2009-03-25 17:49:28 +01001298 * to the (now gone) cpu table.
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001299 */
1300 case 0x02030200: /* Au1100 AB */
1301 case 0x02030201: /* Au1100 BA */
1302 case 0x02030202: /* Au1100 BC */
1303 set_c0_config(1 << 19);
1304 break;
1305 }
1306}
1307
Ralf Baechle89052bd2008-06-12 17:26:02 +01001308/* CP0 hazard avoidance. */
1309#define NXP_BARRIER() \
1310 __asm__ __volatile__( \
1311 ".set noreorder\n\t" \
1312 "nop; nop; nop; nop; nop; nop;\n\t" \
1313 ".set reorder\n\t")
1314
1315static void nxp_pr4450_fixup_config(void)
1316{
1317 unsigned long config0;
1318
1319 config0 = read_c0_config();
1320
1321 /* clear all three cache coherency fields */
1322 config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1323 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) |
1324 ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1325 ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1326 write_c0_config(config0);
1327 NXP_BARRIER();
1328}
1329
Chris Dearman35133692007-09-19 00:58:24 +01001330static int __cpuinitdata cca = -1;
1331
1332static int __init cca_setup(char *str)
1333{
1334 get_option(&str, &cca);
1335
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001336 return 0;
Chris Dearman35133692007-09-19 00:58:24 +01001337}
1338
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001339early_param("cca", cca_setup);
Chris Dearman35133692007-09-19 00:58:24 +01001340
Ralf Baechle234fcd12008-03-08 09:56:28 +00001341static void __cpuinit coherency_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342{
Chris Dearman35133692007-09-19 00:58:24 +01001343 if (cca < 0 || cca > 7)
1344 cca = read_c0_config() & CONF_CM_CMASK;
1345 _page_cachable_default = cca << _CACHE_SHIFT;
1346
1347 pr_debug("Using cache attribute %d\n", cca);
1348 change_c0_config(CONF_CM_CMASK, cca);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
1350 /*
1351 * c0_status.cu=0 specifies that updates by the sc instruction use
1352 * the coherency mode specified by the TLB; 1 means cachable
1353 * coherent update on write will be used. Not all processors have
1354 * this bit and; some wire it to zero, others like Toshiba had the
1355 * silly idea of putting something else there ...
1356 */
Ralf Baechle10cc3522007-10-11 23:46:15 +01001357 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 case CPU_R4000PC:
1359 case CPU_R4000SC:
1360 case CPU_R4000MC:
1361 case CPU_R4400PC:
1362 case CPU_R4400SC:
1363 case CPU_R4400MC:
1364 clear_c0_config(CONF_CU);
1365 break;
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001366 /*
Ralf Baechledf586d52006-08-01 23:42:30 +01001367 * We need to catch the early Alchemy SOCs with
Manuel Lauss270717a2009-03-25 17:49:28 +01001368 * the write-only co_config.od bit and set it back to one on:
1369 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001370 */
Manuel Lauss270717a2009-03-25 17:49:28 +01001371 case CPU_ALCHEMY:
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001372 au1x00_fixup_config_od();
1373 break;
Ralf Baechle89052bd2008-06-12 17:26:02 +01001374
1375 case PRID_IMP_PR4450:
1376 nxp_pr4450_fixup_config();
1377 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 }
1379}
1380
Ralf Baechle39b8d522008-04-28 17:14:26 +01001381#if defined(CONFIG_DMA_NONCOHERENT)
1382
1383static int __cpuinitdata coherentio;
1384
1385static int __init setcoherentio(char *str)
1386{
1387 coherentio = 1;
1388
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001389 return 0;
Ralf Baechle39b8d522008-04-28 17:14:26 +01001390}
1391
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001392early_param("coherentio", setcoherentio);
Ralf Baechle39b8d522008-04-28 17:14:26 +01001393#endif
1394
David Daney9cd9669b2012-05-15 00:04:49 -07001395static void __cpuinit r4k_cache_error_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396{
Ralf Baechle641e97f2007-10-11 23:46:05 +01001397 extern char __weak except_vec2_generic;
1398 extern char __weak except_vec2_sb1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 struct cpuinfo_mips *c = &current_cpu_data;
1400
Ralf Baechle641e97f2007-10-11 23:46:05 +01001401 switch (c->cputype) {
1402 case CPU_SB1:
1403 case CPU_SB1A:
1404 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1405 break;
1406
1407 default:
1408 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1409 break;
1410 }
David Daney9cd9669b2012-05-15 00:04:49 -07001411}
1412
1413void __cpuinit r4k_cache_init(void)
1414{
1415 extern void build_clear_page(void);
1416 extern void build_copy_page(void);
1417 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418
1419 probe_pcache();
1420 setup_scache();
1421
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 r4k_blast_dcache_page_setup();
1423 r4k_blast_dcache_page_indexed_setup();
1424 r4k_blast_dcache_setup();
1425 r4k_blast_icache_page_setup();
1426 r4k_blast_icache_page_indexed_setup();
1427 r4k_blast_icache_setup();
1428 r4k_blast_scache_page_setup();
1429 r4k_blast_scache_page_indexed_setup();
1430 r4k_blast_scache_setup();
1431
1432 /*
1433 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1434 * This code supports virtually indexed processors and will be
1435 * unnecessarily inefficient on physically indexed processors.
1436 */
Chris Dearman73f40352006-06-20 18:06:52 +01001437 if (c->dcache.linesz)
1438 shm_align_mask = max_t( unsigned long,
1439 c->dcache.sets * c->dcache.linesz - 1,
1440 PAGE_SIZE - 1);
1441 else
1442 shm_align_mask = PAGE_SIZE-1;
Ralf Baechle9c5a3d72008-04-05 15:13:23 +01001443
1444 __flush_cache_vmap = r4k__flush_cache_vmap;
1445 __flush_cache_vunmap = r4k__flush_cache_vunmap;
1446
Ralf Baechledb813fe2007-09-27 18:26:43 +01001447 flush_cache_all = cache_noop;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 __flush_cache_all = r4k___flush_cache_all;
1449 flush_cache_mm = r4k_flush_cache_mm;
1450 flush_cache_page = r4k_flush_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451 flush_cache_range = r4k_flush_cache_range;
1452
Ralf Baechled9cdc9012011-06-17 16:20:28 +01001453 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1454
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1456 flush_icache_all = r4k_flush_icache_all;
Ralf Baechle7e3bfc72006-04-05 20:42:04 +01001457 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 flush_data_cache_page = r4k_flush_data_cache_page;
1459 flush_icache_range = r4k_flush_icache_range;
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02001460 local_flush_icache_range = local_r4k_flush_icache_range;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461
Ralf Baechle39b8d522008-04-28 17:14:26 +01001462#if defined(CONFIG_DMA_NONCOHERENT)
1463 if (coherentio) {
1464 _dma_cache_wback_inv = (void *)cache_noop;
1465 _dma_cache_wback = (void *)cache_noop;
1466 _dma_cache_inv = (void *)cache_noop;
1467 } else {
1468 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1469 _dma_cache_wback = r4k_dma_cache_wback_inv;
1470 _dma_cache_inv = r4k_dma_cache_inv;
1471 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472#endif
1473
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 build_clear_page();
1475 build_copy_page();
Ralf Baechle39b8d522008-04-28 17:14:26 +01001476#if !defined(CONFIG_MIPS_CMP)
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001477 local_r4k___flush_cache_all(NULL);
Ralf Baechle39b8d522008-04-28 17:14:26 +01001478#endif
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001479 coherency_setup();
David Daney9cd9669b2012-05-15 00:04:49 -07001480 board_cache_error_setup = r4k_cache_error_setup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481}