blob: 6ea152552e5164ef4d25d6554072cdc9e6ae0b85 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
Thiemo Seufere30ec452008-01-28 20:05:38 +00008 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
David Daney95affdd2009-05-20 11:40:59 -07009 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
Ralf Baechle41c594a2006-04-05 09:45:45 +010010 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
David Daneyfd062c82009-05-27 17:47:44 -070011 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
Steven J. Hill113c62d2012-07-06 23:56:00 +020012 * Copyright (C) 2011 MIPS Technologies, Inc.
Ralf Baechle41c594a2006-04-05 09:45:45 +010013 *
14 * ... and the days got worse and worse and now you see
15 * I've gone completly out of my mind.
16 *
17 * They're coming to take me a away haha
18 * they're coming to take me a away hoho hihi haha
19 * to the funny farm where code is beautiful all the time ...
20 *
21 * (Condolences to Napoleon XIV)
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 */
23
David Daney95affdd2009-05-20 11:40:59 -070024#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/kernel.h>
26#include <linux/types.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010027#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/string.h>
29#include <linux/init.h>
David Daney3d8bfdd2010-12-21 14:19:11 -080030#include <linux/cache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
David Daney3d8bfdd2010-12-21 14:19:11 -080032#include <asm/cacheflush.h>
33#include <asm/pgtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <asm/war.h>
Florian Fainelli3482d712010-01-28 15:21:24 +010035#include <asm/uasm.h>
David Howellsb81947c2012-03-28 18:30:02 +010036#include <asm/setup.h>
Thiemo Seufere30ec452008-01-28 20:05:38 +000037
David Daney1ec56322010-04-28 12:16:18 -070038/*
39 * TLB load/store/modify handlers.
40 *
41 * Only the fastpath gets synthesized at runtime, the slowpath for
42 * do_page_fault remains normal asm.
43 */
44extern void tlb_do_page_fault_0(void);
45extern void tlb_do_page_fault_1(void);
46
David Daneybf286072011-07-05 16:34:46 -070047struct work_registers {
48 int r1;
49 int r2;
50 int r3;
51};
52
53struct tlb_reg_save {
54 unsigned long a;
55 unsigned long b;
56} ____cacheline_aligned_in_smp;
57
58static struct tlb_reg_save handler_reg_save[NR_CPUS];
David Daney1ec56322010-04-28 12:16:18 -070059
Ralf Baechleaeffdbb2007-10-11 23:46:14 +010060static inline int r45k_bvahwbug(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070061{
62 /* XXX: We should probe for the presence of this bug, but we don't. */
63 return 0;
64}
65
Ralf Baechleaeffdbb2007-10-11 23:46:14 +010066static inline int r4k_250MHZhwbug(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070067{
68 /* XXX: We should probe for the presence of this bug, but we don't. */
69 return 0;
70}
71
Ralf Baechleaeffdbb2007-10-11 23:46:14 +010072static inline int __maybe_unused bcm1250_m3_war(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070073{
74 return BCM1250_M3_WAR;
75}
76
Ralf Baechleaeffdbb2007-10-11 23:46:14 +010077static inline int __maybe_unused r10000_llsc_war(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078{
79 return R10000_LLSC_WAR;
80}
81
David Daneycc33ae42010-12-20 15:54:50 -080082static int use_bbit_insns(void)
83{
84 switch (current_cpu_type()) {
85 case CPU_CAVIUM_OCTEON:
86 case CPU_CAVIUM_OCTEON_PLUS:
87 case CPU_CAVIUM_OCTEON2:
88 return 1;
89 default:
90 return 0;
91 }
92}
93
David Daney2c8c53e2010-12-27 18:07:57 -080094static int use_lwx_insns(void)
95{
96 switch (current_cpu_type()) {
97 case CPU_CAVIUM_OCTEON2:
98 return 1;
99 default:
100 return 0;
101 }
102}
103#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
104 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
105static bool scratchpad_available(void)
106{
107 return true;
108}
109static int scratchpad_offset(int i)
110{
111 /*
112 * CVMSEG starts at address -32768 and extends for
113 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
114 */
115 i += 1; /* Kernel use starts at the top and works down. */
116 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
117}
118#else
119static bool scratchpad_available(void)
120{
121 return false;
122}
123static int scratchpad_offset(int i)
124{
125 BUG();
David Daneye1c87d22011-01-19 15:24:42 -0800126 /* Really unreachable, but evidently some GCC want this. */
127 return 0;
David Daney2c8c53e2010-12-27 18:07:57 -0800128}
129#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130/*
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +0100131 * Found by experiment: At least some revisions of the 4kc throw under
132 * some circumstances a machine check exception, triggered by invalid
133 * values in the index register. Delaying the tlbp instruction until
134 * after the next branch, plus adding an additional nop in front of
135 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
136 * why; it's not an issue caused by the core RTL.
137 *
138 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000139static int __cpuinit m4kc_tlbp_war(void)
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +0100140{
141 return (current_cpu_data.processor_id & 0xffff00) ==
142 (PRID_COMP_MIPS | PRID_IMP_4KC);
143}
144
Thiemo Seufere30ec452008-01-28 20:05:38 +0000145/* Handle labels (which must be positive integers). */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146enum label_id {
Thiemo Seufere30ec452008-01-28 20:05:38 +0000147 label_second_part = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 label_leave,
149 label_vmalloc,
150 label_vmalloc_done,
Ralf Baechle02a54172012-10-13 22:46:26 +0200151 label_tlbw_hazard_0,
152 label_split = label_tlbw_hazard_0 + 8,
David Daney6dd93442010-02-10 15:12:47 -0800153 label_tlbl_goaround1,
154 label_tlbl_goaround2,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 label_nopage_tlbl,
156 label_nopage_tlbs,
157 label_nopage_tlbm,
158 label_smp_pgtable_change,
159 label_r3000_write_probe_fail,
David Daney1ec56322010-04-28 12:16:18 -0700160 label_large_segbits_fault,
David Daneyfd062c82009-05-27 17:47:44 -0700161#ifdef CONFIG_HUGETLB_PAGE
162 label_tlb_huge_update,
163#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164};
165
Thiemo Seufere30ec452008-01-28 20:05:38 +0000166UASM_L_LA(_second_part)
167UASM_L_LA(_leave)
Thiemo Seufere30ec452008-01-28 20:05:38 +0000168UASM_L_LA(_vmalloc)
169UASM_L_LA(_vmalloc_done)
Ralf Baechle02a54172012-10-13 22:46:26 +0200170/* _tlbw_hazard_x is handled differently. */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000171UASM_L_LA(_split)
David Daney6dd93442010-02-10 15:12:47 -0800172UASM_L_LA(_tlbl_goaround1)
173UASM_L_LA(_tlbl_goaround2)
Thiemo Seufere30ec452008-01-28 20:05:38 +0000174UASM_L_LA(_nopage_tlbl)
175UASM_L_LA(_nopage_tlbs)
176UASM_L_LA(_nopage_tlbm)
177UASM_L_LA(_smp_pgtable_change)
178UASM_L_LA(_r3000_write_probe_fail)
David Daney1ec56322010-04-28 12:16:18 -0700179UASM_L_LA(_large_segbits_fault)
David Daneyfd062c82009-05-27 17:47:44 -0700180#ifdef CONFIG_HUGETLB_PAGE
181UASM_L_LA(_tlb_huge_update)
182#endif
Atsushi Nemoto656be922006-10-26 00:08:31 +0900183
Ralf Baechle02a54172012-10-13 22:46:26 +0200184static int __cpuinitdata hazard_instance;
185
186static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
187{
188 switch (instance) {
189 case 0 ... 7:
190 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
191 return;
192 default:
193 BUG();
194 }
195}
196
197static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
198{
199 switch (instance) {
200 case 0 ... 7:
201 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
202 break;
203 default:
204 BUG();
205 }
206}
207
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +0200208/*
209 * For debug purposes.
210 */
211static inline void dump_handler(const u32 *handler, int count)
212{
213 int i;
214
215 pr_debug("\t.set push\n");
216 pr_debug("\t.set noreorder\n");
217
218 for (i = 0; i < count; i++)
219 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
220
221 pr_debug("\t.set pop\n");
222}
223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224/* The only general purpose registers allowed in TLB handlers. */
225#define K0 26
226#define K1 27
227
228/* Some CP0 registers */
Ralf Baechle41c594a2006-04-05 09:45:45 +0100229#define C0_INDEX 0, 0
230#define C0_ENTRYLO0 2, 0
231#define C0_TCBIND 2, 2
232#define C0_ENTRYLO1 3, 0
233#define C0_CONTEXT 4, 0
David Daneyfd062c82009-05-27 17:47:44 -0700234#define C0_PAGEMASK 5, 0
Ralf Baechle41c594a2006-04-05 09:45:45 +0100235#define C0_BADVADDR 8, 0
236#define C0_ENTRYHI 10, 0
237#define C0_EPC 14, 0
238#define C0_XCONTEXT 20, 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
Ralf Baechle875d43e2005-09-03 15:56:16 -0700240#ifdef CONFIG_64BIT
Thiemo Seufere30ec452008-01-28 20:05:38 +0000241# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242#else
Thiemo Seufere30ec452008-01-28 20:05:38 +0000243# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244#endif
245
246/* The worst case length of the handler is around 18 instructions for
247 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
248 * Maximum space available is 32 instructions for R3000 and 64
249 * instructions for R4000.
250 *
251 * We deliberately chose a buffer size of 128, so we won't scribble
252 * over anything important on overflow before we panic.
253 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000254static u32 tlb_handler[128] __cpuinitdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255
256/* simply assume worst case size for labels and relocs */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000257static struct uasm_label labels[128] __cpuinitdata;
258static struct uasm_reloc relocs[128] __cpuinitdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
David Daney1ec56322010-04-28 12:16:18 -0700260#ifdef CONFIG_64BIT
261static int check_for_high_segbits __cpuinitdata;
262#endif
263
David Daney2c8c53e2010-12-27 18:07:57 -0800264static int check_for_high_segbits __cpuinitdata;
David Daney3d8bfdd2010-12-21 14:19:11 -0800265
266static unsigned int kscratch_used_mask __cpuinitdata;
267
268static int __cpuinit allocate_kscratch(void)
269{
270 int r;
271 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
272
273 r = ffs(a);
274
275 if (r == 0)
276 return -1;
277
278 r--; /* make it zero based */
279
280 kscratch_used_mask |= (1 << r);
281
282 return r;
283}
284
David Daney2c8c53e2010-12-27 18:07:57 -0800285static int scratch_reg __cpuinitdata;
David Daney3d8bfdd2010-12-21 14:19:11 -0800286static int pgd_reg __cpuinitdata;
David Daney2c8c53e2010-12-27 18:07:57 -0800287enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
David Daney3d8bfdd2010-12-21 14:19:11 -0800288
David Daneybf286072011-07-05 16:34:46 -0700289static struct work_registers __cpuinit build_get_work_registers(u32 **p)
290{
291 struct work_registers r;
292
293 int smp_processor_id_reg;
294 int smp_processor_id_sel;
295 int smp_processor_id_shift;
296
297 if (scratch_reg > 0) {
298 /* Save in CPU local C0_KScratch? */
299 UASM_i_MTC0(p, 1, 31, scratch_reg);
300 r.r1 = K0;
301 r.r2 = K1;
302 r.r3 = 1;
303 return r;
304 }
305
306 if (num_possible_cpus() > 1) {
307#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
308 smp_processor_id_shift = 51;
309 smp_processor_id_reg = 20; /* XContext */
310 smp_processor_id_sel = 0;
311#else
312# ifdef CONFIG_32BIT
313 smp_processor_id_shift = 25;
314 smp_processor_id_reg = 4; /* Context */
315 smp_processor_id_sel = 0;
316# endif
317# ifdef CONFIG_64BIT
318 smp_processor_id_shift = 26;
319 smp_processor_id_reg = 4; /* Context */
320 smp_processor_id_sel = 0;
321# endif
322#endif
323 /* Get smp_processor_id */
324 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel);
325 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift);
326
327 /* handler_reg_save index in K0 */
328 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
329
330 UASM_i_LA(p, K1, (long)&handler_reg_save);
331 UASM_i_ADDU(p, K0, K0, K1);
332 } else {
333 UASM_i_LA(p, K0, (long)&handler_reg_save);
334 }
335 /* K0 now points to save area, save $1 and $2 */
336 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
337 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
338
339 r.r1 = K1;
340 r.r2 = 1;
341 r.r3 = 2;
342 return r;
343}
344
345static void __cpuinit build_restore_work_registers(u32 **p)
346{
347 if (scratch_reg > 0) {
348 UASM_i_MFC0(p, 1, 31, scratch_reg);
349 return;
350 }
351 /* K0 already points to save area, restore $1 and $2 */
352 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
353 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
354}
355
David Daney2c8c53e2010-12-27 18:07:57 -0800356#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
357
David Daney826222842009-10-14 12:16:56 -0700358/*
359 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
360 * we cannot do r3000 under these circumstances.
David Daney3d8bfdd2010-12-21 14:19:11 -0800361 *
362 * Declare pgd_current here instead of including mmu_context.h to avoid type
363 * conflicts for tlbmiss_handler_setup_pgd
David Daney826222842009-10-14 12:16:56 -0700364 */
David Daney3d8bfdd2010-12-21 14:19:11 -0800365extern unsigned long pgd_current[];
David Daney826222842009-10-14 12:16:56 -0700366
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367/*
368 * The R3000 TLB handler is simple.
369 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000370static void __cpuinit build_r3000_tlb_refill_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371{
372 long pgdc = (long)pgd_current;
373 u32 *p;
374
375 memset(tlb_handler, 0, sizeof(tlb_handler));
376 p = tlb_handler;
377
Thiemo Seufere30ec452008-01-28 20:05:38 +0000378 uasm_i_mfc0(&p, K0, C0_BADVADDR);
379 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
380 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
381 uasm_i_srl(&p, K0, K0, 22); /* load delay */
382 uasm_i_sll(&p, K0, K0, 2);
383 uasm_i_addu(&p, K1, K1, K0);
384 uasm_i_mfc0(&p, K0, C0_CONTEXT);
385 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
386 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
387 uasm_i_addu(&p, K1, K1, K0);
388 uasm_i_lw(&p, K0, 0, K1);
389 uasm_i_nop(&p); /* load delay */
390 uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
391 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
392 uasm_i_tlbwr(&p); /* cp0 delay */
393 uasm_i_jr(&p, K1);
394 uasm_i_rfe(&p); /* branch delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
396 if (p > tlb_handler + 32)
397 panic("TLB refill handler space exceeded");
398
Thiemo Seufere30ec452008-01-28 20:05:38 +0000399 pr_debug("Wrote TLB refill handler (%u instructions).\n",
400 (unsigned int)(p - tlb_handler));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
Ralf Baechle91b05e62006-03-29 18:53:00 +0100402 memcpy((void *)ebase, tlb_handler, 0x80);
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +0200403
404 dump_handler((u32 *)ebase, 32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405}
David Daney826222842009-10-14 12:16:56 -0700406#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407
408/*
409 * The R4000 TLB handler is much more complicated. We have two
410 * consecutive handler areas with 32 instructions space each.
411 * Since they aren't used at the same time, we can overflow in the
412 * other one.To keep things simple, we first assume linear space,
413 * then we relocate it to the final handler layout as needed.
414 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000415static u32 final_handler[64] __cpuinitdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416
417/*
418 * Hazards
419 *
420 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
421 * 2. A timing hazard exists for the TLBP instruction.
422 *
423 * stalling_instruction
424 * TLBP
425 *
426 * The JTLB is being read for the TLBP throughout the stall generated by the
427 * previous instruction. This is not really correct as the stalling instruction
428 * can modify the address used to access the JTLB. The failure symptom is that
429 * the TLBP instruction will use an address created for the stalling instruction
430 * and not the address held in C0_ENHI and thus report the wrong results.
431 *
432 * The software work-around is to not allow the instruction preceding the TLBP
433 * to stall - make it an NOP or some other instruction guaranteed not to stall.
434 *
435 * Errata 2 will not be fixed. This errata is also on the R5000.
436 *
437 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
438 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000439static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440{
Ralf Baechle10cc3522007-10-11 23:46:15 +0100441 switch (current_cpu_type()) {
Thomas Bogendoerfer326e2e12008-05-12 13:55:42 +0200442 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
Thiemo Seuferf5b4d952005-09-09 17:11:50 +0000443 case CPU_R4600:
Thomas Bogendoerfer326e2e12008-05-12 13:55:42 +0200444 case CPU_R4700:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 case CPU_R5000:
446 case CPU_R5000A:
447 case CPU_NEVADA:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000448 uasm_i_nop(p);
449 uasm_i_tlbp(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 break;
451
452 default:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000453 uasm_i_tlbp(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 break;
455 }
456}
457
458/*
459 * Write random or indexed TLB entry, and care about the hazards from
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300460 * the preceding mtc0 and for the following eret.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 */
462enum tlb_write_entry { tlb_random, tlb_indexed };
463
Ralf Baechle234fcd12008-03-08 09:56:28 +0000464static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
Thiemo Seufere30ec452008-01-28 20:05:38 +0000465 struct uasm_reloc **r,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 enum tlb_write_entry wmode)
467{
468 void(*tlbw)(u32 **) = NULL;
469
470 switch (wmode) {
Thiemo Seufere30ec452008-01-28 20:05:38 +0000471 case tlb_random: tlbw = uasm_i_tlbwr; break;
472 case tlb_indexed: tlbw = uasm_i_tlbwi; break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 }
474
Ralf Baechle161548b2008-01-29 10:14:54 +0000475 if (cpu_has_mips_r2) {
Steven J. Hill625c0a22012-08-28 23:20:08 -0500476 /*
477 * The architecture spec says an ehb is required here,
478 * but a number of cores do not have the hazard and
479 * using an ehb causes an expensive pipeline stall.
480 */
481 switch (current_cpu_type()) {
482 case CPU_M14KC:
483 case CPU_74K:
484 break;
485
486 default:
David Daney41f0e4d2009-05-12 12:41:53 -0700487 uasm_i_ehb(p);
Steven J. Hill625c0a22012-08-28 23:20:08 -0500488 break;
489 }
Ralf Baechle161548b2008-01-29 10:14:54 +0000490 tlbw(p);
491 return;
492 }
493
Ralf Baechle10cc3522007-10-11 23:46:15 +0100494 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 case CPU_R4000PC:
496 case CPU_R4000SC:
497 case CPU_R4000MC:
498 case CPU_R4400PC:
499 case CPU_R4400SC:
500 case CPU_R4400MC:
501 /*
502 * This branch uses up a mtc0 hazard nop slot and saves
503 * two nops after the tlbw instruction.
504 */
Ralf Baechle02a54172012-10-13 22:46:26 +0200505 uasm_bgezl_hazard(p, r, hazard_instance);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 tlbw(p);
Ralf Baechle02a54172012-10-13 22:46:26 +0200507 uasm_bgezl_label(l, p, hazard_instance);
508 hazard_instance++;
Thiemo Seufere30ec452008-01-28 20:05:38 +0000509 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 break;
511
512 case CPU_R4600:
513 case CPU_R4700:
514 case CPU_R5000:
515 case CPU_R5000A:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000516 uasm_i_nop(p);
Maciej W. Rozycki2c93e122005-06-30 10:51:01 +0000517 tlbw(p);
Thiemo Seufere30ec452008-01-28 20:05:38 +0000518 uasm_i_nop(p);
Maciej W. Rozycki2c93e122005-06-30 10:51:01 +0000519 break;
520
521 case CPU_R4300:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 case CPU_5KC:
523 case CPU_TX49XX:
Pete Popovbdf21b12005-07-14 17:47:57 +0000524 case CPU_PR4450:
Jayachandran Cefa0f812011-05-07 01:36:21 +0530525 case CPU_XLR:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000526 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 tlbw(p);
528 break;
529
530 case CPU_R10000:
531 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400532 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 case CPU_4KC:
Thomas Bogendoerferb1ec4c82008-03-26 16:42:54 +0100534 case CPU_4KEC:
Steven J. Hill113c62d2012-07-06 23:56:00 +0200535 case CPU_M14KC:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 case CPU_SB1:
Andrew Isaacson93ce2f522005-10-19 23:56:20 -0700537 case CPU_SB1A:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538 case CPU_4KSC:
539 case CPU_20KC:
540 case CPU_25KF:
Kevin Cernekee602977b2010-10-16 14:22:30 -0700541 case CPU_BMIPS32:
542 case CPU_BMIPS3300:
543 case CPU_BMIPS4350:
544 case CPU_BMIPS4380:
545 case CPU_BMIPS5000:
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800546 case CPU_LOONGSON2:
Shinya Kuribayashia644b272009-03-03 18:05:51 +0900547 case CPU_R5500:
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +0100548 if (m4kc_tlbp_war())
Thiemo Seufere30ec452008-01-28 20:05:38 +0000549 uasm_i_nop(p);
Manuel Lauss2f794d02009-03-25 17:49:30 +0100550 case CPU_ALCHEMY:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 tlbw(p);
552 break;
553
554 case CPU_NEVADA:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000555 uasm_i_nop(p); /* QED specifies 2 nops hazard */
Ralf Baechle02a54172012-10-13 22:46:26 +0200556 uasm_i_nop(p); /* QED specifies 2 nops hazard */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 /*
558 * This branch uses up a mtc0 hazard nop slot and saves
559 * a nop after the tlbw instruction.
560 */
Ralf Baechle02a54172012-10-13 22:46:26 +0200561 uasm_bgezl_hazard(p, r, hazard_instance);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 tlbw(p);
Ralf Baechle02a54172012-10-13 22:46:26 +0200563 uasm_bgezl_label(l, p, hazard_instance);
564 hazard_instance++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 break;
566
567 case CPU_RM7000:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000568 uasm_i_nop(p);
569 uasm_i_nop(p);
570 uasm_i_nop(p);
571 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 tlbw(p);
573 break;
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 case CPU_RM9000:
576 /*
577 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
578 * use of the JTLB for instructions should not occur for 4
579 * cpu cycles and use for data translations should not occur
580 * for 3 cpu cycles.
581 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000582 uasm_i_ssnop(p);
583 uasm_i_ssnop(p);
584 uasm_i_ssnop(p);
585 uasm_i_ssnop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 tlbw(p);
Thiemo Seufere30ec452008-01-28 20:05:38 +0000587 uasm_i_ssnop(p);
588 uasm_i_ssnop(p);
589 uasm_i_ssnop(p);
590 uasm_i_ssnop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 break;
592
593 case CPU_VR4111:
594 case CPU_VR4121:
595 case CPU_VR4122:
596 case CPU_VR4181:
597 case CPU_VR4181A:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000598 uasm_i_nop(p);
599 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 tlbw(p);
Thiemo Seufere30ec452008-01-28 20:05:38 +0000601 uasm_i_nop(p);
602 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 break;
604
605 case CPU_VR4131:
606 case CPU_VR4133:
Ralf Baechle7623deb2005-08-29 16:49:55 +0000607 case CPU_R5432:
Thiemo Seufere30ec452008-01-28 20:05:38 +0000608 uasm_i_nop(p);
609 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 tlbw(p);
611 break;
612
Lars-Peter Clausen83ccf692010-07-17 11:07:51 +0000613 case CPU_JZRISC:
614 tlbw(p);
615 uasm_i_nop(p);
616 break;
617
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 default:
619 panic("No TLB refill handler yet (CPU type: %d)",
620 current_cpu_data.cputype);
621 break;
622 }
623}
624
David Daney6dd93442010-02-10 15:12:47 -0800625static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
626 unsigned int reg)
627{
Steven J. Hill05857c62012-09-13 16:51:46 -0500628 if (cpu_has_rixi) {
David Daney748e7872012-08-23 10:02:03 -0700629 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -0800630 } else {
631#ifdef CONFIG_64BIT_PHYS_ADDR
David Daney3be60222010-04-28 12:16:17 -0700632 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -0800633#else
634 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
635#endif
636 }
637}
638
David Daneyfd062c82009-05-27 17:47:44 -0700639#ifdef CONFIG_HUGETLB_PAGE
David Daney6dd93442010-02-10 15:12:47 -0800640
641static __cpuinit void build_restore_pagemask(u32 **p,
642 struct uasm_reloc **r,
643 unsigned int tmp,
David Daney2c8c53e2010-12-27 18:07:57 -0800644 enum label_id lid,
645 int restore_scratch)
David Daney6dd93442010-02-10 15:12:47 -0800646{
David Daney2c8c53e2010-12-27 18:07:57 -0800647 if (restore_scratch) {
648 /* Reset default page size */
649 if (PM_DEFAULT_MASK >> 16) {
650 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
651 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
652 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
653 uasm_il_b(p, r, lid);
654 } else if (PM_DEFAULT_MASK) {
655 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
656 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
657 uasm_il_b(p, r, lid);
658 } else {
659 uasm_i_mtc0(p, 0, C0_PAGEMASK);
660 uasm_il_b(p, r, lid);
661 }
662 if (scratch_reg > 0)
663 UASM_i_MFC0(p, 1, 31, scratch_reg);
664 else
665 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
David Daney6dd93442010-02-10 15:12:47 -0800666 } else {
David Daney2c8c53e2010-12-27 18:07:57 -0800667 /* Reset default page size */
668 if (PM_DEFAULT_MASK >> 16) {
669 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
670 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
671 uasm_il_b(p, r, lid);
672 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
673 } else if (PM_DEFAULT_MASK) {
674 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
675 uasm_il_b(p, r, lid);
676 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
677 } else {
678 uasm_il_b(p, r, lid);
679 uasm_i_mtc0(p, 0, C0_PAGEMASK);
680 }
David Daney6dd93442010-02-10 15:12:47 -0800681 }
682}
683
David Daneyfd062c82009-05-27 17:47:44 -0700684static __cpuinit void build_huge_tlb_write_entry(u32 **p,
685 struct uasm_label **l,
686 struct uasm_reloc **r,
687 unsigned int tmp,
David Daney2c8c53e2010-12-27 18:07:57 -0800688 enum tlb_write_entry wmode,
689 int restore_scratch)
David Daneyfd062c82009-05-27 17:47:44 -0700690{
691 /* Set huge page tlb entry size */
692 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
693 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
694 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
695
696 build_tlb_write_entry(p, l, r, wmode);
697
David Daney2c8c53e2010-12-27 18:07:57 -0800698 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
David Daneyfd062c82009-05-27 17:47:44 -0700699}
700
701/*
702 * Check if Huge PTE is present, if so then jump to LABEL.
703 */
704static void __cpuinit
705build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
706 unsigned int pmd, int lid)
707{
708 UASM_i_LW(p, tmp, 0, pmd);
David Daneycc33ae42010-12-20 15:54:50 -0800709 if (use_bbit_insns()) {
710 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
711 } else {
712 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
713 uasm_il_bnez(p, r, tmp, lid);
714 }
David Daneyfd062c82009-05-27 17:47:44 -0700715}
716
717static __cpuinit void build_huge_update_entries(u32 **p,
718 unsigned int pte,
719 unsigned int tmp)
720{
721 int small_sequence;
722
723 /*
724 * A huge PTE describes an area the size of the
725 * configured huge page size. This is twice the
726 * of the large TLB entry size we intend to use.
727 * A TLB entry half the size of the configured
728 * huge page size is configured into entrylo0
729 * and entrylo1 to cover the contiguous huge PTE
730 * address space.
731 */
732 small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
733
734 /* We can clobber tmp. It isn't used after this.*/
735 if (!small_sequence)
736 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
737
David Daney6dd93442010-02-10 15:12:47 -0800738 build_convert_pte_to_entrylo(p, pte);
David Daney9b8c3892010-02-10 15:12:44 -0800739 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
David Daneyfd062c82009-05-27 17:47:44 -0700740 /* convert to entrylo1 */
741 if (small_sequence)
742 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
743 else
744 UASM_i_ADDU(p, pte, pte, tmp);
745
David Daney9b8c3892010-02-10 15:12:44 -0800746 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
David Daneyfd062c82009-05-27 17:47:44 -0700747}
748
749static __cpuinit void build_huge_handler_tail(u32 **p,
750 struct uasm_reloc **r,
751 struct uasm_label **l,
752 unsigned int pte,
753 unsigned int ptr)
754{
755#ifdef CONFIG_SMP
756 UASM_i_SC(p, pte, 0, ptr);
757 uasm_il_beqz(p, r, pte, label_tlb_huge_update);
758 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
759#else
760 UASM_i_SW(p, pte, 0, ptr);
761#endif
762 build_huge_update_entries(p, pte, ptr);
David Daney2c8c53e2010-12-27 18:07:57 -0800763 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
David Daneyfd062c82009-05-27 17:47:44 -0700764}
765#endif /* CONFIG_HUGETLB_PAGE */
766
Ralf Baechle875d43e2005-09-03 15:56:16 -0700767#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768/*
769 * TMP and PTR are scratch.
770 * TMP will be clobbered, PTR will hold the pmd entry.
771 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000772static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +0000773build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 unsigned int tmp, unsigned int ptr)
775{
David Daney826222842009-10-14 12:16:56 -0700776#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 long pgdc = (long)pgd_current;
David Daney826222842009-10-14 12:16:56 -0700778#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 /*
780 * The vmalloc handling is not in the hotpath.
781 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000782 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
David Daney1ec56322010-04-28 12:16:18 -0700783
784 if (check_for_high_segbits) {
785 /*
786 * The kernel currently implicitely assumes that the
787 * MIPS SEGBITS parameter for the processor is
788 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
789 * allocate virtual addresses outside the maximum
790 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
791 * that doesn't prevent user code from accessing the
792 * higher xuseg addresses. Here, we make sure that
793 * everything but the lower xuseg addresses goes down
794 * the module_alloc/vmalloc path.
795 */
796 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
797 uasm_il_bnez(p, r, ptr, label_vmalloc);
798 } else {
799 uasm_il_bltz(p, r, tmp, label_vmalloc);
800 }
Thiemo Seufere30ec452008-01-28 20:05:38 +0000801 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
David Daney826222842009-10-14 12:16:56 -0700803#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
David Daney3d8bfdd2010-12-21 14:19:11 -0800804 if (pgd_reg != -1) {
805 /* pgd is in pgd_reg */
806 UASM_i_MFC0(p, ptr, 31, pgd_reg);
807 } else {
808 /*
809 * &pgd << 11 stored in CONTEXT [23..63].
810 */
811 UASM_i_MFC0(p, ptr, C0_CONTEXT);
812
813 /* Clear lower 23 bits of context. */
814 uasm_i_dins(p, ptr, 0, 0, 23);
815
816 /* 1 0 1 0 1 << 6 xkphys cached */
817 uasm_i_ori(p, ptr, ptr, 0x540);
818 uasm_i_drotr(p, ptr, ptr, 11);
819 }
David Daney826222842009-10-14 12:16:56 -0700820#elif defined(CONFIG_SMP)
Ralf Baechle41c594a2006-04-05 09:45:45 +0100821# ifdef CONFIG_MIPS_MT_SMTC
822 /*
823 * SMTC uses TCBind value as "CPU" index
824 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000825 uasm_i_mfc0(p, ptr, C0_TCBIND);
David Daney3be60222010-04-28 12:16:17 -0700826 uasm_i_dsrl_safe(p, ptr, ptr, 19);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100827# else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 /*
Thiemo Seufer1b3a6e92005-04-01 14:07:13 +0000829 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 * stored in CONTEXT.
831 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000832 uasm_i_dmfc0(p, ptr, C0_CONTEXT);
David Daney3be60222010-04-28 12:16:17 -0700833 uasm_i_dsrl_safe(p, ptr, ptr, 23);
David Daney826222842009-10-14 12:16:56 -0700834# endif
Thiemo Seufere30ec452008-01-28 20:05:38 +0000835 UASM_i_LA_mostly(p, tmp, pgdc);
836 uasm_i_daddu(p, ptr, ptr, tmp);
837 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
838 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839#else
Thiemo Seufere30ec452008-01-28 20:05:38 +0000840 UASM_i_LA_mostly(p, ptr, pgdc);
841 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842#endif
843
Thiemo Seufere30ec452008-01-28 20:05:38 +0000844 uasm_l_vmalloc_done(l, *p);
Ralf Baechle242954b2006-10-24 02:29:01 +0100845
David Daney3be60222010-04-28 12:16:17 -0700846 /* get pgd offset in bytes */
847 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
Ralf Baechle242954b2006-10-24 02:29:01 +0100848
Thiemo Seufere30ec452008-01-28 20:05:38 +0000849 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
850 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
David Daney325f8a02009-12-04 13:52:36 -0800851#ifndef __PAGETABLE_PMD_FOLDED
Thiemo Seufere30ec452008-01-28 20:05:38 +0000852 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
853 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
David Daney3be60222010-04-28 12:16:17 -0700854 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000855 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
856 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
David Daney325f8a02009-12-04 13:52:36 -0800857#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858}
859
860/*
861 * BVADDR is the faulting address, PTR is scratch.
862 * PTR will hold the pgd for vmalloc.
863 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000864static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +0000865build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
David Daney1ec56322010-04-28 12:16:18 -0700866 unsigned int bvaddr, unsigned int ptr,
867 enum vmalloc64_mode mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868{
869 long swpd = (long)swapper_pg_dir;
David Daney1ec56322010-04-28 12:16:18 -0700870 int single_insn_swpd;
871 int did_vmalloc_branch = 0;
872
873 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874
Thiemo Seufere30ec452008-01-28 20:05:38 +0000875 uasm_l_vmalloc(l, *p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876
David Daney2c8c53e2010-12-27 18:07:57 -0800877 if (mode != not_refill && check_for_high_segbits) {
David Daney1ec56322010-04-28 12:16:18 -0700878 if (single_insn_swpd) {
879 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
880 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
881 did_vmalloc_branch = 1;
882 /* fall through */
883 } else {
884 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
885 }
886 }
887 if (!did_vmalloc_branch) {
888 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
889 uasm_il_b(p, r, label_vmalloc_done);
890 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
891 } else {
892 UASM_i_LA_mostly(p, ptr, swpd);
893 uasm_il_b(p, r, label_vmalloc_done);
894 if (uasm_in_compat_space_p(swpd))
895 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
896 else
897 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
898 }
899 }
David Daney2c8c53e2010-12-27 18:07:57 -0800900 if (mode != not_refill && check_for_high_segbits) {
David Daney1ec56322010-04-28 12:16:18 -0700901 uasm_l_large_segbits_fault(l, *p);
902 /*
903 * We get here if we are an xsseg address, or if we are
904 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
905 *
906 * Ignoring xsseg (assume disabled so would generate
907 * (address errors?), the only remaining possibility
908 * is the upper xuseg addresses. On processors with
909 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
910 * addresses would have taken an address error. We try
911 * to mimic that here by taking a load/istream page
912 * fault.
913 */
914 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
915 uasm_i_jr(p, ptr);
David Daney2c8c53e2010-12-27 18:07:57 -0800916
917 if (mode == refill_scratch) {
918 if (scratch_reg > 0)
919 UASM_i_MFC0(p, 1, 31, scratch_reg);
920 else
921 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
922 } else {
923 uasm_i_nop(p);
924 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925 }
926}
927
Ralf Baechle875d43e2005-09-03 15:56:16 -0700928#else /* !CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930/*
931 * TMP and PTR are scratch.
932 * TMP will be clobbered, PTR will hold the pgd entry.
933 */
Ralf Baechle234fcd12008-03-08 09:56:28 +0000934static void __cpuinit __maybe_unused
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
936{
937 long pgdc = (long)pgd_current;
938
939 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
940#ifdef CONFIG_SMP
Ralf Baechle41c594a2006-04-05 09:45:45 +0100941#ifdef CONFIG_MIPS_MT_SMTC
942 /*
943 * SMTC uses TCBind value as "CPU" index
944 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000945 uasm_i_mfc0(p, ptr, C0_TCBIND);
946 UASM_i_LA_mostly(p, tmp, pgdc);
947 uasm_i_srl(p, ptr, ptr, 19);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100948#else
949 /*
950 * smp_processor_id() << 3 is stored in CONTEXT.
951 */
Thiemo Seufere30ec452008-01-28 20:05:38 +0000952 uasm_i_mfc0(p, ptr, C0_CONTEXT);
953 UASM_i_LA_mostly(p, tmp, pgdc);
954 uasm_i_srl(p, ptr, ptr, 23);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100955#endif
Thiemo Seufere30ec452008-01-28 20:05:38 +0000956 uasm_i_addu(p, ptr, tmp, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957#else
Thiemo Seufere30ec452008-01-28 20:05:38 +0000958 UASM_i_LA_mostly(p, ptr, pgdc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959#endif
Thiemo Seufere30ec452008-01-28 20:05:38 +0000960 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
961 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
Steven J. Hillff401e52012-08-28 23:20:39 -0500962
963 if (cpu_has_mips_r2) {
964 uasm_i_ext(p, tmp, tmp, PGDIR_SHIFT, (32 - PGDIR_SHIFT));
965 uasm_i_ins(p, ptr, tmp, PGD_T_LOG2, (32 - PGDIR_SHIFT));
966 return;
967 }
968
Thiemo Seufere30ec452008-01-28 20:05:38 +0000969 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
970 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
971 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972}
973
Ralf Baechle875d43e2005-09-03 15:56:16 -0700974#endif /* !CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975
Ralf Baechle234fcd12008-03-08 09:56:28 +0000976static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977{
Ralf Baechle242954b2006-10-24 02:29:01 +0100978 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
980
Ralf Baechle10cc3522007-10-11 23:46:15 +0100981 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 case CPU_VR41XX:
983 case CPU_VR4111:
984 case CPU_VR4121:
985 case CPU_VR4122:
986 case CPU_VR4131:
987 case CPU_VR4181:
988 case CPU_VR4181A:
989 case CPU_VR4133:
990 shift += 2;
991 break;
992
993 default:
994 break;
995 }
996
997 if (shift)
Thiemo Seufere30ec452008-01-28 20:05:38 +0000998 UASM_i_SRL(p, ctx, ctx, shift);
999 uasm_i_andi(p, ctx, ctx, mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000}
1001
Ralf Baechle234fcd12008-03-08 09:56:28 +00001002static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003{
Steven J. Hillff401e52012-08-28 23:20:39 -05001004 if (cpu_has_mips_r2) {
1005 /* PTE ptr offset is obtained from BadVAddr */
1006 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1007 UASM_i_LW(p, ptr, 0, ptr);
1008 uasm_i_ext(p, tmp, tmp, PAGE_SHIFT+1, PGDIR_SHIFT-PAGE_SHIFT-1);
1009 uasm_i_ins(p, ptr, tmp, PTE_T_LOG2+1, PGDIR_SHIFT-PAGE_SHIFT-1);
1010 return;
1011 }
1012
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 /*
1014 * Bug workaround for the Nevada. It seems as if under certain
1015 * circumstances the move from cp0_context might produce a
1016 * bogus result when the mfc0 instruction and its consumer are
1017 * in a different cacheline or a load instruction, probably any
1018 * memory reference, is between them.
1019 */
Ralf Baechle10cc3522007-10-11 23:46:15 +01001020 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 case CPU_NEVADA:
Thiemo Seufere30ec452008-01-28 20:05:38 +00001022 UASM_i_LW(p, ptr, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 GET_CONTEXT(p, tmp); /* get context reg */
1024 break;
1025
1026 default:
1027 GET_CONTEXT(p, tmp); /* get context reg */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001028 UASM_i_LW(p, ptr, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 break;
1030 }
1031
1032 build_adjust_context(p, tmp);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001033 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034}
1035
Ralf Baechle234fcd12008-03-08 09:56:28 +00001036static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 unsigned int ptep)
1038{
1039 /*
1040 * 64bit address support (36bit on a 32bit CPU) in a 32bit
1041 * Kernel is a special case. Only a few CPUs use it.
1042 */
1043#ifdef CONFIG_64BIT_PHYS_ADDR
1044 if (cpu_has_64bits) {
Thiemo Seufere30ec452008-01-28 20:05:38 +00001045 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
1046 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
Steven J. Hill05857c62012-09-13 16:51:46 -05001047 if (cpu_has_rixi) {
David Daney748e7872012-08-23 10:02:03 -07001048 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -08001049 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
David Daney748e7872012-08-23 10:02:03 -07001050 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -08001051 } else {
David Daney3be60222010-04-28 12:16:17 -07001052 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
David Daney6dd93442010-02-10 15:12:47 -08001053 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
David Daney3be60222010-04-28 12:16:17 -07001054 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
David Daney6dd93442010-02-10 15:12:47 -08001055 }
David Daney9b8c3892010-02-10 15:12:44 -08001056 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 } else {
1058 int pte_off_even = sizeof(pte_t) / 2;
1059 int pte_off_odd = pte_off_even + sizeof(pte_t);
1060
1061 /* The pte entries are pre-shifted */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001062 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
David Daney9b8c3892010-02-10 15:12:44 -08001063 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001064 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
David Daney9b8c3892010-02-10 15:12:44 -08001065 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 }
1067#else
Thiemo Seufere30ec452008-01-28 20:05:38 +00001068 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
1069 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 if (r45k_bvahwbug())
1071 build_tlb_probe_entry(p);
Steven J. Hill05857c62012-09-13 16:51:46 -05001072 if (cpu_has_rixi) {
David Daney748e7872012-08-23 10:02:03 -07001073 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -08001074 if (r4k_250MHZhwbug())
1075 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
1076 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
David Daney748e7872012-08-23 10:02:03 -07001077 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL));
David Daney6dd93442010-02-10 15:12:47 -08001078 } else {
1079 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
1080 if (r4k_250MHZhwbug())
1081 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
1082 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
1083 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
1084 if (r45k_bvahwbug())
1085 uasm_i_mfc0(p, tmp, C0_INDEX);
1086 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087 if (r4k_250MHZhwbug())
David Daney9b8c3892010-02-10 15:12:44 -08001088 UASM_i_MTC0(p, 0, C0_ENTRYLO1);
1089 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090#endif
1091}
1092
David Daney2c8c53e2010-12-27 18:07:57 -08001093struct mips_huge_tlb_info {
1094 int huge_pte;
1095 int restore_scratch;
1096};
1097
1098static struct mips_huge_tlb_info __cpuinit
1099build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1100 struct uasm_reloc **r, unsigned int tmp,
1101 unsigned int ptr, int c0_scratch)
1102{
1103 struct mips_huge_tlb_info rv;
1104 unsigned int even, odd;
1105 int vmalloc_branch_delay_filled = 0;
1106 const int scratch = 1; /* Our extra working register */
1107
1108 rv.huge_pte = scratch;
1109 rv.restore_scratch = 0;
1110
1111 if (check_for_high_segbits) {
1112 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1113
1114 if (pgd_reg != -1)
1115 UASM_i_MFC0(p, ptr, 31, pgd_reg);
1116 else
1117 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1118
1119 if (c0_scratch >= 0)
1120 UASM_i_MTC0(p, scratch, 31, c0_scratch);
1121 else
1122 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1123
1124 uasm_i_dsrl_safe(p, scratch, tmp,
1125 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1126 uasm_il_bnez(p, r, scratch, label_vmalloc);
1127
1128 if (pgd_reg == -1) {
1129 vmalloc_branch_delay_filled = 1;
1130 /* Clear lower 23 bits of context. */
1131 uasm_i_dins(p, ptr, 0, 0, 23);
1132 }
1133 } else {
1134 if (pgd_reg != -1)
1135 UASM_i_MFC0(p, ptr, 31, pgd_reg);
1136 else
1137 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1138
1139 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1140
1141 if (c0_scratch >= 0)
1142 UASM_i_MTC0(p, scratch, 31, c0_scratch);
1143 else
1144 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1145
1146 if (pgd_reg == -1)
1147 /* Clear lower 23 bits of context. */
1148 uasm_i_dins(p, ptr, 0, 0, 23);
1149
1150 uasm_il_bltz(p, r, tmp, label_vmalloc);
1151 }
1152
1153 if (pgd_reg == -1) {
1154 vmalloc_branch_delay_filled = 1;
1155 /* 1 0 1 0 1 << 6 xkphys cached */
1156 uasm_i_ori(p, ptr, ptr, 0x540);
1157 uasm_i_drotr(p, ptr, ptr, 11);
1158 }
1159
1160#ifdef __PAGETABLE_PMD_FOLDED
1161#define LOC_PTEP scratch
1162#else
1163#define LOC_PTEP ptr
1164#endif
1165
1166 if (!vmalloc_branch_delay_filled)
1167 /* get pgd offset in bytes */
1168 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1169
1170 uasm_l_vmalloc_done(l, *p);
1171
1172 /*
1173 * tmp ptr
1174 * fall-through case = badvaddr *pgd_current
1175 * vmalloc case = badvaddr swapper_pg_dir
1176 */
1177
1178 if (vmalloc_branch_delay_filled)
1179 /* get pgd offset in bytes */
1180 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1181
1182#ifdef __PAGETABLE_PMD_FOLDED
1183 GET_CONTEXT(p, tmp); /* get context reg */
1184#endif
1185 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
1186
1187 if (use_lwx_insns()) {
1188 UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
1189 } else {
1190 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
1191 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
1192 }
1193
1194#ifndef __PAGETABLE_PMD_FOLDED
1195 /* get pmd offset in bytes */
1196 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
1197 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
1198 GET_CONTEXT(p, tmp); /* get context reg */
1199
1200 if (use_lwx_insns()) {
1201 UASM_i_LWX(p, scratch, scratch, ptr);
1202 } else {
1203 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1204 UASM_i_LW(p, scratch, 0, ptr);
1205 }
1206#endif
1207 /* Adjust the context during the load latency. */
1208 build_adjust_context(p, tmp);
1209
1210#ifdef CONFIG_HUGETLB_PAGE
1211 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
1212 /*
1213 * The in the LWX case we don't want to do the load in the
1214 * delay slot. It cannot issue in the same cycle and may be
1215 * speculative and unneeded.
1216 */
1217 if (use_lwx_insns())
1218 uasm_i_nop(p);
1219#endif /* CONFIG_HUGETLB_PAGE */
1220
1221
1222 /* build_update_entries */
1223 if (use_lwx_insns()) {
1224 even = ptr;
1225 odd = tmp;
1226 UASM_i_LWX(p, even, scratch, tmp);
1227 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
1228 UASM_i_LWX(p, odd, scratch, tmp);
1229 } else {
1230 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
1231 even = tmp;
1232 odd = ptr;
1233 UASM_i_LW(p, even, 0, ptr); /* get even pte */
1234 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
1235 }
Steven J. Hill05857c62012-09-13 16:51:46 -05001236 if (cpu_has_rixi) {
David Daney748e7872012-08-23 10:02:03 -07001237 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
David Daney2c8c53e2010-12-27 18:07:57 -08001238 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
David Daney748e7872012-08-23 10:02:03 -07001239 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
David Daney2c8c53e2010-12-27 18:07:57 -08001240 } else {
1241 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
1242 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1243 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
1244 }
1245 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1246
1247 if (c0_scratch >= 0) {
1248 UASM_i_MFC0(p, scratch, 31, c0_scratch);
1249 build_tlb_write_entry(p, l, r, tlb_random);
1250 uasm_l_leave(l, *p);
1251 rv.restore_scratch = 1;
1252 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
1253 build_tlb_write_entry(p, l, r, tlb_random);
1254 uasm_l_leave(l, *p);
1255 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1256 } else {
1257 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1258 build_tlb_write_entry(p, l, r, tlb_random);
1259 uasm_l_leave(l, *p);
1260 rv.restore_scratch = 1;
1261 }
1262
1263 uasm_i_eret(p); /* return from trap */
1264
1265 return rv;
1266}
1267
David Daneye6f72d32009-05-20 11:40:58 -07001268/*
1269 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1270 * because EXL == 0. If we wrap, we can also use the 32 instruction
1271 * slots before the XTLB refill exception handler which belong to the
1272 * unused TLB refill exception.
1273 */
1274#define MIPS64_REFILL_INSNS 32
1275
Ralf Baechle234fcd12008-03-08 09:56:28 +00001276static void __cpuinit build_r4000_tlb_refill_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277{
1278 u32 *p = tlb_handler;
Thiemo Seufere30ec452008-01-28 20:05:38 +00001279 struct uasm_label *l = labels;
1280 struct uasm_reloc *r = relocs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 u32 *f;
1282 unsigned int final_len;
Ralf Baechle4a9040f2011-03-29 10:54:54 +02001283 struct mips_huge_tlb_info htlb_info __maybe_unused;
1284 enum vmalloc64_mode vmalloc_mode __maybe_unused;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285
1286 memset(tlb_handler, 0, sizeof(tlb_handler));
1287 memset(labels, 0, sizeof(labels));
1288 memset(relocs, 0, sizeof(relocs));
1289 memset(final_handler, 0, sizeof(final_handler));
1290
David Daney2c8c53e2010-12-27 18:07:57 -08001291 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) {
1292 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1293 scratch_reg);
1294 vmalloc_mode = refill_scratch;
1295 } else {
1296 htlb_info.huge_pte = K0;
1297 htlb_info.restore_scratch = 0;
1298 vmalloc_mode = refill_noscratch;
1299 /*
1300 * create the plain linear handler
1301 */
1302 if (bcm1250_m3_war()) {
1303 unsigned int segbits = 44;
1304
1305 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1306 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1307 uasm_i_xor(&p, K0, K0, K1);
1308 uasm_i_dsrl_safe(&p, K1, K0, 62);
1309 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1310 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1311 uasm_i_or(&p, K0, K0, K1);
1312 uasm_il_bnez(&p, &r, K0, label_leave);
1313 /* No need for uasm_i_nop */
1314 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315
Ralf Baechle875d43e2005-09-03 15:56:16 -07001316#ifdef CONFIG_64BIT
David Daney2c8c53e2010-12-27 18:07:57 -08001317 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318#else
David Daney2c8c53e2010-12-27 18:07:57 -08001319 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320#endif
1321
David Daneyfd062c82009-05-27 17:47:44 -07001322#ifdef CONFIG_HUGETLB_PAGE
David Daney2c8c53e2010-12-27 18:07:57 -08001323 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
David Daneyfd062c82009-05-27 17:47:44 -07001324#endif
1325
David Daney2c8c53e2010-12-27 18:07:57 -08001326 build_get_ptep(&p, K0, K1);
1327 build_update_entries(&p, K0, K1);
1328 build_tlb_write_entry(&p, &l, &r, tlb_random);
1329 uasm_l_leave(&l, p);
1330 uasm_i_eret(&p); /* return from trap */
1331 }
David Daneyfd062c82009-05-27 17:47:44 -07001332#ifdef CONFIG_HUGETLB_PAGE
1333 uasm_l_tlb_huge_update(&l, p);
David Daney2c8c53e2010-12-27 18:07:57 -08001334 build_huge_update_entries(&p, htlb_info.huge_pte, K1);
1335 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
1336 htlb_info.restore_scratch);
David Daneyfd062c82009-05-27 17:47:44 -07001337#endif
1338
Ralf Baechle875d43e2005-09-03 15:56:16 -07001339#ifdef CONFIG_64BIT
David Daney2c8c53e2010-12-27 18:07:57 -08001340 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341#endif
1342
1343 /*
1344 * Overflow check: For the 64bit handler, we need at least one
1345 * free instruction slot for the wrap-around branch. In worst
1346 * case, if the intended insertion point is a delay slot, we
Matt LaPlante4b3f6862006-10-03 22:21:02 +02001347 * need three, with the second nop'ed and the third being
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 * unused.
1349 */
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001350 /* Loongson2 ebase is different than r4k, we have more space */
1351#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 if ((p - tlb_handler) > 64)
1353 panic("TLB refill handler space exceeded");
1354#else
David Daneye6f72d32009-05-20 11:40:58 -07001355 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
1356 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
1357 && uasm_insn_has_bdelay(relocs,
1358 tlb_handler + MIPS64_REFILL_INSNS - 3)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 panic("TLB refill handler space exceeded");
1360#endif
1361
1362 /*
1363 * Now fold the handler in the TLB refill handler space.
1364 */
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001365#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 f = final_handler;
1367 /* Simplest case, just copy the handler. */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001368 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 final_len = p - tlb_handler;
Ralf Baechle875d43e2005-09-03 15:56:16 -07001370#else /* CONFIG_64BIT */
David Daneye6f72d32009-05-20 11:40:58 -07001371 f = final_handler + MIPS64_REFILL_INSNS;
1372 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 /* Just copy the handler. */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001374 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 final_len = p - tlb_handler;
1376 } else {
David Daneyfd062c82009-05-27 17:47:44 -07001377#if defined(CONFIG_HUGETLB_PAGE)
1378 const enum label_id ls = label_tlb_huge_update;
David Daney95affdd2009-05-20 11:40:59 -07001379#else
1380 const enum label_id ls = label_vmalloc;
1381#endif
1382 u32 *split;
1383 int ov = 0;
1384 int i;
1385
1386 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
1387 ;
1388 BUG_ON(i == ARRAY_SIZE(labels));
1389 split = labels[i].addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390
1391 /*
David Daney95affdd2009-05-20 11:40:59 -07001392 * See if we have overflown one way or the other.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 */
David Daney95affdd2009-05-20 11:40:59 -07001394 if (split > tlb_handler + MIPS64_REFILL_INSNS ||
1395 split < p - MIPS64_REFILL_INSNS)
1396 ov = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397
David Daney95affdd2009-05-20 11:40:59 -07001398 if (ov) {
1399 /*
1400 * Split two instructions before the end. One
1401 * for the branch and one for the instruction
1402 * in the delay slot.
1403 */
1404 split = tlb_handler + MIPS64_REFILL_INSNS - 2;
1405
1406 /*
1407 * If the branch would fall in a delay slot,
1408 * we must back up an additional instruction
1409 * so that it is no longer in a delay slot.
1410 */
1411 if (uasm_insn_has_bdelay(relocs, split - 1))
1412 split--;
1413 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 /* Copy first part of the handler. */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001415 uasm_copy_handler(relocs, labels, tlb_handler, split, f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416 f += split - tlb_handler;
1417
David Daney95affdd2009-05-20 11:40:59 -07001418 if (ov) {
1419 /* Insert branch. */
1420 uasm_l_split(&l, final_handler);
1421 uasm_il_b(&f, &r, label_split);
1422 if (uasm_insn_has_bdelay(relocs, split))
1423 uasm_i_nop(&f);
1424 else {
1425 uasm_copy_handler(relocs, labels,
1426 split, split + 1, f);
1427 uasm_move_labels(labels, f, f + 1, -1);
1428 f++;
1429 split++;
1430 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431 }
1432
1433 /* Copy the rest of the handler. */
Thiemo Seufere30ec452008-01-28 20:05:38 +00001434 uasm_copy_handler(relocs, labels, split, p, final_handler);
David Daneye6f72d32009-05-20 11:40:58 -07001435 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
1436 (p - split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437 }
Ralf Baechle875d43e2005-09-03 15:56:16 -07001438#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439
Thiemo Seufere30ec452008-01-28 20:05:38 +00001440 uasm_resolve_relocs(relocs, labels);
1441 pr_debug("Wrote TLB refill handler (%u instructions).\n",
1442 final_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443
Ralf Baechle91b05e62006-03-29 18:53:00 +01001444 memcpy((void *)ebase, final_handler, 0x100);
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02001445
1446 dump_handler((u32 *)ebase, 64);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447}
1448
1449/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 * 128 instructions for the fastpath handler is generous and should
1451 * never be exceeded.
1452 */
1453#define FASTPATH_SIZE 128
1454
Franck Bui-Huucbdbe072007-10-18 09:11:16 +02001455u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
1456u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
1457u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
David Daney3d8bfdd2010-12-21 14:19:11 -08001458#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
1459u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
1460
1461static void __cpuinit build_r4000_setup_pgd(void)
1462{
1463 const int a0 = 4;
1464 const int a1 = 5;
1465 u32 *p = tlbmiss_handler_setup_pgd;
1466 struct uasm_label *l = labels;
1467 struct uasm_reloc *r = relocs;
1468
1469 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
1470 memset(labels, 0, sizeof(labels));
1471 memset(relocs, 0, sizeof(relocs));
1472
1473 pgd_reg = allocate_kscratch();
1474
1475 if (pgd_reg == -1) {
1476 /* PGD << 11 in c0_Context */
1477 /*
1478 * If it is a ckseg0 address, convert to a physical
1479 * address. Shifting right by 29 and adding 4 will
1480 * result in zero for these addresses.
1481 *
1482 */
1483 UASM_i_SRA(&p, a1, a0, 29);
1484 UASM_i_ADDIU(&p, a1, a1, 4);
1485 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
1486 uasm_i_nop(&p);
1487 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
1488 uasm_l_tlbl_goaround1(&l, p);
1489 UASM_i_SLL(&p, a0, a0, 11);
1490 uasm_i_jr(&p, 31);
1491 UASM_i_MTC0(&p, a0, C0_CONTEXT);
1492 } else {
1493 /* PGD in c0_KScratch */
1494 uasm_i_jr(&p, 31);
1495 UASM_i_MTC0(&p, a0, 31, pgd_reg);
1496 }
1497 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
1498 panic("tlbmiss_handler_setup_pgd space exceeded");
1499 uasm_resolve_relocs(relocs, labels);
1500 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1501 (unsigned int)(p - tlbmiss_handler_setup_pgd));
1502
1503 dump_handler(tlbmiss_handler_setup_pgd,
1504 ARRAY_SIZE(tlbmiss_handler_setup_pgd));
1505}
1506#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507
Ralf Baechle234fcd12008-03-08 09:56:28 +00001508static void __cpuinit
David Daneybd1437e2009-05-08 15:10:50 -07001509iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510{
1511#ifdef CONFIG_SMP
1512# ifdef CONFIG_64BIT_PHYS_ADDR
1513 if (cpu_has_64bits)
Thiemo Seufere30ec452008-01-28 20:05:38 +00001514 uasm_i_lld(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 else
1516# endif
Thiemo Seufere30ec452008-01-28 20:05:38 +00001517 UASM_i_LL(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518#else
1519# ifdef CONFIG_64BIT_PHYS_ADDR
1520 if (cpu_has_64bits)
Thiemo Seufere30ec452008-01-28 20:05:38 +00001521 uasm_i_ld(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 else
1523# endif
Thiemo Seufere30ec452008-01-28 20:05:38 +00001524 UASM_i_LW(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525#endif
1526}
1527
Ralf Baechle234fcd12008-03-08 09:56:28 +00001528static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001529iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
Thiemo Seufer63b2d2f2005-04-28 08:52:57 +00001530 unsigned int mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531{
Thiemo Seufer63b2d2f2005-04-28 08:52:57 +00001532#ifdef CONFIG_64BIT_PHYS_ADDR
1533 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1534#endif
1535
Thiemo Seufere30ec452008-01-28 20:05:38 +00001536 uasm_i_ori(p, pte, pte, mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537#ifdef CONFIG_SMP
1538# ifdef CONFIG_64BIT_PHYS_ADDR
1539 if (cpu_has_64bits)
Thiemo Seufere30ec452008-01-28 20:05:38 +00001540 uasm_i_scd(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541 else
1542# endif
Thiemo Seufere30ec452008-01-28 20:05:38 +00001543 UASM_i_SC(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544
1545 if (r10000_llsc_war())
Thiemo Seufere30ec452008-01-28 20:05:38 +00001546 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 else
Thiemo Seufere30ec452008-01-28 20:05:38 +00001548 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
1550# ifdef CONFIG_64BIT_PHYS_ADDR
1551 if (!cpu_has_64bits) {
Thiemo Seufere30ec452008-01-28 20:05:38 +00001552 /* no uasm_i_nop needed */
1553 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1554 uasm_i_ori(p, pte, pte, hwmode);
1555 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1556 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1557 /* no uasm_i_nop needed */
1558 uasm_i_lw(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 } else
Thiemo Seufere30ec452008-01-28 20:05:38 +00001560 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561# else
Thiemo Seufere30ec452008-01-28 20:05:38 +00001562 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563# endif
1564#else
1565# ifdef CONFIG_64BIT_PHYS_ADDR
1566 if (cpu_has_64bits)
Thiemo Seufere30ec452008-01-28 20:05:38 +00001567 uasm_i_sd(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 else
1569# endif
Thiemo Seufere30ec452008-01-28 20:05:38 +00001570 UASM_i_SW(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571
1572# ifdef CONFIG_64BIT_PHYS_ADDR
1573 if (!cpu_has_64bits) {
Thiemo Seufere30ec452008-01-28 20:05:38 +00001574 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1575 uasm_i_ori(p, pte, pte, hwmode);
1576 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1577 uasm_i_lw(p, pte, 0, ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 }
1579# endif
1580#endif
1581}
1582
1583/*
1584 * Check if PTE is present, if not then jump to LABEL. PTR points to
1585 * the page table where this PTE is located, PTE will be re-loaded
1586 * with it's original value.
1587 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001588static void __cpuinit
David Daneybd1437e2009-05-08 15:10:50 -07001589build_pte_present(u32 **p, struct uasm_reloc **r,
David Daneybf286072011-07-05 16:34:46 -07001590 int pte, int ptr, int scratch, enum label_id lid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591{
David Daneybf286072011-07-05 16:34:46 -07001592 int t = scratch >= 0 ? scratch : pte;
1593
Steven J. Hill05857c62012-09-13 16:51:46 -05001594 if (cpu_has_rixi) {
David Daneycc33ae42010-12-20 15:54:50 -08001595 if (use_bbit_insns()) {
1596 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1597 uasm_i_nop(p);
1598 } else {
David Daneybf286072011-07-05 16:34:46 -07001599 uasm_i_andi(p, t, pte, _PAGE_PRESENT);
1600 uasm_il_beqz(p, r, t, lid);
1601 if (pte == t)
1602 /* You lose the SMP race :-(*/
1603 iPTE_LW(p, pte, ptr);
David Daneycc33ae42010-12-20 15:54:50 -08001604 }
David Daney6dd93442010-02-10 15:12:47 -08001605 } else {
David Daneybf286072011-07-05 16:34:46 -07001606 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ);
1607 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ);
1608 uasm_il_bnez(p, r, t, lid);
1609 if (pte == t)
1610 /* You lose the SMP race :-(*/
1611 iPTE_LW(p, pte, ptr);
David Daney6dd93442010-02-10 15:12:47 -08001612 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613}
1614
1615/* Make PTE valid, store result in PTR. */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001616static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001617build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 unsigned int ptr)
1619{
Thiemo Seufer63b2d2f2005-04-28 08:52:57 +00001620 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1621
1622 iPTE_SW(p, r, pte, ptr, mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623}
1624
1625/*
1626 * Check if PTE can be written to, if not branch to LABEL. Regardless
1627 * restore PTE with value from PTR when done.
1628 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001629static void __cpuinit
David Daneybd1437e2009-05-08 15:10:50 -07001630build_pte_writable(u32 **p, struct uasm_reloc **r,
David Daneybf286072011-07-05 16:34:46 -07001631 unsigned int pte, unsigned int ptr, int scratch,
1632 enum label_id lid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633{
David Daneybf286072011-07-05 16:34:46 -07001634 int t = scratch >= 0 ? scratch : pte;
1635
1636 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE);
1637 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE);
1638 uasm_il_bnez(p, r, t, lid);
1639 if (pte == t)
1640 /* You lose the SMP race :-(*/
David Daneycc33ae42010-12-20 15:54:50 -08001641 iPTE_LW(p, pte, ptr);
David Daneybf286072011-07-05 16:34:46 -07001642 else
1643 uasm_i_nop(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644}
1645
1646/* Make PTE writable, update software status bits as well, then store
1647 * at PTR.
1648 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001649static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001650build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 unsigned int ptr)
1652{
Thiemo Seufer63b2d2f2005-04-28 08:52:57 +00001653 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1654 | _PAGE_DIRTY);
1655
1656 iPTE_SW(p, r, pte, ptr, mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657}
1658
1659/*
1660 * Check if PTE can be modified, if not branch to LABEL. Regardless
1661 * restore PTE with value from PTR when done.
1662 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001663static void __cpuinit
David Daneybd1437e2009-05-08 15:10:50 -07001664build_pte_modifiable(u32 **p, struct uasm_reloc **r,
David Daneybf286072011-07-05 16:34:46 -07001665 unsigned int pte, unsigned int ptr, int scratch,
1666 enum label_id lid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667{
David Daneycc33ae42010-12-20 15:54:50 -08001668 if (use_bbit_insns()) {
1669 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1670 uasm_i_nop(p);
1671 } else {
David Daneybf286072011-07-05 16:34:46 -07001672 int t = scratch >= 0 ? scratch : pte;
1673 uasm_i_andi(p, t, pte, _PAGE_WRITE);
1674 uasm_il_beqz(p, r, t, lid);
1675 if (pte == t)
1676 /* You lose the SMP race :-(*/
1677 iPTE_LW(p, pte, ptr);
David Daneycc33ae42010-12-20 15:54:50 -08001678 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679}
1680
David Daney826222842009-10-14 12:16:56 -07001681#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
David Daney3d8bfdd2010-12-21 14:19:11 -08001682
1683
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684/*
1685 * R3000 style TLB load/store/modify handlers.
1686 */
1687
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001688/*
1689 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1690 * Then it returns.
1691 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001692static void __cpuinit
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001693build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694{
Thiemo Seufere30ec452008-01-28 20:05:38 +00001695 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1696 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1697 uasm_i_tlbwi(p);
1698 uasm_i_jr(p, tmp);
1699 uasm_i_rfe(p); /* branch delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700}
1701
1702/*
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001703 * This places the pte into ENTRYLO0 and writes it with tlbwi
1704 * or tlbwr as appropriate. This is because the index register
1705 * may have the probe fail bit set as a result of a trap on a
1706 * kseg2 access, i.e. without refill. Then it returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707 */
Ralf Baechle234fcd12008-03-08 09:56:28 +00001708static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001709build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1710 struct uasm_reloc **r, unsigned int pte,
1711 unsigned int tmp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712{
Thiemo Seufere30ec452008-01-28 20:05:38 +00001713 uasm_i_mfc0(p, tmp, C0_INDEX);
1714 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1715 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1716 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
1717 uasm_i_tlbwi(p); /* cp0 delay */
1718 uasm_i_jr(p, tmp);
1719 uasm_i_rfe(p); /* branch delay */
1720 uasm_l_r3000_write_probe_fail(l, *p);
1721 uasm_i_tlbwr(p); /* cp0 delay */
1722 uasm_i_jr(p, tmp);
1723 uasm_i_rfe(p); /* branch delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724}
1725
Ralf Baechle234fcd12008-03-08 09:56:28 +00001726static void __cpuinit
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1728 unsigned int ptr)
1729{
1730 long pgdc = (long)pgd_current;
1731
Thiemo Seufere30ec452008-01-28 20:05:38 +00001732 uasm_i_mfc0(p, pte, C0_BADVADDR);
1733 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
1734 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
1735 uasm_i_srl(p, pte, pte, 22); /* load delay */
1736 uasm_i_sll(p, pte, pte, 2);
1737 uasm_i_addu(p, ptr, ptr, pte);
1738 uasm_i_mfc0(p, pte, C0_CONTEXT);
1739 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
1740 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
1741 uasm_i_addu(p, ptr, ptr, pte);
1742 uasm_i_lw(p, pte, 0, ptr);
1743 uasm_i_tlbp(p); /* load delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744}
1745
Ralf Baechle234fcd12008-03-08 09:56:28 +00001746static void __cpuinit build_r3000_tlb_load_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747{
1748 u32 *p = handle_tlbl;
Thiemo Seufere30ec452008-01-28 20:05:38 +00001749 struct uasm_label *l = labels;
1750 struct uasm_reloc *r = relocs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
1752 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1753 memset(labels, 0, sizeof(labels));
1754 memset(relocs, 0, sizeof(relocs));
1755
1756 build_r3000_tlbchange_handler_head(&p, K0, K1);
David Daneybf286072011-07-05 16:34:46 -07001757 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001758 uasm_i_nop(&p); /* load delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759 build_make_valid(&p, &r, K0, K1);
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001760 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761
Thiemo Seufere30ec452008-01-28 20:05:38 +00001762 uasm_l_nopage_tlbl(&l, p);
1763 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1764 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765
1766 if ((p - handle_tlbl) > FASTPATH_SIZE)
1767 panic("TLB load handler fastpath space exceeded");
1768
Thiemo Seufere30ec452008-01-28 20:05:38 +00001769 uasm_resolve_relocs(relocs, labels);
1770 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1771 (unsigned int)(p - handle_tlbl));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02001773 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774}
1775
Ralf Baechle234fcd12008-03-08 09:56:28 +00001776static void __cpuinit build_r3000_tlb_store_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777{
1778 u32 *p = handle_tlbs;
Thiemo Seufere30ec452008-01-28 20:05:38 +00001779 struct uasm_label *l = labels;
1780 struct uasm_reloc *r = relocs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781
1782 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1783 memset(labels, 0, sizeof(labels));
1784 memset(relocs, 0, sizeof(relocs));
1785
1786 build_r3000_tlbchange_handler_head(&p, K0, K1);
David Daneybf286072011-07-05 16:34:46 -07001787 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001788 uasm_i_nop(&p); /* load delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789 build_make_write(&p, &r, K0, K1);
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001790 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791
Thiemo Seufere30ec452008-01-28 20:05:38 +00001792 uasm_l_nopage_tlbs(&l, p);
1793 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1794 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795
1796 if ((p - handle_tlbs) > FASTPATH_SIZE)
1797 panic("TLB store handler fastpath space exceeded");
1798
Thiemo Seufere30ec452008-01-28 20:05:38 +00001799 uasm_resolve_relocs(relocs, labels);
1800 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1801 (unsigned int)(p - handle_tlbs));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02001803 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804}
1805
Ralf Baechle234fcd12008-03-08 09:56:28 +00001806static void __cpuinit build_r3000_tlb_modify_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807{
1808 u32 *p = handle_tlbm;
Thiemo Seufere30ec452008-01-28 20:05:38 +00001809 struct uasm_label *l = labels;
1810 struct uasm_reloc *r = relocs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811
1812 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1813 memset(labels, 0, sizeof(labels));
1814 memset(relocs, 0, sizeof(relocs));
1815
1816 build_r3000_tlbchange_handler_head(&p, K0, K1);
Ralf Baechled954ffe2011-08-02 22:52:48 +01001817 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001818 uasm_i_nop(&p); /* load delay */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 build_make_write(&p, &r, K0, K1);
Maciej W. Rozyckifded2e52005-06-13 20:24:00 +00001820 build_r3000_pte_reload_tlbwi(&p, K0, K1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821
Thiemo Seufere30ec452008-01-28 20:05:38 +00001822 uasm_l_nopage_tlbm(&l, p);
1823 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1824 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825
1826 if ((p - handle_tlbm) > FASTPATH_SIZE)
1827 panic("TLB modify handler fastpath space exceeded");
1828
Thiemo Seufere30ec452008-01-28 20:05:38 +00001829 uasm_resolve_relocs(relocs, labels);
1830 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1831 (unsigned int)(p - handle_tlbm));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001832
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02001833 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834}
David Daney826222842009-10-14 12:16:56 -07001835#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836
1837/*
1838 * R4000 style TLB load/store/modify handlers.
1839 */
David Daneybf286072011-07-05 16:34:46 -07001840static struct work_registers __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001841build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
David Daneybf286072011-07-05 16:34:46 -07001842 struct uasm_reloc **r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843{
David Daneybf286072011-07-05 16:34:46 -07001844 struct work_registers wr = build_get_work_registers(p);
1845
Ralf Baechle875d43e2005-09-03 15:56:16 -07001846#ifdef CONFIG_64BIT
David Daneybf286072011-07-05 16:34:46 -07001847 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848#else
David Daneybf286072011-07-05 16:34:46 -07001849 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850#endif
1851
David Daneyfd062c82009-05-27 17:47:44 -07001852#ifdef CONFIG_HUGETLB_PAGE
1853 /*
1854 * For huge tlb entries, pmd doesn't contain an address but
1855 * instead contains the tlb pte. Check the PAGE_HUGE bit and
1856 * see if we need to jump to huge tlb processing.
1857 */
David Daneybf286072011-07-05 16:34:46 -07001858 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
David Daneyfd062c82009-05-27 17:47:44 -07001859#endif
1860
David Daneybf286072011-07-05 16:34:46 -07001861 UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
1862 UASM_i_LW(p, wr.r2, 0, wr.r2);
1863 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1864 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1865 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866
1867#ifdef CONFIG_SMP
Thiemo Seufere30ec452008-01-28 20:05:38 +00001868 uasm_l_smp_pgtable_change(l, *p);
1869#endif
David Daneybf286072011-07-05 16:34:46 -07001870 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +01001871 if (!m4kc_tlbp_war())
1872 build_tlb_probe_entry(p);
David Daneybf286072011-07-05 16:34:46 -07001873 return wr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874}
1875
Ralf Baechle234fcd12008-03-08 09:56:28 +00001876static void __cpuinit
Thiemo Seufere30ec452008-01-28 20:05:38 +00001877build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1878 struct uasm_reloc **r, unsigned int tmp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879 unsigned int ptr)
1880{
Thiemo Seufere30ec452008-01-28 20:05:38 +00001881 uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
1882 uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883 build_update_entries(p, tmp, ptr);
1884 build_tlb_write_entry(p, l, r, tlb_indexed);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001885 uasm_l_leave(l, *p);
David Daneybf286072011-07-05 16:34:46 -07001886 build_restore_work_registers(p);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001887 uasm_i_eret(p); /* return from trap */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888
Ralf Baechle875d43e2005-09-03 15:56:16 -07001889#ifdef CONFIG_64BIT
David Daney1ec56322010-04-28 12:16:18 -07001890 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891#endif
1892}
1893
Ralf Baechle234fcd12008-03-08 09:56:28 +00001894static void __cpuinit build_r4000_tlb_load_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895{
1896 u32 *p = handle_tlbl;
Thiemo Seufere30ec452008-01-28 20:05:38 +00001897 struct uasm_label *l = labels;
1898 struct uasm_reloc *r = relocs;
David Daneybf286072011-07-05 16:34:46 -07001899 struct work_registers wr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900
1901 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1902 memset(labels, 0, sizeof(labels));
1903 memset(relocs, 0, sizeof(relocs));
1904
1905 if (bcm1250_m3_war()) {
Ralf Baechle3d452852010-03-23 17:56:38 +01001906 unsigned int segbits = 44;
1907
1908 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1909 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001910 uasm_i_xor(&p, K0, K0, K1);
David Daney3be60222010-04-28 12:16:17 -07001911 uasm_i_dsrl_safe(&p, K1, K0, 62);
1912 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1913 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
Ralf Baechle3d452852010-03-23 17:56:38 +01001914 uasm_i_or(&p, K0, K0, K1);
Thiemo Seufere30ec452008-01-28 20:05:38 +00001915 uasm_il_bnez(&p, &r, K0, label_leave);
1916 /* No need for uasm_i_nop */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 }
1918
David Daneybf286072011-07-05 16:34:46 -07001919 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
1920 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +01001921 if (m4kc_tlbp_war())
1922 build_tlb_probe_entry(&p);
David Daney6dd93442010-02-10 15:12:47 -08001923
Steven J. Hill05857c62012-09-13 16:51:46 -05001924 if (cpu_has_rixi) {
David Daney6dd93442010-02-10 15:12:47 -08001925 /*
1926 * If the page is not _PAGE_VALID, RI or XI could not
1927 * have triggered it. Skip the expensive test..
1928 */
David Daneycc33ae42010-12-20 15:54:50 -08001929 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07001930 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
David Daneycc33ae42010-12-20 15:54:50 -08001931 label_tlbl_goaround1);
1932 } else {
David Daneybf286072011-07-05 16:34:46 -07001933 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
1934 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
David Daneycc33ae42010-12-20 15:54:50 -08001935 }
David Daney6dd93442010-02-10 15:12:47 -08001936 uasm_i_nop(&p);
1937
1938 uasm_i_tlbr(&p);
1939 /* Examine entrylo 0 or 1 based on ptr. */
David Daneycc33ae42010-12-20 15:54:50 -08001940 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07001941 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
David Daneycc33ae42010-12-20 15:54:50 -08001942 } else {
David Daneybf286072011-07-05 16:34:46 -07001943 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
1944 uasm_i_beqz(&p, wr.r3, 8);
David Daneycc33ae42010-12-20 15:54:50 -08001945 }
David Daneybf286072011-07-05 16:34:46 -07001946 /* load it in the delay slot*/
1947 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
1948 /* load it if ptr is odd */
1949 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
David Daney6dd93442010-02-10 15:12:47 -08001950 /*
David Daneybf286072011-07-05 16:34:46 -07001951 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
David Daney6dd93442010-02-10 15:12:47 -08001952 * XI must have triggered it.
1953 */
David Daneycc33ae42010-12-20 15:54:50 -08001954 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07001955 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
1956 uasm_i_nop(&p);
David Daneycc33ae42010-12-20 15:54:50 -08001957 uasm_l_tlbl_goaround1(&l, p);
1958 } else {
David Daneybf286072011-07-05 16:34:46 -07001959 uasm_i_andi(&p, wr.r3, wr.r3, 2);
1960 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
1961 uasm_i_nop(&p);
David Daneycc33ae42010-12-20 15:54:50 -08001962 }
David Daneybf286072011-07-05 16:34:46 -07001963 uasm_l_tlbl_goaround1(&l, p);
David Daney6dd93442010-02-10 15:12:47 -08001964 }
David Daneybf286072011-07-05 16:34:46 -07001965 build_make_valid(&p, &r, wr.r1, wr.r2);
1966 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967
David Daneyfd062c82009-05-27 17:47:44 -07001968#ifdef CONFIG_HUGETLB_PAGE
1969 /*
1970 * This is the entry point when build_r4000_tlbchange_handler_head
1971 * spots a huge page.
1972 */
1973 uasm_l_tlb_huge_update(&l, p);
David Daneybf286072011-07-05 16:34:46 -07001974 iPTE_LW(&p, wr.r1, wr.r2);
1975 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
David Daneyfd062c82009-05-27 17:47:44 -07001976 build_tlb_probe_entry(&p);
David Daney6dd93442010-02-10 15:12:47 -08001977
Steven J. Hill05857c62012-09-13 16:51:46 -05001978 if (cpu_has_rixi) {
David Daney6dd93442010-02-10 15:12:47 -08001979 /*
1980 * If the page is not _PAGE_VALID, RI or XI could not
1981 * have triggered it. Skip the expensive test..
1982 */
David Daneycc33ae42010-12-20 15:54:50 -08001983 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07001984 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
David Daneycc33ae42010-12-20 15:54:50 -08001985 label_tlbl_goaround2);
1986 } else {
David Daneybf286072011-07-05 16:34:46 -07001987 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
1988 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
David Daneycc33ae42010-12-20 15:54:50 -08001989 }
David Daney6dd93442010-02-10 15:12:47 -08001990 uasm_i_nop(&p);
1991
1992 uasm_i_tlbr(&p);
1993 /* Examine entrylo 0 or 1 based on ptr. */
David Daneycc33ae42010-12-20 15:54:50 -08001994 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07001995 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
David Daneycc33ae42010-12-20 15:54:50 -08001996 } else {
David Daneybf286072011-07-05 16:34:46 -07001997 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
1998 uasm_i_beqz(&p, wr.r3, 8);
David Daneycc33ae42010-12-20 15:54:50 -08001999 }
David Daneybf286072011-07-05 16:34:46 -07002000 /* load it in the delay slot*/
2001 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2002 /* load it if ptr is odd */
2003 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
David Daney6dd93442010-02-10 15:12:47 -08002004 /*
David Daneybf286072011-07-05 16:34:46 -07002005 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
David Daney6dd93442010-02-10 15:12:47 -08002006 * XI must have triggered it.
2007 */
David Daneycc33ae42010-12-20 15:54:50 -08002008 if (use_bbit_insns()) {
David Daneybf286072011-07-05 16:34:46 -07002009 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
David Daneycc33ae42010-12-20 15:54:50 -08002010 } else {
David Daneybf286072011-07-05 16:34:46 -07002011 uasm_i_andi(&p, wr.r3, wr.r3, 2);
2012 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
David Daneycc33ae42010-12-20 15:54:50 -08002013 }
David Daney0f4ccbc2011-09-16 18:06:02 -07002014 if (PM_DEFAULT_MASK == 0)
2015 uasm_i_nop(&p);
David Daney6dd93442010-02-10 15:12:47 -08002016 /*
2017 * We clobbered C0_PAGEMASK, restore it. On the other branch
2018 * it is restored in build_huge_tlb_write_entry.
2019 */
David Daneybf286072011-07-05 16:34:46 -07002020 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
David Daney6dd93442010-02-10 15:12:47 -08002021
2022 uasm_l_tlbl_goaround2(&l, p);
2023 }
David Daneybf286072011-07-05 16:34:46 -07002024 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
2025 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
David Daneyfd062c82009-05-27 17:47:44 -07002026#endif
2027
Thiemo Seufere30ec452008-01-28 20:05:38 +00002028 uasm_l_nopage_tlbl(&l, p);
David Daneybf286072011-07-05 16:34:46 -07002029 build_restore_work_registers(&p);
Thiemo Seufere30ec452008-01-28 20:05:38 +00002030 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
2031 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032
2033 if ((p - handle_tlbl) > FASTPATH_SIZE)
2034 panic("TLB load handler fastpath space exceeded");
2035
Thiemo Seufere30ec452008-01-28 20:05:38 +00002036 uasm_resolve_relocs(relocs, labels);
2037 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
2038 (unsigned int)(p - handle_tlbl));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02002040 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002041}
2042
Ralf Baechle234fcd12008-03-08 09:56:28 +00002043static void __cpuinit build_r4000_tlb_store_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044{
2045 u32 *p = handle_tlbs;
Thiemo Seufere30ec452008-01-28 20:05:38 +00002046 struct uasm_label *l = labels;
2047 struct uasm_reloc *r = relocs;
David Daneybf286072011-07-05 16:34:46 -07002048 struct work_registers wr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049
2050 memset(handle_tlbs, 0, sizeof(handle_tlbs));
2051 memset(labels, 0, sizeof(labels));
2052 memset(relocs, 0, sizeof(relocs));
2053
David Daneybf286072011-07-05 16:34:46 -07002054 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2055 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +01002056 if (m4kc_tlbp_war())
2057 build_tlb_probe_entry(&p);
David Daneybf286072011-07-05 16:34:46 -07002058 build_make_write(&p, &r, wr.r1, wr.r2);
2059 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060
David Daneyfd062c82009-05-27 17:47:44 -07002061#ifdef CONFIG_HUGETLB_PAGE
2062 /*
2063 * This is the entry point when
2064 * build_r4000_tlbchange_handler_head spots a huge page.
2065 */
2066 uasm_l_tlb_huge_update(&l, p);
David Daneybf286072011-07-05 16:34:46 -07002067 iPTE_LW(&p, wr.r1, wr.r2);
2068 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
David Daneyfd062c82009-05-27 17:47:44 -07002069 build_tlb_probe_entry(&p);
David Daneybf286072011-07-05 16:34:46 -07002070 uasm_i_ori(&p, wr.r1, wr.r1,
David Daneyfd062c82009-05-27 17:47:44 -07002071 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
David Daneybf286072011-07-05 16:34:46 -07002072 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
David Daneyfd062c82009-05-27 17:47:44 -07002073#endif
2074
Thiemo Seufere30ec452008-01-28 20:05:38 +00002075 uasm_l_nopage_tlbs(&l, p);
David Daneybf286072011-07-05 16:34:46 -07002076 build_restore_work_registers(&p);
Thiemo Seufere30ec452008-01-28 20:05:38 +00002077 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2078 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079
2080 if ((p - handle_tlbs) > FASTPATH_SIZE)
2081 panic("TLB store handler fastpath space exceeded");
2082
Thiemo Seufere30ec452008-01-28 20:05:38 +00002083 uasm_resolve_relocs(relocs, labels);
2084 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
2085 (unsigned int)(p - handle_tlbs));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02002087 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088}
2089
Ralf Baechle234fcd12008-03-08 09:56:28 +00002090static void __cpuinit build_r4000_tlb_modify_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091{
2092 u32 *p = handle_tlbm;
Thiemo Seufere30ec452008-01-28 20:05:38 +00002093 struct uasm_label *l = labels;
2094 struct uasm_reloc *r = relocs;
David Daneybf286072011-07-05 16:34:46 -07002095 struct work_registers wr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096
2097 memset(handle_tlbm, 0, sizeof(handle_tlbm));
2098 memset(labels, 0, sizeof(labels));
2099 memset(relocs, 0, sizeof(relocs));
2100
David Daneybf286072011-07-05 16:34:46 -07002101 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2102 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
Maciej W. Rozycki8df5bea2006-08-23 14:26:50 +01002103 if (m4kc_tlbp_war())
2104 build_tlb_probe_entry(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002105 /* Present and writable bits set, set accessed and dirty bits. */
David Daneybf286072011-07-05 16:34:46 -07002106 build_make_write(&p, &r, wr.r1, wr.r2);
2107 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108
David Daneyfd062c82009-05-27 17:47:44 -07002109#ifdef CONFIG_HUGETLB_PAGE
2110 /*
2111 * This is the entry point when
2112 * build_r4000_tlbchange_handler_head spots a huge page.
2113 */
2114 uasm_l_tlb_huge_update(&l, p);
David Daneybf286072011-07-05 16:34:46 -07002115 iPTE_LW(&p, wr.r1, wr.r2);
2116 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
David Daneyfd062c82009-05-27 17:47:44 -07002117 build_tlb_probe_entry(&p);
David Daneybf286072011-07-05 16:34:46 -07002118 uasm_i_ori(&p, wr.r1, wr.r1,
David Daneyfd062c82009-05-27 17:47:44 -07002119 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
David Daneybf286072011-07-05 16:34:46 -07002120 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
David Daneyfd062c82009-05-27 17:47:44 -07002121#endif
2122
Thiemo Seufere30ec452008-01-28 20:05:38 +00002123 uasm_l_nopage_tlbm(&l, p);
David Daneybf286072011-07-05 16:34:46 -07002124 build_restore_work_registers(&p);
Thiemo Seufere30ec452008-01-28 20:05:38 +00002125 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2126 uasm_i_nop(&p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127
2128 if ((p - handle_tlbm) > FASTPATH_SIZE)
2129 panic("TLB modify handler fastpath space exceeded");
2130
Thiemo Seufere30ec452008-01-28 20:05:38 +00002131 uasm_resolve_relocs(relocs, labels);
2132 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2133 (unsigned int)(p - handle_tlbm));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002134
Franck Bui-Huu92b1e6a2007-10-18 09:11:17 +02002135 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136}
2137
Ralf Baechle234fcd12008-03-08 09:56:28 +00002138void __cpuinit build_tlb_refill_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139{
2140 /*
2141 * The refill handler is generated per-CPU, multi-node systems
2142 * may have local storage for it. The other handlers are only
2143 * needed once.
2144 */
2145 static int run_once = 0;
2146
David Daney1ec56322010-04-28 12:16:18 -07002147#ifdef CONFIG_64BIT
2148 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
2149#endif
2150
Ralf Baechle10cc3522007-10-11 23:46:15 +01002151 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152 case CPU_R2000:
2153 case CPU_R3000:
2154 case CPU_R3000A:
2155 case CPU_R3081E:
2156 case CPU_TX3912:
2157 case CPU_TX3922:
2158 case CPU_TX3927:
David Daney826222842009-10-14 12:16:56 -07002159#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
Linus Torvalds1da177e2005-04-16 15:20:36 -07002160 build_r3000_tlb_refill_handler();
2161 if (!run_once) {
2162 build_r3000_tlb_load_handler();
2163 build_r3000_tlb_store_handler();
2164 build_r3000_tlb_modify_handler();
2165 run_once++;
2166 }
David Daney826222842009-10-14 12:16:56 -07002167#else
2168 panic("No R3000 TLB refill handler");
2169#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002170 break;
2171
2172 case CPU_R6000:
2173 case CPU_R6000A:
2174 panic("No R6000 TLB refill handler yet");
2175 break;
2176
2177 case CPU_R8000:
2178 panic("No R8000 TLB refill handler yet");
2179 break;
2180
2181 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002182 if (!run_once) {
David Daneybf286072011-07-05 16:34:46 -07002183 scratch_reg = allocate_kscratch();
David Daney3d8bfdd2010-12-21 14:19:11 -08002184#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2185 build_r4000_setup_pgd();
2186#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002187 build_r4000_tlb_load_handler();
2188 build_r4000_tlb_store_handler();
2189 build_r4000_tlb_modify_handler();
2190 run_once++;
2191 }
David Daney3d8bfdd2010-12-21 14:19:11 -08002192 build_r4000_tlb_refill_handler();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002193 }
2194}
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002195
Ralf Baechle234fcd12008-03-08 09:56:28 +00002196void __cpuinit flush_tlb_handlers(void)
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002197{
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02002198 local_flush_icache_range((unsigned long)handle_tlbl,
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002199 (unsigned long)handle_tlbl + sizeof(handle_tlbl));
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02002200 local_flush_icache_range((unsigned long)handle_tlbs,
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002201 (unsigned long)handle_tlbs + sizeof(handle_tlbs));
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02002202 local_flush_icache_range((unsigned long)handle_tlbm,
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002203 (unsigned long)handle_tlbm + sizeof(handle_tlbm));
David Daney3d8bfdd2010-12-21 14:19:11 -08002204#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2205 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2206 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
2207#endif
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00002208}