blob: 493131c81a29b9a1c45bf44a000596ccdf17035f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Justin P. Mattock79add622011-04-04 14:15:29 -07006 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8 * Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
10 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
12#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010013#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/mm.h>
David Daneyfd062c82009-05-27 17:47:44 -070015#include <linux/hugetlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
17#include <asm/cpu.h>
18#include <asm/bootinfo.h>
19#include <asm/mmu_context.h>
20#include <asm/pgtable.h>
Ralf Baechle3d18c982011-11-28 16:11:28 +000021#include <asm/tlbmisc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23extern void build_tlb_refill_handler(void);
24
Thiemo Seufer172546b2005-04-02 10:21:56 +000025/*
26 * Make sure all entries differ. If they're not different
27 * MIPS32 will take revenge ...
28 */
29#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
30
Ralf Baechle41c594a2006-04-05 09:45:45 +010031/* Atomicity and interruptability */
32#ifdef CONFIG_MIPS_MT_SMTC
33
34#include <asm/smtc.h>
35#include <asm/mipsmtregs.h>
36
37#define ENTER_CRITICAL(flags) \
38 { \
39 unsigned int mvpflags; \
40 local_irq_save(flags);\
41 mvpflags = dvpe()
42#define EXIT_CRITICAL(flags) \
43 evpe(mvpflags); \
44 local_irq_restore(flags); \
45 }
46#else
47
48#define ENTER_CRITICAL(flags) local_irq_save(flags)
49#define EXIT_CRITICAL(flags) local_irq_restore(flags)
50
51#endif /* CONFIG_MIPS_MT_SMTC */
52
Fuxin Zhang2a21c732007-06-06 14:52:43 +080053#if defined(CONFIG_CPU_LOONGSON2)
54/*
55 * LOONGSON2 has a 4 entry itlb which is a subset of dtlb,
56 * unfortrunately, itlb is not totally transparent to software.
57 */
58#define FLUSH_ITLB write_c0_diag(4);
59
60#define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); }
61
62#else
63
64#define FLUSH_ITLB
65#define FLUSH_ITLB_VM(vma)
66
67#endif
68
Linus Torvalds1da177e2005-04-16 15:20:36 -070069void local_flush_tlb_all(void)
70{
71 unsigned long flags;
72 unsigned long old_ctx;
73 int entry;
74
Ralf Baechle41c594a2006-04-05 09:45:45 +010075 ENTER_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 /* Save old context and create impossible VPN2 value */
77 old_ctx = read_c0_entryhi();
78 write_c0_entrylo0(0);
79 write_c0_entrylo1(0);
80
81 entry = read_c0_wired();
82
83 /* Blast 'em all away. */
84 while (entry < current_cpu_data.tlbsize) {
Thiemo Seufer172546b2005-04-02 10:21:56 +000085 /* Make sure all entries differ. */
86 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 write_c0_index(entry);
88 mtc0_tlbw_hazard();
89 tlb_write_indexed();
90 entry++;
91 }
92 tlbw_use_hazard();
93 write_c0_entryhi(old_ctx);
Fuxin Zhang2a21c732007-06-06 14:52:43 +080094 FLUSH_ITLB;
Ralf Baechle41c594a2006-04-05 09:45:45 +010095 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070096}
97
Thiemo Seufer172546b2005-04-02 10:21:56 +000098/* All entries common to a mm share an asid. To effectively flush
99 these entries, we just bump the asid. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100void local_flush_tlb_mm(struct mm_struct *mm)
101{
Thiemo Seufer172546b2005-04-02 10:21:56 +0000102 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Thiemo Seufer172546b2005-04-02 10:21:56 +0000104 preempt_disable();
105
106 cpu = smp_processor_id();
107
108 if (cpu_context(cpu, mm) != 0) {
109 drop_mmu_context(mm, cpu);
110 }
111
112 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113}
114
115void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
116 unsigned long end)
117{
118 struct mm_struct *mm = vma->vm_mm;
119 int cpu = smp_processor_id();
120
121 if (cpu_context(cpu, mm) != 0) {
Greg Ungerera5e696e2009-05-20 16:12:32 +1000122 unsigned long size, flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
Ralf Baechle41c594a2006-04-05 09:45:45 +0100124 ENTER_CRITICAL(flags);
David Daneyac53c4f2012-12-03 12:44:26 -0800125 start = round_down(start, PAGE_SIZE << 1);
126 end = round_up(end, PAGE_SIZE << 1);
127 size = (end - start) >> (PAGE_SHIFT + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 if (size <= current_cpu_data.tlbsize/2) {
129 int oldpid = read_c0_entryhi();
130 int newpid = cpu_asid(cpu, mm);
131
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 while (start < end) {
133 int idx;
134
135 write_c0_entryhi(start | newpid);
David Daneyac53c4f2012-12-03 12:44:26 -0800136 start += (PAGE_SIZE << 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 mtc0_tlbw_hazard();
138 tlb_probe();
Ralf Baechle432bef22006-09-08 04:16:21 +0200139 tlb_probe_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 idx = read_c0_index();
141 write_c0_entrylo0(0);
142 write_c0_entrylo1(0);
143 if (idx < 0)
144 continue;
145 /* Make sure all entries differ. */
Thiemo Seufer172546b2005-04-02 10:21:56 +0000146 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 mtc0_tlbw_hazard();
148 tlb_write_indexed();
149 }
150 tlbw_use_hazard();
151 write_c0_entryhi(oldpid);
152 } else {
153 drop_mmu_context(mm, cpu);
154 }
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800155 FLUSH_ITLB;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100156 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 }
158}
159
160void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
161{
Greg Ungerera5e696e2009-05-20 16:12:32 +1000162 unsigned long size, flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
Ralf Baechle41c594a2006-04-05 09:45:45 +0100164 ENTER_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
166 size = (size + 1) >> 1;
167 if (size <= current_cpu_data.tlbsize / 2) {
168 int pid = read_c0_entryhi();
169
170 start &= (PAGE_MASK << 1);
171 end += ((PAGE_SIZE << 1) - 1);
172 end &= (PAGE_MASK << 1);
173
174 while (start < end) {
175 int idx;
176
177 write_c0_entryhi(start);
178 start += (PAGE_SIZE << 1);
179 mtc0_tlbw_hazard();
180 tlb_probe();
Ralf Baechle432bef22006-09-08 04:16:21 +0200181 tlb_probe_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 idx = read_c0_index();
183 write_c0_entrylo0(0);
184 write_c0_entrylo1(0);
185 if (idx < 0)
186 continue;
187 /* Make sure all entries differ. */
Thiemo Seufer172546b2005-04-02 10:21:56 +0000188 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 mtc0_tlbw_hazard();
190 tlb_write_indexed();
191 }
192 tlbw_use_hazard();
193 write_c0_entryhi(pid);
194 } else {
195 local_flush_tlb_all();
196 }
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800197 FLUSH_ITLB;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100198 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199}
200
201void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
202{
203 int cpu = smp_processor_id();
204
205 if (cpu_context(cpu, vma->vm_mm) != 0) {
206 unsigned long flags;
207 int oldpid, newpid, idx;
208
209 newpid = cpu_asid(cpu, vma->vm_mm);
210 page &= (PAGE_MASK << 1);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100211 ENTER_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 oldpid = read_c0_entryhi();
213 write_c0_entryhi(page | newpid);
214 mtc0_tlbw_hazard();
215 tlb_probe();
Ralf Baechle432bef22006-09-08 04:16:21 +0200216 tlb_probe_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 idx = read_c0_index();
218 write_c0_entrylo0(0);
219 write_c0_entrylo1(0);
220 if (idx < 0)
221 goto finish;
222 /* Make sure all entries differ. */
Thiemo Seufer172546b2005-04-02 10:21:56 +0000223 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 mtc0_tlbw_hazard();
225 tlb_write_indexed();
226 tlbw_use_hazard();
227
228 finish:
229 write_c0_entryhi(oldpid);
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800230 FLUSH_ITLB_VM(vma);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100231 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 }
233}
234
235/*
236 * This one is only used for pages with the global bit set so we don't care
237 * much about the ASID.
238 */
239void local_flush_tlb_one(unsigned long page)
240{
241 unsigned long flags;
242 int oldpid, idx;
243
Ralf Baechle41c594a2006-04-05 09:45:45 +0100244 ENTER_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 oldpid = read_c0_entryhi();
Thiemo Seufer172546b2005-04-02 10:21:56 +0000246 page &= (PAGE_MASK << 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 write_c0_entryhi(page);
248 mtc0_tlbw_hazard();
249 tlb_probe();
Ralf Baechle432bef22006-09-08 04:16:21 +0200250 tlb_probe_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 idx = read_c0_index();
252 write_c0_entrylo0(0);
253 write_c0_entrylo1(0);
254 if (idx >= 0) {
255 /* Make sure all entries differ. */
Thiemo Seufer172546b2005-04-02 10:21:56 +0000256 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 mtc0_tlbw_hazard();
258 tlb_write_indexed();
259 tlbw_use_hazard();
260 }
261 write_c0_entryhi(oldpid);
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800262 FLUSH_ITLB;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100263 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264}
265
266/*
267 * We will need multiple versions of update_mmu_cache(), one that just
268 * updates the TLB with the new pte(s), and another which also checks
269 * for the R4k "end of page" hardware bug and does the needy.
270 */
271void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
272{
273 unsigned long flags;
274 pgd_t *pgdp;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000275 pud_t *pudp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 pmd_t *pmdp;
277 pte_t *ptep;
278 int idx, pid;
279
280 /*
281 * Handle debugger faulting in for debugee.
282 */
283 if (current->active_mm != vma->vm_mm)
284 return;
285
Ralf Baechle41c594a2006-04-05 09:45:45 +0100286 ENTER_CRITICAL(flags);
Thiemo Seufer172546b2005-04-02 10:21:56 +0000287
288 pid = read_c0_entryhi() & ASID_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 address &= (PAGE_MASK << 1);
290 write_c0_entryhi(address | pid);
291 pgdp = pgd_offset(vma->vm_mm, address);
292 mtc0_tlbw_hazard();
293 tlb_probe();
Ralf Baechle432bef22006-09-08 04:16:21 +0200294 tlb_probe_hazard();
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000295 pudp = pud_offset(pgdp, address);
296 pmdp = pmd_offset(pudp, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 idx = read_c0_index();
David Daneyaa1762f2012-10-17 00:48:10 +0200298#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
David Daneyfd062c82009-05-27 17:47:44 -0700299 /* this could be a huge page */
300 if (pmd_huge(*pmdp)) {
301 unsigned long lo;
302 write_c0_pagemask(PM_HUGE_MASK);
303 ptep = (pte_t *)pmdp;
David Daney6dd93442010-02-10 15:12:47 -0800304 lo = pte_to_entrylo(pte_val(*ptep));
David Daneyfd062c82009-05-27 17:47:44 -0700305 write_c0_entrylo0(lo);
306 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
307
308 mtc0_tlbw_hazard();
309 if (idx < 0)
310 tlb_write_random();
311 else
312 tlb_write_indexed();
Ralf Baechlefb944c92012-10-17 01:01:21 +0200313 tlbw_use_hazard();
David Daneyfd062c82009-05-27 17:47:44 -0700314 write_c0_pagemask(PM_DEFAULT_MASK);
315 } else
316#endif
317 {
318 ptep = pte_offset_map(pmdp, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Chris Dearman962f4802007-09-19 00:46:32 +0100320#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
David Daneyfd062c82009-05-27 17:47:44 -0700321 write_c0_entrylo0(ptep->pte_high);
322 ptep++;
323 write_c0_entrylo1(ptep->pte_high);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324#else
David Daney6dd93442010-02-10 15:12:47 -0800325 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
326 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327#endif
David Daneyfd062c82009-05-27 17:47:44 -0700328 mtc0_tlbw_hazard();
329 if (idx < 0)
330 tlb_write_random();
331 else
332 tlb_write_indexed();
333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 tlbw_use_hazard();
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800335 FLUSH_ITLB_VM(vma);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100336 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
Manuel Lauss694b8c32011-08-02 19:51:08 +0200339void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
340 unsigned long entryhi, unsigned long pagemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341{
342 unsigned long flags;
343 unsigned long wired;
344 unsigned long old_pagemask;
345 unsigned long old_ctx;
346
Ralf Baechle41c594a2006-04-05 09:45:45 +0100347 ENTER_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 /* Save old context and create impossible VPN2 value */
349 old_ctx = read_c0_entryhi();
350 old_pagemask = read_c0_pagemask();
351 wired = read_c0_wired();
352 write_c0_wired(wired + 1);
353 write_c0_index(wired);
Ralf Baechle432bef22006-09-08 04:16:21 +0200354 tlbw_use_hazard(); /* What is the hazard here? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 write_c0_pagemask(pagemask);
356 write_c0_entryhi(entryhi);
357 write_c0_entrylo0(entrylo0);
358 write_c0_entrylo1(entrylo1);
359 mtc0_tlbw_hazard();
360 tlb_write_indexed();
361 tlbw_use_hazard();
362
363 write_c0_entryhi(old_ctx);
Ralf Baechle432bef22006-09-08 04:16:21 +0200364 tlbw_use_hazard(); /* What is the hazard here? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 write_c0_pagemask(old_pagemask);
366 local_flush_tlb_all();
Ralf Baechle41c594a2006-04-05 09:45:45 +0100367 EXIT_CRITICAL(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368}
369
Ralf Baechle970d0322012-10-18 13:54:15 +0200370#ifdef CONFIG_TRANSPARENT_HUGEPAGE
371
372int __init has_transparent_hugepage(void)
373{
374 unsigned int mask;
375 unsigned long flags;
376
377 ENTER_CRITICAL(flags);
378 write_c0_pagemask(PM_HUGE_MASK);
379 back_to_back_c0_hazard();
380 mask = read_c0_pagemask();
381 write_c0_pagemask(PM_DEFAULT_MASK);
382
383 EXIT_CRITICAL(flags);
384
385 return mask == PM_HUGE_MASK;
386}
387
388#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
389
Ralf Baechle982f6ff2009-09-17 02:25:07 +0200390static int __cpuinitdata ntlb;
Ralf Baechle41c594a2006-04-05 09:45:45 +0100391static int __init set_ntlb(char *str)
392{
393 get_option(&str, &ntlb);
394 return 1;
395}
396
397__setup("ntlb=", set_ntlb);
398
Ralf Baechle234fcd12008-03-08 09:56:28 +0000399void __cpuinit tlb_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 /*
402 * You should never change this register:
403 * - On R4600 1.7 the tlbp never hits for pages smaller than
404 * the value in the c0_pagemask register.
405 * - The entire mm handling assumes the c0_pagemask register to
Thiemo Seufera7c29962008-02-29 00:43:47 +0000406 * be set to fixed-size pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 write_c0_pagemask(PM_DEFAULT_MASK);
409 write_c0_wired(0);
Ralf Baechlecde15b52009-01-06 23:07:20 +0000410 if (current_cpu_type() == CPU_R10000 ||
411 current_cpu_type() == CPU_R12000 ||
412 current_cpu_type() == CPU_R14000)
413 write_c0_framemask(0);
David Daney6dd93442010-02-10 15:12:47 -0800414
Steven J. Hill05857c62012-09-13 16:51:46 -0500415 if (cpu_has_rixi) {
David Daney6dd93442010-02-10 15:12:47 -0800416 /*
417 * Enable the no read, no exec bits, and enable large virtual
418 * address.
419 */
420 u32 pg = PG_RIE | PG_XIE;
421#ifdef CONFIG_64BIT
422 pg |= PG_ELPA;
423#endif
424 write_c0_pagegrain(pg);
425 }
426
Ralf Baechle70342282013-01-22 12:59:30 +0100427 /* From this point on the ARC firmware is dead. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 local_flush_tlb_all();
429
Thiemo Seuferc6281ed2006-03-14 14:35:27 +0000430 /* Did I tell you that ARC SUCKS? */
431
Ralf Baechle41c594a2006-04-05 09:45:45 +0100432 if (ntlb) {
433 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
434 int wired = current_cpu_data.tlbsize - ntlb;
435 write_c0_wired(wired);
436 write_c0_index(wired-1);
Ralf Baechle49a89ef2007-10-11 23:46:15 +0100437 printk("Restricting TLB to %d entries\n", ntlb);
Ralf Baechle41c594a2006-04-05 09:45:45 +0100438 } else
439 printk("Ignoring invalid argument ntlb=%d\n", ntlb);
440 }
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 build_tlb_refill_handler();
443}