blob: 36cf474b3d365f4dba63c93834683ba17f5e5c2a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * PowerPC64 port by Mike Corrigan and Dave Engebretsen
3 * {mikejc|engebret}@us.ibm.com
4 *
5 * Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
6 *
7 * SMP scalability work:
8 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
9 *
10 * Module name: htab.c
11 *
12 * Description:
13 * PowerPC Hashed Page Table functions
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
21#undef DEBUG
22
23#include <linux/config.h>
24#include <linux/spinlock.h>
25#include <linux/errno.h>
26#include <linux/sched.h>
27#include <linux/proc_fs.h>
28#include <linux/stat.h>
29#include <linux/sysctl.h>
30#include <linux/ctype.h>
31#include <linux/cache.h>
32#include <linux/init.h>
33#include <linux/signal.h>
34
35#include <asm/ppcdebug.h>
36#include <asm/processor.h>
37#include <asm/pgtable.h>
38#include <asm/mmu.h>
39#include <asm/mmu_context.h>
40#include <asm/page.h>
41#include <asm/types.h>
42#include <asm/system.h>
43#include <asm/uaccess.h>
44#include <asm/machdep.h>
45#include <asm/lmb.h>
46#include <asm/abs_addr.h>
47#include <asm/tlbflush.h>
48#include <asm/io.h>
49#include <asm/eeh.h>
50#include <asm/tlb.h>
51#include <asm/cacheflush.h>
52#include <asm/cputable.h>
53#include <asm/abs_addr.h>
54#include <asm/sections.h>
55
56#ifdef DEBUG
57#define DBG(fmt...) udbg_printf(fmt)
58#else
59#define DBG(fmt...)
60#endif
61
62/*
63 * Note: pte --> Linux PTE
64 * HPTE --> PowerPC Hashed Page Table Entry
65 *
66 * Execution context:
67 * htab_initialize is called with the MMU off (of course), but
68 * the kernel has been copied down to zero so it can directly
69 * reference global data. At this point it is very difficult
70 * to print debug info.
71 *
72 */
73
74#ifdef CONFIG_U3_DART
75extern unsigned long dart_tablebase;
76#endif /* CONFIG_U3_DART */
77
David Gibson96e28442005-07-13 01:11:42 -070078hpte_t *htab_address;
79unsigned long htab_hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
81extern unsigned long _SDR1;
82
83#define KB (1024)
84#define MB (1024*KB)
85
86static inline void loop_forever(void)
87{
88 volatile unsigned long x = 1;
89 for(;x;x|=1)
90 ;
91}
92
93#ifdef CONFIG_PPC_MULTIPLATFORM
94static inline void create_pte_mapping(unsigned long start, unsigned long end,
95 unsigned long mode, int large)
96{
97 unsigned long addr;
98 unsigned int step;
99 unsigned long tmp_mode;
David Gibson96e28442005-07-13 01:11:42 -0700100 unsigned long vflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101
David Gibson96e28442005-07-13 01:11:42 -0700102 if (large) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 step = 16*MB;
David Gibson96e28442005-07-13 01:11:42 -0700104 vflags = HPTE_V_BOLTED | HPTE_V_LARGE;
105 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 step = 4*KB;
David Gibson96e28442005-07-13 01:11:42 -0700107 vflags = HPTE_V_BOLTED;
108 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110 for (addr = start; addr < end; addr += step) {
111 unsigned long vpn, hash, hpteg;
112 unsigned long vsid = get_kernel_vsid(addr);
113 unsigned long va = (vsid << 28) | (addr & 0xfffffff);
114 int ret;
115
116 if (large)
117 vpn = va >> HPAGE_SHIFT;
118 else
119 vpn = va >> PAGE_SHIFT;
120
121
122 tmp_mode = mode;
123
124 /* Make non-kernel text non-executable */
125 if (!in_kernel_text(addr))
126 tmp_mode = mode | HW_NO_EXEC;
127
128 hash = hpt_hash(vpn, large);
129
130 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
131
132#ifdef CONFIG_PPC_PSERIES
133 if (systemcfg->platform & PLATFORM_LPAR)
134 ret = pSeries_lpar_hpte_insert(hpteg, va,
135 virt_to_abs(addr) >> PAGE_SHIFT,
David Gibson96e28442005-07-13 01:11:42 -0700136 vflags, tmp_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 else
138#endif /* CONFIG_PPC_PSERIES */
139 ret = native_hpte_insert(hpteg, va,
140 virt_to_abs(addr) >> PAGE_SHIFT,
David Gibson96e28442005-07-13 01:11:42 -0700141 vflags, tmp_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
143 if (ret == -1) {
144 ppc64_terminate_msg(0x20, "create_pte_mapping");
145 loop_forever();
146 }
147 }
148}
149
150void __init htab_initialize(void)
151{
152 unsigned long table, htab_size_bytes;
153 unsigned long pteg_count;
154 unsigned long mode_rw;
155 int i, use_largepages = 0;
156 unsigned long base = 0, size = 0;
157 extern unsigned long tce_alloc_start, tce_alloc_end;
158
159 DBG(" -> htab_initialize()\n");
160
161 /*
162 * Calculate the required size of the htab. We want the number of
163 * PTEGs to equal one half the number of real pages.
164 */
165 htab_size_bytes = 1UL << ppc64_pft_size;
166 pteg_count = htab_size_bytes >> 7;
167
168 /* For debug, make the HTAB 1/8 as big as it normally would be. */
169 ifppcdebug(PPCDBG_HTABSIZE) {
170 pteg_count >>= 3;
171 htab_size_bytes = pteg_count << 7;
172 }
173
174 htab_hash_mask = pteg_count - 1;
175
176 if (systemcfg->platform & PLATFORM_LPAR) {
177 /* Using a hypervisor which owns the htab */
178 htab_address = NULL;
179 _SDR1 = 0;
180 } else {
181 /* Find storage for the HPT. Must be contiguous in
182 * the absolute address space.
183 */
184 table = lmb_alloc(htab_size_bytes, htab_size_bytes);
185
186 DBG("Hash table allocated at %lx, size: %lx\n", table,
187 htab_size_bytes);
188
189 if ( !table ) {
190 ppc64_terminate_msg(0x20, "hpt space");
191 loop_forever();
192 }
193 htab_address = abs_to_virt(table);
194
195 /* htab absolute addr + encoded htabsize */
196 _SDR1 = table + __ilog2(pteg_count) - 11;
197
198 /* Initialize the HPT with no entries */
199 memset((void *)table, 0, htab_size_bytes);
200 }
201
Anton Blanchard515bae92005-06-21 17:15:55 -0700202 mode_rw = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
204 /* On U3 based machines, we need to reserve the DART area and
205 * _NOT_ map it to avoid cache paradoxes as it's remapped non
206 * cacheable later on
207 */
208 if (cpu_has_feature(CPU_FTR_16M_PAGE))
209 use_largepages = 1;
210
211 /* create bolted the linear mapping in the hash table */
212 for (i=0; i < lmb.memory.cnt; i++) {
Michael Ellerman180379d2005-08-03 20:21:26 +1000213 base = lmb.memory.region[i].base + KERNELBASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 size = lmb.memory.region[i].size;
215
216 DBG("creating mapping for region: %lx : %lx\n", base, size);
217
218#ifdef CONFIG_U3_DART
219 /* Do not map the DART space. Fortunately, it will be aligned
220 * in such a way that it will not cross two lmb regions and will
221 * fit within a single 16Mb page.
222 * The DART space is assumed to be a full 16Mb region even if we
223 * only use 2Mb of that space. We will use more of it later for
224 * AGP GART. We have to use a full 16Mb large page.
225 */
226 DBG("DART base: %lx\n", dart_tablebase);
227
228 if (dart_tablebase != 0 && dart_tablebase >= base
229 && dart_tablebase < (base + size)) {
230 if (base != dart_tablebase)
231 create_pte_mapping(base, dart_tablebase, mode_rw,
232 use_largepages);
233 if ((base + size) > (dart_tablebase + 16*MB))
234 create_pte_mapping(dart_tablebase + 16*MB, base + size,
235 mode_rw, use_largepages);
236 continue;
237 }
238#endif /* CONFIG_U3_DART */
239 create_pte_mapping(base, base + size, mode_rw, use_largepages);
240 }
241
242 /*
243 * If we have a memory_limit and we've allocated TCEs then we need to
244 * explicitly map the TCE area at the top of RAM. We also cope with the
245 * case that the TCEs start below memory_limit.
246 * tce_alloc_start/end are 16MB aligned so the mapping should work
247 * for either 4K or 16MB pages.
248 */
249 if (tce_alloc_start) {
250 tce_alloc_start += KERNELBASE;
251 tce_alloc_end += KERNELBASE;
252
253 if (base + size >= tce_alloc_start)
254 tce_alloc_start = base + size + 1;
255
256 create_pte_mapping(tce_alloc_start, tce_alloc_end,
257 mode_rw, use_largepages);
258 }
259
260 DBG(" <- htab_initialize()\n");
261}
262#undef KB
263#undef MB
264#endif /* CONFIG_PPC_MULTIPLATFORM */
265
266/*
267 * Called by asm hashtable.S for doing lazy icache flush
268 */
269unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
270{
271 struct page *page;
272
273 if (!pfn_valid(pte_pfn(pte)))
274 return pp;
275
276 page = pte_page(pte);
277
278 /* page is dirty */
279 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
280 if (trap == 0x400) {
281 __flush_dcache_icache(page_address(page));
282 set_bit(PG_arch_1, &page->flags);
283 } else
284 pp |= HW_NO_EXEC;
285 }
286 return pp;
287}
288
289/* Result code is:
290 * 0 - handled
291 * 1 - normal page fault
292 * -1 - critical hash insertion error
293 */
294int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
295{
296 void *pgdir;
297 unsigned long vsid;
298 struct mm_struct *mm;
299 pte_t *ptep;
300 int ret;
301 int user_region = 0;
302 int local = 0;
303 cpumask_t tmp;
304
David Gibsone28f7fa2005-08-05 19:39:06 +1000305 if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
David Gibson1f8d4192005-05-05 16:15:13 -0700306 return 1;
307
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 switch (REGION_ID(ea)) {
309 case USER_REGION_ID:
310 user_region = 1;
311 mm = current->mm;
David Gibson1f8d4192005-05-05 16:15:13 -0700312 if (! mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 return 1;
314
315 vsid = get_vsid(mm->context.id, ea);
316 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 case VMALLOC_REGION_ID:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 mm = &init_mm;
319 vsid = get_kernel_vsid(ea);
320 break;
321#if 0
322 case KERNEL_REGION_ID:
323 /*
324 * Should never get here - entire 0xC0... region is bolted.
325 * Send the problem up to do_page_fault
326 */
327#endif
328 default:
329 /* Not a valid range
330 * Send the problem up to do_page_fault
331 */
332 return 1;
333 break;
334 }
335
336 pgdir = mm->pgd;
337
338 if (pgdir == NULL)
339 return 1;
340
341 tmp = cpumask_of_cpu(smp_processor_id());
342 if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
343 local = 1;
344
345 /* Is this a huge page ? */
346 if (unlikely(in_hugepage_area(mm->context, ea)))
347 ret = hash_huge_page(mm, access, ea, vsid, local);
348 else {
349 ptep = find_linux_pte(pgdir, ea);
350 if (ptep == NULL)
351 return 1;
352 ret = __hash_page(ea, access, vsid, ptep, trap, local);
353 }
354
355 return ret;
356}
357
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000358void flush_hash_page(unsigned long va, pte_t pte, int local)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359{
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000360 unsigned long vpn, hash, secondary, slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 unsigned long huge = pte_huge(pte);
362
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 if (huge)
364 vpn = va >> HPAGE_SHIFT;
365 else
366 vpn = va >> PAGE_SHIFT;
367 hash = hpt_hash(vpn, huge);
368 secondary = (pte_val(pte) & _PAGE_SECONDARY) >> 15;
369 if (secondary)
370 hash = ~hash;
371 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
372 slot += (pte_val(pte) & _PAGE_GROUP_IX) >> 12;
373
374 ppc_md.hpte_invalidate(slot, va, huge, local);
375}
376
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000377void flush_hash_range(unsigned long number, int local)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378{
379 if (ppc_md.flush_hash_range) {
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000380 ppc_md.flush_hash_range(number, local);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 } else {
382 int i;
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000383 struct ppc64_tlb_batch *batch =
384 &__get_cpu_var(ppc64_tlb_batch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
386 for (i = 0; i < number; i++)
Benjamin Herrenschmidt61b1a942005-09-20 13:52:50 +1000387 flush_hash_page(batch->vaddr[i], batch->pte[i], local);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 }
389}
390
391static inline void make_bl(unsigned int *insn_addr, void *func)
392{
393 unsigned long funcp = *((unsigned long *)func);
394 int offset = funcp - (unsigned long)insn_addr;
395
396 *insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc));
397 flush_icache_range((unsigned long)insn_addr, 4+
398 (unsigned long)insn_addr);
399}
400
401/*
402 * low_hash_fault is called when we the low level hash code failed
403 * to instert a PTE due to an hypervisor error
404 */
405void low_hash_fault(struct pt_regs *regs, unsigned long address)
406{
407 if (user_mode(regs)) {
408 siginfo_t info;
409
410 info.si_signo = SIGBUS;
411 info.si_errno = 0;
412 info.si_code = BUS_ADRERR;
413 info.si_addr = (void __user *)address;
414 force_sig_info(SIGBUS, &info, current);
415 return;
416 }
417 bad_page_fault(regs, address, SIGBUS);
418}
419
420void __init htab_finish_init(void)
421{
422 extern unsigned int *htab_call_hpte_insert1;
423 extern unsigned int *htab_call_hpte_insert2;
424 extern unsigned int *htab_call_hpte_remove;
425 extern unsigned int *htab_call_hpte_updatepp;
426
427 make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert);
428 make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert);
429 make_bl(htab_call_hpte_remove, ppc_md.hpte_remove);
430 make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
431}