Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame^] | 1 | /* |
| 2 | * PowerPC64 port by Mike Corrigan and Dave Engebretsen |
| 3 | * {mikejc|engebret}@us.ibm.com |
| 4 | * |
| 5 | * Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com> |
| 6 | * |
| 7 | * SMP scalability work: |
| 8 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM |
| 9 | * |
| 10 | * Module name: htab.c |
| 11 | * |
| 12 | * Description: |
| 13 | * PowerPC Hashed Page Table functions |
| 14 | * |
| 15 | * This program is free software; you can redistribute it and/or |
| 16 | * modify it under the terms of the GNU General Public License |
| 17 | * as published by the Free Software Foundation; either version |
| 18 | * 2 of the License, or (at your option) any later version. |
| 19 | */ |
| 20 | |
| 21 | #undef DEBUG |
| 22 | |
| 23 | #include <linux/config.h> |
| 24 | #include <linux/spinlock.h> |
| 25 | #include <linux/errno.h> |
| 26 | #include <linux/sched.h> |
| 27 | #include <linux/proc_fs.h> |
| 28 | #include <linux/stat.h> |
| 29 | #include <linux/sysctl.h> |
| 30 | #include <linux/ctype.h> |
| 31 | #include <linux/cache.h> |
| 32 | #include <linux/init.h> |
| 33 | #include <linux/signal.h> |
| 34 | |
| 35 | #include <asm/ppcdebug.h> |
| 36 | #include <asm/processor.h> |
| 37 | #include <asm/pgtable.h> |
| 38 | #include <asm/mmu.h> |
| 39 | #include <asm/mmu_context.h> |
| 40 | #include <asm/page.h> |
| 41 | #include <asm/types.h> |
| 42 | #include <asm/system.h> |
| 43 | #include <asm/uaccess.h> |
| 44 | #include <asm/machdep.h> |
| 45 | #include <asm/lmb.h> |
| 46 | #include <asm/abs_addr.h> |
| 47 | #include <asm/tlbflush.h> |
| 48 | #include <asm/io.h> |
| 49 | #include <asm/eeh.h> |
| 50 | #include <asm/tlb.h> |
| 51 | #include <asm/cacheflush.h> |
| 52 | #include <asm/cputable.h> |
| 53 | #include <asm/abs_addr.h> |
| 54 | #include <asm/sections.h> |
| 55 | |
| 56 | #ifdef DEBUG |
| 57 | #define DBG(fmt...) udbg_printf(fmt) |
| 58 | #else |
| 59 | #define DBG(fmt...) |
| 60 | #endif |
| 61 | |
| 62 | /* |
| 63 | * Note: pte --> Linux PTE |
| 64 | * HPTE --> PowerPC Hashed Page Table Entry |
| 65 | * |
| 66 | * Execution context: |
| 67 | * htab_initialize is called with the MMU off (of course), but |
| 68 | * the kernel has been copied down to zero so it can directly |
| 69 | * reference global data. At this point it is very difficult |
| 70 | * to print debug info. |
| 71 | * |
| 72 | */ |
| 73 | |
| 74 | #ifdef CONFIG_U3_DART |
| 75 | extern unsigned long dart_tablebase; |
| 76 | #endif /* CONFIG_U3_DART */ |
| 77 | |
| 78 | HPTE *htab_address; |
| 79 | unsigned long htab_hash_mask; |
| 80 | |
| 81 | extern unsigned long _SDR1; |
| 82 | |
| 83 | #define KB (1024) |
| 84 | #define MB (1024*KB) |
| 85 | |
| 86 | static inline void loop_forever(void) |
| 87 | { |
| 88 | volatile unsigned long x = 1; |
| 89 | for(;x;x|=1) |
| 90 | ; |
| 91 | } |
| 92 | |
| 93 | #ifdef CONFIG_PPC_MULTIPLATFORM |
| 94 | static inline void create_pte_mapping(unsigned long start, unsigned long end, |
| 95 | unsigned long mode, int large) |
| 96 | { |
| 97 | unsigned long addr; |
| 98 | unsigned int step; |
| 99 | unsigned long tmp_mode; |
| 100 | |
| 101 | if (large) |
| 102 | step = 16*MB; |
| 103 | else |
| 104 | step = 4*KB; |
| 105 | |
| 106 | for (addr = start; addr < end; addr += step) { |
| 107 | unsigned long vpn, hash, hpteg; |
| 108 | unsigned long vsid = get_kernel_vsid(addr); |
| 109 | unsigned long va = (vsid << 28) | (addr & 0xfffffff); |
| 110 | int ret; |
| 111 | |
| 112 | if (large) |
| 113 | vpn = va >> HPAGE_SHIFT; |
| 114 | else |
| 115 | vpn = va >> PAGE_SHIFT; |
| 116 | |
| 117 | |
| 118 | tmp_mode = mode; |
| 119 | |
| 120 | /* Make non-kernel text non-executable */ |
| 121 | if (!in_kernel_text(addr)) |
| 122 | tmp_mode = mode | HW_NO_EXEC; |
| 123 | |
| 124 | hash = hpt_hash(vpn, large); |
| 125 | |
| 126 | hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); |
| 127 | |
| 128 | #ifdef CONFIG_PPC_PSERIES |
| 129 | if (systemcfg->platform & PLATFORM_LPAR) |
| 130 | ret = pSeries_lpar_hpte_insert(hpteg, va, |
| 131 | virt_to_abs(addr) >> PAGE_SHIFT, |
| 132 | 0, tmp_mode, 1, large); |
| 133 | else |
| 134 | #endif /* CONFIG_PPC_PSERIES */ |
| 135 | ret = native_hpte_insert(hpteg, va, |
| 136 | virt_to_abs(addr) >> PAGE_SHIFT, |
| 137 | 0, tmp_mode, 1, large); |
| 138 | |
| 139 | if (ret == -1) { |
| 140 | ppc64_terminate_msg(0x20, "create_pte_mapping"); |
| 141 | loop_forever(); |
| 142 | } |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | void __init htab_initialize(void) |
| 147 | { |
| 148 | unsigned long table, htab_size_bytes; |
| 149 | unsigned long pteg_count; |
| 150 | unsigned long mode_rw; |
| 151 | int i, use_largepages = 0; |
| 152 | unsigned long base = 0, size = 0; |
| 153 | extern unsigned long tce_alloc_start, tce_alloc_end; |
| 154 | |
| 155 | DBG(" -> htab_initialize()\n"); |
| 156 | |
| 157 | /* |
| 158 | * Calculate the required size of the htab. We want the number of |
| 159 | * PTEGs to equal one half the number of real pages. |
| 160 | */ |
| 161 | htab_size_bytes = 1UL << ppc64_pft_size; |
| 162 | pteg_count = htab_size_bytes >> 7; |
| 163 | |
| 164 | /* For debug, make the HTAB 1/8 as big as it normally would be. */ |
| 165 | ifppcdebug(PPCDBG_HTABSIZE) { |
| 166 | pteg_count >>= 3; |
| 167 | htab_size_bytes = pteg_count << 7; |
| 168 | } |
| 169 | |
| 170 | htab_hash_mask = pteg_count - 1; |
| 171 | |
| 172 | if (systemcfg->platform & PLATFORM_LPAR) { |
| 173 | /* Using a hypervisor which owns the htab */ |
| 174 | htab_address = NULL; |
| 175 | _SDR1 = 0; |
| 176 | } else { |
| 177 | /* Find storage for the HPT. Must be contiguous in |
| 178 | * the absolute address space. |
| 179 | */ |
| 180 | table = lmb_alloc(htab_size_bytes, htab_size_bytes); |
| 181 | |
| 182 | DBG("Hash table allocated at %lx, size: %lx\n", table, |
| 183 | htab_size_bytes); |
| 184 | |
| 185 | if ( !table ) { |
| 186 | ppc64_terminate_msg(0x20, "hpt space"); |
| 187 | loop_forever(); |
| 188 | } |
| 189 | htab_address = abs_to_virt(table); |
| 190 | |
| 191 | /* htab absolute addr + encoded htabsize */ |
| 192 | _SDR1 = table + __ilog2(pteg_count) - 11; |
| 193 | |
| 194 | /* Initialize the HPT with no entries */ |
| 195 | memset((void *)table, 0, htab_size_bytes); |
| 196 | } |
| 197 | |
| 198 | mode_rw = _PAGE_ACCESSED | _PAGE_COHERENT | PP_RWXX; |
| 199 | |
| 200 | /* On U3 based machines, we need to reserve the DART area and |
| 201 | * _NOT_ map it to avoid cache paradoxes as it's remapped non |
| 202 | * cacheable later on |
| 203 | */ |
| 204 | if (cpu_has_feature(CPU_FTR_16M_PAGE)) |
| 205 | use_largepages = 1; |
| 206 | |
| 207 | /* create bolted the linear mapping in the hash table */ |
| 208 | for (i=0; i < lmb.memory.cnt; i++) { |
| 209 | base = lmb.memory.region[i].physbase + KERNELBASE; |
| 210 | size = lmb.memory.region[i].size; |
| 211 | |
| 212 | DBG("creating mapping for region: %lx : %lx\n", base, size); |
| 213 | |
| 214 | #ifdef CONFIG_U3_DART |
| 215 | /* Do not map the DART space. Fortunately, it will be aligned |
| 216 | * in such a way that it will not cross two lmb regions and will |
| 217 | * fit within a single 16Mb page. |
| 218 | * The DART space is assumed to be a full 16Mb region even if we |
| 219 | * only use 2Mb of that space. We will use more of it later for |
| 220 | * AGP GART. We have to use a full 16Mb large page. |
| 221 | */ |
| 222 | DBG("DART base: %lx\n", dart_tablebase); |
| 223 | |
| 224 | if (dart_tablebase != 0 && dart_tablebase >= base |
| 225 | && dart_tablebase < (base + size)) { |
| 226 | if (base != dart_tablebase) |
| 227 | create_pte_mapping(base, dart_tablebase, mode_rw, |
| 228 | use_largepages); |
| 229 | if ((base + size) > (dart_tablebase + 16*MB)) |
| 230 | create_pte_mapping(dart_tablebase + 16*MB, base + size, |
| 231 | mode_rw, use_largepages); |
| 232 | continue; |
| 233 | } |
| 234 | #endif /* CONFIG_U3_DART */ |
| 235 | create_pte_mapping(base, base + size, mode_rw, use_largepages); |
| 236 | } |
| 237 | |
| 238 | /* |
| 239 | * If we have a memory_limit and we've allocated TCEs then we need to |
| 240 | * explicitly map the TCE area at the top of RAM. We also cope with the |
| 241 | * case that the TCEs start below memory_limit. |
| 242 | * tce_alloc_start/end are 16MB aligned so the mapping should work |
| 243 | * for either 4K or 16MB pages. |
| 244 | */ |
| 245 | if (tce_alloc_start) { |
| 246 | tce_alloc_start += KERNELBASE; |
| 247 | tce_alloc_end += KERNELBASE; |
| 248 | |
| 249 | if (base + size >= tce_alloc_start) |
| 250 | tce_alloc_start = base + size + 1; |
| 251 | |
| 252 | create_pte_mapping(tce_alloc_start, tce_alloc_end, |
| 253 | mode_rw, use_largepages); |
| 254 | } |
| 255 | |
| 256 | DBG(" <- htab_initialize()\n"); |
| 257 | } |
| 258 | #undef KB |
| 259 | #undef MB |
| 260 | #endif /* CONFIG_PPC_MULTIPLATFORM */ |
| 261 | |
| 262 | /* |
| 263 | * Called by asm hashtable.S for doing lazy icache flush |
| 264 | */ |
| 265 | unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) |
| 266 | { |
| 267 | struct page *page; |
| 268 | |
| 269 | if (!pfn_valid(pte_pfn(pte))) |
| 270 | return pp; |
| 271 | |
| 272 | page = pte_page(pte); |
| 273 | |
| 274 | /* page is dirty */ |
| 275 | if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { |
| 276 | if (trap == 0x400) { |
| 277 | __flush_dcache_icache(page_address(page)); |
| 278 | set_bit(PG_arch_1, &page->flags); |
| 279 | } else |
| 280 | pp |= HW_NO_EXEC; |
| 281 | } |
| 282 | return pp; |
| 283 | } |
| 284 | |
| 285 | /* Result code is: |
| 286 | * 0 - handled |
| 287 | * 1 - normal page fault |
| 288 | * -1 - critical hash insertion error |
| 289 | */ |
| 290 | int hash_page(unsigned long ea, unsigned long access, unsigned long trap) |
| 291 | { |
| 292 | void *pgdir; |
| 293 | unsigned long vsid; |
| 294 | struct mm_struct *mm; |
| 295 | pte_t *ptep; |
| 296 | int ret; |
| 297 | int user_region = 0; |
| 298 | int local = 0; |
| 299 | cpumask_t tmp; |
| 300 | |
| 301 | switch (REGION_ID(ea)) { |
| 302 | case USER_REGION_ID: |
| 303 | user_region = 1; |
| 304 | mm = current->mm; |
| 305 | if ((ea > USER_END) || (! mm)) |
| 306 | return 1; |
| 307 | |
| 308 | vsid = get_vsid(mm->context.id, ea); |
| 309 | break; |
| 310 | case IO_REGION_ID: |
| 311 | if (ea > IMALLOC_END) |
| 312 | return 1; |
| 313 | mm = &ioremap_mm; |
| 314 | vsid = get_kernel_vsid(ea); |
| 315 | break; |
| 316 | case VMALLOC_REGION_ID: |
| 317 | if (ea > VMALLOC_END) |
| 318 | return 1; |
| 319 | mm = &init_mm; |
| 320 | vsid = get_kernel_vsid(ea); |
| 321 | break; |
| 322 | #if 0 |
| 323 | case KERNEL_REGION_ID: |
| 324 | /* |
| 325 | * Should never get here - entire 0xC0... region is bolted. |
| 326 | * Send the problem up to do_page_fault |
| 327 | */ |
| 328 | #endif |
| 329 | default: |
| 330 | /* Not a valid range |
| 331 | * Send the problem up to do_page_fault |
| 332 | */ |
| 333 | return 1; |
| 334 | break; |
| 335 | } |
| 336 | |
| 337 | pgdir = mm->pgd; |
| 338 | |
| 339 | if (pgdir == NULL) |
| 340 | return 1; |
| 341 | |
| 342 | tmp = cpumask_of_cpu(smp_processor_id()); |
| 343 | if (user_region && cpus_equal(mm->cpu_vm_mask, tmp)) |
| 344 | local = 1; |
| 345 | |
| 346 | /* Is this a huge page ? */ |
| 347 | if (unlikely(in_hugepage_area(mm->context, ea))) |
| 348 | ret = hash_huge_page(mm, access, ea, vsid, local); |
| 349 | else { |
| 350 | ptep = find_linux_pte(pgdir, ea); |
| 351 | if (ptep == NULL) |
| 352 | return 1; |
| 353 | ret = __hash_page(ea, access, vsid, ptep, trap, local); |
| 354 | } |
| 355 | |
| 356 | return ret; |
| 357 | } |
| 358 | |
| 359 | void flush_hash_page(unsigned long context, unsigned long ea, pte_t pte, |
| 360 | int local) |
| 361 | { |
| 362 | unsigned long vsid, vpn, va, hash, secondary, slot; |
| 363 | unsigned long huge = pte_huge(pte); |
| 364 | |
| 365 | if ((ea >= USER_START) && (ea <= USER_END)) |
| 366 | vsid = get_vsid(context, ea); |
| 367 | else |
| 368 | vsid = get_kernel_vsid(ea); |
| 369 | |
| 370 | va = (vsid << 28) | (ea & 0x0fffffff); |
| 371 | if (huge) |
| 372 | vpn = va >> HPAGE_SHIFT; |
| 373 | else |
| 374 | vpn = va >> PAGE_SHIFT; |
| 375 | hash = hpt_hash(vpn, huge); |
| 376 | secondary = (pte_val(pte) & _PAGE_SECONDARY) >> 15; |
| 377 | if (secondary) |
| 378 | hash = ~hash; |
| 379 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; |
| 380 | slot += (pte_val(pte) & _PAGE_GROUP_IX) >> 12; |
| 381 | |
| 382 | ppc_md.hpte_invalidate(slot, va, huge, local); |
| 383 | } |
| 384 | |
| 385 | void flush_hash_range(unsigned long context, unsigned long number, int local) |
| 386 | { |
| 387 | if (ppc_md.flush_hash_range) { |
| 388 | ppc_md.flush_hash_range(context, number, local); |
| 389 | } else { |
| 390 | int i; |
| 391 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
| 392 | |
| 393 | for (i = 0; i < number; i++) |
| 394 | flush_hash_page(context, batch->addr[i], batch->pte[i], |
| 395 | local); |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | static inline void make_bl(unsigned int *insn_addr, void *func) |
| 400 | { |
| 401 | unsigned long funcp = *((unsigned long *)func); |
| 402 | int offset = funcp - (unsigned long)insn_addr; |
| 403 | |
| 404 | *insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc)); |
| 405 | flush_icache_range((unsigned long)insn_addr, 4+ |
| 406 | (unsigned long)insn_addr); |
| 407 | } |
| 408 | |
| 409 | /* |
| 410 | * low_hash_fault is called when we the low level hash code failed |
| 411 | * to instert a PTE due to an hypervisor error |
| 412 | */ |
| 413 | void low_hash_fault(struct pt_regs *regs, unsigned long address) |
| 414 | { |
| 415 | if (user_mode(regs)) { |
| 416 | siginfo_t info; |
| 417 | |
| 418 | info.si_signo = SIGBUS; |
| 419 | info.si_errno = 0; |
| 420 | info.si_code = BUS_ADRERR; |
| 421 | info.si_addr = (void __user *)address; |
| 422 | force_sig_info(SIGBUS, &info, current); |
| 423 | return; |
| 424 | } |
| 425 | bad_page_fault(regs, address, SIGBUS); |
| 426 | } |
| 427 | |
| 428 | void __init htab_finish_init(void) |
| 429 | { |
| 430 | extern unsigned int *htab_call_hpte_insert1; |
| 431 | extern unsigned int *htab_call_hpte_insert2; |
| 432 | extern unsigned int *htab_call_hpte_remove; |
| 433 | extern unsigned int *htab_call_hpte_updatepp; |
| 434 | |
| 435 | make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert); |
| 436 | make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert); |
| 437 | make_bl(htab_call_hpte_remove, ppc_md.hpte_remove); |
| 438 | make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp); |
| 439 | } |