Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * PPC64 (POWER4) Huge TLB Page Support for Kernel. |
| 3 | * |
| 4 | * Copyright (C) 2003 David Gibson, IBM Corporation. |
| 5 | * |
| 6 | * Based on the IA-32 version: |
| 7 | * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> |
| 8 | */ |
| 9 | |
| 10 | #include <linux/init.h> |
| 11 | #include <linux/fs.h> |
| 12 | #include <linux/mm.h> |
| 13 | #include <linux/hugetlb.h> |
| 14 | #include <linux/pagemap.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/slab.h> |
| 16 | #include <linux/err.h> |
| 17 | #include <linux/sysctl.h> |
| 18 | #include <asm/mman.h> |
| 19 | #include <asm/pgalloc.h> |
| 20 | #include <asm/tlb.h> |
| 21 | #include <asm/tlbflush.h> |
| 22 | #include <asm/mmu_context.h> |
| 23 | #include <asm/machdep.h> |
| 24 | #include <asm/cputable.h> |
Benjamin Herrenschmidt | 94b2a43 | 2007-03-10 00:05:37 +0100 | [diff] [blame] | 25 | #include <asm/spu.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 27 | #define HPAGE_SHIFT_64K 16 |
| 28 | #define HPAGE_SHIFT_16M 24 |
| 29 | |
David Gibson | c594ada | 2005-08-11 16:55:21 +1000 | [diff] [blame] | 30 | #define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT) |
| 31 | #define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT) |
Jon Tollefson | ec4b2c0 | 2008-07-23 21:27:53 -0700 | [diff] [blame] | 32 | #define MAX_NUMBER_GPAGES 1024 |
| 33 | |
| 34 | /* Tracks the 16G pages after the device tree is scanned and before the |
| 35 | * huge_boot_pages list is ready. */ |
| 36 | static unsigned long gpage_freearray[MAX_NUMBER_GPAGES]; |
| 37 | static unsigned nr_gpages; |
David Gibson | c594ada | 2005-08-11 16:55:21 +1000 | [diff] [blame] | 38 | |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 39 | unsigned int hugepte_shift; |
| 40 | #define PTRS_PER_HUGEPTE (1 << hugepte_shift) |
| 41 | #define HUGEPTE_TABLE_SIZE (sizeof(pte_t) << hugepte_shift) |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 42 | |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 43 | #define HUGEPD_SHIFT (HPAGE_SHIFT + hugepte_shift) |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 44 | #define HUGEPD_SIZE (1UL << HUGEPD_SHIFT) |
| 45 | #define HUGEPD_MASK (~(HUGEPD_SIZE-1)) |
| 46 | |
| 47 | #define huge_pgtable_cache (pgtable_cache[HUGEPTE_CACHE_NUM]) |
| 48 | |
| 49 | /* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad() |
| 50 | * will choke on pointers to hugepte tables, which is handy for |
| 51 | * catching screwups early. */ |
| 52 | #define HUGEPD_OK 0x1 |
| 53 | |
| 54 | typedef struct { unsigned long pd; } hugepd_t; |
| 55 | |
| 56 | #define hugepd_none(hpd) ((hpd).pd == 0) |
| 57 | |
| 58 | static inline pte_t *hugepd_page(hugepd_t hpd) |
| 59 | { |
| 60 | BUG_ON(!(hpd.pd & HUGEPD_OK)); |
| 61 | return (pte_t *)(hpd.pd & ~HUGEPD_OK); |
| 62 | } |
| 63 | |
| 64 | static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr) |
| 65 | { |
| 66 | unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1)); |
| 67 | pte_t *dir = hugepd_page(*hpdp); |
| 68 | |
| 69 | return dir + idx; |
| 70 | } |
| 71 | |
| 72 | static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, |
| 73 | unsigned long address) |
| 74 | { |
| 75 | pte_t *new = kmem_cache_alloc(huge_pgtable_cache, |
| 76 | GFP_KERNEL|__GFP_REPEAT); |
| 77 | |
| 78 | if (! new) |
| 79 | return -ENOMEM; |
| 80 | |
| 81 | spin_lock(&mm->page_table_lock); |
| 82 | if (!hugepd_none(*hpdp)) |
| 83 | kmem_cache_free(huge_pgtable_cache, new); |
| 84 | else |
| 85 | hpdp->pd = (unsigned long)new | HUGEPD_OK; |
| 86 | spin_unlock(&mm->page_table_lock); |
| 87 | return 0; |
| 88 | } |
| 89 | |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 90 | /* Base page size affects how we walk hugetlb page tables */ |
| 91 | #ifdef CONFIG_PPC_64K_PAGES |
| 92 | #define hpmd_offset(pud, addr) pmd_offset(pud, addr) |
| 93 | #define hpmd_alloc(mm, pud, addr) pmd_alloc(mm, pud, addr) |
| 94 | #else |
| 95 | static inline |
| 96 | pmd_t *hpmd_offset(pud_t *pud, unsigned long addr) |
| 97 | { |
| 98 | if (HPAGE_SHIFT == HPAGE_SHIFT_64K) |
| 99 | return pmd_offset(pud, addr); |
| 100 | else |
| 101 | return (pmd_t *) pud; |
| 102 | } |
| 103 | static inline |
| 104 | pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr) |
| 105 | { |
| 106 | if (HPAGE_SHIFT == HPAGE_SHIFT_64K) |
| 107 | return pmd_alloc(mm, pud, addr); |
| 108 | else |
| 109 | return (pmd_t *) pud; |
| 110 | } |
| 111 | #endif |
| 112 | |
Jon Tollefson | 658013e | 2008-07-23 21:27:54 -0700 | [diff] [blame^] | 113 | /* Build list of addresses of gigantic pages. This function is used in early |
| 114 | * boot before the buddy or bootmem allocator is setup. |
| 115 | */ |
| 116 | void add_gpage(unsigned long addr, unsigned long page_size, |
| 117 | unsigned long number_of_pages) |
| 118 | { |
| 119 | if (!addr) |
| 120 | return; |
| 121 | while (number_of_pages > 0) { |
| 122 | gpage_freearray[nr_gpages] = addr; |
| 123 | nr_gpages++; |
| 124 | number_of_pages--; |
| 125 | addr += page_size; |
| 126 | } |
| 127 | } |
| 128 | |
Jon Tollefson | ec4b2c0 | 2008-07-23 21:27:53 -0700 | [diff] [blame] | 129 | /* Moves the gigantic page addresses from the temporary list to the |
| 130 | * huge_boot_pages list. */ |
| 131 | int alloc_bootmem_huge_page(struct hstate *h) |
| 132 | { |
| 133 | struct huge_bootmem_page *m; |
| 134 | if (nr_gpages == 0) |
| 135 | return 0; |
| 136 | m = phys_to_virt(gpage_freearray[--nr_gpages]); |
| 137 | gpage_freearray[nr_gpages] = 0; |
| 138 | list_add(&m->list, &huge_boot_pages); |
| 139 | m->hstate = h; |
| 140 | return 1; |
| 141 | } |
| 142 | |
| 143 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 144 | /* Modelled after find_linux_pte() */ |
| 145 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | { |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 147 | pgd_t *pg; |
| 148 | pud_t *pu; |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 149 | pmd_t *pm; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | |
Benjamin Herrenschmidt | d0f13e3 | 2007-05-08 16:27:27 +1000 | [diff] [blame] | 151 | BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 153 | addr &= HPAGE_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 154 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 155 | pg = pgd_offset(mm, addr); |
| 156 | if (!pgd_none(*pg)) { |
| 157 | pu = pud_offset(pg, addr); |
| 158 | if (!pud_none(*pu)) { |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 159 | pm = hpmd_offset(pu, addr); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 160 | if (!pmd_none(*pm)) |
| 161 | return hugepte_offset((hugepd_t *)pm, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 162 | } |
| 163 | } |
| 164 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 165 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | } |
| 167 | |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 168 | pte_t *huge_pte_alloc(struct mm_struct *mm, |
| 169 | unsigned long addr, unsigned long sz) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | { |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 171 | pgd_t *pg; |
| 172 | pud_t *pu; |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 173 | pmd_t *pm; |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 174 | hugepd_t *hpdp = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | |
Benjamin Herrenschmidt | d0f13e3 | 2007-05-08 16:27:27 +1000 | [diff] [blame] | 176 | BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 178 | addr &= HPAGE_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 180 | pg = pgd_offset(mm, addr); |
| 181 | pu = pud_alloc(mm, pg, addr); |
| 182 | |
| 183 | if (pu) { |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 184 | pm = hpmd_alloc(mm, pu, addr); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 185 | if (pm) |
| 186 | hpdp = (hugepd_t *)pm; |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 187 | } |
| 188 | |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 189 | if (! hpdp) |
| 190 | return NULL; |
| 191 | |
| 192 | if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr)) |
| 193 | return NULL; |
| 194 | |
| 195 | return hugepte_offset(hpdp, addr); |
| 196 | } |
| 197 | |
Chen, Kenneth W | 39dde65 | 2006-12-06 20:32:03 -0800 | [diff] [blame] | 198 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) |
| 199 | { |
| 200 | return 0; |
| 201 | } |
| 202 | |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 203 | static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp) |
| 204 | { |
| 205 | pte_t *hugepte = hugepd_page(*hpdp); |
| 206 | |
| 207 | hpdp->pd = 0; |
| 208 | tlb->need_flush = 1; |
| 209 | pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, HUGEPTE_CACHE_NUM, |
Adam Litke | c9169f8 | 2006-08-18 11:22:21 -0700 | [diff] [blame] | 210 | PGF_CACHENUM_MASK)); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 211 | } |
| 212 | |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 213 | static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, |
| 214 | unsigned long addr, unsigned long end, |
| 215 | unsigned long floor, unsigned long ceiling) |
| 216 | { |
| 217 | pmd_t *pmd; |
| 218 | unsigned long next; |
| 219 | unsigned long start; |
| 220 | |
| 221 | start = addr; |
| 222 | pmd = pmd_offset(pud, addr); |
| 223 | do { |
| 224 | next = pmd_addr_end(addr, end); |
| 225 | if (pmd_none(*pmd)) |
| 226 | continue; |
| 227 | free_hugepte_range(tlb, (hugepd_t *)pmd); |
| 228 | } while (pmd++, addr = next, addr != end); |
| 229 | |
| 230 | start &= PUD_MASK; |
| 231 | if (start < floor) |
| 232 | return; |
| 233 | if (ceiling) { |
| 234 | ceiling &= PUD_MASK; |
| 235 | if (!ceiling) |
| 236 | return; |
| 237 | } |
| 238 | if (end - 1 > ceiling - 1) |
| 239 | return; |
| 240 | |
| 241 | pmd = pmd_offset(pud, start); |
| 242 | pud_clear(pud); |
| 243 | pmd_free_tlb(tlb, pmd); |
| 244 | } |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 245 | |
| 246 | static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, |
| 247 | unsigned long addr, unsigned long end, |
| 248 | unsigned long floor, unsigned long ceiling) |
| 249 | { |
| 250 | pud_t *pud; |
| 251 | unsigned long next; |
| 252 | unsigned long start; |
| 253 | |
| 254 | start = addr; |
| 255 | pud = pud_offset(pgd, addr); |
| 256 | do { |
| 257 | next = pud_addr_end(addr, end); |
| 258 | #ifdef CONFIG_PPC_64K_PAGES |
| 259 | if (pud_none_or_clear_bad(pud)) |
| 260 | continue; |
| 261 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); |
| 262 | #else |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 263 | if (HPAGE_SHIFT == HPAGE_SHIFT_64K) { |
| 264 | if (pud_none_or_clear_bad(pud)) |
| 265 | continue; |
| 266 | hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling); |
| 267 | } else { |
| 268 | if (pud_none(*pud)) |
| 269 | continue; |
| 270 | free_hugepte_range(tlb, (hugepd_t *)pud); |
| 271 | } |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 272 | #endif |
| 273 | } while (pud++, addr = next, addr != end); |
| 274 | |
| 275 | start &= PGDIR_MASK; |
| 276 | if (start < floor) |
| 277 | return; |
| 278 | if (ceiling) { |
| 279 | ceiling &= PGDIR_MASK; |
| 280 | if (!ceiling) |
| 281 | return; |
| 282 | } |
| 283 | if (end - 1 > ceiling - 1) |
| 284 | return; |
| 285 | |
| 286 | pud = pud_offset(pgd, start); |
| 287 | pgd_clear(pgd); |
| 288 | pud_free_tlb(tlb, pud); |
| 289 | } |
| 290 | |
| 291 | /* |
| 292 | * This function frees user-level page tables of a process. |
| 293 | * |
| 294 | * Must be called with pagetable lock held. |
| 295 | */ |
Jan Beulich | 42b7772 | 2008-07-23 21:27:10 -0700 | [diff] [blame] | 296 | void hugetlb_free_pgd_range(struct mmu_gather *tlb, |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 297 | unsigned long addr, unsigned long end, |
| 298 | unsigned long floor, unsigned long ceiling) |
| 299 | { |
| 300 | pgd_t *pgd; |
| 301 | unsigned long next; |
| 302 | unsigned long start; |
| 303 | |
| 304 | /* |
| 305 | * Comments below take from the normal free_pgd_range(). They |
| 306 | * apply here too. The tests against HUGEPD_MASK below are |
| 307 | * essential, because we *don't* test for this at the bottom |
| 308 | * level. Without them we'll attempt to free a hugepte table |
| 309 | * when we unmap just part of it, even if there are other |
| 310 | * active mappings using it. |
| 311 | * |
| 312 | * The next few lines have given us lots of grief... |
| 313 | * |
| 314 | * Why are we testing HUGEPD* at this top level? Because |
| 315 | * often there will be no work to do at all, and we'd prefer |
| 316 | * not to go all the way down to the bottom just to discover |
| 317 | * that. |
| 318 | * |
| 319 | * Why all these "- 1"s? Because 0 represents both the bottom |
| 320 | * of the address space and the top of it (using -1 for the |
| 321 | * top wouldn't help much: the masks would do the wrong thing). |
| 322 | * The rule is that addr 0 and floor 0 refer to the bottom of |
| 323 | * the address space, but end 0 and ceiling 0 refer to the top |
| 324 | * Comparisons need to use "end - 1" and "ceiling - 1" (though |
| 325 | * that end 0 case should be mythical). |
| 326 | * |
| 327 | * Wherever addr is brought up or ceiling brought down, we |
| 328 | * must be careful to reject "the opposite 0" before it |
| 329 | * confuses the subsequent tests. But what about where end is |
| 330 | * brought down by HUGEPD_SIZE below? no, end can't go down to |
| 331 | * 0 there. |
| 332 | * |
| 333 | * Whereas we round start (addr) and ceiling down, by different |
| 334 | * masks at different levels, in order to test whether a table |
| 335 | * now has no other vmas using it, so can be freed, we don't |
| 336 | * bother to round floor or end up - the tests don't need that. |
| 337 | */ |
| 338 | |
| 339 | addr &= HUGEPD_MASK; |
| 340 | if (addr < floor) { |
| 341 | addr += HUGEPD_SIZE; |
| 342 | if (!addr) |
| 343 | return; |
| 344 | } |
| 345 | if (ceiling) { |
| 346 | ceiling &= HUGEPD_MASK; |
| 347 | if (!ceiling) |
| 348 | return; |
| 349 | } |
| 350 | if (end - 1 > ceiling - 1) |
| 351 | end -= HUGEPD_SIZE; |
| 352 | if (addr > end - 1) |
| 353 | return; |
| 354 | |
| 355 | start = addr; |
Jan Beulich | 42b7772 | 2008-07-23 21:27:10 -0700 | [diff] [blame] | 356 | pgd = pgd_offset(tlb->mm, addr); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 357 | do { |
Jan Beulich | 42b7772 | 2008-07-23 21:27:10 -0700 | [diff] [blame] | 358 | BUG_ON(get_slice_psize(tlb->mm, addr) != mmu_huge_psize); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 359 | next = pgd_addr_end(addr, end); |
| 360 | if (pgd_none_or_clear_bad(pgd)) |
| 361 | continue; |
Jan Beulich | 42b7772 | 2008-07-23 21:27:10 -0700 | [diff] [blame] | 362 | hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 363 | } while (pgd++, addr = next, addr != end); |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 364 | } |
| 365 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 366 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 367 | pte_t *ptep, pte_t pte) |
| 368 | { |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 369 | if (pte_present(*ptep)) { |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 370 | /* We open-code pte_clear because we need to pass the right |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame] | 371 | * argument to hpte_need_flush (huge / !huge). Might not be |
| 372 | * necessary anymore if we make hpte_need_flush() get the |
| 373 | * page size from the slices |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 374 | */ |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame] | 375 | pte_update(mm, addr & HPAGE_MASK, ptep, ~0UL, 1); |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 376 | } |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 377 | *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 378 | } |
| 379 | |
| 380 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
| 381 | pte_t *ptep) |
| 382 | { |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame] | 383 | unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1); |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 384 | return __pte(old); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 385 | } |
| 386 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 387 | struct page * |
| 388 | follow_huge_addr(struct mm_struct *mm, unsigned long address, int write) |
| 389 | { |
| 390 | pte_t *ptep; |
| 391 | struct page *page; |
| 392 | |
Benjamin Herrenschmidt | d0f13e3 | 2007-05-08 16:27:27 +1000 | [diff] [blame] | 393 | if (get_slice_psize(mm, address) != mmu_huge_psize) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 394 | return ERR_PTR(-EINVAL); |
| 395 | |
| 396 | ptep = huge_pte_offset(mm, address); |
| 397 | page = pte_page(*ptep); |
| 398 | if (page) |
| 399 | page += (address % HPAGE_SIZE) / PAGE_SIZE; |
| 400 | |
| 401 | return page; |
| 402 | } |
| 403 | |
| 404 | int pmd_huge(pmd_t pmd) |
| 405 | { |
| 406 | return 0; |
| 407 | } |
| 408 | |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 409 | int pud_huge(pud_t pud) |
| 410 | { |
| 411 | return 0; |
| 412 | } |
| 413 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | struct page * |
| 415 | follow_huge_pmd(struct mm_struct *mm, unsigned long address, |
| 416 | pmd_t *pmd, int write) |
| 417 | { |
| 418 | BUG(); |
| 419 | return NULL; |
| 420 | } |
| 421 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 422 | |
| 423 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, |
| 424 | unsigned long len, unsigned long pgoff, |
| 425 | unsigned long flags) |
| 426 | { |
Benjamin Herrenschmidt | d0f13e3 | 2007-05-08 16:27:27 +1000 | [diff] [blame] | 427 | return slice_get_unmapped_area(addr, len, flags, |
| 428 | mmu_huge_psize, 1, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 429 | } |
| 430 | |
David Gibson | cbf52af | 2005-12-09 14:20:52 +1100 | [diff] [blame] | 431 | /* |
| 432 | * Called by asm hashtable.S for doing lazy icache flush |
| 433 | */ |
| 434 | static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags, |
| 435 | pte_t pte, int trap) |
| 436 | { |
| 437 | struct page *page; |
| 438 | int i; |
| 439 | |
| 440 | if (!pfn_valid(pte_pfn(pte))) |
| 441 | return rflags; |
| 442 | |
| 443 | page = pte_page(pte); |
| 444 | |
| 445 | /* page is dirty */ |
| 446 | if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { |
| 447 | if (trap == 0x400) { |
| 448 | for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) |
| 449 | __flush_dcache_icache(page_address(page+i)); |
| 450 | set_bit(PG_arch_1, &page->flags); |
| 451 | } else { |
| 452 | rflags |= HPTE_R_N; |
| 453 | } |
| 454 | } |
| 455 | return rflags; |
| 456 | } |
| 457 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 458 | int hash_huge_page(struct mm_struct *mm, unsigned long access, |
David Gibson | cbf52af | 2005-12-09 14:20:52 +1100 | [diff] [blame] | 459 | unsigned long ea, unsigned long vsid, int local, |
| 460 | unsigned long trap) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 461 | { |
| 462 | pte_t *ptep; |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 463 | unsigned long old_pte, new_pte; |
| 464 | unsigned long va, rflags, pa; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | long slot; |
| 466 | int err = 1; |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 467 | int ssize = user_segment_size(ea); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | ptep = huge_pte_offset(mm, ea); |
| 470 | |
| 471 | /* Search the Linux page table for a match with va */ |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 472 | va = hpt_va(ea, vsid, ssize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 473 | |
| 474 | /* |
| 475 | * If no pte found or not present, send the problem up to |
| 476 | * do_page_fault |
| 477 | */ |
| 478 | if (unlikely(!ptep || pte_none(*ptep))) |
| 479 | goto out; |
| 480 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 481 | /* |
| 482 | * Check the user's access rights to the page. If access should be |
| 483 | * prevented then send the problem up to do_page_fault. |
| 484 | */ |
| 485 | if (unlikely(access & ~pte_val(*ptep))) |
| 486 | goto out; |
| 487 | /* |
| 488 | * At this point, we have a pte (old_pte) which can be used to build |
| 489 | * or update an HPTE. There are 2 cases: |
| 490 | * |
| 491 | * 1. There is a valid (present) pte with no associated HPTE (this is |
| 492 | * the most common case) |
| 493 | * 2. There is a valid (present) pte with an associated HPTE. The |
| 494 | * current values of the pp bits in the HPTE prevent access |
| 495 | * because we are doing software DIRTY bit management and the |
| 496 | * page is currently not DIRTY. |
| 497 | */ |
| 498 | |
| 499 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 500 | do { |
| 501 | old_pte = pte_val(*ptep); |
| 502 | if (old_pte & _PAGE_BUSY) |
| 503 | goto out; |
Benjamin Herrenschmidt | 41743a4 | 2008-06-11 15:37:10 +1000 | [diff] [blame] | 504 | new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED; |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 505 | } while(old_pte != __cmpxchg_u64((unsigned long *)ptep, |
| 506 | old_pte, new_pte)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 507 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 508 | rflags = 0x2 | (!(new_pte & _PAGE_RW)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 509 | /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */ |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 510 | rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N); |
David Gibson | cbf52af | 2005-12-09 14:20:52 +1100 | [diff] [blame] | 511 | if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) |
| 512 | /* No CPU has hugepages but lacks no execute, so we |
| 513 | * don't need to worry about that case */ |
| 514 | rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte), |
| 515 | trap); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 516 | |
| 517 | /* Check if pte already has an hpte (case 2) */ |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 518 | if (unlikely(old_pte & _PAGE_HASHPTE)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 519 | /* There MIGHT be an HPTE for this pte */ |
| 520 | unsigned long hash, slot; |
| 521 | |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 522 | hash = hpt_hash(va, HPAGE_SHIFT, ssize); |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 523 | if (old_pte & _PAGE_F_SECOND) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 524 | hash = ~hash; |
| 525 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 526 | slot += (old_pte & _PAGE_F_GIX) >> 12; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 527 | |
Benjamin Herrenschmidt | 325c82a | 2005-12-08 16:51:44 +1100 | [diff] [blame] | 528 | if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize, |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 529 | ssize, local) == -1) |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 530 | old_pte &= ~_PAGE_HPTEFLAGS; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 531 | } |
| 532 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 533 | if (likely(!(old_pte & _PAGE_HASHPTE))) { |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 534 | unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 535 | unsigned long hpte_group; |
| 536 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 537 | pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 538 | |
| 539 | repeat: |
| 540 | hpte_group = ((hash & htab_hash_mask) * |
| 541 | HPTES_PER_GROUP) & ~0x7UL; |
| 542 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 543 | /* clear HPTE slot informations in new PTE */ |
Benjamin Herrenschmidt | 41743a4 | 2008-06-11 15:37:10 +1000 | [diff] [blame] | 544 | #ifdef CONFIG_PPC_64K_PAGES |
| 545 | new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0; |
| 546 | #else |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 547 | new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE; |
Benjamin Herrenschmidt | 41743a4 | 2008-06-11 15:37:10 +1000 | [diff] [blame] | 548 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 549 | /* Add in WIMG bits */ |
Dave Kleikamp | 87e9ab1 | 2008-06-19 08:32:56 +1000 | [diff] [blame] | 550 | rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | |
| 551 | _PAGE_COHERENT | _PAGE_GUARDED)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 552 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 553 | /* Insert into the hash table, primary slot */ |
| 554 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 555 | mmu_huge_psize, ssize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 556 | |
| 557 | /* Primary is full, try the secondary */ |
| 558 | if (unlikely(slot == -1)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 559 | hpte_group = ((~hash & htab_hash_mask) * |
| 560 | HPTES_PER_GROUP) & ~0x7UL; |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 561 | slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, |
Benjamin Herrenschmidt | 67b1081 | 2005-09-23 13:24:07 -0700 | [diff] [blame] | 562 | HPTE_V_SECONDARY, |
Paul Mackerras | 1189be6 | 2007-10-11 20:37:10 +1000 | [diff] [blame] | 563 | mmu_huge_psize, ssize); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 564 | if (slot == -1) { |
| 565 | if (mftb() & 0x1) |
Benjamin Herrenschmidt | 67b1081 | 2005-09-23 13:24:07 -0700 | [diff] [blame] | 566 | hpte_group = ((hash & htab_hash_mask) * |
| 567 | HPTES_PER_GROUP)&~0x7UL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 568 | |
| 569 | ppc_md.hpte_remove(hpte_group); |
| 570 | goto repeat; |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | if (unlikely(slot == -2)) |
| 575 | panic("hash_huge_page: pte_insert failed\n"); |
| 576 | |
Ishizaki Kou | d649bd7 | 2007-01-12 09:54:39 +0900 | [diff] [blame] | 577 | new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 578 | } |
| 579 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 580 | /* |
Hugh Dickins | 01edcd8 | 2005-11-23 13:37:39 -0800 | [diff] [blame] | 581 | * No need to use ldarx/stdcx here |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 582 | */ |
| 583 | *ptep = __pte(new_pte & ~_PAGE_BUSY); |
| 584 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 585 | err = 0; |
| 586 | |
| 587 | out: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 588 | return err; |
| 589 | } |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 590 | |
Jon Tollefson | 4ec161c | 2008-01-04 09:59:50 +1100 | [diff] [blame] | 591 | void set_huge_psize(int psize) |
| 592 | { |
| 593 | /* Check that it is a page size supported by the hardware and |
| 594 | * that it fits within pagetable limits. */ |
| 595 | if (mmu_psize_defs[psize].shift && mmu_psize_defs[psize].shift < SID_SHIFT && |
| 596 | (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT || |
| 597 | mmu_psize_defs[psize].shift == HPAGE_SHIFT_64K)) { |
| 598 | HPAGE_SHIFT = mmu_psize_defs[psize].shift; |
| 599 | mmu_huge_psize = psize; |
| 600 | #ifdef CONFIG_PPC_64K_PAGES |
| 601 | hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT); |
| 602 | #else |
| 603 | if (HPAGE_SHIFT == HPAGE_SHIFT_64K) |
| 604 | hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT); |
| 605 | else |
| 606 | hugepte_shift = (PUD_SHIFT-HPAGE_SHIFT); |
| 607 | #endif |
| 608 | |
| 609 | } else |
| 610 | HPAGE_SHIFT = 0; |
| 611 | } |
| 612 | |
| 613 | static int __init hugepage_setup_sz(char *str) |
| 614 | { |
| 615 | unsigned long long size; |
| 616 | int mmu_psize = -1; |
| 617 | int shift; |
| 618 | |
| 619 | size = memparse(str, &str); |
| 620 | |
| 621 | shift = __ffs(size); |
| 622 | switch (shift) { |
| 623 | #ifndef CONFIG_PPC_64K_PAGES |
| 624 | case HPAGE_SHIFT_64K: |
| 625 | mmu_psize = MMU_PAGE_64K; |
| 626 | break; |
| 627 | #endif |
| 628 | case HPAGE_SHIFT_16M: |
| 629 | mmu_psize = MMU_PAGE_16M; |
| 630 | break; |
| 631 | } |
| 632 | |
| 633 | if (mmu_psize >=0 && mmu_psize_defs[mmu_psize].shift) |
| 634 | set_huge_psize(mmu_psize); |
| 635 | else |
| 636 | printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size); |
| 637 | |
| 638 | return 1; |
| 639 | } |
| 640 | __setup("hugepagesz=", hugepage_setup_sz); |
| 641 | |
Christoph Lameter | 4ba9b9d | 2007-10-16 23:25:51 -0700 | [diff] [blame] | 642 | static void zero_ctor(struct kmem_cache *cache, void *addr) |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 643 | { |
| 644 | memset(addr, 0, kmem_cache_size(cache)); |
| 645 | } |
| 646 | |
| 647 | static int __init hugetlbpage_init(void) |
| 648 | { |
| 649 | if (!cpu_has_feature(CPU_FTR_16M_PAGE)) |
| 650 | return -ENODEV; |
| 651 | |
| 652 | huge_pgtable_cache = kmem_cache_create("hugepte_cache", |
| 653 | HUGEPTE_TABLE_SIZE, |
| 654 | HUGEPTE_TABLE_SIZE, |
Christoph Lameter | f0f3980 | 2007-05-06 14:49:58 -0700 | [diff] [blame] | 655 | 0, |
Paul Mundt | 20c2df8 | 2007-07-20 10:11:58 +0900 | [diff] [blame] | 656 | zero_ctor); |
David Gibson | f10a04c | 2006-04-28 15:02:51 +1000 | [diff] [blame] | 657 | if (! huge_pgtable_cache) |
| 658 | panic("hugetlbpage_init(): could not create hugepte cache\n"); |
| 659 | |
| 660 | return 0; |
| 661 | } |
| 662 | |
| 663 | module_init(hugetlbpage_init); |