blob: 7bbf4e4ed4306b3a395906b6307808df326fb7cf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 *
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */
9
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
14#include <linux/pagemap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/slab.h>
16#include <linux/err.h>
17#include <linux/sysctl.h>
18#include <asm/mman.h>
19#include <asm/pgalloc.h>
20#include <asm/tlb.h>
21#include <asm/tlbflush.h>
22#include <asm/mmu_context.h>
23#include <asm/machdep.h>
24#include <asm/cputable.h>
Benjamin Herrenschmidt94b2a432007-03-10 00:05:37 +010025#include <asm/spu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Jon Tollefson91224342008-07-23 21:27:55 -070027#define PAGE_SHIFT_64K 16
28#define PAGE_SHIFT_16M 24
29#define PAGE_SHIFT_16G 34
Jon Tollefson4ec161c2008-01-04 09:59:50 +110030
David Gibsonc594ada2005-08-11 16:55:21 +100031#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
32#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -070033#define MAX_NUMBER_GPAGES 1024
34
35/* Tracks the 16G pages after the device tree is scanned and before the
36 * huge_boot_pages list is ready. */
37static unsigned long gpage_freearray[MAX_NUMBER_GPAGES];
38static unsigned nr_gpages;
David Gibsonc594ada2005-08-11 16:55:21 +100039
Jon Tollefson0d9ea752008-07-23 21:27:56 -070040/* Array of valid huge page sizes - non-zero value(hugepte_shift) is
41 * stored for the huge page sizes that are valid.
42 */
43unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */
David Gibsonf10a04c2006-04-28 15:02:51 +100044
Jon Tollefson0d9ea752008-07-23 21:27:56 -070045#define hugepte_shift mmu_huge_psizes
46#define PTRS_PER_HUGEPTE(psize) (1 << hugepte_shift[psize])
47#define HUGEPTE_TABLE_SIZE(psize) (sizeof(pte_t) << hugepte_shift[psize])
David Gibsonf10a04c2006-04-28 15:02:51 +100048
Jon Tollefson0d9ea752008-07-23 21:27:56 -070049#define HUGEPD_SHIFT(psize) (mmu_psize_to_shift(psize) \
50 + hugepte_shift[psize])
51#define HUGEPD_SIZE(psize) (1UL << HUGEPD_SHIFT(psize))
52#define HUGEPD_MASK(psize) (~(HUGEPD_SIZE(psize)-1))
53
54/* Subtract one from array size because we don't need a cache for 4K since
55 * is not a huge page size */
56#define huge_pgtable_cache(psize) (pgtable_cache[HUGEPTE_CACHE_NUM \
57 + psize-1])
58#define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize])
59
60static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = {
61 "unused_4K", "hugepte_cache_64K", "unused_64K_AP",
62 "hugepte_cache_1M", "hugepte_cache_16M", "hugepte_cache_16G"
63};
David Gibsonf10a04c2006-04-28 15:02:51 +100064
65/* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad()
66 * will choke on pointers to hugepte tables, which is handy for
67 * catching screwups early. */
68#define HUGEPD_OK 0x1
69
70typedef struct { unsigned long pd; } hugepd_t;
71
72#define hugepd_none(hpd) ((hpd).pd == 0)
73
Jon Tollefson0d9ea752008-07-23 21:27:56 -070074static inline int shift_to_mmu_psize(unsigned int shift)
75{
76 switch (shift) {
77#ifndef CONFIG_PPC_64K_PAGES
78 case PAGE_SHIFT_64K:
79 return MMU_PAGE_64K;
80#endif
81 case PAGE_SHIFT_16M:
82 return MMU_PAGE_16M;
83 case PAGE_SHIFT_16G:
84 return MMU_PAGE_16G;
85 }
86 return -1;
87}
88
89static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
90{
91 if (mmu_psize_defs[mmu_psize].shift)
92 return mmu_psize_defs[mmu_psize].shift;
93 BUG();
94}
95
David Gibsonf10a04c2006-04-28 15:02:51 +100096static inline pte_t *hugepd_page(hugepd_t hpd)
97{
98 BUG_ON(!(hpd.pd & HUGEPD_OK));
99 return (pte_t *)(hpd.pd & ~HUGEPD_OK);
100}
101
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700102static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr,
103 struct hstate *hstate)
David Gibsonf10a04c2006-04-28 15:02:51 +1000104{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700105 unsigned int shift = huge_page_shift(hstate);
106 int psize = shift_to_mmu_psize(shift);
107 unsigned long idx = ((addr >> shift) & (PTRS_PER_HUGEPTE(psize)-1));
David Gibsonf10a04c2006-04-28 15:02:51 +1000108 pte_t *dir = hugepd_page(*hpdp);
109
110 return dir + idx;
111}
112
113static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700114 unsigned long address, unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000115{
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700116 pte_t *new = kmem_cache_zalloc(huge_pgtable_cache(psize),
David Gibsonf10a04c2006-04-28 15:02:51 +1000117 GFP_KERNEL|__GFP_REPEAT);
118
119 if (! new)
120 return -ENOMEM;
121
122 spin_lock(&mm->page_table_lock);
123 if (!hugepd_none(*hpdp))
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700124 kmem_cache_free(huge_pgtable_cache(psize), new);
David Gibsonf10a04c2006-04-28 15:02:51 +1000125 else
126 hpdp->pd = (unsigned long)new | HUGEPD_OK;
127 spin_unlock(&mm->page_table_lock);
128 return 0;
129}
130
David Gibson0b264252008-09-05 11:49:54 +1000131
132static pud_t *hpud_offset(pgd_t *pgd, unsigned long addr, struct hstate *hstate)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100133{
David Gibson0b264252008-09-05 11:49:54 +1000134 if (huge_page_shift(hstate) < PUD_SHIFT)
135 return pud_offset(pgd, addr);
136 else
137 return (pud_t *) pgd;
138}
139static pud_t *hpud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long addr,
140 struct hstate *hstate)
141{
142 if (huge_page_shift(hstate) < PUD_SHIFT)
143 return pud_alloc(mm, pgd, addr);
144 else
145 return (pud_t *) pgd;
146}
147static pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
148{
149 if (huge_page_shift(hstate) < PMD_SHIFT)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100150 return pmd_offset(pud, addr);
151 else
152 return (pmd_t *) pud;
153}
David Gibson0b264252008-09-05 11:49:54 +1000154static pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
155 struct hstate *hstate)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100156{
David Gibson0b264252008-09-05 11:49:54 +1000157 if (huge_page_shift(hstate) < PMD_SHIFT)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100158 return pmd_alloc(mm, pud, addr);
159 else
160 return (pmd_t *) pud;
161}
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100162
Jon Tollefson658013e2008-07-23 21:27:54 -0700163/* Build list of addresses of gigantic pages. This function is used in early
164 * boot before the buddy or bootmem allocator is setup.
165 */
166void add_gpage(unsigned long addr, unsigned long page_size,
167 unsigned long number_of_pages)
168{
169 if (!addr)
170 return;
171 while (number_of_pages > 0) {
172 gpage_freearray[nr_gpages] = addr;
173 nr_gpages++;
174 number_of_pages--;
175 addr += page_size;
176 }
177}
178
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700179/* Moves the gigantic page addresses from the temporary list to the
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700180 * huge_boot_pages list.
181 */
182int alloc_bootmem_huge_page(struct hstate *hstate)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700183{
184 struct huge_bootmem_page *m;
185 if (nr_gpages == 0)
186 return 0;
187 m = phys_to_virt(gpage_freearray[--nr_gpages]);
188 gpage_freearray[nr_gpages] = 0;
189 list_add(&m->list, &huge_boot_pages);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700190 m->hstate = hstate;
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700191 return 1;
192}
193
194
David Gibsone28f7fa2005-08-05 19:39:06 +1000195/* Modelled after find_linux_pte() */
196pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197{
David Gibsone28f7fa2005-08-05 19:39:06 +1000198 pgd_t *pg;
199 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100200 pmd_t *pm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700202 unsigned int psize;
203 unsigned int shift;
204 unsigned long sz;
205 struct hstate *hstate;
206 psize = get_slice_psize(mm, addr);
207 shift = mmu_psize_to_shift(psize);
208 sz = ((1UL) << shift);
209 hstate = size_to_hstate(sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700211 addr &= hstate->mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212
David Gibsone28f7fa2005-08-05 19:39:06 +1000213 pg = pgd_offset(mm, addr);
214 if (!pgd_none(*pg)) {
David Gibson0b264252008-09-05 11:49:54 +1000215 pu = hpud_offset(pg, addr, hstate);
David Gibsone28f7fa2005-08-05 19:39:06 +1000216 if (!pud_none(*pu)) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700217 pm = hpmd_offset(pu, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000218 if (!pmd_none(*pm))
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700219 return hugepte_offset((hugepd_t *)pm, addr,
220 hstate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 }
222 }
223
David Gibsone28f7fa2005-08-05 19:39:06 +1000224 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225}
226
Andi Kleena5516432008-07-23 21:27:41 -0700227pte_t *huge_pte_alloc(struct mm_struct *mm,
228 unsigned long addr, unsigned long sz)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229{
David Gibsone28f7fa2005-08-05 19:39:06 +1000230 pgd_t *pg;
231 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100232 pmd_t *pm;
David Gibsonf10a04c2006-04-28 15:02:51 +1000233 hugepd_t *hpdp = NULL;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700234 struct hstate *hstate;
235 unsigned int psize;
236 hstate = size_to_hstate(sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700238 psize = get_slice_psize(mm, addr);
239 BUG_ON(!mmu_huge_psizes[psize]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700241 addr &= hstate->mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
David Gibsone28f7fa2005-08-05 19:39:06 +1000243 pg = pgd_offset(mm, addr);
David Gibson0b264252008-09-05 11:49:54 +1000244 pu = hpud_alloc(mm, pg, addr, hstate);
David Gibsone28f7fa2005-08-05 19:39:06 +1000245
246 if (pu) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700247 pm = hpmd_alloc(mm, pu, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000248 if (pm)
249 hpdp = (hugepd_t *)pm;
David Gibsone28f7fa2005-08-05 19:39:06 +1000250 }
251
David Gibsonf10a04c2006-04-28 15:02:51 +1000252 if (! hpdp)
253 return NULL;
254
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700255 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, psize))
David Gibsonf10a04c2006-04-28 15:02:51 +1000256 return NULL;
257
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700258 return hugepte_offset(hpdp, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000259}
260
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800261int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
262{
263 return 0;
264}
265
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700266static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp,
267 unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000268{
269 pte_t *hugepte = hugepd_page(*hpdp);
270
271 hpdp->pd = 0;
272 tlb->need_flush = 1;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700273 pgtable_free_tlb(tlb, pgtable_free_cache(hugepte,
274 HUGEPTE_CACHE_NUM+psize-1,
Adam Litkec9169f82006-08-18 11:22:21 -0700275 PGF_CACHENUM_MASK));
David Gibsonf10a04c2006-04-28 15:02:51 +1000276}
277
David Gibsonf10a04c2006-04-28 15:02:51 +1000278static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
279 unsigned long addr, unsigned long end,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700280 unsigned long floor, unsigned long ceiling,
281 unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000282{
283 pmd_t *pmd;
284 unsigned long next;
285 unsigned long start;
286
287 start = addr;
288 pmd = pmd_offset(pud, addr);
289 do {
290 next = pmd_addr_end(addr, end);
291 if (pmd_none(*pmd))
292 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700293 free_hugepte_range(tlb, (hugepd_t *)pmd, psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000294 } while (pmd++, addr = next, addr != end);
295
296 start &= PUD_MASK;
297 if (start < floor)
298 return;
299 if (ceiling) {
300 ceiling &= PUD_MASK;
301 if (!ceiling)
302 return;
303 }
304 if (end - 1 > ceiling - 1)
305 return;
306
307 pmd = pmd_offset(pud, start);
308 pud_clear(pud);
309 pmd_free_tlb(tlb, pmd);
310}
David Gibsonf10a04c2006-04-28 15:02:51 +1000311
312static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
313 unsigned long addr, unsigned long end,
314 unsigned long floor, unsigned long ceiling)
315{
316 pud_t *pud;
317 unsigned long next;
318 unsigned long start;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700319 unsigned int shift;
320 unsigned int psize = get_slice_psize(tlb->mm, addr);
321 shift = mmu_psize_to_shift(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000322
323 start = addr;
324 pud = pud_offset(pgd, addr);
325 do {
326 next = pud_addr_end(addr, end);
David Gibson0b264252008-09-05 11:49:54 +1000327 if (shift < PMD_SHIFT) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100328 if (pud_none_or_clear_bad(pud))
329 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700330 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
331 ceiling, psize);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100332 } else {
333 if (pud_none(*pud))
334 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700335 free_hugepte_range(tlb, (hugepd_t *)pud, psize);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100336 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000337 } while (pud++, addr = next, addr != end);
338
339 start &= PGDIR_MASK;
340 if (start < floor)
341 return;
342 if (ceiling) {
343 ceiling &= PGDIR_MASK;
344 if (!ceiling)
345 return;
346 }
347 if (end - 1 > ceiling - 1)
348 return;
349
350 pud = pud_offset(pgd, start);
351 pgd_clear(pgd);
352 pud_free_tlb(tlb, pud);
353}
354
355/*
356 * This function frees user-level page tables of a process.
357 *
358 * Must be called with pagetable lock held.
359 */
Jan Beulich42b77722008-07-23 21:27:10 -0700360void hugetlb_free_pgd_range(struct mmu_gather *tlb,
David Gibsonf10a04c2006-04-28 15:02:51 +1000361 unsigned long addr, unsigned long end,
362 unsigned long floor, unsigned long ceiling)
363{
364 pgd_t *pgd;
365 unsigned long next;
366 unsigned long start;
367
368 /*
369 * Comments below take from the normal free_pgd_range(). They
370 * apply here too. The tests against HUGEPD_MASK below are
371 * essential, because we *don't* test for this at the bottom
372 * level. Without them we'll attempt to free a hugepte table
373 * when we unmap just part of it, even if there are other
374 * active mappings using it.
375 *
376 * The next few lines have given us lots of grief...
377 *
378 * Why are we testing HUGEPD* at this top level? Because
379 * often there will be no work to do at all, and we'd prefer
380 * not to go all the way down to the bottom just to discover
381 * that.
382 *
383 * Why all these "- 1"s? Because 0 represents both the bottom
384 * of the address space and the top of it (using -1 for the
385 * top wouldn't help much: the masks would do the wrong thing).
386 * The rule is that addr 0 and floor 0 refer to the bottom of
387 * the address space, but end 0 and ceiling 0 refer to the top
388 * Comparisons need to use "end - 1" and "ceiling - 1" (though
389 * that end 0 case should be mythical).
390 *
391 * Wherever addr is brought up or ceiling brought down, we
392 * must be careful to reject "the opposite 0" before it
393 * confuses the subsequent tests. But what about where end is
394 * brought down by HUGEPD_SIZE below? no, end can't go down to
395 * 0 there.
396 *
397 * Whereas we round start (addr) and ceiling down, by different
398 * masks at different levels, in order to test whether a table
399 * now has no other vmas using it, so can be freed, we don't
400 * bother to round floor or end up - the tests don't need that.
401 */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700402 unsigned int psize = get_slice_psize(tlb->mm, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000403
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700404 addr &= HUGEPD_MASK(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000405 if (addr < floor) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700406 addr += HUGEPD_SIZE(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000407 if (!addr)
408 return;
409 }
410 if (ceiling) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700411 ceiling &= HUGEPD_MASK(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000412 if (!ceiling)
413 return;
414 }
415 if (end - 1 > ceiling - 1)
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700416 end -= HUGEPD_SIZE(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000417 if (addr > end - 1)
418 return;
419
420 start = addr;
Jan Beulich42b77722008-07-23 21:27:10 -0700421 pgd = pgd_offset(tlb->mm, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000422 do {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700423 psize = get_slice_psize(tlb->mm, addr);
424 BUG_ON(!mmu_huge_psizes[psize]);
David Gibsonf10a04c2006-04-28 15:02:51 +1000425 next = pgd_addr_end(addr, end);
David Gibson0b264252008-09-05 11:49:54 +1000426 if (mmu_psize_to_shift(psize) < PUD_SHIFT) {
427 if (pgd_none_or_clear_bad(pgd))
428 continue;
429 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
430 } else {
431 if (pgd_none(*pgd))
432 continue;
433 free_hugepte_range(tlb, (hugepd_t *)pgd, psize);
434 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000435 } while (pgd++, addr = next, addr != end);
David Gibsone28f7fa2005-08-05 19:39:06 +1000436}
437
David Gibsone28f7fa2005-08-05 19:39:06 +1000438void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
439 pte_t *ptep, pte_t pte)
440{
David Gibsone28f7fa2005-08-05 19:39:06 +1000441 if (pte_present(*ptep)) {
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100442 /* We open-code pte_clear because we need to pass the right
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000443 * argument to hpte_need_flush (huge / !huge). Might not be
444 * necessary anymore if we make hpte_need_flush() get the
445 * page size from the slices
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100446 */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700447 unsigned int psize = get_slice_psize(mm, addr);
448 unsigned int shift = mmu_psize_to_shift(psize);
449 unsigned long sz = ((1UL) << shift);
450 struct hstate *hstate = size_to_hstate(sz);
451 pte_update(mm, addr & hstate->mask, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000452 }
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100453 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
David Gibsone28f7fa2005-08-05 19:39:06 +1000454}
455
456pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
457 pte_t *ptep)
458{
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000459 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000460 return __pte(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461}
462
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463struct page *
464follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
465{
466 pte_t *ptep;
467 struct page *page;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700468 unsigned int mmu_psize = get_slice_psize(mm, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700470 /* Verify it is a huge page else bail. */
471 if (!mmu_huge_psizes[mmu_psize])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 return ERR_PTR(-EINVAL);
473
474 ptep = huge_pte_offset(mm, address);
475 page = pte_page(*ptep);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700476 if (page) {
477 unsigned int shift = mmu_psize_to_shift(mmu_psize);
478 unsigned long sz = ((1UL) << shift);
479 page += (address % sz) / PAGE_SIZE;
480 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481
482 return page;
483}
484
485int pmd_huge(pmd_t pmd)
486{
487 return 0;
488}
489
Andi Kleenceb86872008-07-23 21:27:50 -0700490int pud_huge(pud_t pud)
491{
492 return 0;
493}
494
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495struct page *
496follow_huge_pmd(struct mm_struct *mm, unsigned long address,
497 pmd_t *pmd, int write)
498{
499 BUG();
500 return NULL;
501}
502
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
504unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
505 unsigned long len, unsigned long pgoff,
506 unsigned long flags)
507{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700508 struct hstate *hstate = hstate_file(file);
509 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
510 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511}
512
David Gibsoncbf52af2005-12-09 14:20:52 +1100513/*
514 * Called by asm hashtable.S for doing lazy icache flush
515 */
516static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700517 pte_t pte, int trap, unsigned long sz)
David Gibsoncbf52af2005-12-09 14:20:52 +1100518{
519 struct page *page;
520 int i;
521
522 if (!pfn_valid(pte_pfn(pte)))
523 return rflags;
524
525 page = pte_page(pte);
526
527 /* page is dirty */
528 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
529 if (trap == 0x400) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700530 for (i = 0; i < (sz / PAGE_SIZE); i++)
David Gibsoncbf52af2005-12-09 14:20:52 +1100531 __flush_dcache_icache(page_address(page+i));
532 set_bit(PG_arch_1, &page->flags);
533 } else {
534 rflags |= HPTE_R_N;
535 }
536 }
537 return rflags;
538}
539
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540int hash_huge_page(struct mm_struct *mm, unsigned long access,
David Gibsoncbf52af2005-12-09 14:20:52 +1100541 unsigned long ea, unsigned long vsid, int local,
542 unsigned long trap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543{
544 pte_t *ptep;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100545 unsigned long old_pte, new_pte;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700546 unsigned long va, rflags, pa, sz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 long slot;
548 int err = 1;
Paul Mackerras1189be62007-10-11 20:37:10 +1000549 int ssize = user_segment_size(ea);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700550 unsigned int mmu_psize;
551 int shift;
552 mmu_psize = get_slice_psize(mm, ea);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700554 if (!mmu_huge_psizes[mmu_psize])
555 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 ptep = huge_pte_offset(mm, ea);
557
558 /* Search the Linux page table for a match with va */
Paul Mackerras1189be62007-10-11 20:37:10 +1000559 va = hpt_va(ea, vsid, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560
561 /*
562 * If no pte found or not present, send the problem up to
563 * do_page_fault
564 */
565 if (unlikely(!ptep || pte_none(*ptep)))
566 goto out;
567
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 /*
569 * Check the user's access rights to the page. If access should be
570 * prevented then send the problem up to do_page_fault.
571 */
572 if (unlikely(access & ~pte_val(*ptep)))
573 goto out;
574 /*
575 * At this point, we have a pte (old_pte) which can be used to build
576 * or update an HPTE. There are 2 cases:
577 *
578 * 1. There is a valid (present) pte with no associated HPTE (this is
579 * the most common case)
580 * 2. There is a valid (present) pte with an associated HPTE. The
581 * current values of the pp bits in the HPTE prevent access
582 * because we are doing software DIRTY bit management and the
583 * page is currently not DIRTY.
584 */
585
586
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100587 do {
588 old_pte = pte_val(*ptep);
589 if (old_pte & _PAGE_BUSY)
590 goto out;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000591 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100592 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
593 old_pte, new_pte));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100595 rflags = 0x2 | (!(new_pte & _PAGE_RW));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100597 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700598 shift = mmu_psize_to_shift(mmu_psize);
599 sz = ((1UL) << shift);
David Gibsoncbf52af2005-12-09 14:20:52 +1100600 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
601 /* No CPU has hugepages but lacks no execute, so we
602 * don't need to worry about that case */
603 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700604 trap, sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
606 /* Check if pte already has an hpte (case 2) */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100607 if (unlikely(old_pte & _PAGE_HASHPTE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 /* There MIGHT be an HPTE for this pte */
609 unsigned long hash, slot;
610
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700611 hash = hpt_hash(va, shift, ssize);
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100612 if (old_pte & _PAGE_F_SECOND)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 hash = ~hash;
614 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100615 slot += (old_pte & _PAGE_F_GIX) >> 12;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700617 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
Paul Mackerras1189be62007-10-11 20:37:10 +1000618 ssize, local) == -1)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100619 old_pte &= ~_PAGE_HPTEFLAGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 }
621
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100622 if (likely(!(old_pte & _PAGE_HASHPTE))) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700623 unsigned long hash = hpt_hash(va, shift, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 unsigned long hpte_group;
625
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100626 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
628repeat:
629 hpte_group = ((hash & htab_hash_mask) *
630 HPTES_PER_GROUP) & ~0x7UL;
631
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100632 /* clear HPTE slot informations in new PTE */
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000633#ifdef CONFIG_PPC_64K_PAGES
634 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
635#else
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100636 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000637#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 /* Add in WIMG bits */
Dave Kleikamp87e9ab12008-06-19 08:32:56 +1000639 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
640 _PAGE_COHERENT | _PAGE_GUARDED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100642 /* Insert into the hash table, primary slot */
643 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700644 mmu_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645
646 /* Primary is full, try the secondary */
647 if (unlikely(slot == -1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 hpte_group = ((~hash & htab_hash_mask) *
649 HPTES_PER_GROUP) & ~0x7UL;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100650 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700651 HPTE_V_SECONDARY,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700652 mmu_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 if (slot == -1) {
654 if (mftb() & 0x1)
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700655 hpte_group = ((hash & htab_hash_mask) *
656 HPTES_PER_GROUP)&~0x7UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657
658 ppc_md.hpte_remove(hpte_group);
659 goto repeat;
660 }
661 }
662
663 if (unlikely(slot == -2))
664 panic("hash_huge_page: pte_insert failed\n");
665
Ishizaki Koud649bd72007-01-12 09:54:39 +0900666 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667 }
668
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100669 /*
Hugh Dickins01edcd82005-11-23 13:37:39 -0800670 * No need to use ldarx/stdcx here
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100671 */
672 *ptep = __pte(new_pte & ~_PAGE_BUSY);
673
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 err = 0;
675
676 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 return err;
678}
David Gibsonf10a04c2006-04-28 15:02:51 +1000679
Al Viro4ea8fb92008-11-22 17:33:44 +0000680static void __init set_huge_psize(int psize)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100681{
682 /* Check that it is a page size supported by the hardware and
683 * that it fits within pagetable limits. */
Jon Tollefson91224342008-07-23 21:27:55 -0700684 if (mmu_psize_defs[psize].shift &&
685 mmu_psize_defs[psize].shift < SID_SHIFT_1T &&
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100686 (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT ||
Jon Tollefson91224342008-07-23 21:27:55 -0700687 mmu_psize_defs[psize].shift == PAGE_SHIFT_64K ||
688 mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700689 /* Return if huge page size has already been setup or is the
690 * same as the base page size. */
691 if (mmu_huge_psizes[psize] ||
692 mmu_psize_defs[psize].shift == PAGE_SHIFT)
Jon Tollefson91224342008-07-23 21:27:55 -0700693 return;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700694 hugetlb_add_hstate(mmu_psize_defs[psize].shift - PAGE_SHIFT);
Jon Tollefson91224342008-07-23 21:27:55 -0700695
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700696 switch (mmu_psize_defs[psize].shift) {
Jon Tollefson91224342008-07-23 21:27:55 -0700697 case PAGE_SHIFT_64K:
698 /* We only allow 64k hpages with 4k base page,
699 * which was checked above, and always put them
700 * at the PMD */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700701 hugepte_shift[psize] = PMD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700702 break;
703 case PAGE_SHIFT_16M:
704 /* 16M pages can be at two different levels
705 * of pagestables based on base page size */
706 if (PAGE_SHIFT == PAGE_SHIFT_64K)
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700707 hugepte_shift[psize] = PMD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700708 else /* 4k base page */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700709 hugepte_shift[psize] = PUD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700710 break;
711 case PAGE_SHIFT_16G:
712 /* 16G pages are always at PGD level */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700713 hugepte_shift[psize] = PGDIR_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700714 break;
715 }
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700716 hugepte_shift[psize] -= mmu_psize_defs[psize].shift;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100717 } else
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700718 hugepte_shift[psize] = 0;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100719}
720
721static int __init hugepage_setup_sz(char *str)
722{
723 unsigned long long size;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700724 int mmu_psize;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100725 int shift;
726
727 size = memparse(str, &str);
728
729 shift = __ffs(size);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700730 mmu_psize = shift_to_mmu_psize(shift);
731 if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100732 set_huge_psize(mmu_psize);
733 else
734 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
735
736 return 1;
737}
738__setup("hugepagesz=", hugepage_setup_sz);
739
David Gibsonf10a04c2006-04-28 15:02:51 +1000740static int __init hugetlbpage_init(void)
741{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700742 unsigned int psize;
743
David Gibsonf10a04c2006-04-28 15:02:51 +1000744 if (!cpu_has_feature(CPU_FTR_16M_PAGE))
745 return -ENODEV;
Benjamin Herrenschmidt00df4382008-07-28 16:13:18 +1000746
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700747 /* Add supported huge page sizes. Need to change HUGE_MAX_HSTATE
748 * and adjust PTE_NONCACHE_NUM if the number of supported huge page
749 * sizes changes.
750 */
751 set_huge_psize(MMU_PAGE_16M);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700752 set_huge_psize(MMU_PAGE_16G);
David Gibsonf10a04c2006-04-28 15:02:51 +1000753
Benjamin Herrenschmidt00df4382008-07-28 16:13:18 +1000754 /* Temporarily disable support for 64K huge pages when 64K SPU local
755 * store support is enabled as the current implementation conflicts.
756 */
757#ifndef CONFIG_SPU_FS_64K_LS
758 set_huge_psize(MMU_PAGE_64K);
759#endif
760
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700761 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
762 if (mmu_huge_psizes[psize]) {
763 huge_pgtable_cache(psize) = kmem_cache_create(
764 HUGEPTE_CACHE_NAME(psize),
765 HUGEPTE_TABLE_SIZE(psize),
766 HUGEPTE_TABLE_SIZE(psize),
767 0,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700768 NULL);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700769 if (!huge_pgtable_cache(psize))
770 panic("hugetlbpage_init(): could not create %s"\
771 "\n", HUGEPTE_CACHE_NAME(psize));
772 }
773 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000774
775 return 0;
776}
777
778module_init(hugetlbpage_init);