blob: 9920d6a7cf290cc02339da13d337fa6eddbf021a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 *
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */
9
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
14#include <linux/pagemap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/slab.h>
16#include <linux/err.h>
17#include <linux/sysctl.h>
18#include <asm/mman.h>
19#include <asm/pgalloc.h>
20#include <asm/tlb.h>
21#include <asm/tlbflush.h>
22#include <asm/mmu_context.h>
23#include <asm/machdep.h>
24#include <asm/cputable.h>
Benjamin Herrenschmidt94b2a432007-03-10 00:05:37 +010025#include <asm/spu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Jon Tollefson91224342008-07-23 21:27:55 -070027#define PAGE_SHIFT_64K 16
28#define PAGE_SHIFT_16M 24
29#define PAGE_SHIFT_16G 34
Jon Tollefson4ec161c2008-01-04 09:59:50 +110030
David Gibsonc594ada2005-08-11 16:55:21 +100031#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
32#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -070033#define MAX_NUMBER_GPAGES 1024
34
35/* Tracks the 16G pages after the device tree is scanned and before the
36 * huge_boot_pages list is ready. */
37static unsigned long gpage_freearray[MAX_NUMBER_GPAGES];
38static unsigned nr_gpages;
David Gibsonc594ada2005-08-11 16:55:21 +100039
Jon Tollefson0d9ea752008-07-23 21:27:56 -070040/* Array of valid huge page sizes - non-zero value(hugepte_shift) is
41 * stored for the huge page sizes that are valid.
42 */
43unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */
David Gibsonf10a04c2006-04-28 15:02:51 +100044
Jon Tollefson0d9ea752008-07-23 21:27:56 -070045#define hugepte_shift mmu_huge_psizes
46#define PTRS_PER_HUGEPTE(psize) (1 << hugepte_shift[psize])
47#define HUGEPTE_TABLE_SIZE(psize) (sizeof(pte_t) << hugepte_shift[psize])
David Gibsonf10a04c2006-04-28 15:02:51 +100048
Jon Tollefson0d9ea752008-07-23 21:27:56 -070049#define HUGEPD_SHIFT(psize) (mmu_psize_to_shift(psize) \
50 + hugepte_shift[psize])
51#define HUGEPD_SIZE(psize) (1UL << HUGEPD_SHIFT(psize))
52#define HUGEPD_MASK(psize) (~(HUGEPD_SIZE(psize)-1))
53
54/* Subtract one from array size because we don't need a cache for 4K since
55 * is not a huge page size */
Jon Tollefson7d4320f2008-10-30 12:03:57 +000056#define HUGE_PGTABLE_INDEX(psize) (HUGEPTE_CACHE_NUM + psize - 1)
Jon Tollefson0d9ea752008-07-23 21:27:56 -070057#define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize])
58
59static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = {
60 "unused_4K", "hugepte_cache_64K", "unused_64K_AP",
61 "hugepte_cache_1M", "hugepte_cache_16M", "hugepte_cache_16G"
62};
David Gibsonf10a04c2006-04-28 15:02:51 +100063
64/* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad()
65 * will choke on pointers to hugepte tables, which is handy for
66 * catching screwups early. */
67#define HUGEPD_OK 0x1
68
69typedef struct { unsigned long pd; } hugepd_t;
70
71#define hugepd_none(hpd) ((hpd).pd == 0)
72
Jon Tollefson0d9ea752008-07-23 21:27:56 -070073static inline int shift_to_mmu_psize(unsigned int shift)
74{
75 switch (shift) {
76#ifndef CONFIG_PPC_64K_PAGES
77 case PAGE_SHIFT_64K:
78 return MMU_PAGE_64K;
79#endif
80 case PAGE_SHIFT_16M:
81 return MMU_PAGE_16M;
82 case PAGE_SHIFT_16G:
83 return MMU_PAGE_16G;
84 }
85 return -1;
86}
87
88static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
89{
90 if (mmu_psize_defs[mmu_psize].shift)
91 return mmu_psize_defs[mmu_psize].shift;
92 BUG();
93}
94
David Gibsonf10a04c2006-04-28 15:02:51 +100095static inline pte_t *hugepd_page(hugepd_t hpd)
96{
97 BUG_ON(!(hpd.pd & HUGEPD_OK));
98 return (pte_t *)(hpd.pd & ~HUGEPD_OK);
99}
100
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700101static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr,
102 struct hstate *hstate)
David Gibsonf10a04c2006-04-28 15:02:51 +1000103{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700104 unsigned int shift = huge_page_shift(hstate);
105 int psize = shift_to_mmu_psize(shift);
106 unsigned long idx = ((addr >> shift) & (PTRS_PER_HUGEPTE(psize)-1));
David Gibsonf10a04c2006-04-28 15:02:51 +1000107 pte_t *dir = hugepd_page(*hpdp);
108
109 return dir + idx;
110}
111
112static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700113 unsigned long address, unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000114{
Jon Tollefson7d4320f2008-10-30 12:03:57 +0000115 pte_t *new = kmem_cache_zalloc(pgtable_cache[HUGE_PGTABLE_INDEX(psize)],
David Gibsonf10a04c2006-04-28 15:02:51 +1000116 GFP_KERNEL|__GFP_REPEAT);
117
118 if (! new)
119 return -ENOMEM;
120
121 spin_lock(&mm->page_table_lock);
122 if (!hugepd_none(*hpdp))
Jon Tollefson7d4320f2008-10-30 12:03:57 +0000123 kmem_cache_free(pgtable_cache[HUGE_PGTABLE_INDEX(psize)], new);
David Gibsonf10a04c2006-04-28 15:02:51 +1000124 else
125 hpdp->pd = (unsigned long)new | HUGEPD_OK;
126 spin_unlock(&mm->page_table_lock);
127 return 0;
128}
129
David Gibson0b264252008-09-05 11:49:54 +1000130
131static pud_t *hpud_offset(pgd_t *pgd, unsigned long addr, struct hstate *hstate)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100132{
David Gibson0b264252008-09-05 11:49:54 +1000133 if (huge_page_shift(hstate) < PUD_SHIFT)
134 return pud_offset(pgd, addr);
135 else
136 return (pud_t *) pgd;
137}
138static pud_t *hpud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long addr,
139 struct hstate *hstate)
140{
141 if (huge_page_shift(hstate) < PUD_SHIFT)
142 return pud_alloc(mm, pgd, addr);
143 else
144 return (pud_t *) pgd;
145}
146static pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
147{
148 if (huge_page_shift(hstate) < PMD_SHIFT)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100149 return pmd_offset(pud, addr);
150 else
151 return (pmd_t *) pud;
152}
David Gibson0b264252008-09-05 11:49:54 +1000153static pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
154 struct hstate *hstate)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100155{
David Gibson0b264252008-09-05 11:49:54 +1000156 if (huge_page_shift(hstate) < PMD_SHIFT)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100157 return pmd_alloc(mm, pud, addr);
158 else
159 return (pmd_t *) pud;
160}
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100161
Jon Tollefson658013e2008-07-23 21:27:54 -0700162/* Build list of addresses of gigantic pages. This function is used in early
163 * boot before the buddy or bootmem allocator is setup.
164 */
165void add_gpage(unsigned long addr, unsigned long page_size,
166 unsigned long number_of_pages)
167{
168 if (!addr)
169 return;
170 while (number_of_pages > 0) {
171 gpage_freearray[nr_gpages] = addr;
172 nr_gpages++;
173 number_of_pages--;
174 addr += page_size;
175 }
176}
177
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700178/* Moves the gigantic page addresses from the temporary list to the
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700179 * huge_boot_pages list.
180 */
181int alloc_bootmem_huge_page(struct hstate *hstate)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700182{
183 struct huge_bootmem_page *m;
184 if (nr_gpages == 0)
185 return 0;
186 m = phys_to_virt(gpage_freearray[--nr_gpages]);
187 gpage_freearray[nr_gpages] = 0;
188 list_add(&m->list, &huge_boot_pages);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700189 m->hstate = hstate;
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700190 return 1;
191}
192
193
David Gibsone28f7fa2005-08-05 19:39:06 +1000194/* Modelled after find_linux_pte() */
195pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196{
David Gibsone28f7fa2005-08-05 19:39:06 +1000197 pgd_t *pg;
198 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100199 pmd_t *pm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700201 unsigned int psize;
202 unsigned int shift;
203 unsigned long sz;
204 struct hstate *hstate;
205 psize = get_slice_psize(mm, addr);
206 shift = mmu_psize_to_shift(psize);
207 sz = ((1UL) << shift);
208 hstate = size_to_hstate(sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700210 addr &= hstate->mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
David Gibsone28f7fa2005-08-05 19:39:06 +1000212 pg = pgd_offset(mm, addr);
213 if (!pgd_none(*pg)) {
David Gibson0b264252008-09-05 11:49:54 +1000214 pu = hpud_offset(pg, addr, hstate);
David Gibsone28f7fa2005-08-05 19:39:06 +1000215 if (!pud_none(*pu)) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700216 pm = hpmd_offset(pu, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000217 if (!pmd_none(*pm))
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700218 return hugepte_offset((hugepd_t *)pm, addr,
219 hstate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 }
221 }
222
David Gibsone28f7fa2005-08-05 19:39:06 +1000223 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224}
225
Andi Kleena5516432008-07-23 21:27:41 -0700226pte_t *huge_pte_alloc(struct mm_struct *mm,
227 unsigned long addr, unsigned long sz)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228{
David Gibsone28f7fa2005-08-05 19:39:06 +1000229 pgd_t *pg;
230 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100231 pmd_t *pm;
David Gibsonf10a04c2006-04-28 15:02:51 +1000232 hugepd_t *hpdp = NULL;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700233 struct hstate *hstate;
234 unsigned int psize;
235 hstate = size_to_hstate(sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700237 psize = get_slice_psize(mm, addr);
238 BUG_ON(!mmu_huge_psizes[psize]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700240 addr &= hstate->mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241
David Gibsone28f7fa2005-08-05 19:39:06 +1000242 pg = pgd_offset(mm, addr);
David Gibson0b264252008-09-05 11:49:54 +1000243 pu = hpud_alloc(mm, pg, addr, hstate);
David Gibsone28f7fa2005-08-05 19:39:06 +1000244
245 if (pu) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700246 pm = hpmd_alloc(mm, pu, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000247 if (pm)
248 hpdp = (hugepd_t *)pm;
David Gibsone28f7fa2005-08-05 19:39:06 +1000249 }
250
David Gibsonf10a04c2006-04-28 15:02:51 +1000251 if (! hpdp)
252 return NULL;
253
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700254 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, psize))
David Gibsonf10a04c2006-04-28 15:02:51 +1000255 return NULL;
256
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700257 return hugepte_offset(hpdp, addr, hstate);
David Gibsonf10a04c2006-04-28 15:02:51 +1000258}
259
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800260int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
261{
262 return 0;
263}
264
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700265static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp,
266 unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000267{
268 pte_t *hugepte = hugepd_page(*hpdp);
269
270 hpdp->pd = 0;
271 tlb->need_flush = 1;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700272 pgtable_free_tlb(tlb, pgtable_free_cache(hugepte,
273 HUGEPTE_CACHE_NUM+psize-1,
Adam Litkec9169f82006-08-18 11:22:21 -0700274 PGF_CACHENUM_MASK));
David Gibsonf10a04c2006-04-28 15:02:51 +1000275}
276
David Gibsonf10a04c2006-04-28 15:02:51 +1000277static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
278 unsigned long addr, unsigned long end,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700279 unsigned long floor, unsigned long ceiling,
280 unsigned int psize)
David Gibsonf10a04c2006-04-28 15:02:51 +1000281{
282 pmd_t *pmd;
283 unsigned long next;
284 unsigned long start;
285
286 start = addr;
287 pmd = pmd_offset(pud, addr);
288 do {
289 next = pmd_addr_end(addr, end);
290 if (pmd_none(*pmd))
291 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700292 free_hugepte_range(tlb, (hugepd_t *)pmd, psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000293 } while (pmd++, addr = next, addr != end);
294
295 start &= PUD_MASK;
296 if (start < floor)
297 return;
298 if (ceiling) {
299 ceiling &= PUD_MASK;
300 if (!ceiling)
301 return;
302 }
303 if (end - 1 > ceiling - 1)
304 return;
305
306 pmd = pmd_offset(pud, start);
307 pud_clear(pud);
308 pmd_free_tlb(tlb, pmd);
309}
David Gibsonf10a04c2006-04-28 15:02:51 +1000310
311static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
312 unsigned long addr, unsigned long end,
313 unsigned long floor, unsigned long ceiling)
314{
315 pud_t *pud;
316 unsigned long next;
317 unsigned long start;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700318 unsigned int shift;
319 unsigned int psize = get_slice_psize(tlb->mm, addr);
320 shift = mmu_psize_to_shift(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000321
322 start = addr;
323 pud = pud_offset(pgd, addr);
324 do {
325 next = pud_addr_end(addr, end);
David Gibson0b264252008-09-05 11:49:54 +1000326 if (shift < PMD_SHIFT) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100327 if (pud_none_or_clear_bad(pud))
328 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700329 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
330 ceiling, psize);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100331 } else {
332 if (pud_none(*pud))
333 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700334 free_hugepte_range(tlb, (hugepd_t *)pud, psize);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100335 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000336 } while (pud++, addr = next, addr != end);
337
338 start &= PGDIR_MASK;
339 if (start < floor)
340 return;
341 if (ceiling) {
342 ceiling &= PGDIR_MASK;
343 if (!ceiling)
344 return;
345 }
346 if (end - 1 > ceiling - 1)
347 return;
348
349 pud = pud_offset(pgd, start);
350 pgd_clear(pgd);
351 pud_free_tlb(tlb, pud);
352}
353
354/*
355 * This function frees user-level page tables of a process.
356 *
357 * Must be called with pagetable lock held.
358 */
Jan Beulich42b77722008-07-23 21:27:10 -0700359void hugetlb_free_pgd_range(struct mmu_gather *tlb,
David Gibsonf10a04c2006-04-28 15:02:51 +1000360 unsigned long addr, unsigned long end,
361 unsigned long floor, unsigned long ceiling)
362{
363 pgd_t *pgd;
364 unsigned long next;
365 unsigned long start;
366
367 /*
368 * Comments below take from the normal free_pgd_range(). They
369 * apply here too. The tests against HUGEPD_MASK below are
370 * essential, because we *don't* test for this at the bottom
371 * level. Without them we'll attempt to free a hugepte table
372 * when we unmap just part of it, even if there are other
373 * active mappings using it.
374 *
375 * The next few lines have given us lots of grief...
376 *
377 * Why are we testing HUGEPD* at this top level? Because
378 * often there will be no work to do at all, and we'd prefer
379 * not to go all the way down to the bottom just to discover
380 * that.
381 *
382 * Why all these "- 1"s? Because 0 represents both the bottom
383 * of the address space and the top of it (using -1 for the
384 * top wouldn't help much: the masks would do the wrong thing).
385 * The rule is that addr 0 and floor 0 refer to the bottom of
386 * the address space, but end 0 and ceiling 0 refer to the top
387 * Comparisons need to use "end - 1" and "ceiling - 1" (though
388 * that end 0 case should be mythical).
389 *
390 * Wherever addr is brought up or ceiling brought down, we
391 * must be careful to reject "the opposite 0" before it
392 * confuses the subsequent tests. But what about where end is
393 * brought down by HUGEPD_SIZE below? no, end can't go down to
394 * 0 there.
395 *
396 * Whereas we round start (addr) and ceiling down, by different
397 * masks at different levels, in order to test whether a table
398 * now has no other vmas using it, so can be freed, we don't
399 * bother to round floor or end up - the tests don't need that.
400 */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700401 unsigned int psize = get_slice_psize(tlb->mm, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000402
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700403 addr &= HUGEPD_MASK(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000404 if (addr < floor) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700405 addr += HUGEPD_SIZE(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000406 if (!addr)
407 return;
408 }
409 if (ceiling) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700410 ceiling &= HUGEPD_MASK(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000411 if (!ceiling)
412 return;
413 }
414 if (end - 1 > ceiling - 1)
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700415 end -= HUGEPD_SIZE(psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000416 if (addr > end - 1)
417 return;
418
419 start = addr;
Jan Beulich42b77722008-07-23 21:27:10 -0700420 pgd = pgd_offset(tlb->mm, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000421 do {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700422 psize = get_slice_psize(tlb->mm, addr);
423 BUG_ON(!mmu_huge_psizes[psize]);
David Gibsonf10a04c2006-04-28 15:02:51 +1000424 next = pgd_addr_end(addr, end);
David Gibson0b264252008-09-05 11:49:54 +1000425 if (mmu_psize_to_shift(psize) < PUD_SHIFT) {
426 if (pgd_none_or_clear_bad(pgd))
427 continue;
428 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
429 } else {
430 if (pgd_none(*pgd))
431 continue;
432 free_hugepte_range(tlb, (hugepd_t *)pgd, psize);
433 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000434 } while (pgd++, addr = next, addr != end);
David Gibsone28f7fa2005-08-05 19:39:06 +1000435}
436
David Gibsone28f7fa2005-08-05 19:39:06 +1000437void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
438 pte_t *ptep, pte_t pte)
439{
David Gibsone28f7fa2005-08-05 19:39:06 +1000440 if (pte_present(*ptep)) {
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100441 /* We open-code pte_clear because we need to pass the right
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000442 * argument to hpte_need_flush (huge / !huge). Might not be
443 * necessary anymore if we make hpte_need_flush() get the
444 * page size from the slices
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100445 */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700446 unsigned int psize = get_slice_psize(mm, addr);
447 unsigned int shift = mmu_psize_to_shift(psize);
448 unsigned long sz = ((1UL) << shift);
449 struct hstate *hstate = size_to_hstate(sz);
450 pte_update(mm, addr & hstate->mask, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000451 }
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100452 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
David Gibsone28f7fa2005-08-05 19:39:06 +1000453}
454
455pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
456 pte_t *ptep)
457{
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000458 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000459 return __pte(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460}
461
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462struct page *
463follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
464{
465 pte_t *ptep;
466 struct page *page;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700467 unsigned int mmu_psize = get_slice_psize(mm, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700469 /* Verify it is a huge page else bail. */
470 if (!mmu_huge_psizes[mmu_psize])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 return ERR_PTR(-EINVAL);
472
473 ptep = huge_pte_offset(mm, address);
474 page = pte_page(*ptep);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700475 if (page) {
476 unsigned int shift = mmu_psize_to_shift(mmu_psize);
477 unsigned long sz = ((1UL) << shift);
478 page += (address % sz) / PAGE_SIZE;
479 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480
481 return page;
482}
483
484int pmd_huge(pmd_t pmd)
485{
486 return 0;
487}
488
Andi Kleenceb86872008-07-23 21:27:50 -0700489int pud_huge(pud_t pud)
490{
491 return 0;
492}
493
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494struct page *
495follow_huge_pmd(struct mm_struct *mm, unsigned long address,
496 pmd_t *pmd, int write)
497{
498 BUG();
499 return NULL;
500}
501
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
503unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
504 unsigned long len, unsigned long pgoff,
505 unsigned long flags)
506{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700507 struct hstate *hstate = hstate_file(file);
508 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
Brian King48f797d2008-12-04 04:07:54 +0000509
510 if (!mmu_huge_psizes[mmu_psize])
511 return -EINVAL;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700512 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513}
514
Mel Gorman33402892009-01-06 14:38:54 -0800515unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
516{
517 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
518
519 return 1UL << mmu_psize_to_shift(psize);
520}
521
David Gibsoncbf52af2005-12-09 14:20:52 +1100522/*
523 * Called by asm hashtable.S for doing lazy icache flush
524 */
525static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700526 pte_t pte, int trap, unsigned long sz)
David Gibsoncbf52af2005-12-09 14:20:52 +1100527{
528 struct page *page;
529 int i;
530
531 if (!pfn_valid(pte_pfn(pte)))
532 return rflags;
533
534 page = pte_page(pte);
535
536 /* page is dirty */
537 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
538 if (trap == 0x400) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700539 for (i = 0; i < (sz / PAGE_SIZE); i++)
David Gibsoncbf52af2005-12-09 14:20:52 +1100540 __flush_dcache_icache(page_address(page+i));
541 set_bit(PG_arch_1, &page->flags);
542 } else {
543 rflags |= HPTE_R_N;
544 }
545 }
546 return rflags;
547}
548
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549int hash_huge_page(struct mm_struct *mm, unsigned long access,
David Gibsoncbf52af2005-12-09 14:20:52 +1100550 unsigned long ea, unsigned long vsid, int local,
551 unsigned long trap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
553 pte_t *ptep;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100554 unsigned long old_pte, new_pte;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700555 unsigned long va, rflags, pa, sz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 long slot;
557 int err = 1;
Paul Mackerras1189be62007-10-11 20:37:10 +1000558 int ssize = user_segment_size(ea);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700559 unsigned int mmu_psize;
560 int shift;
561 mmu_psize = get_slice_psize(mm, ea);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700563 if (!mmu_huge_psizes[mmu_psize])
564 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 ptep = huge_pte_offset(mm, ea);
566
567 /* Search the Linux page table for a match with va */
Paul Mackerras1189be62007-10-11 20:37:10 +1000568 va = hpt_va(ea, vsid, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569
570 /*
571 * If no pte found or not present, send the problem up to
572 * do_page_fault
573 */
574 if (unlikely(!ptep || pte_none(*ptep)))
575 goto out;
576
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 /*
578 * Check the user's access rights to the page. If access should be
579 * prevented then send the problem up to do_page_fault.
580 */
581 if (unlikely(access & ~pte_val(*ptep)))
582 goto out;
583 /*
584 * At this point, we have a pte (old_pte) which can be used to build
585 * or update an HPTE. There are 2 cases:
586 *
587 * 1. There is a valid (present) pte with no associated HPTE (this is
588 * the most common case)
589 * 2. There is a valid (present) pte with an associated HPTE. The
590 * current values of the pp bits in the HPTE prevent access
591 * because we are doing software DIRTY bit management and the
592 * page is currently not DIRTY.
593 */
594
595
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100596 do {
597 old_pte = pte_val(*ptep);
598 if (old_pte & _PAGE_BUSY)
599 goto out;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000600 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100601 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
602 old_pte, new_pte));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100604 rflags = 0x2 | (!(new_pte & _PAGE_RW));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100606 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700607 shift = mmu_psize_to_shift(mmu_psize);
608 sz = ((1UL) << shift);
David Gibsoncbf52af2005-12-09 14:20:52 +1100609 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
610 /* No CPU has hugepages but lacks no execute, so we
611 * don't need to worry about that case */
612 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700613 trap, sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
615 /* Check if pte already has an hpte (case 2) */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100616 if (unlikely(old_pte & _PAGE_HASHPTE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 /* There MIGHT be an HPTE for this pte */
618 unsigned long hash, slot;
619
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700620 hash = hpt_hash(va, shift, ssize);
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100621 if (old_pte & _PAGE_F_SECOND)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 hash = ~hash;
623 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100624 slot += (old_pte & _PAGE_F_GIX) >> 12;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700626 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
Paul Mackerras1189be62007-10-11 20:37:10 +1000627 ssize, local) == -1)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100628 old_pte &= ~_PAGE_HPTEFLAGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100631 if (likely(!(old_pte & _PAGE_HASHPTE))) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700632 unsigned long hash = hpt_hash(va, shift, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 unsigned long hpte_group;
634
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100635 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636
637repeat:
638 hpte_group = ((hash & htab_hash_mask) *
639 HPTES_PER_GROUP) & ~0x7UL;
640
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100641 /* clear HPTE slot informations in new PTE */
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000642#ifdef CONFIG_PPC_64K_PAGES
643 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
644#else
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100645 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000646#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 /* Add in WIMG bits */
Dave Kleikamp87e9ab12008-06-19 08:32:56 +1000648 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
649 _PAGE_COHERENT | _PAGE_GUARDED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100651 /* Insert into the hash table, primary slot */
652 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700653 mmu_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
655 /* Primary is full, try the secondary */
656 if (unlikely(slot == -1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 hpte_group = ((~hash & htab_hash_mask) *
658 HPTES_PER_GROUP) & ~0x7UL;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100659 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700660 HPTE_V_SECONDARY,
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700661 mmu_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 if (slot == -1) {
663 if (mftb() & 0x1)
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700664 hpte_group = ((hash & htab_hash_mask) *
665 HPTES_PER_GROUP)&~0x7UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
667 ppc_md.hpte_remove(hpte_group);
668 goto repeat;
669 }
670 }
671
672 if (unlikely(slot == -2))
673 panic("hash_huge_page: pte_insert failed\n");
674
Ishizaki Koud649bd72007-01-12 09:54:39 +0900675 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 }
677
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100678 /*
Hugh Dickins01edcd82005-11-23 13:37:39 -0800679 * No need to use ldarx/stdcx here
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100680 */
681 *ptep = __pte(new_pte & ~_PAGE_BUSY);
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 err = 0;
684
685 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 return err;
687}
David Gibsonf10a04c2006-04-28 15:02:51 +1000688
Al Viro4ea8fb92008-11-22 17:33:44 +0000689static void __init set_huge_psize(int psize)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100690{
691 /* Check that it is a page size supported by the hardware and
692 * that it fits within pagetable limits. */
Jon Tollefson91224342008-07-23 21:27:55 -0700693 if (mmu_psize_defs[psize].shift &&
694 mmu_psize_defs[psize].shift < SID_SHIFT_1T &&
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100695 (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT ||
Jon Tollefson91224342008-07-23 21:27:55 -0700696 mmu_psize_defs[psize].shift == PAGE_SHIFT_64K ||
697 mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) {
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700698 /* Return if huge page size has already been setup or is the
699 * same as the base page size. */
700 if (mmu_huge_psizes[psize] ||
701 mmu_psize_defs[psize].shift == PAGE_SHIFT)
Jon Tollefson91224342008-07-23 21:27:55 -0700702 return;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700703 hugetlb_add_hstate(mmu_psize_defs[psize].shift - PAGE_SHIFT);
Jon Tollefson91224342008-07-23 21:27:55 -0700704
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700705 switch (mmu_psize_defs[psize].shift) {
Jon Tollefson91224342008-07-23 21:27:55 -0700706 case PAGE_SHIFT_64K:
707 /* We only allow 64k hpages with 4k base page,
708 * which was checked above, and always put them
709 * at the PMD */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700710 hugepte_shift[psize] = PMD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700711 break;
712 case PAGE_SHIFT_16M:
713 /* 16M pages can be at two different levels
714 * of pagestables based on base page size */
715 if (PAGE_SHIFT == PAGE_SHIFT_64K)
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700716 hugepte_shift[psize] = PMD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700717 else /* 4k base page */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700718 hugepte_shift[psize] = PUD_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700719 break;
720 case PAGE_SHIFT_16G:
721 /* 16G pages are always at PGD level */
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700722 hugepte_shift[psize] = PGDIR_SHIFT;
Jon Tollefson91224342008-07-23 21:27:55 -0700723 break;
724 }
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700725 hugepte_shift[psize] -= mmu_psize_defs[psize].shift;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100726 } else
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700727 hugepte_shift[psize] = 0;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100728}
729
730static int __init hugepage_setup_sz(char *str)
731{
732 unsigned long long size;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700733 int mmu_psize;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100734 int shift;
735
736 size = memparse(str, &str);
737
738 shift = __ffs(size);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700739 mmu_psize = shift_to_mmu_psize(shift);
740 if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100741 set_huge_psize(mmu_psize);
742 else
743 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
744
745 return 1;
746}
747__setup("hugepagesz=", hugepage_setup_sz);
748
David Gibsonf10a04c2006-04-28 15:02:51 +1000749static int __init hugetlbpage_init(void)
750{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700751 unsigned int psize;
752
David Gibsonf10a04c2006-04-28 15:02:51 +1000753 if (!cpu_has_feature(CPU_FTR_16M_PAGE))
754 return -ENODEV;
Benjamin Herrenschmidt00df4382008-07-28 16:13:18 +1000755
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700756 /* Add supported huge page sizes. Need to change HUGE_MAX_HSTATE
757 * and adjust PTE_NONCACHE_NUM if the number of supported huge page
758 * sizes changes.
759 */
760 set_huge_psize(MMU_PAGE_16M);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700761 set_huge_psize(MMU_PAGE_16G);
David Gibsonf10a04c2006-04-28 15:02:51 +1000762
Benjamin Herrenschmidt00df4382008-07-28 16:13:18 +1000763 /* Temporarily disable support for 64K huge pages when 64K SPU local
764 * store support is enabled as the current implementation conflicts.
765 */
766#ifndef CONFIG_SPU_FS_64K_LS
767 set_huge_psize(MMU_PAGE_64K);
768#endif
769
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700770 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
771 if (mmu_huge_psizes[psize]) {
Jon Tollefson7d4320f2008-10-30 12:03:57 +0000772 pgtable_cache[HUGE_PGTABLE_INDEX(psize)] =
773 kmem_cache_create(
774 HUGEPTE_CACHE_NAME(psize),
775 HUGEPTE_TABLE_SIZE(psize),
776 HUGEPTE_TABLE_SIZE(psize),
777 0,
778 NULL);
779 if (!pgtable_cache[HUGE_PGTABLE_INDEX(psize)])
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700780 panic("hugetlbpage_init(): could not create %s"\
781 "\n", HUGEPTE_CACHE_NAME(psize));
782 }
783 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000784
785 return 0;
786}
787
788module_init(hugetlbpage_init);