blob: 0d12fba31bc596d13ceaf58ea8d04c23663082ac [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 *
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
8 */
9
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
14#include <linux/pagemap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/slab.h>
16#include <linux/err.h>
17#include <linux/sysctl.h>
18#include <asm/mman.h>
19#include <asm/pgalloc.h>
20#include <asm/tlb.h>
21#include <asm/tlbflush.h>
22#include <asm/mmu_context.h>
23#include <asm/machdep.h>
24#include <asm/cputable.h>
Benjamin Herrenschmidt94b2a432007-03-10 00:05:37 +010025#include <asm/spu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Jon Tollefson4ec161c2008-01-04 09:59:50 +110027#define HPAGE_SHIFT_64K 16
28#define HPAGE_SHIFT_16M 24
29
David Gibsonc594ada2005-08-11 16:55:21 +100030#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
31#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
32
Jon Tollefson4ec161c2008-01-04 09:59:50 +110033unsigned int hugepte_shift;
34#define PTRS_PER_HUGEPTE (1 << hugepte_shift)
35#define HUGEPTE_TABLE_SIZE (sizeof(pte_t) << hugepte_shift)
David Gibsonf10a04c2006-04-28 15:02:51 +100036
Jon Tollefson4ec161c2008-01-04 09:59:50 +110037#define HUGEPD_SHIFT (HPAGE_SHIFT + hugepte_shift)
David Gibsonf10a04c2006-04-28 15:02:51 +100038#define HUGEPD_SIZE (1UL << HUGEPD_SHIFT)
39#define HUGEPD_MASK (~(HUGEPD_SIZE-1))
40
41#define huge_pgtable_cache (pgtable_cache[HUGEPTE_CACHE_NUM])
42
43/* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad()
44 * will choke on pointers to hugepte tables, which is handy for
45 * catching screwups early. */
46#define HUGEPD_OK 0x1
47
48typedef struct { unsigned long pd; } hugepd_t;
49
50#define hugepd_none(hpd) ((hpd).pd == 0)
51
52static inline pte_t *hugepd_page(hugepd_t hpd)
53{
54 BUG_ON(!(hpd.pd & HUGEPD_OK));
55 return (pte_t *)(hpd.pd & ~HUGEPD_OK);
56}
57
58static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr)
59{
60 unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1));
61 pte_t *dir = hugepd_page(*hpdp);
62
63 return dir + idx;
64}
65
66static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
67 unsigned long address)
68{
69 pte_t *new = kmem_cache_alloc(huge_pgtable_cache,
70 GFP_KERNEL|__GFP_REPEAT);
71
72 if (! new)
73 return -ENOMEM;
74
75 spin_lock(&mm->page_table_lock);
76 if (!hugepd_none(*hpdp))
77 kmem_cache_free(huge_pgtable_cache, new);
78 else
79 hpdp->pd = (unsigned long)new | HUGEPD_OK;
80 spin_unlock(&mm->page_table_lock);
81 return 0;
82}
83
Jon Tollefson4ec161c2008-01-04 09:59:50 +110084/* Base page size affects how we walk hugetlb page tables */
85#ifdef CONFIG_PPC_64K_PAGES
86#define hpmd_offset(pud, addr) pmd_offset(pud, addr)
87#define hpmd_alloc(mm, pud, addr) pmd_alloc(mm, pud, addr)
88#else
89static inline
90pmd_t *hpmd_offset(pud_t *pud, unsigned long addr)
91{
92 if (HPAGE_SHIFT == HPAGE_SHIFT_64K)
93 return pmd_offset(pud, addr);
94 else
95 return (pmd_t *) pud;
96}
97static inline
98pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr)
99{
100 if (HPAGE_SHIFT == HPAGE_SHIFT_64K)
101 return pmd_alloc(mm, pud, addr);
102 else
103 return (pmd_t *) pud;
104}
105#endif
106
David Gibsone28f7fa2005-08-05 19:39:06 +1000107/* Modelled after find_linux_pte() */
108pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109{
David Gibsone28f7fa2005-08-05 19:39:06 +1000110 pgd_t *pg;
111 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100112 pmd_t *pm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000114 BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
David Gibsone28f7fa2005-08-05 19:39:06 +1000116 addr &= HPAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117
David Gibsone28f7fa2005-08-05 19:39:06 +1000118 pg = pgd_offset(mm, addr);
119 if (!pgd_none(*pg)) {
120 pu = pud_offset(pg, addr);
121 if (!pud_none(*pu)) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100122 pm = hpmd_offset(pu, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000123 if (!pmd_none(*pm))
124 return hugepte_offset((hugepd_t *)pm, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 }
126 }
127
David Gibsone28f7fa2005-08-05 19:39:06 +1000128 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129}
130
David Gibson63551ae2005-06-21 17:14:44 -0700131pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132{
David Gibsone28f7fa2005-08-05 19:39:06 +1000133 pgd_t *pg;
134 pud_t *pu;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100135 pmd_t *pm;
David Gibsonf10a04c2006-04-28 15:02:51 +1000136 hugepd_t *hpdp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000138 BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
David Gibsone28f7fa2005-08-05 19:39:06 +1000140 addr &= HPAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
David Gibsone28f7fa2005-08-05 19:39:06 +1000142 pg = pgd_offset(mm, addr);
143 pu = pud_alloc(mm, pg, addr);
144
145 if (pu) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100146 pm = hpmd_alloc(mm, pu, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000147 if (pm)
148 hpdp = (hugepd_t *)pm;
David Gibsone28f7fa2005-08-05 19:39:06 +1000149 }
150
David Gibsonf10a04c2006-04-28 15:02:51 +1000151 if (! hpdp)
152 return NULL;
153
154 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr))
155 return NULL;
156
157 return hugepte_offset(hpdp, addr);
158}
159
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800160int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
161{
162 return 0;
163}
164
David Gibsonf10a04c2006-04-28 15:02:51 +1000165static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp)
166{
167 pte_t *hugepte = hugepd_page(*hpdp);
168
169 hpdp->pd = 0;
170 tlb->need_flush = 1;
171 pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, HUGEPTE_CACHE_NUM,
Adam Litkec9169f82006-08-18 11:22:21 -0700172 PGF_CACHENUM_MASK));
David Gibsonf10a04c2006-04-28 15:02:51 +1000173}
174
David Gibsonf10a04c2006-04-28 15:02:51 +1000175static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
176 unsigned long addr, unsigned long end,
177 unsigned long floor, unsigned long ceiling)
178{
179 pmd_t *pmd;
180 unsigned long next;
181 unsigned long start;
182
183 start = addr;
184 pmd = pmd_offset(pud, addr);
185 do {
186 next = pmd_addr_end(addr, end);
187 if (pmd_none(*pmd))
188 continue;
189 free_hugepte_range(tlb, (hugepd_t *)pmd);
190 } while (pmd++, addr = next, addr != end);
191
192 start &= PUD_MASK;
193 if (start < floor)
194 return;
195 if (ceiling) {
196 ceiling &= PUD_MASK;
197 if (!ceiling)
198 return;
199 }
200 if (end - 1 > ceiling - 1)
201 return;
202
203 pmd = pmd_offset(pud, start);
204 pud_clear(pud);
205 pmd_free_tlb(tlb, pmd);
206}
David Gibsonf10a04c2006-04-28 15:02:51 +1000207
208static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
209 unsigned long addr, unsigned long end,
210 unsigned long floor, unsigned long ceiling)
211{
212 pud_t *pud;
213 unsigned long next;
214 unsigned long start;
215
216 start = addr;
217 pud = pud_offset(pgd, addr);
218 do {
219 next = pud_addr_end(addr, end);
220#ifdef CONFIG_PPC_64K_PAGES
221 if (pud_none_or_clear_bad(pud))
222 continue;
223 hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling);
224#else
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100225 if (HPAGE_SHIFT == HPAGE_SHIFT_64K) {
226 if (pud_none_or_clear_bad(pud))
227 continue;
228 hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling);
229 } else {
230 if (pud_none(*pud))
231 continue;
232 free_hugepte_range(tlb, (hugepd_t *)pud);
233 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000234#endif
235 } while (pud++, addr = next, addr != end);
236
237 start &= PGDIR_MASK;
238 if (start < floor)
239 return;
240 if (ceiling) {
241 ceiling &= PGDIR_MASK;
242 if (!ceiling)
243 return;
244 }
245 if (end - 1 > ceiling - 1)
246 return;
247
248 pud = pud_offset(pgd, start);
249 pgd_clear(pgd);
250 pud_free_tlb(tlb, pud);
251}
252
253/*
254 * This function frees user-level page tables of a process.
255 *
256 * Must be called with pagetable lock held.
257 */
258void hugetlb_free_pgd_range(struct mmu_gather **tlb,
259 unsigned long addr, unsigned long end,
260 unsigned long floor, unsigned long ceiling)
261{
262 pgd_t *pgd;
263 unsigned long next;
264 unsigned long start;
265
266 /*
267 * Comments below take from the normal free_pgd_range(). They
268 * apply here too. The tests against HUGEPD_MASK below are
269 * essential, because we *don't* test for this at the bottom
270 * level. Without them we'll attempt to free a hugepte table
271 * when we unmap just part of it, even if there are other
272 * active mappings using it.
273 *
274 * The next few lines have given us lots of grief...
275 *
276 * Why are we testing HUGEPD* at this top level? Because
277 * often there will be no work to do at all, and we'd prefer
278 * not to go all the way down to the bottom just to discover
279 * that.
280 *
281 * Why all these "- 1"s? Because 0 represents both the bottom
282 * of the address space and the top of it (using -1 for the
283 * top wouldn't help much: the masks would do the wrong thing).
284 * The rule is that addr 0 and floor 0 refer to the bottom of
285 * the address space, but end 0 and ceiling 0 refer to the top
286 * Comparisons need to use "end - 1" and "ceiling - 1" (though
287 * that end 0 case should be mythical).
288 *
289 * Wherever addr is brought up or ceiling brought down, we
290 * must be careful to reject "the opposite 0" before it
291 * confuses the subsequent tests. But what about where end is
292 * brought down by HUGEPD_SIZE below? no, end can't go down to
293 * 0 there.
294 *
295 * Whereas we round start (addr) and ceiling down, by different
296 * masks at different levels, in order to test whether a table
297 * now has no other vmas using it, so can be freed, we don't
298 * bother to round floor or end up - the tests don't need that.
299 */
300
301 addr &= HUGEPD_MASK;
302 if (addr < floor) {
303 addr += HUGEPD_SIZE;
304 if (!addr)
305 return;
306 }
307 if (ceiling) {
308 ceiling &= HUGEPD_MASK;
309 if (!ceiling)
310 return;
311 }
312 if (end - 1 > ceiling - 1)
313 end -= HUGEPD_SIZE;
314 if (addr > end - 1)
315 return;
316
317 start = addr;
318 pgd = pgd_offset((*tlb)->mm, addr);
319 do {
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000320 BUG_ON(get_slice_psize((*tlb)->mm, addr) != mmu_huge_psize);
David Gibsonf10a04c2006-04-28 15:02:51 +1000321 next = pgd_addr_end(addr, end);
322 if (pgd_none_or_clear_bad(pgd))
323 continue;
324 hugetlb_free_pud_range(*tlb, pgd, addr, next, floor, ceiling);
325 } while (pgd++, addr = next, addr != end);
David Gibsone28f7fa2005-08-05 19:39:06 +1000326}
327
David Gibsone28f7fa2005-08-05 19:39:06 +1000328void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
329 pte_t *ptep, pte_t pte)
330{
David Gibsone28f7fa2005-08-05 19:39:06 +1000331 if (pte_present(*ptep)) {
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100332 /* We open-code pte_clear because we need to pass the right
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000333 * argument to hpte_need_flush (huge / !huge). Might not be
334 * necessary anymore if we make hpte_need_flush() get the
335 * page size from the slices
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100336 */
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000337 pte_update(mm, addr & HPAGE_MASK, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000338 }
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100339 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
David Gibsone28f7fa2005-08-05 19:39:06 +1000340}
341
342pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
343 pte_t *ptep)
344{
Benjamin Herrenschmidta741e672007-04-10 17:09:37 +1000345 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
David Gibsone28f7fa2005-08-05 19:39:06 +1000346 return __pte(old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347}
348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349struct page *
350follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
351{
352 pte_t *ptep;
353 struct page *page;
354
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000355 if (get_slice_psize(mm, address) != mmu_huge_psize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 return ERR_PTR(-EINVAL);
357
358 ptep = huge_pte_offset(mm, address);
359 page = pte_page(*ptep);
360 if (page)
361 page += (address % HPAGE_SIZE) / PAGE_SIZE;
362
363 return page;
364}
365
366int pmd_huge(pmd_t pmd)
367{
368 return 0;
369}
370
371struct page *
372follow_huge_pmd(struct mm_struct *mm, unsigned long address,
373 pmd_t *pmd, int write)
374{
375 BUG();
376 return NULL;
377}
378
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379
380unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
381 unsigned long len, unsigned long pgoff,
382 unsigned long flags)
383{
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000384 return slice_get_unmapped_area(addr, len, flags,
385 mmu_huge_psize, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
David Gibsoncbf52af2005-12-09 14:20:52 +1100388/*
389 * Called by asm hashtable.S for doing lazy icache flush
390 */
391static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
392 pte_t pte, int trap)
393{
394 struct page *page;
395 int i;
396
397 if (!pfn_valid(pte_pfn(pte)))
398 return rflags;
399
400 page = pte_page(pte);
401
402 /* page is dirty */
403 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
404 if (trap == 0x400) {
405 for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++)
406 __flush_dcache_icache(page_address(page+i));
407 set_bit(PG_arch_1, &page->flags);
408 } else {
409 rflags |= HPTE_R_N;
410 }
411 }
412 return rflags;
413}
414
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415int hash_huge_page(struct mm_struct *mm, unsigned long access,
David Gibsoncbf52af2005-12-09 14:20:52 +1100416 unsigned long ea, unsigned long vsid, int local,
417 unsigned long trap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418{
419 pte_t *ptep;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100420 unsigned long old_pte, new_pte;
421 unsigned long va, rflags, pa;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 long slot;
423 int err = 1;
Paul Mackerras1189be62007-10-11 20:37:10 +1000424 int ssize = user_segment_size(ea);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 ptep = huge_pte_offset(mm, ea);
427
428 /* Search the Linux page table for a match with va */
Paul Mackerras1189be62007-10-11 20:37:10 +1000429 va = hpt_va(ea, vsid, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430
431 /*
432 * If no pte found or not present, send the problem up to
433 * do_page_fault
434 */
435 if (unlikely(!ptep || pte_none(*ptep)))
436 goto out;
437
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 /*
439 * Check the user's access rights to the page. If access should be
440 * prevented then send the problem up to do_page_fault.
441 */
442 if (unlikely(access & ~pte_val(*ptep)))
443 goto out;
444 /*
445 * At this point, we have a pte (old_pte) which can be used to build
446 * or update an HPTE. There are 2 cases:
447 *
448 * 1. There is a valid (present) pte with no associated HPTE (this is
449 * the most common case)
450 * 2. There is a valid (present) pte with an associated HPTE. The
451 * current values of the pp bits in the HPTE prevent access
452 * because we are doing software DIRTY bit management and the
453 * page is currently not DIRTY.
454 */
455
456
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100457 do {
458 old_pte = pte_val(*ptep);
459 if (old_pte & _PAGE_BUSY)
460 goto out;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000461 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100462 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
463 old_pte, new_pte));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100465 rflags = 0x2 | (!(new_pte & _PAGE_RW));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100467 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
David Gibsoncbf52af2005-12-09 14:20:52 +1100468 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
469 /* No CPU has hugepages but lacks no execute, so we
470 * don't need to worry about that case */
471 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
472 trap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473
474 /* Check if pte already has an hpte (case 2) */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100475 if (unlikely(old_pte & _PAGE_HASHPTE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 /* There MIGHT be an HPTE for this pte */
477 unsigned long hash, slot;
478
Paul Mackerras1189be62007-10-11 20:37:10 +1000479 hash = hpt_hash(va, HPAGE_SHIFT, ssize);
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100480 if (old_pte & _PAGE_F_SECOND)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 hash = ~hash;
482 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100483 slot += (old_pte & _PAGE_F_GIX) >> 12;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484
Benjamin Herrenschmidt325c82a2005-12-08 16:51:44 +1100485 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize,
Paul Mackerras1189be62007-10-11 20:37:10 +1000486 ssize, local) == -1)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100487 old_pte &= ~_PAGE_HPTEFLAGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 }
489
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100490 if (likely(!(old_pte & _PAGE_HASHPTE))) {
Paul Mackerras1189be62007-10-11 20:37:10 +1000491 unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 unsigned long hpte_group;
493
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100494 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495
496repeat:
497 hpte_group = ((hash & htab_hash_mask) *
498 HPTES_PER_GROUP) & ~0x7UL;
499
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100500 /* clear HPTE slot informations in new PTE */
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000501#ifdef CONFIG_PPC_64K_PAGES
502 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
503#else
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100504 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
Benjamin Herrenschmidt41743a42008-06-11 15:37:10 +1000505#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 /* Add in WIMG bits */
Dave Kleikamp87e9ab12008-06-19 08:32:56 +1000507 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
508 _PAGE_COHERENT | _PAGE_GUARDED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100510 /* Insert into the hash table, primary slot */
511 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
Paul Mackerras1189be62007-10-11 20:37:10 +1000512 mmu_huge_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
514 /* Primary is full, try the secondary */
515 if (unlikely(slot == -1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 hpte_group = ((~hash & htab_hash_mask) *
517 HPTES_PER_GROUP) & ~0x7UL;
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100518 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700519 HPTE_V_SECONDARY,
Paul Mackerras1189be62007-10-11 20:37:10 +1000520 mmu_huge_psize, ssize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 if (slot == -1) {
522 if (mftb() & 0x1)
Benjamin Herrenschmidt67b10812005-09-23 13:24:07 -0700523 hpte_group = ((hash & htab_hash_mask) *
524 HPTES_PER_GROUP)&~0x7UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
526 ppc_md.hpte_remove(hpte_group);
527 goto repeat;
528 }
529 }
530
531 if (unlikely(slot == -2))
532 panic("hash_huge_page: pte_insert failed\n");
533
Ishizaki Koud649bd72007-01-12 09:54:39 +0900534 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 }
536
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100537 /*
Hugh Dickins01edcd82005-11-23 13:37:39 -0800538 * No need to use ldarx/stdcx here
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100539 */
540 *ptep = __pte(new_pte & ~_PAGE_BUSY);
541
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 err = 0;
543
544 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 return err;
546}
David Gibsonf10a04c2006-04-28 15:02:51 +1000547
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100548void set_huge_psize(int psize)
549{
550 /* Check that it is a page size supported by the hardware and
551 * that it fits within pagetable limits. */
552 if (mmu_psize_defs[psize].shift && mmu_psize_defs[psize].shift < SID_SHIFT &&
553 (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT ||
554 mmu_psize_defs[psize].shift == HPAGE_SHIFT_64K)) {
555 HPAGE_SHIFT = mmu_psize_defs[psize].shift;
556 mmu_huge_psize = psize;
557#ifdef CONFIG_PPC_64K_PAGES
558 hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT);
559#else
560 if (HPAGE_SHIFT == HPAGE_SHIFT_64K)
561 hugepte_shift = (PMD_SHIFT-HPAGE_SHIFT);
562 else
563 hugepte_shift = (PUD_SHIFT-HPAGE_SHIFT);
564#endif
565
566 } else
567 HPAGE_SHIFT = 0;
568}
569
570static int __init hugepage_setup_sz(char *str)
571{
572 unsigned long long size;
573 int mmu_psize = -1;
574 int shift;
575
576 size = memparse(str, &str);
577
578 shift = __ffs(size);
579 switch (shift) {
580#ifndef CONFIG_PPC_64K_PAGES
581 case HPAGE_SHIFT_64K:
582 mmu_psize = MMU_PAGE_64K;
583 break;
584#endif
585 case HPAGE_SHIFT_16M:
586 mmu_psize = MMU_PAGE_16M;
587 break;
588 }
589
590 if (mmu_psize >=0 && mmu_psize_defs[mmu_psize].shift)
591 set_huge_psize(mmu_psize);
592 else
593 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
594
595 return 1;
596}
597__setup("hugepagesz=", hugepage_setup_sz);
598
Christoph Lameter4ba9b9d2007-10-16 23:25:51 -0700599static void zero_ctor(struct kmem_cache *cache, void *addr)
David Gibsonf10a04c2006-04-28 15:02:51 +1000600{
601 memset(addr, 0, kmem_cache_size(cache));
602}
603
604static int __init hugetlbpage_init(void)
605{
606 if (!cpu_has_feature(CPU_FTR_16M_PAGE))
607 return -ENODEV;
608
609 huge_pgtable_cache = kmem_cache_create("hugepte_cache",
610 HUGEPTE_TABLE_SIZE,
611 HUGEPTE_TABLE_SIZE,
Christoph Lameterf0f39802007-05-06 14:49:58 -0700612 0,
Paul Mundt20c2df82007-07-20 10:11:58 +0900613 zero_ctor);
David Gibsonf10a04c2006-04-28 15:02:51 +1000614 if (! huge_pgtable_cache)
615 panic("hugetlbpage_init(): could not create hugepte cache\n");
616
617 return 0;
618}
619
620module_init(hugetlbpage_init);