blob: e9e6882231daf8fac6eb00c3b4b3e88714224923 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Becky Bruce41151e72011-06-28 09:54:48 +00002 * PPC Huge TLB Page Support for Kernel.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
Becky Bruce41151e72011-06-28 09:54:48 +00005 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/mm.h>
David Gibson883a3e52009-10-26 19:24:31 +000012#include <linux/io.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/hugetlb.h>
Paul Mackerras342d3db2011-12-12 12:38:05 +000015#include <linux/export.h>
Becky Bruce41151e72011-06-28 09:54:48 +000016#include <linux/of_fdt.h>
17#include <linux/memblock.h>
18#include <linux/bootmem.h>
Kumar Gala13020be2011-11-24 09:40:07 +000019#include <linux/moduleparam.h>
David Gibson883a3e52009-10-26 19:24:31 +000020#include <asm/pgtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/pgalloc.h>
22#include <asm/tlb.h>
Becky Bruce41151e72011-06-28 09:54:48 +000023#include <asm/setup.h>
Aneesh Kumar K.V29409992013-06-20 14:30:16 +053024#include <asm/hugetlb.h>
25
26#ifdef CONFIG_HUGETLB_PAGE
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Jon Tollefson91224342008-07-23 21:27:55 -070028#define PAGE_SHIFT_64K 16
29#define PAGE_SHIFT_16M 24
30#define PAGE_SHIFT_16G 34
Jon Tollefson4ec161c2008-01-04 09:59:50 +110031
Becky Bruce41151e72011-06-28 09:54:48 +000032unsigned int HPAGE_SHIFT;
33
34/*
35 * Tracks gpages after the device tree is scanned and before the
Becky Brucea6146882011-10-10 10:50:43 +000036 * huge_boot_pages list is ready. On non-Freescale implementations, this is
37 * just used to track 16G pages and so is a single array. FSL-based
38 * implementations may have more than one gpage size, so we need multiple
39 * arrays
Becky Bruce41151e72011-06-28 09:54:48 +000040 */
Becky Bruce881fde12011-10-10 10:50:40 +000041#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +000042#define MAX_NUMBER_GPAGES 128
43struct psize_gpages {
44 u64 gpage_list[MAX_NUMBER_GPAGES];
45 unsigned int nr_gpages;
46};
47static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
Becky Bruce881fde12011-10-10 10:50:40 +000048#else
49#define MAX_NUMBER_GPAGES 1024
50static u64 gpage_freearray[MAX_NUMBER_GPAGES];
51static unsigned nr_gpages;
Becky Bruce41151e72011-06-28 09:54:48 +000052#endif
David Gibsonf10a04c2006-04-28 15:02:51 +100053
David Gibsona4fe3ce2009-10-26 19:24:31 +000054#define hugepd_none(hpd) ((hpd).pd == 0)
55
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +000056#ifdef CONFIG_PPC_BOOK3S_64
57/*
58 * At this point we do the placement change only for BOOK3S 64. This would
59 * possibly work on other subarchs.
60 */
61
62/*
63 * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
64 * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
65 */
66int pmd_huge(pmd_t pmd)
67{
68 /*
69 * leaf pte for huge page, bottom two bits != 00
70 */
71 return ((pmd_val(pmd) & 0x3) != 0x0);
72}
73
74int pud_huge(pud_t pud)
75{
76 /*
77 * leaf pte for huge page, bottom two bits != 00
78 */
79 return ((pud_val(pud) & 0x3) != 0x0);
80}
81
82int pgd_huge(pgd_t pgd)
83{
84 /*
85 * leaf pte for huge page, bottom two bits != 00
86 */
87 return ((pgd_val(pgd) & 0x3) != 0x0);
88}
89#else
90int pmd_huge(pmd_t pmd)
91{
92 return 0;
93}
94
95int pud_huge(pud_t pud)
96{
97 return 0;
98}
99
100int pgd_huge(pgd_t pgd)
101{
102 return 0;
103}
104#endif
105
David Gibsona4fe3ce2009-10-26 19:24:31 +0000106pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
107{
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +0530108 /* Only called for hugetlbfs pages, hence can ignore THP */
David Gibsona4fe3ce2009-10-26 19:24:31 +0000109 return find_linux_pte_or_hugepte(mm->pgd, addr, NULL);
110}
111
112static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
113 unsigned long address, unsigned pdshift, unsigned pshift)
114{
Becky Bruce41151e72011-06-28 09:54:48 +0000115 struct kmem_cache *cachep;
116 pte_t *new;
117
Becky Bruce881fde12011-10-10 10:50:40 +0000118#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000119 int i;
120 int num_hugepd = 1 << (pshift - pdshift);
121 cachep = hugepte_cache;
Becky Bruce881fde12011-10-10 10:50:40 +0000122#else
123 cachep = PGT_CACHE(pdshift - pshift);
Becky Bruce41151e72011-06-28 09:54:48 +0000124#endif
125
126 new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT);
David Gibsonf10a04c2006-04-28 15:02:51 +1000127
David Gibsona4fe3ce2009-10-26 19:24:31 +0000128 BUG_ON(pshift > HUGEPD_SHIFT_MASK);
129 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
130
David Gibsonf10a04c2006-04-28 15:02:51 +1000131 if (! new)
132 return -ENOMEM;
133
134 spin_lock(&mm->page_table_lock);
Becky Bruce881fde12011-10-10 10:50:40 +0000135#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000136 /*
137 * We have multiple higher-level entries that point to the same
138 * actual pte location. Fill in each as we go and backtrack on error.
139 * We need all of these so the DTLB pgtable walk code can find the
140 * right higher-level entry without knowing if it's a hugepage or not.
141 */
142 for (i = 0; i < num_hugepd; i++, hpdp++) {
143 if (unlikely(!hugepd_none(*hpdp)))
144 break;
145 else
Aneesh Kumar K.Vcf9427b2013-04-28 09:37:29 +0000146 /* We use the old format for PPC_FSL_BOOK3E */
Becky Bruce41151e72011-06-28 09:54:48 +0000147 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
148 }
149 /* If we bailed from the for loop early, an error occurred, clean up */
150 if (i < num_hugepd) {
151 for (i = i - 1 ; i >= 0; i--, hpdp--)
152 hpdp->pd = 0;
153 kmem_cache_free(cachep, new);
154 }
Becky Brucea1cd5412011-10-10 10:50:39 +0000155#else
156 if (!hugepd_none(*hpdp))
157 kmem_cache_free(cachep, new);
Aneesh Kumar K.Vcf9427b2013-04-28 09:37:29 +0000158 else {
159#ifdef CONFIG_PPC_BOOK3S_64
160 hpdp->pd = (unsigned long)new |
161 (shift_to_mmu_psize(pshift) << 2);
162#else
Becky Brucea1cd5412011-10-10 10:50:39 +0000163 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
Becky Bruce41151e72011-06-28 09:54:48 +0000164#endif
Aneesh Kumar K.Vcf9427b2013-04-28 09:37:29 +0000165 }
166#endif
David Gibsonf10a04c2006-04-28 15:02:51 +1000167 spin_unlock(&mm->page_table_lock);
168 return 0;
169}
170
Becky Brucea1cd5412011-10-10 10:50:39 +0000171/*
172 * These macros define how to determine which level of the page table holds
173 * the hpdp.
174 */
175#ifdef CONFIG_PPC_FSL_BOOK3E
176#define HUGEPD_PGD_SHIFT PGDIR_SHIFT
177#define HUGEPD_PUD_SHIFT PUD_SHIFT
178#else
179#define HUGEPD_PGD_SHIFT PUD_SHIFT
180#define HUGEPD_PUD_SHIFT PMD_SHIFT
181#endif
182
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000183#ifdef CONFIG_PPC_BOOK3S_64
184/*
185 * At this point we do the placement change only for BOOK3S 64. This would
186 * possibly work on other subarchs.
187 */
188pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
189{
190 pgd_t *pg;
191 pud_t *pu;
192 pmd_t *pm;
193 hugepd_t *hpdp = NULL;
194 unsigned pshift = __ffs(sz);
195 unsigned pdshift = PGDIR_SHIFT;
196
197 addr &= ~(sz-1);
198 pg = pgd_offset(mm, addr);
199
200 if (pshift == PGDIR_SHIFT)
201 /* 16GB huge page */
202 return (pte_t *) pg;
203 else if (pshift > PUD_SHIFT)
204 /*
205 * We need to use hugepd table
206 */
207 hpdp = (hugepd_t *)pg;
208 else {
209 pdshift = PUD_SHIFT;
210 pu = pud_alloc(mm, pg, addr);
211 if (pshift == PUD_SHIFT)
212 return (pte_t *)pu;
213 else if (pshift > PMD_SHIFT)
214 hpdp = (hugepd_t *)pu;
215 else {
216 pdshift = PMD_SHIFT;
217 pm = pmd_alloc(mm, pu, addr);
218 if (pshift == PMD_SHIFT)
219 /* 16MB hugepage */
220 return (pte_t *)pm;
221 else
222 hpdp = (hugepd_t *)pm;
223 }
224 }
225 if (!hpdp)
226 return NULL;
227
228 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
229
230 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
231 return NULL;
232
233 return hugepte_offset(hpdp, addr, pdshift);
234}
235
236#else
237
David Gibsona4fe3ce2009-10-26 19:24:31 +0000238pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
239{
240 pgd_t *pg;
241 pud_t *pu;
242 pmd_t *pm;
243 hugepd_t *hpdp = NULL;
244 unsigned pshift = __ffs(sz);
245 unsigned pdshift = PGDIR_SHIFT;
David Gibson0b264252008-09-05 11:49:54 +1000246
David Gibsona4fe3ce2009-10-26 19:24:31 +0000247 addr &= ~(sz-1);
248
249 pg = pgd_offset(mm, addr);
Becky Brucea1cd5412011-10-10 10:50:39 +0000250
251 if (pshift >= HUGEPD_PGD_SHIFT) {
David Gibsona4fe3ce2009-10-26 19:24:31 +0000252 hpdp = (hugepd_t *)pg;
253 } else {
254 pdshift = PUD_SHIFT;
255 pu = pud_alloc(mm, pg, addr);
Becky Brucea1cd5412011-10-10 10:50:39 +0000256 if (pshift >= HUGEPD_PUD_SHIFT) {
David Gibsona4fe3ce2009-10-26 19:24:31 +0000257 hpdp = (hugepd_t *)pu;
258 } else {
259 pdshift = PMD_SHIFT;
260 pm = pmd_alloc(mm, pu, addr);
261 hpdp = (hugepd_t *)pm;
262 }
263 }
264
265 if (!hpdp)
266 return NULL;
267
268 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
269
270 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
271 return NULL;
272
273 return hugepte_offset(hpdp, addr, pdshift);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100274}
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000275#endif
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100276
Becky Bruce881fde12011-10-10 10:50:40 +0000277#ifdef CONFIG_PPC_FSL_BOOK3E
Jon Tollefson658013e2008-07-23 21:27:54 -0700278/* Build list of addresses of gigantic pages. This function is used in early
279 * boot before the buddy or bootmem allocator is setup.
280 */
Becky Bruce41151e72011-06-28 09:54:48 +0000281void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
282{
283 unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
284 int i;
285
286 if (addr == 0)
287 return;
288
289 gpage_freearray[idx].nr_gpages = number_of_pages;
290
291 for (i = 0; i < number_of_pages; i++) {
292 gpage_freearray[idx].gpage_list[i] = addr;
293 addr += page_size;
294 }
295}
296
297/*
298 * Moves the gigantic page addresses from the temporary list to the
299 * huge_boot_pages list.
300 */
301int alloc_bootmem_huge_page(struct hstate *hstate)
302{
303 struct huge_bootmem_page *m;
304 int idx = shift_to_mmu_psize(hstate->order + PAGE_SHIFT);
305 int nr_gpages = gpage_freearray[idx].nr_gpages;
306
307 if (nr_gpages == 0)
308 return 0;
309
310#ifdef CONFIG_HIGHMEM
311 /*
312 * If gpages can be in highmem we can't use the trick of storing the
313 * data structure in the page; allocate space for this
314 */
315 m = alloc_bootmem(sizeof(struct huge_bootmem_page));
316 m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
317#else
318 m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
319#endif
320
321 list_add(&m->list, &huge_boot_pages);
322 gpage_freearray[idx].nr_gpages = nr_gpages;
323 gpage_freearray[idx].gpage_list[nr_gpages] = 0;
324 m->hstate = hstate;
325
326 return 1;
327}
328/*
329 * Scan the command line hugepagesz= options for gigantic pages; store those in
330 * a list that we use to allocate the memory once all options are parsed.
331 */
332
333unsigned long gpage_npages[MMU_PAGE_COUNT];
334
Paul Gortmaker89528122012-05-07 10:32:22 -0400335static int __init do_gpage_early_setup(char *param, char *val,
336 const char *unused)
Becky Bruce41151e72011-06-28 09:54:48 +0000337{
338 static phys_addr_t size;
339 unsigned long npages;
340
341 /*
342 * The hugepagesz and hugepages cmdline options are interleaved. We
343 * use the size variable to keep track of whether or not this was done
344 * properly and skip over instances where it is incorrect. Other
345 * command-line parsing code will issue warnings, so we don't need to.
346 *
347 */
348 if ((strcmp(param, "default_hugepagesz") == 0) ||
349 (strcmp(param, "hugepagesz") == 0)) {
350 size = memparse(val, NULL);
351 } else if (strcmp(param, "hugepages") == 0) {
352 if (size != 0) {
353 if (sscanf(val, "%lu", &npages) <= 0)
354 npages = 0;
355 gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
356 size = 0;
357 }
358 }
359 return 0;
360}
361
362
363/*
364 * This function allocates physical space for pages that are larger than the
365 * buddy allocator can handle. We want to allocate these in highmem because
366 * the amount of lowmem is limited. This means that this function MUST be
367 * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
368 * allocate to grab highmem.
369 */
370void __init reserve_hugetlb_gpages(void)
371{
372 static __initdata char cmdline[COMMAND_LINE_SIZE];
373 phys_addr_t size, base;
374 int i;
375
376 strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
Pawel Moll026cee02012-03-26 12:50:51 +1030377 parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
378 &do_gpage_early_setup);
Becky Bruce41151e72011-06-28 09:54:48 +0000379
380 /*
381 * Walk gpage list in reverse, allocating larger page sizes first.
382 * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
383 * When we reach the point in the list where pages are no longer
384 * considered gpages, we're done.
385 */
386 for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
387 if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
388 continue;
389 else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
390 break;
391
392 size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
393 base = memblock_alloc_base(size * gpage_npages[i], size,
394 MEMBLOCK_ALLOC_ANYWHERE);
395 add_gpage(base, size, gpage_npages[i]);
396 }
397}
398
Becky Bruce881fde12011-10-10 10:50:40 +0000399#else /* !PPC_FSL_BOOK3E */
Becky Bruce41151e72011-06-28 09:54:48 +0000400
401/* Build list of addresses of gigantic pages. This function is used in early
402 * boot before the buddy or bootmem allocator is setup.
403 */
404void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
Jon Tollefson658013e2008-07-23 21:27:54 -0700405{
406 if (!addr)
407 return;
408 while (number_of_pages > 0) {
409 gpage_freearray[nr_gpages] = addr;
410 nr_gpages++;
411 number_of_pages--;
412 addr += page_size;
413 }
414}
415
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700416/* Moves the gigantic page addresses from the temporary list to the
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700417 * huge_boot_pages list.
418 */
419int alloc_bootmem_huge_page(struct hstate *hstate)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700420{
421 struct huge_bootmem_page *m;
422 if (nr_gpages == 0)
423 return 0;
424 m = phys_to_virt(gpage_freearray[--nr_gpages]);
425 gpage_freearray[nr_gpages] = 0;
426 list_add(&m->list, &huge_boot_pages);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700427 m->hstate = hstate;
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700428 return 1;
429}
Becky Bruce41151e72011-06-28 09:54:48 +0000430#endif
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700431
Chen, Kenneth W39dde652006-12-06 20:32:03 -0800432int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
433{
434 return 0;
435}
436
Becky Bruce881fde12011-10-10 10:50:40 +0000437#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000438#define HUGEPD_FREELIST_SIZE \
439 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
440
441struct hugepd_freelist {
442 struct rcu_head rcu;
443 unsigned int index;
444 void *ptes[0];
445};
446
447static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
448
449static void hugepd_free_rcu_callback(struct rcu_head *head)
450{
451 struct hugepd_freelist *batch =
452 container_of(head, struct hugepd_freelist, rcu);
453 unsigned int i;
454
455 for (i = 0; i < batch->index; i++)
456 kmem_cache_free(hugepte_cache, batch->ptes[i]);
457
458 free_page((unsigned long)batch);
459}
460
461static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
462{
463 struct hugepd_freelist **batchp;
464
465 batchp = &__get_cpu_var(hugepd_freelist_cur);
466
467 if (atomic_read(&tlb->mm->mm_users) < 2 ||
468 cpumask_equal(mm_cpumask(tlb->mm),
469 cpumask_of(smp_processor_id()))) {
470 kmem_cache_free(hugepte_cache, hugepte);
471 return;
472 }
473
474 if (*batchp == NULL) {
475 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
476 (*batchp)->index = 0;
477 }
478
479 (*batchp)->ptes[(*batchp)->index++] = hugepte;
480 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
481 call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
482 *batchp = NULL;
483 }
484}
485#endif
486
David Gibsona4fe3ce2009-10-26 19:24:31 +0000487static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
488 unsigned long start, unsigned long end,
489 unsigned long floor, unsigned long ceiling)
David Gibsonf10a04c2006-04-28 15:02:51 +1000490{
491 pte_t *hugepte = hugepd_page(*hpdp);
Becky Bruce41151e72011-06-28 09:54:48 +0000492 int i;
493
David Gibsona4fe3ce2009-10-26 19:24:31 +0000494 unsigned long pdmask = ~((1UL << pdshift) - 1);
Becky Bruce41151e72011-06-28 09:54:48 +0000495 unsigned int num_hugepd = 1;
496
Becky Bruce881fde12011-10-10 10:50:40 +0000497#ifdef CONFIG_PPC_FSL_BOOK3E
498 /* Note: On fsl the hpdp may be the first of several */
Becky Bruce41151e72011-06-28 09:54:48 +0000499 num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
Becky Bruce881fde12011-10-10 10:50:40 +0000500#else
501 unsigned int shift = hugepd_shift(*hpdp);
Becky Bruce41151e72011-06-28 09:54:48 +0000502#endif
David Gibsona4fe3ce2009-10-26 19:24:31 +0000503
504 start &= pdmask;
505 if (start < floor)
506 return;
507 if (ceiling) {
508 ceiling &= pdmask;
509 if (! ceiling)
510 return;
511 }
512 if (end - 1 > ceiling - 1)
513 return;
David Gibsonf10a04c2006-04-28 15:02:51 +1000514
Becky Bruce41151e72011-06-28 09:54:48 +0000515 for (i = 0; i < num_hugepd; i++, hpdp++)
516 hpdp->pd = 0;
517
David Gibsonf10a04c2006-04-28 15:02:51 +1000518 tlb->need_flush = 1;
Becky Bruce881fde12011-10-10 10:50:40 +0000519
520#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000521 hugepd_free(tlb, hugepte);
Becky Bruce881fde12011-10-10 10:50:40 +0000522#else
523 pgtable_free_tlb(tlb, hugepte, pdshift - shift);
Becky Bruce41151e72011-06-28 09:54:48 +0000524#endif
David Gibsonf10a04c2006-04-28 15:02:51 +1000525}
526
David Gibsonf10a04c2006-04-28 15:02:51 +1000527static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
528 unsigned long addr, unsigned long end,
David Gibsona4fe3ce2009-10-26 19:24:31 +0000529 unsigned long floor, unsigned long ceiling)
David Gibsonf10a04c2006-04-28 15:02:51 +1000530{
531 pmd_t *pmd;
532 unsigned long next;
533 unsigned long start;
534
535 start = addr;
David Gibsonf10a04c2006-04-28 15:02:51 +1000536 do {
Becky Brucea1cd5412011-10-10 10:50:39 +0000537 pmd = pmd_offset(pud, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000538 next = pmd_addr_end(addr, end);
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000539 if (pmd_none_or_clear_bad(pmd))
David Gibsonf10a04c2006-04-28 15:02:51 +1000540 continue;
Becky Brucea1cd5412011-10-10 10:50:39 +0000541#ifdef CONFIG_PPC_FSL_BOOK3E
542 /*
543 * Increment next by the size of the huge mapping since
544 * there may be more than one entry at this level for a
545 * single hugepage, but all of them point to
546 * the same kmem cache that holds the hugepte.
547 */
548 next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
549#endif
David Gibsona4fe3ce2009-10-26 19:24:31 +0000550 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
551 addr, next, floor, ceiling);
Becky Brucea1cd5412011-10-10 10:50:39 +0000552 } while (addr = next, addr != end);
David Gibsonf10a04c2006-04-28 15:02:51 +1000553
554 start &= PUD_MASK;
555 if (start < floor)
556 return;
557 if (ceiling) {
558 ceiling &= PUD_MASK;
559 if (!ceiling)
560 return;
561 }
562 if (end - 1 > ceiling - 1)
563 return;
564
565 pmd = pmd_offset(pud, start);
566 pud_clear(pud);
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +1000567 pmd_free_tlb(tlb, pmd, start);
David Gibsonf10a04c2006-04-28 15:02:51 +1000568}
David Gibsonf10a04c2006-04-28 15:02:51 +1000569
570static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
571 unsigned long addr, unsigned long end,
572 unsigned long floor, unsigned long ceiling)
573{
574 pud_t *pud;
575 unsigned long next;
576 unsigned long start;
577
578 start = addr;
David Gibsonf10a04c2006-04-28 15:02:51 +1000579 do {
Becky Brucea1cd5412011-10-10 10:50:39 +0000580 pud = pud_offset(pgd, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000581 next = pud_addr_end(addr, end);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000582 if (!is_hugepd(pud)) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100583 if (pud_none_or_clear_bad(pud))
584 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700585 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
David Gibsona4fe3ce2009-10-26 19:24:31 +0000586 ceiling);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100587 } else {
Becky Brucea1cd5412011-10-10 10:50:39 +0000588#ifdef CONFIG_PPC_FSL_BOOK3E
589 /*
590 * Increment next by the size of the huge mapping since
591 * there may be more than one entry at this level for a
592 * single hugepage, but all of them point to
593 * the same kmem cache that holds the hugepte.
594 */
595 next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
596#endif
David Gibsona4fe3ce2009-10-26 19:24:31 +0000597 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
598 addr, next, floor, ceiling);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100599 }
Becky Brucea1cd5412011-10-10 10:50:39 +0000600 } while (addr = next, addr != end);
David Gibsonf10a04c2006-04-28 15:02:51 +1000601
602 start &= PGDIR_MASK;
603 if (start < floor)
604 return;
605 if (ceiling) {
606 ceiling &= PGDIR_MASK;
607 if (!ceiling)
608 return;
609 }
610 if (end - 1 > ceiling - 1)
611 return;
612
613 pud = pud_offset(pgd, start);
614 pgd_clear(pgd);
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +1000615 pud_free_tlb(tlb, pud, start);
David Gibsonf10a04c2006-04-28 15:02:51 +1000616}
617
618/*
619 * This function frees user-level page tables of a process.
620 *
621 * Must be called with pagetable lock held.
622 */
Jan Beulich42b77722008-07-23 21:27:10 -0700623void hugetlb_free_pgd_range(struct mmu_gather *tlb,
David Gibsonf10a04c2006-04-28 15:02:51 +1000624 unsigned long addr, unsigned long end,
625 unsigned long floor, unsigned long ceiling)
626{
627 pgd_t *pgd;
628 unsigned long next;
David Gibsonf10a04c2006-04-28 15:02:51 +1000629
630 /*
David Gibsona4fe3ce2009-10-26 19:24:31 +0000631 * Because there are a number of different possible pagetable
632 * layouts for hugepage ranges, we limit knowledge of how
633 * things should be laid out to the allocation path
634 * (huge_pte_alloc(), above). Everything else works out the
635 * structure as it goes from information in the hugepd
636 * pointers. That means that we can't here use the
637 * optimization used in the normal page free_pgd_range(), of
638 * checking whether we're actually covering a large enough
639 * range to have to do anything at the top level of the walk
640 * instead of at the bottom.
David Gibsonf10a04c2006-04-28 15:02:51 +1000641 *
David Gibsona4fe3ce2009-10-26 19:24:31 +0000642 * To make sense of this, you should probably go read the big
643 * block comment at the top of the normal free_pgd_range(),
644 * too.
David Gibsonf10a04c2006-04-28 15:02:51 +1000645 */
646
David Gibsonf10a04c2006-04-28 15:02:51 +1000647 do {
David Gibsonf10a04c2006-04-28 15:02:51 +1000648 next = pgd_addr_end(addr, end);
Becky Bruce41151e72011-06-28 09:54:48 +0000649 pgd = pgd_offset(tlb->mm, addr);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000650 if (!is_hugepd(pgd)) {
David Gibson0b264252008-09-05 11:49:54 +1000651 if (pgd_none_or_clear_bad(pgd))
652 continue;
653 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
654 } else {
Becky Bruce881fde12011-10-10 10:50:40 +0000655#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000656 /*
657 * Increment next by the size of the huge mapping since
Becky Bruce881fde12011-10-10 10:50:40 +0000658 * there may be more than one entry at the pgd level
659 * for a single hugepage, but all of them point to the
660 * same kmem cache that holds the hugepte.
Becky Bruce41151e72011-06-28 09:54:48 +0000661 */
662 next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
663#endif
David Gibsona4fe3ce2009-10-26 19:24:31 +0000664 free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
665 addr, next, floor, ceiling);
David Gibson0b264252008-09-05 11:49:54 +1000666 }
Becky Bruce41151e72011-06-28 09:54:48 +0000667 } while (addr = next, addr != end);
David Gibsone28f7fa2005-08-05 19:39:06 +1000668}
669
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670struct page *
671follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
672{
673 pte_t *ptep;
674 struct page *page;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000675 unsigned shift;
676 unsigned long mask;
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +0530677 /*
678 * Transparent hugepages are handled by generic code. We can skip them
679 * here.
680 */
David Gibsona4fe3ce2009-10-26 19:24:31 +0000681 ptep = find_linux_pte_or_hugepte(mm->pgd, address, &shift);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700683 /* Verify it is a huge page else bail. */
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +0530684 if (!ptep || !shift || pmd_trans_huge(*(pmd_t *)ptep))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 return ERR_PTR(-EINVAL);
686
David Gibsona4fe3ce2009-10-26 19:24:31 +0000687 mask = (1UL << shift) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 page = pte_page(*ptep);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000689 if (page)
690 page += (address & mask) / PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
692 return page;
693}
694
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695struct page *
696follow_huge_pmd(struct mm_struct *mm, unsigned long address,
697 pmd_t *pmd, int write)
698{
699 BUG();
700 return NULL;
701}
702
David Gibson39adfa52009-11-23 20:03:40 +0000703static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
704 unsigned long sz)
705{
706 unsigned long __boundary = (addr + sz) & ~(sz-1);
707 return (__boundary - 1 < end - 1) ? __boundary : end;
708}
709
David Gibsona4fe3ce2009-10-26 19:24:31 +0000710int gup_hugepd(hugepd_t *hugepd, unsigned pdshift,
711 unsigned long addr, unsigned long end,
712 int write, struct page **pages, int *nr)
713{
714 pte_t *ptep;
715 unsigned long sz = 1UL << hugepd_shift(*hugepd);
David Gibson39adfa52009-11-23 20:03:40 +0000716 unsigned long next;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000717
718 ptep = hugepte_offset(hugepd, addr, pdshift);
719 do {
David Gibson39adfa52009-11-23 20:03:40 +0000720 next = hugepte_addr_end(addr, end, sz);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000721 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
722 return 0;
David Gibson39adfa52009-11-23 20:03:40 +0000723 } while (ptep++, addr = next, addr != end);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000724
725 return 1;
726}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Becky Bruce76512952011-10-10 10:50:36 +0000728#ifdef CONFIG_PPC_MM_SLICES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
730 unsigned long len, unsigned long pgoff,
731 unsigned long flags)
732{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700733 struct hstate *hstate = hstate_file(file);
734 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
Brian King48f797d2008-12-04 04:07:54 +0000735
Michel Lespinasse34d07172013-04-29 11:53:52 -0700736 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737}
Becky Bruce76512952011-10-10 10:50:36 +0000738#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Mel Gorman33402892009-01-06 14:38:54 -0800740unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
741{
Paul Mackerras25c29f92011-09-20 19:58:10 +0000742#ifdef CONFIG_PPC_MM_SLICES
Mel Gorman33402892009-01-06 14:38:54 -0800743 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
744
745 return 1UL << mmu_psize_to_shift(psize);
Becky Bruce41151e72011-06-28 09:54:48 +0000746#else
747 if (!is_vm_hugetlb_page(vma))
748 return PAGE_SIZE;
749
750 return huge_page_size(hstate_vma(vma));
751#endif
752}
753
754static inline bool is_power_of_4(unsigned long x)
755{
756 if (is_power_of_2(x))
757 return (__ilog2(x) % 2) ? false : true;
758 return false;
Mel Gorman33402892009-01-06 14:38:54 -0800759}
760
David Gibsond1837cb2009-10-26 19:24:31 +0000761static int __init add_huge_page_size(unsigned long long size)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100762{
David Gibsond1837cb2009-10-26 19:24:31 +0000763 int shift = __ffs(size);
764 int mmu_psize;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000765
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100766 /* Check that it is a page size supported by the hardware and
David Gibsond1837cb2009-10-26 19:24:31 +0000767 * that it fits within pagetable and slice limits. */
Becky Bruce41151e72011-06-28 09:54:48 +0000768#ifdef CONFIG_PPC_FSL_BOOK3E
769 if ((size < PAGE_SIZE) || !is_power_of_4(size))
770 return -EINVAL;
771#else
David Gibsond1837cb2009-10-26 19:24:31 +0000772 if (!is_power_of_2(size)
773 || (shift > SLICE_HIGH_SHIFT) || (shift <= PAGE_SHIFT))
774 return -EINVAL;
Becky Bruce41151e72011-06-28 09:54:48 +0000775#endif
Jon Tollefson91224342008-07-23 21:27:55 -0700776
David Gibsond1837cb2009-10-26 19:24:31 +0000777 if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
778 return -EINVAL;
779
780#ifdef CONFIG_SPU_FS_64K_LS
781 /* Disable support for 64K huge pages when 64K SPU local store
782 * support is enabled as the current implementation conflicts.
783 */
784 if (shift == PAGE_SHIFT_64K)
785 return -EINVAL;
786#endif /* CONFIG_SPU_FS_64K_LS */
787
788 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
789
790 /* Return if huge page size has already been setup */
791 if (size_to_hstate(size))
792 return 0;
793
794 hugetlb_add_hstate(shift - PAGE_SHIFT);
795
796 return 0;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100797}
798
799static int __init hugepage_setup_sz(char *str)
800{
801 unsigned long long size;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100802
803 size = memparse(str, &str);
804
David Gibsond1837cb2009-10-26 19:24:31 +0000805 if (add_huge_page_size(size) != 0)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100806 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
807
808 return 1;
809}
810__setup("hugepagesz=", hugepage_setup_sz);
811
Becky Bruce881fde12011-10-10 10:50:40 +0000812#ifdef CONFIG_PPC_FSL_BOOK3E
Becky Bruce41151e72011-06-28 09:54:48 +0000813struct kmem_cache *hugepte_cache;
814static int __init hugetlbpage_init(void)
815{
816 int psize;
817
818 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
819 unsigned shift;
820
821 if (!mmu_psize_defs[psize].shift)
822 continue;
823
824 shift = mmu_psize_to_shift(psize);
825
826 /* Don't treat normal page sizes as huge... */
827 if (shift != PAGE_SHIFT)
828 if (add_huge_page_size(1ULL << shift) < 0)
829 continue;
830 }
831
832 /*
833 * Create a kmem cache for hugeptes. The bottom bits in the pte have
834 * size information encoded in them, so align them to allow this
835 */
836 hugepte_cache = kmem_cache_create("hugepte-cache", sizeof(pte_t),
837 HUGEPD_SHIFT_MASK + 1, 0, NULL);
838 if (hugepte_cache == NULL)
839 panic("%s: Unable to create kmem cache for hugeptes\n",
840 __func__);
841
842 /* Default hpage size = 4M */
843 if (mmu_psize_defs[MMU_PAGE_4M].shift)
844 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
845 else
846 panic("%s: Unable to set default huge page size\n", __func__);
847
848
849 return 0;
850}
851#else
David Gibsonf10a04c2006-04-28 15:02:51 +1000852static int __init hugetlbpage_init(void)
853{
David Gibsona4fe3ce2009-10-26 19:24:31 +0000854 int psize;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700855
Matt Evans44ae3ab2011-04-06 19:48:50 +0000856 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
David Gibsonf10a04c2006-04-28 15:02:51 +1000857 return -ENODEV;
Benjamin Herrenschmidt00df4382008-07-28 16:13:18 +1000858
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700859 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
David Gibsond1837cb2009-10-26 19:24:31 +0000860 unsigned shift;
861 unsigned pdshift;
862
863 if (!mmu_psize_defs[psize].shift)
864 continue;
865
866 shift = mmu_psize_to_shift(psize);
867
868 if (add_huge_page_size(1ULL << shift) < 0)
869 continue;
870
871 if (shift < PMD_SHIFT)
872 pdshift = PMD_SHIFT;
873 else if (shift < PUD_SHIFT)
874 pdshift = PUD_SHIFT;
875 else
876 pdshift = PGDIR_SHIFT;
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000877 /*
878 * if we have pdshift and shift value same, we don't
879 * use pgt cache for hugepd.
880 */
881 if (pdshift != shift) {
882 pgtable_cache_add(pdshift - shift, NULL);
883 if (!PGT_CACHE(pdshift - shift))
884 panic("hugetlbpage_init(): could not create "
885 "pgtable cache for %d bit pagesize\n", shift);
886 }
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700887 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000888
David Gibsond1837cb2009-10-26 19:24:31 +0000889 /* Set default large page size. Currently, we pick 16M or 1M
890 * depending on what is available
891 */
892 if (mmu_psize_defs[MMU_PAGE_16M].shift)
893 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
894 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
895 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
896
David Gibsonf10a04c2006-04-28 15:02:51 +1000897 return 0;
898}
Becky Bruce41151e72011-06-28 09:54:48 +0000899#endif
David Gibsonf10a04c2006-04-28 15:02:51 +1000900module_init(hugetlbpage_init);
David Gibson0895ecd2009-10-26 19:24:31 +0000901
902void flush_dcache_icache_hugepage(struct page *page)
903{
904 int i;
Becky Bruce41151e72011-06-28 09:54:48 +0000905 void *start;
David Gibson0895ecd2009-10-26 19:24:31 +0000906
907 BUG_ON(!PageCompound(page));
908
Becky Bruce41151e72011-06-28 09:54:48 +0000909 for (i = 0; i < (1UL << compound_order(page)); i++) {
910 if (!PageHighMem(page)) {
911 __flush_dcache_icache(page_address(page+i));
912 } else {
Cong Wang2480b202011-11-25 23:14:16 +0800913 start = kmap_atomic(page+i);
Becky Bruce41151e72011-06-28 09:54:48 +0000914 __flush_dcache_icache(start);
Cong Wang2480b202011-11-25 23:14:16 +0800915 kunmap_atomic(start);
Becky Bruce41151e72011-06-28 09:54:48 +0000916 }
917 }
David Gibson0895ecd2009-10-26 19:24:31 +0000918}
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530919
920#endif /* CONFIG_HUGETLB_PAGE */
921
922/*
923 * We have 4 cases for pgds and pmds:
924 * (1) invalid (all zeroes)
925 * (2) pointer to next table, as normal; bottom 6 bits == 0
926 * (3) leaf pte for huge page, bottom two bits != 00
927 * (4) hugepd pointer, bottom two bits == 00, next 4 bits indicate size of table
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530928 *
929 * So long as we atomically load page table pointers we are safe against teardown,
930 * we can follow the address down to the the page and take a ref on it.
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530931 */
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530932
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530933pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift)
934{
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530935 pgd_t pgd, *pgdp;
936 pud_t pud, *pudp;
937 pmd_t pmd, *pmdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530938 pte_t *ret_pte;
939 hugepd_t *hpdp = NULL;
940 unsigned pdshift = PGDIR_SHIFT;
941
942 if (shift)
943 *shift = 0;
944
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530945 pgdp = pgdir + pgd_index(ea);
946 pgd = ACCESS_ONCE(*pgdp);
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530947 /*
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530948 * Always operate on the local stack value. This make sure the
949 * value don't get updated by a parallel THP split/collapse,
950 * page fault or a page unmap. The return pte_t * is still not
951 * stable. So should be checked there for above conditions.
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530952 */
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530953 if (pgd_none(pgd))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530954 return NULL;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530955 else if (pgd_huge(pgd)) {
956 ret_pte = (pte_t *) pgdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530957 goto out;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530958 } else if (is_hugepd(&pgd))
959 hpdp = (hugepd_t *)&pgd;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530960 else {
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530961 /*
962 * Even if we end up with an unmap, the pgtable will not
963 * be freed, because we do an rcu free and here we are
964 * irq disabled
965 */
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530966 pdshift = PUD_SHIFT;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530967 pudp = pud_offset(&pgd, ea);
968 pud = ACCESS_ONCE(*pudp);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530969
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530970 if (pud_none(pud))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530971 return NULL;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530972 else if (pud_huge(pud)) {
973 ret_pte = (pte_t *) pudp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530974 goto out;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530975 } else if (is_hugepd(&pud))
976 hpdp = (hugepd_t *)&pud;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530977 else {
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530978 pdshift = PMD_SHIFT;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530979 pmdp = pmd_offset(&pud, ea);
980 pmd = ACCESS_ONCE(*pmdp);
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530981 /*
982 * A hugepage collapse is captured by pmd_none, because
983 * it mark the pmd none and do a hpte invalidate.
984 *
985 * A hugepage split is captured by pmd_trans_splitting
986 * because we mark the pmd trans splitting and do a
987 * hpte invalidate
988 *
989 */
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530990 if (pmd_none(pmd) || pmd_trans_splitting(pmd))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530991 return NULL;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530992
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530993 if (pmd_huge(pmd) || pmd_large(pmd)) {
994 ret_pte = (pte_t *) pmdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530995 goto out;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530996 } else if (is_hugepd(&pmd))
997 hpdp = (hugepd_t *)&pmd;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530998 else
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530999 return pte_offset_kernel(&pmd, ea);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301000 }
1001 }
1002 if (!hpdp)
1003 return NULL;
1004
1005 ret_pte = hugepte_offset(hpdp, ea, pdshift);
1006 pdshift = hugepd_shift(*hpdp);
1007out:
1008 if (shift)
1009 *shift = pdshift;
1010 return ret_pte;
1011}
1012EXPORT_SYMBOL_GPL(find_linux_pte_or_hugepte);
1013
1014int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
1015 unsigned long end, int write, struct page **pages, int *nr)
1016{
1017 unsigned long mask;
1018 unsigned long pte_end;
1019 struct page *head, *page, *tail;
1020 pte_t pte;
1021 int refs;
1022
1023 pte_end = (addr + sz) & ~(sz-1);
1024 if (pte_end < end)
1025 end = pte_end;
1026
1027 pte = *ptep;
1028 mask = _PAGE_PRESENT | _PAGE_USER;
1029 if (write)
1030 mask |= _PAGE_RW;
1031
1032 if ((pte_val(pte) & mask) != mask)
1033 return 0;
1034
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +05301035#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1036 /*
1037 * check for splitting here
1038 */
1039 if (pmd_trans_splitting(pte_pmd(pte)))
1040 return 0;
1041#endif
1042
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301043 /* hugepages are never "special" */
1044 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1045
1046 refs = 0;
1047 head = pte_page(pte);
1048
1049 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
1050 tail = page;
1051 do {
1052 VM_BUG_ON(compound_head(page) != head);
1053 pages[*nr] = page;
1054 (*nr)++;
1055 page++;
1056 refs++;
1057 } while (addr += PAGE_SIZE, addr != end);
1058
1059 if (!page_cache_add_speculative(head, refs)) {
1060 *nr -= refs;
1061 return 0;
1062 }
1063
1064 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1065 /* Could be optimized better */
1066 *nr -= refs;
1067 while (refs--)
1068 put_page(head);
1069 return 0;
1070 }
1071
1072 /*
1073 * Any tail page need their mapcount reference taken before we
1074 * return.
1075 */
1076 while (refs--) {
1077 if (PageTail(tail))
1078 get_huge_page_tail(tail);
1079 tail++;
1080 }
1081
1082 return 1;
1083}