blob: a4f33de4008eeed9829eb1ceef01e72d7cedfd8f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Becky Bruce41151e72011-06-28 09:54:48 +00002 * PPC Huge TLB Page Support for Kernel.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
Becky Bruce41151e72011-06-28 09:54:48 +00005 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/mm.h>
David Gibson883a3e52009-10-26 19:24:31 +000012#include <linux/io.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/hugetlb.h>
Paul Mackerras342d3db2011-12-12 12:38:05 +000015#include <linux/export.h>
Becky Bruce41151e72011-06-28 09:54:48 +000016#include <linux/of_fdt.h>
17#include <linux/memblock.h>
18#include <linux/bootmem.h>
Kumar Gala13020be2011-11-24 09:40:07 +000019#include <linux/moduleparam.h>
David Gibson883a3e52009-10-26 19:24:31 +000020#include <asm/pgtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/pgalloc.h>
22#include <asm/tlb.h>
Becky Bruce41151e72011-06-28 09:54:48 +000023#include <asm/setup.h>
Aneesh Kumar K.V29409992013-06-20 14:30:16 +053024#include <asm/hugetlb.h>
25
26#ifdef CONFIG_HUGETLB_PAGE
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Jon Tollefson91224342008-07-23 21:27:55 -070028#define PAGE_SHIFT_64K 16
Christophe Leroy4b9142862016-12-07 08:47:28 +010029#define PAGE_SHIFT_512K 19
30#define PAGE_SHIFT_8M 23
Jon Tollefson91224342008-07-23 21:27:55 -070031#define PAGE_SHIFT_16M 24
32#define PAGE_SHIFT_16G 34
Jon Tollefson4ec161c2008-01-04 09:59:50 +110033
Becky Bruce41151e72011-06-28 09:54:48 +000034unsigned int HPAGE_SHIFT;
35
36/*
37 * Tracks gpages after the device tree is scanned and before the
Becky Brucea6146882011-10-10 10:50:43 +000038 * huge_boot_pages list is ready. On non-Freescale implementations, this is
39 * just used to track 16G pages and so is a single array. FSL-based
40 * implementations may have more than one gpage size, so we need multiple
41 * arrays
Becky Bruce41151e72011-06-28 09:54:48 +000042 */
Christophe Leroy4b9142862016-12-07 08:47:28 +010043#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
Becky Bruce41151e72011-06-28 09:54:48 +000044#define MAX_NUMBER_GPAGES 128
45struct psize_gpages {
46 u64 gpage_list[MAX_NUMBER_GPAGES];
47 unsigned int nr_gpages;
48};
49static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
Becky Bruce881fde12011-10-10 10:50:40 +000050#else
51#define MAX_NUMBER_GPAGES 1024
52static u64 gpage_freearray[MAX_NUMBER_GPAGES];
53static unsigned nr_gpages;
Becky Bruce41151e72011-06-28 09:54:48 +000054#endif
David Gibsonf10a04c2006-04-28 15:02:51 +100055
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +053056#define hugepd_none(hpd) (hpd_val(hpd) == 0)
David Gibsona4fe3ce2009-10-26 19:24:31 +000057
David Gibsona4fe3ce2009-10-26 19:24:31 +000058pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
59{
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +053060 /* Only called for hugetlbfs pages, hence can ignore THP */
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +053061 return __find_linux_pte_or_hugepte(mm->pgd, addr, NULL, NULL);
David Gibsona4fe3ce2009-10-26 19:24:31 +000062}
63
64static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
65 unsigned long address, unsigned pdshift, unsigned pshift)
66{
Becky Bruce41151e72011-06-28 09:54:48 +000067 struct kmem_cache *cachep;
68 pte_t *new;
Becky Bruce41151e72011-06-28 09:54:48 +000069 int i;
Christophe Leroy03bb2d62016-12-07 08:47:26 +010070 int num_hugepd;
71
72 if (pshift >= pdshift) {
73 cachep = hugepte_cache;
74 num_hugepd = 1 << (pshift - pdshift);
75 } else {
76 cachep = PGT_CACHE(pdshift - pshift);
77 num_hugepd = 1;
78 }
Becky Bruce41151e72011-06-28 09:54:48 +000079
Michal Hocko2379a232016-06-24 14:49:12 -070080 new = kmem_cache_zalloc(cachep, GFP_KERNEL);
David Gibsonf10a04c2006-04-28 15:02:51 +100081
David Gibsona4fe3ce2009-10-26 19:24:31 +000082 BUG_ON(pshift > HUGEPD_SHIFT_MASK);
83 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
84
David Gibsonf10a04c2006-04-28 15:02:51 +100085 if (! new)
86 return -ENOMEM;
87
Sukadev Bhattiprolu0eab46b2016-03-24 02:07:57 -040088 /*
89 * Make sure other cpus find the hugepd set only after a
90 * properly initialized page table is visible to them.
91 * For more details look for comment in __pte_alloc().
92 */
93 smp_wmb();
94
David Gibsonf10a04c2006-04-28 15:02:51 +100095 spin_lock(&mm->page_table_lock);
Christophe Leroy03bb2d62016-12-07 08:47:26 +010096
Becky Bruce41151e72011-06-28 09:54:48 +000097 /*
98 * We have multiple higher-level entries that point to the same
99 * actual pte location. Fill in each as we go and backtrack on error.
100 * We need all of these so the DTLB pgtable walk code can find the
101 * right higher-level entry without knowing if it's a hugepage or not.
102 */
103 for (i = 0; i < num_hugepd; i++, hpdp++) {
104 if (unlikely(!hugepd_none(*hpdp)))
105 break;
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530106 else {
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100107#ifdef CONFIG_PPC_BOOK3S_64
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530108 *hpdp = __hugepd(__pa(new) |
109 (shift_to_mmu_psize(pshift) << 2));
Christophe Leroy4b9142862016-12-07 08:47:28 +0100110#elif defined(CONFIG_PPC_8xx)
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530111 *hpdp = __hugepd(__pa(new) |
112 (pshift == PAGE_SHIFT_8M ? _PMD_PAGE_8M :
113 _PMD_PAGE_512K) | _PMD_PRESENT);
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100114#else
Aneesh Kumar K.Vcf9427b2013-04-28 09:37:29 +0000115 /* We use the old format for PPC_FSL_BOOK3E */
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530116 *hpdp = __hugepd(((unsigned long)new & ~PD_HUGE) | pshift);
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100117#endif
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530118 }
Becky Bruce41151e72011-06-28 09:54:48 +0000119 }
120 /* If we bailed from the for loop early, an error occurred, clean up */
121 if (i < num_hugepd) {
122 for (i = i - 1 ; i >= 0; i--, hpdp--)
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530123 *hpdp = __hugepd(0);
Becky Bruce41151e72011-06-28 09:54:48 +0000124 kmem_cache_free(cachep, new);
125 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000126 spin_unlock(&mm->page_table_lock);
127 return 0;
128}
129
Becky Brucea1cd5412011-10-10 10:50:39 +0000130/*
131 * These macros define how to determine which level of the page table holds
132 * the hpdp.
133 */
Christophe Leroy4b9142862016-12-07 08:47:28 +0100134#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
Becky Brucea1cd5412011-10-10 10:50:39 +0000135#define HUGEPD_PGD_SHIFT PGDIR_SHIFT
136#define HUGEPD_PUD_SHIFT PUD_SHIFT
137#else
138#define HUGEPD_PGD_SHIFT PUD_SHIFT
139#define HUGEPD_PUD_SHIFT PMD_SHIFT
140#endif
141
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000142/*
143 * At this point we do the placement change only for BOOK3S 64. This would
144 * possibly work on other subarchs.
145 */
146pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
147{
148 pgd_t *pg;
149 pud_t *pu;
150 pmd_t *pm;
151 hugepd_t *hpdp = NULL;
152 unsigned pshift = __ffs(sz);
153 unsigned pdshift = PGDIR_SHIFT;
154
155 addr &= ~(sz-1);
156 pg = pgd_offset(mm, addr);
157
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100158#ifdef CONFIG_PPC_BOOK3S_64
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000159 if (pshift == PGDIR_SHIFT)
160 /* 16GB huge page */
161 return (pte_t *) pg;
162 else if (pshift > PUD_SHIFT)
163 /*
164 * We need to use hugepd table
165 */
166 hpdp = (hugepd_t *)pg;
167 else {
168 pdshift = PUD_SHIFT;
169 pu = pud_alloc(mm, pg, addr);
170 if (pshift == PUD_SHIFT)
171 return (pte_t *)pu;
172 else if (pshift > PMD_SHIFT)
173 hpdp = (hugepd_t *)pu;
174 else {
175 pdshift = PMD_SHIFT;
176 pm = pmd_alloc(mm, pu, addr);
177 if (pshift == PMD_SHIFT)
178 /* 16MB hugepage */
179 return (pte_t *)pm;
180 else
181 hpdp = (hugepd_t *)pm;
182 }
183 }
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000184#else
Becky Brucea1cd5412011-10-10 10:50:39 +0000185 if (pshift >= HUGEPD_PGD_SHIFT) {
David Gibsona4fe3ce2009-10-26 19:24:31 +0000186 hpdp = (hugepd_t *)pg;
187 } else {
188 pdshift = PUD_SHIFT;
189 pu = pud_alloc(mm, pg, addr);
Becky Brucea1cd5412011-10-10 10:50:39 +0000190 if (pshift >= HUGEPD_PUD_SHIFT) {
David Gibsona4fe3ce2009-10-26 19:24:31 +0000191 hpdp = (hugepd_t *)pu;
192 } else {
193 pdshift = PMD_SHIFT;
194 pm = pmd_alloc(mm, pu, addr);
195 hpdp = (hugepd_t *)pm;
196 }
197 }
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100198#endif
David Gibsona4fe3ce2009-10-26 19:24:31 +0000199 if (!hpdp)
200 return NULL;
201
202 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
203
204 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
205 return NULL;
206
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530207 return hugepte_offset(*hpdp, addr, pdshift);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100208}
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100209
Christophe Leroy4b9142862016-12-07 08:47:28 +0100210#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
Jon Tollefson658013e2008-07-23 21:27:54 -0700211/* Build list of addresses of gigantic pages. This function is used in early
Anton Blanchard14ed7402014-09-17 22:15:34 +1000212 * boot before the buddy allocator is setup.
Jon Tollefson658013e2008-07-23 21:27:54 -0700213 */
Becky Bruce41151e72011-06-28 09:54:48 +0000214void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
215{
216 unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
217 int i;
218
219 if (addr == 0)
220 return;
221
222 gpage_freearray[idx].nr_gpages = number_of_pages;
223
224 for (i = 0; i < number_of_pages; i++) {
225 gpage_freearray[idx].gpage_list[i] = addr;
226 addr += page_size;
227 }
228}
229
230/*
231 * Moves the gigantic page addresses from the temporary list to the
232 * huge_boot_pages list.
233 */
234int alloc_bootmem_huge_page(struct hstate *hstate)
235{
236 struct huge_bootmem_page *m;
Wanpeng Li2415cf12013-07-03 15:02:43 -0700237 int idx = shift_to_mmu_psize(huge_page_shift(hstate));
Becky Bruce41151e72011-06-28 09:54:48 +0000238 int nr_gpages = gpage_freearray[idx].nr_gpages;
239
240 if (nr_gpages == 0)
241 return 0;
242
243#ifdef CONFIG_HIGHMEM
244 /*
245 * If gpages can be in highmem we can't use the trick of storing the
246 * data structure in the page; allocate space for this
247 */
Michael Ellermane39f223f2014-11-18 16:47:35 +1100248 m = memblock_virt_alloc(sizeof(struct huge_bootmem_page), 0);
Becky Bruce41151e72011-06-28 09:54:48 +0000249 m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
250#else
251 m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
252#endif
253
254 list_add(&m->list, &huge_boot_pages);
255 gpage_freearray[idx].nr_gpages = nr_gpages;
256 gpage_freearray[idx].gpage_list[nr_gpages] = 0;
257 m->hstate = hstate;
258
259 return 1;
260}
261/*
262 * Scan the command line hugepagesz= options for gigantic pages; store those in
263 * a list that we use to allocate the memory once all options are parsed.
264 */
265
266unsigned long gpage_npages[MMU_PAGE_COUNT];
267
Paul Gortmaker89528122012-05-07 10:32:22 -0400268static int __init do_gpage_early_setup(char *param, char *val,
Luis R. Rodriguezecc86172015-03-30 16:20:03 -0700269 const char *unused, void *arg)
Becky Bruce41151e72011-06-28 09:54:48 +0000270{
271 static phys_addr_t size;
272 unsigned long npages;
273
274 /*
275 * The hugepagesz and hugepages cmdline options are interleaved. We
276 * use the size variable to keep track of whether or not this was done
277 * properly and skip over instances where it is incorrect. Other
278 * command-line parsing code will issue warnings, so we don't need to.
279 *
280 */
281 if ((strcmp(param, "default_hugepagesz") == 0) ||
282 (strcmp(param, "hugepagesz") == 0)) {
283 size = memparse(val, NULL);
284 } else if (strcmp(param, "hugepages") == 0) {
285 if (size != 0) {
286 if (sscanf(val, "%lu", &npages) <= 0)
287 npages = 0;
James Yangc4f3eb52014-11-14 12:32:24 -0600288 if (npages > MAX_NUMBER_GPAGES) {
289 pr_warn("MMU: %lu pages requested for page "
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100290#ifdef CONFIG_PHYS_ADDR_T_64BIT
James Yangc4f3eb52014-11-14 12:32:24 -0600291 "size %llu KB, limiting to "
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100292#else
293 "size %u KB, limiting to "
294#endif
James Yangc4f3eb52014-11-14 12:32:24 -0600295 __stringify(MAX_NUMBER_GPAGES) "\n",
296 npages, size / 1024);
297 npages = MAX_NUMBER_GPAGES;
298 }
Becky Bruce41151e72011-06-28 09:54:48 +0000299 gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
300 size = 0;
301 }
302 }
303 return 0;
304}
305
306
307/*
308 * This function allocates physical space for pages that are larger than the
309 * buddy allocator can handle. We want to allocate these in highmem because
310 * the amount of lowmem is limited. This means that this function MUST be
311 * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
312 * allocate to grab highmem.
313 */
314void __init reserve_hugetlb_gpages(void)
315{
316 static __initdata char cmdline[COMMAND_LINE_SIZE];
317 phys_addr_t size, base;
318 int i;
319
320 strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
Pawel Moll026cee02012-03-26 12:50:51 +1030321 parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
Luis R. Rodriguezecc86172015-03-30 16:20:03 -0700322 NULL, &do_gpage_early_setup);
Becky Bruce41151e72011-06-28 09:54:48 +0000323
324 /*
325 * Walk gpage list in reverse, allocating larger page sizes first.
326 * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
327 * When we reach the point in the list where pages are no longer
328 * considered gpages, we're done.
329 */
330 for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
331 if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
332 continue;
333 else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
334 break;
335
336 size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
337 base = memblock_alloc_base(size * gpage_npages[i], size,
338 MEMBLOCK_ALLOC_ANYWHERE);
339 add_gpage(base, size, gpage_npages[i]);
340 }
341}
342
Becky Bruce881fde12011-10-10 10:50:40 +0000343#else /* !PPC_FSL_BOOK3E */
Becky Bruce41151e72011-06-28 09:54:48 +0000344
345/* Build list of addresses of gigantic pages. This function is used in early
Anton Blanchard14ed7402014-09-17 22:15:34 +1000346 * boot before the buddy allocator is setup.
Becky Bruce41151e72011-06-28 09:54:48 +0000347 */
348void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
Jon Tollefson658013e2008-07-23 21:27:54 -0700349{
350 if (!addr)
351 return;
352 while (number_of_pages > 0) {
353 gpage_freearray[nr_gpages] = addr;
354 nr_gpages++;
355 number_of_pages--;
356 addr += page_size;
357 }
358}
359
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700360/* Moves the gigantic page addresses from the temporary list to the
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700361 * huge_boot_pages list.
362 */
363int alloc_bootmem_huge_page(struct hstate *hstate)
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700364{
365 struct huge_bootmem_page *m;
366 if (nr_gpages == 0)
367 return 0;
368 m = phys_to_virt(gpage_freearray[--nr_gpages]);
369 gpage_freearray[nr_gpages] = 0;
370 list_add(&m->list, &huge_boot_pages);
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700371 m->hstate = hstate;
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700372 return 1;
373}
Becky Bruce41151e72011-06-28 09:54:48 +0000374#endif
Jon Tollefsonec4b2c02008-07-23 21:27:53 -0700375
Christophe Leroy4b9142862016-12-07 08:47:28 +0100376#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
Becky Bruce41151e72011-06-28 09:54:48 +0000377#define HUGEPD_FREELIST_SIZE \
378 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
379
380struct hugepd_freelist {
381 struct rcu_head rcu;
382 unsigned int index;
383 void *ptes[0];
384};
385
386static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
387
388static void hugepd_free_rcu_callback(struct rcu_head *head)
389{
390 struct hugepd_freelist *batch =
391 container_of(head, struct hugepd_freelist, rcu);
392 unsigned int i;
393
394 for (i = 0; i < batch->index; i++)
395 kmem_cache_free(hugepte_cache, batch->ptes[i]);
396
397 free_page((unsigned long)batch);
398}
399
400static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
401{
402 struct hugepd_freelist **batchp;
403
Sebastian Siewior08a5bb22016-03-08 10:03:56 +0100404 batchp = &get_cpu_var(hugepd_freelist_cur);
Becky Bruce41151e72011-06-28 09:54:48 +0000405
406 if (atomic_read(&tlb->mm->mm_users) < 2 ||
407 cpumask_equal(mm_cpumask(tlb->mm),
408 cpumask_of(smp_processor_id()))) {
409 kmem_cache_free(hugepte_cache, hugepte);
Sebastian Siewior08a5bb22016-03-08 10:03:56 +0100410 put_cpu_var(hugepd_freelist_cur);
Becky Bruce41151e72011-06-28 09:54:48 +0000411 return;
412 }
413
414 if (*batchp == NULL) {
415 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
416 (*batchp)->index = 0;
417 }
418
419 (*batchp)->ptes[(*batchp)->index++] = hugepte;
420 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
421 call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
422 *batchp = NULL;
423 }
Tiejun Chen94b09d72014-01-20 16:39:34 +0800424 put_cpu_var(hugepd_freelist_cur);
Becky Bruce41151e72011-06-28 09:54:48 +0000425}
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100426#else
427static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {}
Becky Bruce41151e72011-06-28 09:54:48 +0000428#endif
429
David Gibsona4fe3ce2009-10-26 19:24:31 +0000430static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
431 unsigned long start, unsigned long end,
432 unsigned long floor, unsigned long ceiling)
David Gibsonf10a04c2006-04-28 15:02:51 +1000433{
434 pte_t *hugepte = hugepd_page(*hpdp);
Becky Bruce41151e72011-06-28 09:54:48 +0000435 int i;
436
David Gibsona4fe3ce2009-10-26 19:24:31 +0000437 unsigned long pdmask = ~((1UL << pdshift) - 1);
Becky Bruce41151e72011-06-28 09:54:48 +0000438 unsigned int num_hugepd = 1;
Becky Bruce881fde12011-10-10 10:50:40 +0000439 unsigned int shift = hugepd_shift(*hpdp);
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100440
441 /* Note: On fsl the hpdp may be the first of several */
442 if (shift > pdshift)
443 num_hugepd = 1 << (shift - pdshift);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000444
445 start &= pdmask;
446 if (start < floor)
447 return;
448 if (ceiling) {
449 ceiling &= pdmask;
450 if (! ceiling)
451 return;
452 }
453 if (end - 1 > ceiling - 1)
454 return;
David Gibsonf10a04c2006-04-28 15:02:51 +1000455
Becky Bruce41151e72011-06-28 09:54:48 +0000456 for (i = 0; i < num_hugepd; i++, hpdp++)
Aneesh Kumar K.V20717e12016-12-14 10:07:53 +0530457 *hpdp = __hugepd(0);
Becky Bruce41151e72011-06-28 09:54:48 +0000458
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100459 if (shift >= pdshift)
460 hugepd_free(tlb, hugepte);
461 else
462 pgtable_free_tlb(tlb, hugepte, pdshift - shift);
David Gibsonf10a04c2006-04-28 15:02:51 +1000463}
464
David Gibsonf10a04c2006-04-28 15:02:51 +1000465static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
466 unsigned long addr, unsigned long end,
David Gibsona4fe3ce2009-10-26 19:24:31 +0000467 unsigned long floor, unsigned long ceiling)
David Gibsonf10a04c2006-04-28 15:02:51 +1000468{
469 pmd_t *pmd;
470 unsigned long next;
471 unsigned long start;
472
473 start = addr;
David Gibsonf10a04c2006-04-28 15:02:51 +1000474 do {
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100475 unsigned long more;
476
Becky Brucea1cd5412011-10-10 10:50:39 +0000477 pmd = pmd_offset(pud, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000478 next = pmd_addr_end(addr, end);
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530479 if (!is_hugepd(__hugepd(pmd_val(*pmd)))) {
Aneesh Kumar K.V8bbd9f02013-06-19 12:04:26 +0530480 /*
481 * if it is not hugepd pointer, we should already find
482 * it cleared.
483 */
484 WARN_ON(!pmd_none_or_clear_bad(pmd));
David Gibsonf10a04c2006-04-28 15:02:51 +1000485 continue;
Aneesh Kumar K.V8bbd9f02013-06-19 12:04:26 +0530486 }
Becky Brucea1cd5412011-10-10 10:50:39 +0000487 /*
488 * Increment next by the size of the huge mapping since
489 * there may be more than one entry at this level for a
490 * single hugepage, but all of them point to
491 * the same kmem cache that holds the hugepte.
492 */
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100493 more = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
494 if (more > next)
495 next = more;
496
David Gibsona4fe3ce2009-10-26 19:24:31 +0000497 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
498 addr, next, floor, ceiling);
Becky Brucea1cd5412011-10-10 10:50:39 +0000499 } while (addr = next, addr != end);
David Gibsonf10a04c2006-04-28 15:02:51 +1000500
501 start &= PUD_MASK;
502 if (start < floor)
503 return;
504 if (ceiling) {
505 ceiling &= PUD_MASK;
506 if (!ceiling)
507 return;
508 }
509 if (end - 1 > ceiling - 1)
510 return;
511
512 pmd = pmd_offset(pud, start);
513 pud_clear(pud);
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +1000514 pmd_free_tlb(tlb, pmd, start);
Scott Wood50c6a662015-04-10 19:37:34 -0500515 mm_dec_nr_pmds(tlb->mm);
David Gibsonf10a04c2006-04-28 15:02:51 +1000516}
David Gibsonf10a04c2006-04-28 15:02:51 +1000517
518static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
519 unsigned long addr, unsigned long end,
520 unsigned long floor, unsigned long ceiling)
521{
522 pud_t *pud;
523 unsigned long next;
524 unsigned long start;
525
526 start = addr;
David Gibsonf10a04c2006-04-28 15:02:51 +1000527 do {
Becky Brucea1cd5412011-10-10 10:50:39 +0000528 pud = pud_offset(pgd, addr);
David Gibsonf10a04c2006-04-28 15:02:51 +1000529 next = pud_addr_end(addr, end);
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530530 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100531 if (pud_none_or_clear_bad(pud))
532 continue;
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700533 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
David Gibsona4fe3ce2009-10-26 19:24:31 +0000534 ceiling);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100535 } else {
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100536 unsigned long more;
Becky Brucea1cd5412011-10-10 10:50:39 +0000537 /*
538 * Increment next by the size of the huge mapping since
539 * there may be more than one entry at this level for a
540 * single hugepage, but all of them point to
541 * the same kmem cache that holds the hugepte.
542 */
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100543 more = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
544 if (more > next)
545 next = more;
546
David Gibsona4fe3ce2009-10-26 19:24:31 +0000547 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
548 addr, next, floor, ceiling);
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100549 }
Becky Brucea1cd5412011-10-10 10:50:39 +0000550 } while (addr = next, addr != end);
David Gibsonf10a04c2006-04-28 15:02:51 +1000551
552 start &= PGDIR_MASK;
553 if (start < floor)
554 return;
555 if (ceiling) {
556 ceiling &= PGDIR_MASK;
557 if (!ceiling)
558 return;
559 }
560 if (end - 1 > ceiling - 1)
561 return;
562
563 pud = pud_offset(pgd, start);
564 pgd_clear(pgd);
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +1000565 pud_free_tlb(tlb, pud, start);
David Gibsonf10a04c2006-04-28 15:02:51 +1000566}
567
568/*
569 * This function frees user-level page tables of a process.
David Gibsonf10a04c2006-04-28 15:02:51 +1000570 */
Jan Beulich42b77722008-07-23 21:27:10 -0700571void hugetlb_free_pgd_range(struct mmu_gather *tlb,
David Gibsonf10a04c2006-04-28 15:02:51 +1000572 unsigned long addr, unsigned long end,
573 unsigned long floor, unsigned long ceiling)
574{
575 pgd_t *pgd;
576 unsigned long next;
David Gibsonf10a04c2006-04-28 15:02:51 +1000577
578 /*
David Gibsona4fe3ce2009-10-26 19:24:31 +0000579 * Because there are a number of different possible pagetable
580 * layouts for hugepage ranges, we limit knowledge of how
581 * things should be laid out to the allocation path
582 * (huge_pte_alloc(), above). Everything else works out the
583 * structure as it goes from information in the hugepd
584 * pointers. That means that we can't here use the
585 * optimization used in the normal page free_pgd_range(), of
586 * checking whether we're actually covering a large enough
587 * range to have to do anything at the top level of the walk
588 * instead of at the bottom.
David Gibsonf10a04c2006-04-28 15:02:51 +1000589 *
David Gibsona4fe3ce2009-10-26 19:24:31 +0000590 * To make sense of this, you should probably go read the big
591 * block comment at the top of the normal free_pgd_range(),
592 * too.
David Gibsonf10a04c2006-04-28 15:02:51 +1000593 */
594
David Gibsonf10a04c2006-04-28 15:02:51 +1000595 do {
David Gibsonf10a04c2006-04-28 15:02:51 +1000596 next = pgd_addr_end(addr, end);
Becky Bruce41151e72011-06-28 09:54:48 +0000597 pgd = pgd_offset(tlb->mm, addr);
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530598 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
David Gibson0b264252008-09-05 11:49:54 +1000599 if (pgd_none_or_clear_bad(pgd))
600 continue;
601 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
602 } else {
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100603 unsigned long more;
Becky Bruce41151e72011-06-28 09:54:48 +0000604 /*
605 * Increment next by the size of the huge mapping since
Becky Bruce881fde12011-10-10 10:50:40 +0000606 * there may be more than one entry at the pgd level
607 * for a single hugepage, but all of them point to the
608 * same kmem cache that holds the hugepte.
Becky Bruce41151e72011-06-28 09:54:48 +0000609 */
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100610 more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
611 if (more > next)
612 next = more;
613
David Gibsona4fe3ce2009-10-26 19:24:31 +0000614 free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
615 addr, next, floor, ceiling);
David Gibson0b264252008-09-05 11:49:54 +1000616 }
Becky Bruce41151e72011-06-28 09:54:48 +0000617 } while (addr = next, addr != end);
David Gibsone28f7fa2005-08-05 19:39:06 +1000618}
619
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530620/*
621 * We are holding mmap_sem, so a parallel huge page collapse cannot run.
622 * To prevent hugepage split, disable irq.
623 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624struct page *
625follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
626{
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530627 bool is_thp;
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530628 pte_t *ptep, pte;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000629 unsigned shift;
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530630 unsigned long mask, flags;
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530631 struct page *page = ERR_PTR(-EINVAL);
632
633 local_irq_save(flags);
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530634 ptep = find_linux_pte_or_hugepte(mm->pgd, address, &is_thp, &shift);
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530635 if (!ptep)
636 goto no_page;
637 pte = READ_ONCE(*ptep);
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +0530638 /*
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530639 * Verify it is a huge page else bail.
Aneesh Kumar K.V12bc9f62013-06-20 14:30:18 +0530640 * Transparent hugepages are handled by generic code. We can skip them
641 * here.
642 */
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530643 if (!shift || is_thp)
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530644 goto no_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530646 if (!pte_present(pte)) {
647 page = NULL;
648 goto no_page;
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530649 }
David Gibsona4fe3ce2009-10-26 19:24:31 +0000650 mask = (1UL << shift) - 1;
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530651 page = pte_page(pte);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000652 if (page)
653 page += (address & mask) / PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
Aneesh Kumar K.V7b868e82015-05-11 11:58:29 +0530655no_page:
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530656 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 return page;
658}
659
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660struct page *
661follow_huge_pmd(struct mm_struct *mm, unsigned long address,
662 pmd_t *pmd, int write)
663{
664 BUG();
665 return NULL;
666}
667
Naoya Horiguchi61f77ed2015-02-11 15:25:15 -0800668struct page *
669follow_huge_pud(struct mm_struct *mm, unsigned long address,
670 pud_t *pud, int write)
671{
672 BUG();
673 return NULL;
674}
675
David Gibson39adfa52009-11-23 20:03:40 +0000676static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
677 unsigned long sz)
678{
679 unsigned long __boundary = (addr + sz) & ~(sz-1);
680 return (__boundary - 1 < end - 1) ? __boundary : end;
681}
682
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530683int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned pdshift,
684 unsigned long end, int write, struct page **pages, int *nr)
David Gibsona4fe3ce2009-10-26 19:24:31 +0000685{
686 pte_t *ptep;
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530687 unsigned long sz = 1UL << hugepd_shift(hugepd);
David Gibson39adfa52009-11-23 20:03:40 +0000688 unsigned long next;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000689
690 ptep = hugepte_offset(hugepd, addr, pdshift);
691 do {
David Gibson39adfa52009-11-23 20:03:40 +0000692 next = hugepte_addr_end(addr, end, sz);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000693 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
694 return 0;
David Gibson39adfa52009-11-23 20:03:40 +0000695 } while (ptep++, addr = next, addr != end);
David Gibsona4fe3ce2009-10-26 19:24:31 +0000696
697 return 1;
698}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699
Becky Bruce76512952011-10-10 10:50:36 +0000700#ifdef CONFIG_PPC_MM_SLICES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
702 unsigned long len, unsigned long pgoff,
703 unsigned long flags)
704{
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700705 struct hstate *hstate = hstate_file(file);
706 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
Brian King48f797d2008-12-04 04:07:54 +0000707
Aneesh Kumar K.V48483762016-04-29 23:26:25 +1000708 if (radix_enabled())
709 return radix__hugetlb_get_unmapped_area(file, addr, len,
710 pgoff, flags);
Michel Lespinasse34d07172013-04-29 11:53:52 -0700711 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712}
Becky Bruce76512952011-10-10 10:50:36 +0000713#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714
Mel Gorman33402892009-01-06 14:38:54 -0800715unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
716{
Paul Mackerras25c29f92011-09-20 19:58:10 +0000717#ifdef CONFIG_PPC_MM_SLICES
Mel Gorman33402892009-01-06 14:38:54 -0800718 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
Aneesh Kumar K.V2f5f0df2016-04-29 23:26:24 +1000719 /* With radix we don't use slice, so derive it from vma*/
720 if (!radix_enabled())
721 return 1UL << mmu_psize_to_shift(psize);
722#endif
Becky Bruce41151e72011-06-28 09:54:48 +0000723 if (!is_vm_hugetlb_page(vma))
724 return PAGE_SIZE;
725
726 return huge_page_size(hstate_vma(vma));
Becky Bruce41151e72011-06-28 09:54:48 +0000727}
728
729static inline bool is_power_of_4(unsigned long x)
730{
731 if (is_power_of_2(x))
732 return (__ilog2(x) % 2) ? false : true;
733 return false;
Mel Gorman33402892009-01-06 14:38:54 -0800734}
735
David Gibsond1837cb2009-10-26 19:24:31 +0000736static int __init add_huge_page_size(unsigned long long size)
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100737{
David Gibsond1837cb2009-10-26 19:24:31 +0000738 int shift = __ffs(size);
739 int mmu_psize;
David Gibsona4fe3ce2009-10-26 19:24:31 +0000740
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100741 /* Check that it is a page size supported by the hardware and
David Gibsond1837cb2009-10-26 19:24:31 +0000742 * that it fits within pagetable and slice limits. */
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100743 if (size <= PAGE_SIZE)
744 return -EINVAL;
Christophe Leroy4b9142862016-12-07 08:47:28 +0100745#if defined(CONFIG_PPC_FSL_BOOK3E)
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100746 if (!is_power_of_4(size))
Becky Bruce41151e72011-06-28 09:54:48 +0000747 return -EINVAL;
Christophe Leroy4b9142862016-12-07 08:47:28 +0100748#elif !defined(CONFIG_PPC_8xx)
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100749 if (!is_power_of_2(size) || (shift > SLICE_HIGH_SHIFT))
David Gibsond1837cb2009-10-26 19:24:31 +0000750 return -EINVAL;
Becky Bruce41151e72011-06-28 09:54:48 +0000751#endif
Jon Tollefson91224342008-07-23 21:27:55 -0700752
David Gibsond1837cb2009-10-26 19:24:31 +0000753 if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
754 return -EINVAL;
755
Aneesh Kumar K.Va5251082017-03-21 22:59:56 +0530756#ifdef CONFIG_PPC_BOOK3S_64
757 /*
758 * We need to make sure that for different page sizes reported by
759 * firmware we only add hugetlb support for page sizes that can be
760 * supported by linux page table layout.
761 * For now we have
762 * Radix: 2M
763 * Hash: 16M and 16G
764 */
765 if (radix_enabled()) {
766 if (mmu_psize != MMU_PAGE_2M)
767 return -EINVAL;
768 } else {
769 if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G)
770 return -EINVAL;
771 }
772#endif
773
David Gibsond1837cb2009-10-26 19:24:31 +0000774 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
775
776 /* Return if huge page size has already been setup */
777 if (size_to_hstate(size))
778 return 0;
779
780 hugetlb_add_hstate(shift - PAGE_SHIFT);
781
782 return 0;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100783}
784
785static int __init hugepage_setup_sz(char *str)
786{
787 unsigned long long size;
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100788
789 size = memparse(str, &str);
790
Vaishali Thakkar71bf79c2016-05-19 17:11:14 -0700791 if (add_huge_page_size(size) != 0) {
792 hugetlb_bad_size();
793 pr_err("Invalid huge page size specified(%llu)\n", size);
794 }
Jon Tollefson4ec161c2008-01-04 09:59:50 +1100795
796 return 1;
797}
798__setup("hugepagesz=", hugepage_setup_sz);
799
Becky Bruce41151e72011-06-28 09:54:48 +0000800struct kmem_cache *hugepte_cache;
801static int __init hugetlbpage_init(void)
802{
803 int psize;
804
Christophe Leroy4b9142862016-12-07 08:47:28 +0100805#if !defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_PPC_8xx)
Aneesh Kumar K.V48483762016-04-29 23:26:25 +1000806 if (!radix_enabled() && !mmu_has_feature(MMU_FTR_16M_PAGE))
David Gibsonf10a04c2006-04-28 15:02:51 +1000807 return -ENODEV;
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100808#endif
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700809 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
David Gibsond1837cb2009-10-26 19:24:31 +0000810 unsigned shift;
811 unsigned pdshift;
812
813 if (!mmu_psize_defs[psize].shift)
814 continue;
815
816 shift = mmu_psize_to_shift(psize);
817
818 if (add_huge_page_size(1ULL << shift) < 0)
819 continue;
820
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100821 if (shift < HUGEPD_PUD_SHIFT)
David Gibsond1837cb2009-10-26 19:24:31 +0000822 pdshift = PMD_SHIFT;
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100823 else if (shift < HUGEPD_PGD_SHIFT)
David Gibsond1837cb2009-10-26 19:24:31 +0000824 pdshift = PUD_SHIFT;
825 else
826 pdshift = PGDIR_SHIFT;
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000827 /*
828 * if we have pdshift and shift value same, we don't
829 * use pgt cache for hugepd.
830 */
Nicholas Pigginbf5ca682017-01-04 01:55:17 +1000831 if (pdshift > shift)
Aneesh Kumar K.Ve2b3d202013-04-28 09:37:30 +0000832 pgtable_cache_add(pdshift - shift, NULL);
Christophe Leroy4b9142862016-12-07 08:47:28 +0100833#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100834 else if (!hugepte_cache) {
835 /*
836 * Create a kmem cache for hugeptes. The bottom bits in
837 * the pte have size information encoded in them, so
838 * align them to allow this
839 */
840 hugepte_cache = kmem_cache_create("hugepte-cache",
841 sizeof(pte_t),
842 HUGEPD_SHIFT_MASK + 1,
843 0, NULL);
844 if (hugepte_cache == NULL)
845 panic("%s: Unable to create kmem cache "
846 "for hugeptes\n", __func__);
847
848 }
849#endif
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700850 }
David Gibsonf10a04c2006-04-28 15:02:51 +1000851
Christophe Leroy4b9142862016-12-07 08:47:28 +0100852#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
853 /* Default hpage size = 4M on FSL_BOOK3E and 512k on 8xx */
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100854 if (mmu_psize_defs[MMU_PAGE_4M].shift)
855 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
Christophe Leroy4b9142862016-12-07 08:47:28 +0100856 else if (mmu_psize_defs[MMU_PAGE_512K].shift)
857 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_512K].shift;
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100858#else
David Gibsond1837cb2009-10-26 19:24:31 +0000859 /* Set default large page size. Currently, we pick 16M or 1M
860 * depending on what is available
861 */
862 if (mmu_psize_defs[MMU_PAGE_16M].shift)
863 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
864 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
865 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
Aneesh Kumar K.V48483762016-04-29 23:26:25 +1000866 else if (mmu_psize_defs[MMU_PAGE_2M].shift)
867 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_2M].shift;
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100868#endif
David Gibsonf10a04c2006-04-28 15:02:51 +1000869 return 0;
870}
Christophe Leroy03bb2d62016-12-07 08:47:26 +0100871
Paul Gortmaker6f114282015-05-01 20:08:21 -0400872arch_initcall(hugetlbpage_init);
David Gibson0895ecd2009-10-26 19:24:31 +0000873
874void flush_dcache_icache_hugepage(struct page *page)
875{
876 int i;
Becky Bruce41151e72011-06-28 09:54:48 +0000877 void *start;
David Gibson0895ecd2009-10-26 19:24:31 +0000878
879 BUG_ON(!PageCompound(page));
880
Becky Bruce41151e72011-06-28 09:54:48 +0000881 for (i = 0; i < (1UL << compound_order(page)); i++) {
882 if (!PageHighMem(page)) {
883 __flush_dcache_icache(page_address(page+i));
884 } else {
Cong Wang2480b202011-11-25 23:14:16 +0800885 start = kmap_atomic(page+i);
Becky Bruce41151e72011-06-28 09:54:48 +0000886 __flush_dcache_icache(start);
Cong Wang2480b202011-11-25 23:14:16 +0800887 kunmap_atomic(start);
Becky Bruce41151e72011-06-28 09:54:48 +0000888 }
889 }
David Gibson0895ecd2009-10-26 19:24:31 +0000890}
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530891
892#endif /* CONFIG_HUGETLB_PAGE */
893
894/*
895 * We have 4 cases for pgds and pmds:
896 * (1) invalid (all zeroes)
897 * (2) pointer to next table, as normal; bottom 6 bits == 0
Aneesh Kumar K.V6a119ea2015-12-01 09:06:54 +0530898 * (3) leaf pte for huge page _PAGE_PTE set
899 * (4) hugepd pointer, _PAGE_PTE = 0 and bits [2..6] indicate size of table
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530900 *
901 * So long as we atomically load page table pointers we are safe against teardown,
902 * we can follow the address down to the the page and take a ref on it.
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530903 * This function need to be called with interrupts disabled. We use this variant
904 * when we have MSR[EE] = 0 but the paca->soft_enabled = 1
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530905 */
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530906
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530907pte_t *__find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea,
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530908 bool *is_thp, unsigned *shift)
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530909{
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530910 pgd_t pgd, *pgdp;
911 pud_t pud, *pudp;
912 pmd_t pmd, *pmdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530913 pte_t *ret_pte;
914 hugepd_t *hpdp = NULL;
915 unsigned pdshift = PGDIR_SHIFT;
916
917 if (shift)
918 *shift = 0;
919
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530920 if (is_thp)
921 *is_thp = false;
922
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530923 pgdp = pgdir + pgd_index(ea);
Michael Ellerman4f9c53c2015-03-25 20:11:57 +1100924 pgd = READ_ONCE(*pgdp);
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530925 /*
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530926 * Always operate on the local stack value. This make sure the
927 * value don't get updated by a parallel THP split/collapse,
928 * page fault or a page unmap. The return pte_t * is still not
929 * stable. So should be checked there for above conditions.
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530930 */
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530931 if (pgd_none(pgd))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530932 return NULL;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530933 else if (pgd_huge(pgd)) {
934 ret_pte = (pte_t *) pgdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530935 goto out;
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530936 } else if (is_hugepd(__hugepd(pgd_val(pgd))))
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530937 hpdp = (hugepd_t *)&pgd;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530938 else {
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530939 /*
940 * Even if we end up with an unmap, the pgtable will not
941 * be freed, because we do an rcu free and here we are
942 * irq disabled
943 */
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530944 pdshift = PUD_SHIFT;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530945 pudp = pud_offset(&pgd, ea);
Christian Borntraegerda1a2882015-01-06 22:47:41 +0100946 pud = READ_ONCE(*pudp);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530947
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530948 if (pud_none(pud))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530949 return NULL;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530950 else if (pud_huge(pud)) {
951 ret_pte = (pte_t *) pudp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530952 goto out;
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530953 } else if (is_hugepd(__hugepd(pud_val(pud))))
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530954 hpdp = (hugepd_t *)&pud;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530955 else {
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530956 pdshift = PMD_SHIFT;
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530957 pmdp = pmd_offset(&pud, ea);
Christian Borntraegerda1a2882015-01-06 22:47:41 +0100958 pmd = READ_ONCE(*pmdp);
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530959 /*
960 * A hugepage collapse is captured by pmd_none, because
961 * it mark the pmd none and do a hpte invalidate.
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530962 */
Aneesh Kumar K.V7d6e7f72015-03-30 10:41:04 +0530963 if (pmd_none(pmd))
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530964 return NULL;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530965
Aneesh Kumar K.V891121e2015-10-09 08:32:21 +0530966 if (pmd_trans_huge(pmd)) {
967 if (is_thp)
968 *is_thp = true;
969 ret_pte = (pte_t *) pmdp;
970 goto out;
971 }
972
973 if (pmd_huge(pmd)) {
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530974 ret_pte = (pte_t *) pmdp;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530975 goto out;
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530976 } else if (is_hugepd(__hugepd(pmd_val(pmd))))
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530977 hpdp = (hugepd_t *)&pmd;
Aneesh Kumar K.Vac52ae42013-06-20 14:30:17 +0530978 else
Aneesh Kumar K.V0ac52dd2013-06-20 14:30:22 +0530979 return pte_offset_kernel(&pmd, ea);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530980 }
981 }
982 if (!hpdp)
983 return NULL;
984
Aneesh Kumar K.Vb30e7592014-11-05 21:57:41 +0530985 ret_pte = hugepte_offset(*hpdp, ea, pdshift);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530986 pdshift = hugepd_shift(*hpdp);
987out:
988 if (shift)
989 *shift = pdshift;
990 return ret_pte;
991}
Aneesh Kumar K.V691e95f2015-03-30 10:41:03 +0530992EXPORT_SYMBOL_GPL(__find_linux_pte_or_hugepte);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +0530993
994int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
995 unsigned long end, int write, struct page **pages, int *nr)
996{
997 unsigned long mask;
998 unsigned long pte_end;
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -0800999 struct page *head, *page;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301000 pte_t pte;
1001 int refs;
1002
1003 pte_end = (addr + sz) & ~(sz-1);
1004 if (pte_end < end)
1005 end = pte_end;
1006
Michael Ellerman4f9c53c2015-03-25 20:11:57 +11001007 pte = READ_ONCE(*ptep);
Aneesh Kumar K.Vac29c642016-04-29 23:25:34 +10001008 mask = _PAGE_PRESENT | _PAGE_READ;
Christophe Leroy6b8cb662016-09-19 12:58:54 +02001009
1010 /*
1011 * On some CPUs like the 8xx, _PAGE_RW hence _PAGE_WRITE is defined
1012 * as 0 and _PAGE_RO has to be set when a page is not writable
1013 */
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301014 if (write)
Aneesh Kumar K.Vc7d54842016-04-29 23:25:30 +10001015 mask |= _PAGE_WRITE;
Christophe Leroy6b8cb662016-09-19 12:58:54 +02001016 else
1017 mask |= _PAGE_RO;
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301018
1019 if ((pte_val(pte) & mask) != mask)
1020 return 0;
1021
1022 /* hugepages are never "special" */
1023 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1024
1025 refs = 0;
1026 head = pte_page(pte);
1027
1028 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301029 do {
1030 VM_BUG_ON(compound_head(page) != head);
1031 pages[*nr] = page;
1032 (*nr)++;
1033 page++;
1034 refs++;
1035 } while (addr += PAGE_SIZE, addr != end);
1036
1037 if (!page_cache_add_speculative(head, refs)) {
1038 *nr -= refs;
1039 return 0;
1040 }
1041
1042 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1043 /* Could be optimized better */
1044 *nr -= refs;
1045 while (refs--)
1046 put_page(head);
1047 return 0;
1048 }
1049
Aneesh Kumar K.V29409992013-06-20 14:30:16 +05301050 return 1;
1051}