blob: afa057a1d3fe6806eb19dcb73b4ae1fe327e47b6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Generic hugetlb support.
3 * (C) William Irwin, April 2004
4 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/list.h>
6#include <linux/init.h>
7#include <linux/module.h>
8#include <linux/mm.h>
Alexey Dobriyane1759c22008-10-15 23:50:22 +04009#include <linux/seq_file.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/sysctl.h>
11#include <linux/highmem.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070012#include <linux/mmu_notifier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/nodemask.h>
David Gibson63551ae2005-06-21 17:14:44 -070014#include <linux/pagemap.h>
Christoph Lameter5da7ca82006-01-06 00:10:46 -080015#include <linux/mempolicy.h>
Christoph Lameteraea47ff2006-01-08 01:00:57 -080016#include <linux/cpuset.h>
David Gibson3935baa2006-03-22 00:08:53 -080017#include <linux/mutex.h>
Andi Kleenaa888a72008-07-23 21:27:47 -070018#include <linux/bootmem.h>
Nishanth Aravamudana3437872008-07-23 21:27:44 -070019#include <linux/sysfs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +090021#include <linux/rmap.h>
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +090022#include <linux/swap.h>
23#include <linux/swapops.h>
Linus Torvaldsd6606682008-08-06 12:04:54 -070024
David Gibson63551ae2005-06-21 17:14:44 -070025#include <asm/page.h>
26#include <asm/pgtable.h>
Chris Forbes32f84522011-07-25 17:12:14 -070027#include <linux/io.h>
David Gibson63551ae2005-06-21 17:14:44 -070028
29#include <linux/hugetlb.h>
Lee Schermerhorn9a305232009-12-14 17:58:25 -080030#include <linux/node.h>
Nick Piggin7835e982006-03-22 00:08:40 -080031#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
Mel Gorman396faf02007-07-17 04:03:13 -070034static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
35unsigned long hugepages_treat_as_movable;
Andi Kleena5516432008-07-23 21:27:41 -070036
Andi Kleene5ff2152008-07-23 21:27:42 -070037static int max_hstate;
38unsigned int default_hstate_idx;
39struct hstate hstates[HUGE_MAX_HSTATE];
40
Jon Tollefson53ba51d2008-07-23 21:27:52 -070041__initdata LIST_HEAD(huge_boot_pages);
42
Andi Kleene5ff2152008-07-23 21:27:42 -070043/* for command line parsing */
44static struct hstate * __initdata parsed_hstate;
45static unsigned long __initdata default_hstate_max_huge_pages;
Nick Piggine11bfbf2008-07-23 21:27:52 -070046static unsigned long __initdata default_hstate_size;
Andi Kleene5ff2152008-07-23 21:27:42 -070047
48#define for_each_hstate(h) \
49 for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
Mel Gorman396faf02007-07-17 04:03:13 -070050
David Gibson3935baa2006-03-22 00:08:53 -080051/*
52 * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
53 */
54static DEFINE_SPINLOCK(hugetlb_lock);
Eric Paris0bd0f9f2005-11-21 21:32:28 -080055
David Gibson90481622012-03-21 16:34:12 -070056static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
57{
58 bool free = (spool->count == 0) && (spool->used_hpages == 0);
59
60 spin_unlock(&spool->lock);
61
62 /* If no pages are used, and no other handles to the subpool
63 * remain, free the subpool the subpool remain */
64 if (free)
65 kfree(spool);
66}
67
68struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
69{
70 struct hugepage_subpool *spool;
71
72 spool = kmalloc(sizeof(*spool), GFP_KERNEL);
73 if (!spool)
74 return NULL;
75
76 spin_lock_init(&spool->lock);
77 spool->count = 1;
78 spool->max_hpages = nr_blocks;
79 spool->used_hpages = 0;
80
81 return spool;
82}
83
84void hugepage_put_subpool(struct hugepage_subpool *spool)
85{
86 spin_lock(&spool->lock);
87 BUG_ON(!spool->count);
88 spool->count--;
89 unlock_or_release_subpool(spool);
90}
91
92static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
93 long delta)
94{
95 int ret = 0;
96
97 if (!spool)
98 return 0;
99
100 spin_lock(&spool->lock);
101 if ((spool->used_hpages + delta) <= spool->max_hpages) {
102 spool->used_hpages += delta;
103 } else {
104 ret = -ENOMEM;
105 }
106 spin_unlock(&spool->lock);
107
108 return ret;
109}
110
111static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
112 long delta)
113{
114 if (!spool)
115 return;
116
117 spin_lock(&spool->lock);
118 spool->used_hpages -= delta;
119 /* If hugetlbfs_put_super couldn't free spool due to
120 * an outstanding quota reference, free it now. */
121 unlock_or_release_subpool(spool);
122}
123
124static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
125{
126 return HUGETLBFS_SB(inode->i_sb)->spool;
127}
128
129static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
130{
131 return subpool_inode(vma->vm_file->f_dentry->d_inode);
132}
133
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700134/*
Andy Whitcroft96822902008-07-23 21:27:29 -0700135 * Region tracking -- allows tracking of reservations and instantiated pages
136 * across the pages in a mapping.
Andy Whitcroft84afd992008-07-23 21:27:32 -0700137 *
138 * The region data structures are protected by a combination of the mmap_sem
139 * and the hugetlb_instantion_mutex. To access or modify a region the caller
140 * must either hold the mmap_sem for write, or the mmap_sem for read and
141 * the hugetlb_instantiation mutex:
142 *
Chris Forbes32f84522011-07-25 17:12:14 -0700143 * down_write(&mm->mmap_sem);
Andy Whitcroft84afd992008-07-23 21:27:32 -0700144 * or
Chris Forbes32f84522011-07-25 17:12:14 -0700145 * down_read(&mm->mmap_sem);
146 * mutex_lock(&hugetlb_instantiation_mutex);
Andy Whitcroft96822902008-07-23 21:27:29 -0700147 */
148struct file_region {
149 struct list_head link;
150 long from;
151 long to;
152};
153
154static long region_add(struct list_head *head, long f, long t)
155{
156 struct file_region *rg, *nrg, *trg;
157
158 /* Locate the region we are either in or before. */
159 list_for_each_entry(rg, head, link)
160 if (f <= rg->to)
161 break;
162
163 /* Round our left edge to the current segment if it encloses us. */
164 if (f > rg->from)
165 f = rg->from;
166
167 /* Check for and consume any regions we now overlap with. */
168 nrg = rg;
169 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
170 if (&rg->link == head)
171 break;
172 if (rg->from > t)
173 break;
174
175 /* If this area reaches higher then extend our area to
176 * include it completely. If this is not the first area
177 * which we intend to reuse, free it. */
178 if (rg->to > t)
179 t = rg->to;
180 if (rg != nrg) {
181 list_del(&rg->link);
182 kfree(rg);
183 }
184 }
185 nrg->from = f;
186 nrg->to = t;
187 return 0;
188}
189
190static long region_chg(struct list_head *head, long f, long t)
191{
192 struct file_region *rg, *nrg;
193 long chg = 0;
194
195 /* Locate the region we are before or in. */
196 list_for_each_entry(rg, head, link)
197 if (f <= rg->to)
198 break;
199
200 /* If we are below the current region then a new region is required.
201 * Subtle, allocate a new region at the position but make it zero
202 * size such that we can guarantee to record the reservation. */
203 if (&rg->link == head || t < rg->from) {
204 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
205 if (!nrg)
206 return -ENOMEM;
207 nrg->from = f;
208 nrg->to = f;
209 INIT_LIST_HEAD(&nrg->link);
210 list_add(&nrg->link, rg->link.prev);
211
212 return t - f;
213 }
214
215 /* Round our left edge to the current segment if it encloses us. */
216 if (f > rg->from)
217 f = rg->from;
218 chg = t - f;
219
220 /* Check for and consume any regions we now overlap with. */
221 list_for_each_entry(rg, rg->link.prev, link) {
222 if (&rg->link == head)
223 break;
224 if (rg->from > t)
225 return chg;
226
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300227 /* We overlap with this area, if it extends further than
Andy Whitcroft96822902008-07-23 21:27:29 -0700228 * us then we must extend ourselves. Account for its
229 * existing reservation. */
230 if (rg->to > t) {
231 chg += rg->to - t;
232 t = rg->to;
233 }
234 chg -= rg->to - rg->from;
235 }
236 return chg;
237}
238
239static long region_truncate(struct list_head *head, long end)
240{
241 struct file_region *rg, *trg;
242 long chg = 0;
243
244 /* Locate the region we are either in or before. */
245 list_for_each_entry(rg, head, link)
246 if (end <= rg->to)
247 break;
248 if (&rg->link == head)
249 return 0;
250
251 /* If we are in the middle of a region then adjust it. */
252 if (end > rg->from) {
253 chg = rg->to - end;
254 rg->to = end;
255 rg = list_entry(rg->link.next, typeof(*rg), link);
256 }
257
258 /* Drop any remaining regions. */
259 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
260 if (&rg->link == head)
261 break;
262 chg += rg->to - rg->from;
263 list_del(&rg->link);
264 kfree(rg);
265 }
266 return chg;
267}
268
Andy Whitcroft84afd992008-07-23 21:27:32 -0700269static long region_count(struct list_head *head, long f, long t)
270{
271 struct file_region *rg;
272 long chg = 0;
273
274 /* Locate each segment we overlap with, and count that overlap. */
275 list_for_each_entry(rg, head, link) {
276 int seg_from;
277 int seg_to;
278
279 if (rg->to <= f)
280 continue;
281 if (rg->from >= t)
282 break;
283
284 seg_from = max(rg->from, f);
285 seg_to = min(rg->to, t);
286
287 chg += seg_to - seg_from;
288 }
289
290 return chg;
291}
292
Andy Whitcroft96822902008-07-23 21:27:29 -0700293/*
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700294 * Convert the address within this vma to the page offset within
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700295 * the mapping, in pagecache page units; huge pages here.
296 */
Andi Kleena5516432008-07-23 21:27:41 -0700297static pgoff_t vma_hugecache_offset(struct hstate *h,
298 struct vm_area_struct *vma, unsigned long address)
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700299{
Andi Kleena5516432008-07-23 21:27:41 -0700300 return ((address - vma->vm_start) >> huge_page_shift(h)) +
301 (vma->vm_pgoff >> huge_page_order(h));
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700302}
303
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900304pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
305 unsigned long address)
306{
307 return vma_hugecache_offset(hstate_vma(vma), vma, address);
308}
309
Andy Whitcroft84afd992008-07-23 21:27:32 -0700310/*
Mel Gorman08fba692009-01-06 14:38:53 -0800311 * Return the size of the pages allocated when backing a VMA. In the majority
312 * cases this will be same size as used by the page table entries.
313 */
314unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
315{
316 struct hstate *hstate;
317
318 if (!is_vm_hugetlb_page(vma))
319 return PAGE_SIZE;
320
321 hstate = hstate_vma(vma);
322
323 return 1UL << (hstate->order + PAGE_SHIFT);
324}
Joerg Roedelf340ca02009-06-19 15:16:22 +0200325EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
Mel Gorman08fba692009-01-06 14:38:53 -0800326
327/*
Mel Gorman33402892009-01-06 14:38:54 -0800328 * Return the page size being used by the MMU to back a VMA. In the majority
329 * of cases, the page size used by the kernel matches the MMU size. On
330 * architectures where it differs, an architecture-specific version of this
331 * function is required.
332 */
333#ifndef vma_mmu_pagesize
334unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
335{
336 return vma_kernel_pagesize(vma);
337}
338#endif
339
340/*
Andy Whitcroft84afd992008-07-23 21:27:32 -0700341 * Flags for MAP_PRIVATE reservations. These are stored in the bottom
342 * bits of the reservation map pointer, which are always clear due to
343 * alignment.
344 */
345#define HPAGE_RESV_OWNER (1UL << 0)
346#define HPAGE_RESV_UNMAPPED (1UL << 1)
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700347#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
Andy Whitcroft84afd992008-07-23 21:27:32 -0700348
Mel Gormana1e78772008-07-23 21:27:23 -0700349/*
350 * These helpers are used to track how many pages are reserved for
351 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
352 * is guaranteed to have their future faults succeed.
353 *
354 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
355 * the reserve counters are updated with the hugetlb_lock held. It is safe
356 * to reset the VMA at fork() time as it is not in use yet and there is no
357 * chance of the global counters getting corrupted as a result of the values.
Andy Whitcroft84afd992008-07-23 21:27:32 -0700358 *
359 * The private mapping reservation is represented in a subtly different
360 * manner to a shared mapping. A shared mapping has a region map associated
361 * with the underlying file, this region map represents the backing file
362 * pages which have ever had a reservation assigned which this persists even
363 * after the page is instantiated. A private mapping has a region map
364 * associated with the original mmap which is attached to all VMAs which
365 * reference it, this region map represents those offsets which have consumed
366 * reservation ie. where pages have been instantiated.
Mel Gormana1e78772008-07-23 21:27:23 -0700367 */
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700368static unsigned long get_vma_private_data(struct vm_area_struct *vma)
369{
370 return (unsigned long)vma->vm_private_data;
371}
372
373static void set_vma_private_data(struct vm_area_struct *vma,
374 unsigned long value)
375{
376 vma->vm_private_data = (void *)value;
377}
378
Andy Whitcroft84afd992008-07-23 21:27:32 -0700379struct resv_map {
380 struct kref refs;
381 struct list_head regions;
382};
383
Harvey Harrison2a4b3de2008-10-18 20:27:06 -0700384static struct resv_map *resv_map_alloc(void)
Andy Whitcroft84afd992008-07-23 21:27:32 -0700385{
386 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
387 if (!resv_map)
388 return NULL;
389
390 kref_init(&resv_map->refs);
391 INIT_LIST_HEAD(&resv_map->regions);
392
393 return resv_map;
394}
395
Harvey Harrison2a4b3de2008-10-18 20:27:06 -0700396static void resv_map_release(struct kref *ref)
Andy Whitcroft84afd992008-07-23 21:27:32 -0700397{
398 struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
399
400 /* Clear out any active regions before we release the map. */
401 region_truncate(&resv_map->regions, 0);
402 kfree(resv_map);
403}
404
405static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
Mel Gormana1e78772008-07-23 21:27:23 -0700406{
407 VM_BUG_ON(!is_vm_hugetlb_page(vma));
Mel Gormanf83a2752009-05-28 14:34:40 -0700408 if (!(vma->vm_flags & VM_MAYSHARE))
Andy Whitcroft84afd992008-07-23 21:27:32 -0700409 return (struct resv_map *)(get_vma_private_data(vma) &
410 ~HPAGE_RESV_MASK);
Harvey Harrison2a4b3de2008-10-18 20:27:06 -0700411 return NULL;
Mel Gormana1e78772008-07-23 21:27:23 -0700412}
413
Andy Whitcroft84afd992008-07-23 21:27:32 -0700414static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
Mel Gormana1e78772008-07-23 21:27:23 -0700415{
416 VM_BUG_ON(!is_vm_hugetlb_page(vma));
Mel Gormanf83a2752009-05-28 14:34:40 -0700417 VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
Mel Gormana1e78772008-07-23 21:27:23 -0700418
Andy Whitcroft84afd992008-07-23 21:27:32 -0700419 set_vma_private_data(vma, (get_vma_private_data(vma) &
420 HPAGE_RESV_MASK) | (unsigned long)map);
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700421}
422
423static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
424{
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700425 VM_BUG_ON(!is_vm_hugetlb_page(vma));
Mel Gormanf83a2752009-05-28 14:34:40 -0700426 VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700427
428 set_vma_private_data(vma, get_vma_private_data(vma) | flags);
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700429}
430
431static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
432{
433 VM_BUG_ON(!is_vm_hugetlb_page(vma));
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -0700434
435 return (get_vma_private_data(vma) & flag) != 0;
Mel Gormana1e78772008-07-23 21:27:23 -0700436}
437
438/* Decrement the reserved pages in the hugepage pool by one */
Andi Kleena5516432008-07-23 21:27:41 -0700439static void decrement_hugepage_resv_vma(struct hstate *h,
440 struct vm_area_struct *vma)
Mel Gormana1e78772008-07-23 21:27:23 -0700441{
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -0700442 if (vma->vm_flags & VM_NORESERVE)
443 return;
444
Mel Gormanf83a2752009-05-28 14:34:40 -0700445 if (vma->vm_flags & VM_MAYSHARE) {
Mel Gormana1e78772008-07-23 21:27:23 -0700446 /* Shared mappings always use reserves */
Andi Kleena5516432008-07-23 21:27:41 -0700447 h->resv_huge_pages--;
Andy Whitcroft84afd992008-07-23 21:27:32 -0700448 } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
Mel Gormana1e78772008-07-23 21:27:23 -0700449 /*
450 * Only the process that called mmap() has reserves for
451 * private mappings.
452 */
Andi Kleena5516432008-07-23 21:27:41 -0700453 h->resv_huge_pages--;
Mel Gormana1e78772008-07-23 21:27:23 -0700454 }
455}
456
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700457/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
Mel Gormana1e78772008-07-23 21:27:23 -0700458void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
459{
460 VM_BUG_ON(!is_vm_hugetlb_page(vma));
Mel Gormanf83a2752009-05-28 14:34:40 -0700461 if (!(vma->vm_flags & VM_MAYSHARE))
Mel Gormana1e78772008-07-23 21:27:23 -0700462 vma->vm_private_data = (void *)0;
463}
464
465/* Returns true if the VMA has associated reserve pages */
Mel Gorman7f09ca52008-07-23 21:27:58 -0700466static int vma_has_reserves(struct vm_area_struct *vma)
Mel Gormana1e78772008-07-23 21:27:23 -0700467{
Mel Gormanf83a2752009-05-28 14:34:40 -0700468 if (vma->vm_flags & VM_MAYSHARE)
Mel Gorman7f09ca52008-07-23 21:27:58 -0700469 return 1;
470 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
471 return 1;
472 return 0;
Mel Gormana1e78772008-07-23 21:27:23 -0700473}
474
Naoya Horiguchi0ebabb42010-09-08 10:19:34 +0900475static void copy_gigantic_page(struct page *dst, struct page *src)
476{
477 int i;
478 struct hstate *h = page_hstate(src);
479 struct page *dst_base = dst;
480 struct page *src_base = src;
481
482 for (i = 0; i < pages_per_huge_page(h); ) {
483 cond_resched();
484 copy_highpage(dst, src);
485
486 i++;
487 dst = mem_map_next(dst, dst_base, i);
488 src = mem_map_next(src, src_base, i);
489 }
490}
491
492void copy_huge_page(struct page *dst, struct page *src)
493{
494 int i;
495 struct hstate *h = page_hstate(src);
496
497 if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
498 copy_gigantic_page(dst, src);
499 return;
500 }
501
502 might_sleep();
503 for (i = 0; i < pages_per_huge_page(h); i++) {
504 cond_resched();
505 copy_highpage(dst + i, src + i);
506 }
507}
508
Andi Kleena5516432008-07-23 21:27:41 -0700509static void enqueue_huge_page(struct hstate *h, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
511 int nid = page_to_nid(page);
Andi Kleena5516432008-07-23 21:27:41 -0700512 list_add(&page->lru, &h->hugepage_freelists[nid]);
513 h->free_huge_pages++;
514 h->free_huge_pages_node[nid]++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515}
516
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900517static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
518{
519 struct page *page;
520
521 if (list_empty(&h->hugepage_freelists[nid]))
522 return NULL;
523 page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
524 list_del(&page->lru);
Naoya Horiguchia9869b82010-09-08 10:19:37 +0900525 set_page_refcounted(page);
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900526 h->free_huge_pages--;
527 h->free_huge_pages_node[nid]--;
528 return page;
529}
530
Andi Kleena5516432008-07-23 21:27:41 -0700531static struct page *dequeue_huge_page_vma(struct hstate *h,
532 struct vm_area_struct *vma,
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700533 unsigned long address, int avoid_reserve)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534{
Mel Gormancc9a6c82012-03-21 16:34:11 -0700535 struct page *page;
Lee Schermerhorn480eccf2007-09-18 22:46:47 -0700536 struct mempolicy *mpol;
Mel Gorman19770b32008-04-28 02:12:18 -0700537 nodemask_t *nodemask;
Miao Xiec0ff7452010-05-24 14:32:08 -0700538 struct zonelist *zonelist;
Mel Gormandd1a2392008-04-28 02:12:17 -0700539 struct zone *zone;
540 struct zoneref *z;
Mel Gormancc9a6c82012-03-21 16:34:11 -0700541 unsigned int cpuset_mems_cookie;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542
Mel Gormancc9a6c82012-03-21 16:34:11 -0700543retry_cpuset:
544 cpuset_mems_cookie = get_mems_allowed();
Miao Xiec0ff7452010-05-24 14:32:08 -0700545 zonelist = huge_zonelist(vma, address,
546 htlb_alloc_mask, &mpol, &nodemask);
Mel Gormana1e78772008-07-23 21:27:23 -0700547 /*
548 * A child process with MAP_PRIVATE mappings created by their parent
549 * have no page reserves. This check ensures that reservations are
550 * not "stolen". The child may still get SIGKILLed
551 */
Mel Gorman7f09ca52008-07-23 21:27:58 -0700552 if (!vma_has_reserves(vma) &&
Andi Kleena5516432008-07-23 21:27:41 -0700553 h->free_huge_pages - h->resv_huge_pages == 0)
Miao Xiec0ff7452010-05-24 14:32:08 -0700554 goto err;
Mel Gormana1e78772008-07-23 21:27:23 -0700555
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700556 /* If reserves cannot be used, ensure enough pages are in the pool */
Andi Kleena5516432008-07-23 21:27:41 -0700557 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
Justin P. Mattock6eab04a2011-04-08 19:49:08 -0700558 goto err;
Mel Gorman04f2cbe2008-07-23 21:27:25 -0700559
Mel Gorman19770b32008-04-28 02:12:18 -0700560 for_each_zone_zonelist_nodemask(zone, z, zonelist,
561 MAX_NR_ZONES - 1, nodemask) {
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900562 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) {
563 page = dequeue_huge_page_node(h, zone_to_nid(zone));
564 if (page) {
565 if (!avoid_reserve)
566 decrement_hugepage_resv_vma(h, vma);
567 break;
568 }
Andrew Morton3abf7af2007-07-19 01:49:08 -0700569 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 }
Mel Gormancc9a6c82012-03-21 16:34:11 -0700571
572 mpol_cond_put(mpol);
573 if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
574 goto retry_cpuset;
575 return page;
576
Miao Xiec0ff7452010-05-24 14:32:08 -0700577err:
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700578 mpol_cond_put(mpol);
Mel Gormancc9a6c82012-03-21 16:34:11 -0700579 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580}
581
Andi Kleena5516432008-07-23 21:27:41 -0700582static void update_and_free_page(struct hstate *h, struct page *page)
Adam Litke6af2acb2007-10-16 01:26:16 -0700583{
584 int i;
Andi Kleena5516432008-07-23 21:27:41 -0700585
Andy Whitcroft18229df2008-11-06 12:53:27 -0800586 VM_BUG_ON(h->order >= MAX_ORDER);
587
Andi Kleena5516432008-07-23 21:27:41 -0700588 h->nr_huge_pages--;
589 h->nr_huge_pages_node[page_to_nid(page)]--;
590 for (i = 0; i < pages_per_huge_page(h); i++) {
Chris Forbes32f84522011-07-25 17:12:14 -0700591 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
592 1 << PG_referenced | 1 << PG_dirty |
593 1 << PG_active | 1 << PG_reserved |
594 1 << PG_private | 1 << PG_writeback);
Adam Litke6af2acb2007-10-16 01:26:16 -0700595 }
596 set_compound_page_dtor(page, NULL);
597 set_page_refcounted(page);
Gerald Schaefer7f2e9522008-04-28 02:13:29 -0700598 arch_release_hugepage(page);
Andi Kleena5516432008-07-23 21:27:41 -0700599 __free_pages(page, huge_page_order(h));
Adam Litke6af2acb2007-10-16 01:26:16 -0700600}
601
Andi Kleene5ff2152008-07-23 21:27:42 -0700602struct hstate *size_to_hstate(unsigned long size)
603{
604 struct hstate *h;
605
606 for_each_hstate(h) {
607 if (huge_page_size(h) == size)
608 return h;
609 }
610 return NULL;
611}
612
David Gibson27a85ef2006-03-22 00:08:56 -0800613static void free_huge_page(struct page *page)
614{
Andi Kleena5516432008-07-23 21:27:41 -0700615 /*
616 * Can't pass hstate in here because it is called from the
617 * compound page destructor.
618 */
Andi Kleene5ff2152008-07-23 21:27:42 -0700619 struct hstate *h = page_hstate(page);
Adam Litke7893d1d2007-10-16 01:26:18 -0700620 int nid = page_to_nid(page);
David Gibson90481622012-03-21 16:34:12 -0700621 struct hugepage_subpool *spool =
622 (struct hugepage_subpool *)page_private(page);
David Gibson27a85ef2006-03-22 00:08:56 -0800623
Andy Whitcrofte5df70a2008-02-23 15:23:32 -0800624 set_page_private(page, 0);
Mel Gorman23be7462010-04-23 13:17:56 -0400625 page->mapping = NULL;
Adam Litke7893d1d2007-10-16 01:26:18 -0700626 BUG_ON(page_count(page));
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900627 BUG_ON(page_mapcount(page));
David Gibson27a85ef2006-03-22 00:08:56 -0800628 INIT_LIST_HEAD(&page->lru);
629
630 spin_lock(&hugetlb_lock);
Andi Kleenaa888a72008-07-23 21:27:47 -0700631 if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
Andi Kleena5516432008-07-23 21:27:41 -0700632 update_and_free_page(h, page);
633 h->surplus_huge_pages--;
634 h->surplus_huge_pages_node[nid]--;
Adam Litke7893d1d2007-10-16 01:26:18 -0700635 } else {
Andi Kleena5516432008-07-23 21:27:41 -0700636 enqueue_huge_page(h, page);
Adam Litke7893d1d2007-10-16 01:26:18 -0700637 }
David Gibson27a85ef2006-03-22 00:08:56 -0800638 spin_unlock(&hugetlb_lock);
David Gibson90481622012-03-21 16:34:12 -0700639 hugepage_subpool_put_pages(spool, 1);
David Gibson27a85ef2006-03-22 00:08:56 -0800640}
641
Andi Kleena5516432008-07-23 21:27:41 -0700642static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
Andi Kleenb7ba30c2008-07-23 21:27:40 -0700643{
644 set_compound_page_dtor(page, free_huge_page);
645 spin_lock(&hugetlb_lock);
Andi Kleena5516432008-07-23 21:27:41 -0700646 h->nr_huge_pages++;
647 h->nr_huge_pages_node[nid]++;
Andi Kleenb7ba30c2008-07-23 21:27:40 -0700648 spin_unlock(&hugetlb_lock);
649 put_page(page); /* free it into the hugepage allocator */
650}
651
Wu Fengguang20a03072009-06-16 15:32:22 -0700652static void prep_compound_gigantic_page(struct page *page, unsigned long order)
653{
654 int i;
655 int nr_pages = 1 << order;
656 struct page *p = page + 1;
657
658 /* we rely on prep_new_huge_page to set the destructor */
659 set_compound_order(page, order);
660 __SetPageHead(page);
661 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
662 __SetPageTail(p);
Youquan Song58a84aa2011-12-08 14:34:18 -0800663 set_page_count(p, 0);
Wu Fengguang20a03072009-06-16 15:32:22 -0700664 p->first_page = page;
665 }
666}
667
668int PageHuge(struct page *page)
669{
670 compound_page_dtor *dtor;
671
672 if (!PageCompound(page))
673 return 0;
674
675 page = compound_head(page);
676 dtor = get_compound_page_dtor(page);
677
678 return dtor == free_huge_page;
679}
Naoya Horiguchi43131e12010-05-28 09:29:22 +0900680EXPORT_SYMBOL_GPL(PageHuge);
681
Andi Kleena5516432008-07-23 21:27:41 -0700682static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 struct page *page;
Joe Jinf96efd52007-07-15 23:38:12 -0700685
Andi Kleenaa888a72008-07-23 21:27:47 -0700686 if (h->order >= MAX_ORDER)
687 return NULL;
688
Mel Gorman6484eb32009-06-16 15:31:54 -0700689 page = alloc_pages_exact_node(nid,
Nishanth Aravamudan551883a2008-04-29 00:58:26 -0700690 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
691 __GFP_REPEAT|__GFP_NOWARN,
Andi Kleena5516432008-07-23 21:27:41 -0700692 huge_page_order(h));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 if (page) {
Gerald Schaefer7f2e9522008-04-28 02:13:29 -0700694 if (arch_prepare_hugepage(page)) {
Gerald Schaefercaff3a22008-08-12 15:08:38 -0700695 __free_pages(page, huge_page_order(h));
Harvey Harrison7b8ee842008-04-28 14:13:19 -0700696 return NULL;
Gerald Schaefer7f2e9522008-04-28 02:13:29 -0700697 }
Andi Kleena5516432008-07-23 21:27:41 -0700698 prep_new_huge_page(h, page, nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 }
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700700
701 return page;
702}
703
Andi Kleen5ced66c2008-07-23 21:27:45 -0700704/*
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800705 * common helper functions for hstate_next_node_to_{alloc|free}.
706 * We may have allocated or freed a huge page based on a different
707 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
708 * be outside of *nodes_allowed. Ensure that we use an allowed
709 * node for alloc or free.
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800710 */
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800711static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800712{
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800713 nid = next_node(nid, *nodes_allowed);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800714 if (nid == MAX_NUMNODES)
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800715 nid = first_node(*nodes_allowed);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800716 VM_BUG_ON(nid >= MAX_NUMNODES);
717
718 return nid;
719}
720
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800721static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
Andi Kleen5ced66c2008-07-23 21:27:45 -0700722{
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800723 if (!node_isset(nid, *nodes_allowed))
724 nid = next_node_allowed(nid, nodes_allowed);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800725 return nid;
Andi Kleen5ced66c2008-07-23 21:27:45 -0700726}
727
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800728/*
729 * returns the previously saved node ["this node"] from which to
730 * allocate a persistent huge page for the pool and advance the
731 * next node from which to allocate, handling wrap at end of node
732 * mask.
733 */
734static int hstate_next_node_to_alloc(struct hstate *h,
735 nodemask_t *nodes_allowed)
736{
737 int nid;
738
739 VM_BUG_ON(!nodes_allowed);
740
741 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
742 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
743
744 return nid;
745}
746
747static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700748{
749 struct page *page;
750 int start_nid;
751 int next_nid;
752 int ret = 0;
753
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800754 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700755 next_nid = start_nid;
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700756
757 do {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700758 page = alloc_fresh_huge_page_node(h, next_nid);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800759 if (page) {
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700760 ret = 1;
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800761 break;
762 }
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800763 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800764 } while (next_nid != start_nid);
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700765
Adam Litke3b116302008-04-28 02:13:06 -0700766 if (ret)
767 count_vm_event(HTLB_BUDDY_PGALLOC);
768 else
769 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
770
Nishanth Aravamudan63b46132007-10-16 01:26:24 -0700771 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772}
773
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700774/*
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800775 * helper for free_pool_huge_page() - return the previously saved
776 * node ["this node"] from which to free a huge page. Advance the
777 * next node id whether or not we find a free huge page to free so
778 * that the next attempt to free addresses the next node.
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700779 */
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800780static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700781{
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800782 int nid;
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800783
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800784 VM_BUG_ON(!nodes_allowed);
785
786 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
787 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
788
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800789 return nid;
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700790}
791
792/*
793 * Free huge page from pool from next node to free.
794 * Attempt to keep persistent huge pages more or less
795 * balanced over allowed nodes.
796 * Called with hugetlb_lock locked.
797 */
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800798static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
799 bool acct_surplus)
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700800{
801 int start_nid;
802 int next_nid;
803 int ret = 0;
804
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800805 start_nid = hstate_next_node_to_free(h, nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700806 next_nid = start_nid;
807
808 do {
Lee Schermerhorn685f3452009-09-21 17:01:23 -0700809 /*
810 * If we're returning unused surplus pages, only examine
811 * nodes with surplus pages.
812 */
813 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
814 !list_empty(&h->hugepage_freelists[next_nid])) {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700815 struct page *page =
816 list_entry(h->hugepage_freelists[next_nid].next,
817 struct page, lru);
818 list_del(&page->lru);
819 h->free_huge_pages--;
820 h->free_huge_pages_node[next_nid]--;
Lee Schermerhorn685f3452009-09-21 17:01:23 -0700821 if (acct_surplus) {
822 h->surplus_huge_pages--;
823 h->surplus_huge_pages_node[next_nid]--;
824 }
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700825 update_and_free_page(h, page);
826 ret = 1;
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800827 break;
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700828 }
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -0800829 next_nid = hstate_next_node_to_free(h, nodes_allowed);
Lee Schermerhorn9a76db02009-12-14 17:58:15 -0800830 } while (next_nid != start_nid);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -0700831
832 return ret;
833}
834
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900835static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
Adam Litke7893d1d2007-10-16 01:26:18 -0700836{
837 struct page *page;
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900838 unsigned int r_nid;
Adam Litke7893d1d2007-10-16 01:26:18 -0700839
Andi Kleenaa888a72008-07-23 21:27:47 -0700840 if (h->order >= MAX_ORDER)
841 return NULL;
842
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800843 /*
844 * Assume we will successfully allocate the surplus page to
845 * prevent racing processes from causing the surplus to exceed
846 * overcommit
847 *
848 * This however introduces a different race, where a process B
849 * tries to grow the static hugepage pool while alloc_pages() is
850 * called by process A. B will only examine the per-node
851 * counters in determining if surplus huge pages can be
852 * converted to normal huge pages in adjust_pool_surplus(). A
853 * won't be able to increment the per-node counter, until the
854 * lock is dropped by B, but B doesn't drop hugetlb_lock until
855 * no more huge pages can be converted from surplus to normal
856 * state (and doesn't try to convert again). Thus, we have a
857 * case where a surplus huge page exists, the pool is grown, and
858 * the surplus huge page still exists after, even though it
859 * should just have been converted to a normal huge page. This
860 * does not leak memory, though, as the hugepage will be freed
861 * once it is out of use. It also does not allow the counters to
862 * go out of whack in adjust_pool_surplus() as we don't modify
863 * the node values until we've gotten the hugepage and only the
864 * per-node value is checked there.
865 */
866 spin_lock(&hugetlb_lock);
Andi Kleena5516432008-07-23 21:27:41 -0700867 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800868 spin_unlock(&hugetlb_lock);
869 return NULL;
870 } else {
Andi Kleena5516432008-07-23 21:27:41 -0700871 h->nr_huge_pages++;
872 h->surplus_huge_pages++;
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800873 }
874 spin_unlock(&hugetlb_lock);
875
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900876 if (nid == NUMA_NO_NODE)
877 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
878 __GFP_REPEAT|__GFP_NOWARN,
879 huge_page_order(h));
880 else
881 page = alloc_pages_exact_node(nid,
882 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
883 __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800884
Gerald Schaefercaff3a22008-08-12 15:08:38 -0700885 if (page && arch_prepare_hugepage(page)) {
886 __free_pages(page, huge_page_order(h));
Hillf Dantonea5768c2012-01-10 15:08:30 -0800887 page = NULL;
Gerald Schaefercaff3a22008-08-12 15:08:38 -0700888 }
889
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800890 spin_lock(&hugetlb_lock);
Adam Litke7893d1d2007-10-16 01:26:18 -0700891 if (page) {
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900892 r_nid = page_to_nid(page);
Adam Litke7893d1d2007-10-16 01:26:18 -0700893 set_compound_page_dtor(page, free_huge_page);
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800894 /*
895 * We incremented the global counters already
896 */
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900897 h->nr_huge_pages_node[r_nid]++;
898 h->surplus_huge_pages_node[r_nid]++;
Adam Litke3b116302008-04-28 02:13:06 -0700899 __count_vm_event(HTLB_BUDDY_PGALLOC);
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800900 } else {
Andi Kleena5516432008-07-23 21:27:41 -0700901 h->nr_huge_pages--;
902 h->surplus_huge_pages--;
Adam Litke3b116302008-04-28 02:13:06 -0700903 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
Adam Litke7893d1d2007-10-16 01:26:18 -0700904 }
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -0800905 spin_unlock(&hugetlb_lock);
Adam Litke7893d1d2007-10-16 01:26:18 -0700906
907 return page;
908}
909
Adam Litkee4e574b2007-10-16 01:26:19 -0700910/*
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900911 * This allocation function is useful in the context where vma is irrelevant.
912 * E.g. soft-offlining uses this function because it only cares physical
913 * address of error page.
914 */
915struct page *alloc_huge_page_node(struct hstate *h, int nid)
916{
917 struct page *page;
918
919 spin_lock(&hugetlb_lock);
920 page = dequeue_huge_page_node(h, nid);
921 spin_unlock(&hugetlb_lock);
922
923 if (!page)
924 page = alloc_buddy_huge_page(h, nid);
925
926 return page;
927}
928
929/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300930 * Increase the hugetlb pool such that it can accommodate a reservation
Adam Litkee4e574b2007-10-16 01:26:19 -0700931 * of size 'delta'.
932 */
Andi Kleena5516432008-07-23 21:27:41 -0700933static int gather_surplus_pages(struct hstate *h, int delta)
Adam Litkee4e574b2007-10-16 01:26:19 -0700934{
935 struct list_head surplus_list;
936 struct page *page, *tmp;
937 int ret, i;
938 int needed, allocated;
Hillf Danton28073b02012-03-21 16:34:00 -0700939 bool alloc_ok = true;
Adam Litkee4e574b2007-10-16 01:26:19 -0700940
Andi Kleena5516432008-07-23 21:27:41 -0700941 needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
Adam Litkeac09b3a2008-03-04 14:29:38 -0800942 if (needed <= 0) {
Andi Kleena5516432008-07-23 21:27:41 -0700943 h->resv_huge_pages += delta;
Adam Litkee4e574b2007-10-16 01:26:19 -0700944 return 0;
Adam Litkeac09b3a2008-03-04 14:29:38 -0800945 }
Adam Litkee4e574b2007-10-16 01:26:19 -0700946
947 allocated = 0;
948 INIT_LIST_HEAD(&surplus_list);
949
950 ret = -ENOMEM;
951retry:
952 spin_unlock(&hugetlb_lock);
953 for (i = 0; i < needed; i++) {
Naoya Horiguchibf50bab2010-09-08 10:19:33 +0900954 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
Hillf Danton28073b02012-03-21 16:34:00 -0700955 if (!page) {
956 alloc_ok = false;
957 break;
958 }
Adam Litkee4e574b2007-10-16 01:26:19 -0700959 list_add(&page->lru, &surplus_list);
960 }
Hillf Danton28073b02012-03-21 16:34:00 -0700961 allocated += i;
Adam Litkee4e574b2007-10-16 01:26:19 -0700962
963 /*
964 * After retaking hugetlb_lock, we need to recalculate 'needed'
965 * because either resv_huge_pages or free_huge_pages may have changed.
966 */
967 spin_lock(&hugetlb_lock);
Andi Kleena5516432008-07-23 21:27:41 -0700968 needed = (h->resv_huge_pages + delta) -
969 (h->free_huge_pages + allocated);
Hillf Danton28073b02012-03-21 16:34:00 -0700970 if (needed > 0) {
971 if (alloc_ok)
972 goto retry;
973 /*
974 * We were not able to allocate enough pages to
975 * satisfy the entire reservation so we free what
976 * we've allocated so far.
977 */
978 goto free;
979 }
Adam Litkee4e574b2007-10-16 01:26:19 -0700980 /*
981 * The surplus_list now contains _at_least_ the number of extra pages
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300982 * needed to accommodate the reservation. Add the appropriate number
Adam Litkee4e574b2007-10-16 01:26:19 -0700983 * of pages to the hugetlb pool and free the extras back to the buddy
Adam Litkeac09b3a2008-03-04 14:29:38 -0800984 * allocator. Commit the entire reservation here to prevent another
985 * process from stealing the pages as they are added to the pool but
986 * before they are reserved.
Adam Litkee4e574b2007-10-16 01:26:19 -0700987 */
988 needed += allocated;
Andi Kleena5516432008-07-23 21:27:41 -0700989 h->resv_huge_pages += delta;
Adam Litkee4e574b2007-10-16 01:26:19 -0700990 ret = 0;
Naoya Horiguchia9869b82010-09-08 10:19:37 +0900991
Adam Litke19fc3f02008-04-28 02:12:20 -0700992 /* Free the needed pages to the hugetlb pool */
Adam Litkee4e574b2007-10-16 01:26:19 -0700993 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
Adam Litke19fc3f02008-04-28 02:12:20 -0700994 if ((--needed) < 0)
995 break;
Adam Litkee4e574b2007-10-16 01:26:19 -0700996 list_del(&page->lru);
Naoya Horiguchia9869b82010-09-08 10:19:37 +0900997 /*
998 * This page is now managed by the hugetlb allocator and has
999 * no users -- drop the buddy allocator's reference.
1000 */
1001 put_page_testzero(page);
1002 VM_BUG_ON(page_count(page));
Andi Kleena5516432008-07-23 21:27:41 -07001003 enqueue_huge_page(h, page);
Adam Litke19fc3f02008-04-28 02:12:20 -07001004 }
Hillf Danton28073b02012-03-21 16:34:00 -07001005free:
Hillf Dantonb0365c82011-12-28 15:57:16 -08001006 spin_unlock(&hugetlb_lock);
Adam Litke19fc3f02008-04-28 02:12:20 -07001007
1008 /* Free unnecessary surplus pages to the buddy allocator */
1009 if (!list_empty(&surplus_list)) {
Adam Litke19fc3f02008-04-28 02:12:20 -07001010 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1011 list_del(&page->lru);
Naoya Horiguchia9869b82010-09-08 10:19:37 +09001012 put_page(page);
Adam Litkeaf767cb2007-10-16 01:26:25 -07001013 }
Adam Litkee4e574b2007-10-16 01:26:19 -07001014 }
Naoya Horiguchia9869b82010-09-08 10:19:37 +09001015 spin_lock(&hugetlb_lock);
Adam Litkee4e574b2007-10-16 01:26:19 -07001016
1017 return ret;
1018}
1019
1020/*
1021 * When releasing a hugetlb pool reservation, any surplus pages that were
1022 * allocated to satisfy the reservation must be explicitly freed if they were
1023 * never used.
Lee Schermerhorn685f3452009-09-21 17:01:23 -07001024 * Called with hugetlb_lock held.
Adam Litkee4e574b2007-10-16 01:26:19 -07001025 */
Andi Kleena5516432008-07-23 21:27:41 -07001026static void return_unused_surplus_pages(struct hstate *h,
1027 unsigned long unused_resv_pages)
Adam Litkee4e574b2007-10-16 01:26:19 -07001028{
Adam Litkee4e574b2007-10-16 01:26:19 -07001029 unsigned long nr_pages;
1030
Adam Litkeac09b3a2008-03-04 14:29:38 -08001031 /* Uncommit the reservation */
Andi Kleena5516432008-07-23 21:27:41 -07001032 h->resv_huge_pages -= unused_resv_pages;
Adam Litkeac09b3a2008-03-04 14:29:38 -08001033
Andi Kleenaa888a72008-07-23 21:27:47 -07001034 /* Cannot return gigantic pages currently */
1035 if (h->order >= MAX_ORDER)
1036 return;
1037
Andi Kleena5516432008-07-23 21:27:41 -07001038 nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
Adam Litkee4e574b2007-10-16 01:26:19 -07001039
Lee Schermerhorn685f3452009-09-21 17:01:23 -07001040 /*
1041 * We want to release as many surplus pages as possible, spread
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001042 * evenly across all nodes with memory. Iterate across these nodes
1043 * until we can no longer free unreserved surplus pages. This occurs
1044 * when the nodes with surplus pages have no free pages.
1045 * free_pool_huge_page() will balance the the freed pages across the
1046 * on-line nodes with memory and will handle the hstate accounting.
Lee Schermerhorn685f3452009-09-21 17:01:23 -07001047 */
1048 while (nr_pages--) {
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001049 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
Lee Schermerhorn685f3452009-09-21 17:01:23 -07001050 break;
Adam Litkee4e574b2007-10-16 01:26:19 -07001051 }
1052}
1053
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001054/*
1055 * Determine if the huge page at addr within the vma has an associated
1056 * reservation. Where it does not we will need to logically increase
David Gibson90481622012-03-21 16:34:12 -07001057 * reservation and actually increase subpool usage before an allocation
1058 * can occur. Where any new reservation would be required the
1059 * reservation change is prepared, but not committed. Once the page
1060 * has been allocated from the subpool and instantiated the change should
1061 * be committed via vma_commit_reservation. No action is required on
1062 * failure.
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001063 */
Roel Kluine2f17d92009-03-31 15:23:15 -07001064static long vma_needs_reservation(struct hstate *h,
Andi Kleena5516432008-07-23 21:27:41 -07001065 struct vm_area_struct *vma, unsigned long addr)
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001066{
1067 struct address_space *mapping = vma->vm_file->f_mapping;
1068 struct inode *inode = mapping->host;
1069
Mel Gormanf83a2752009-05-28 14:34:40 -07001070 if (vma->vm_flags & VM_MAYSHARE) {
Andi Kleena5516432008-07-23 21:27:41 -07001071 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001072 return region_chg(&inode->i_mapping->private_list,
1073 idx, idx + 1);
1074
Andy Whitcroft84afd992008-07-23 21:27:32 -07001075 } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1076 return 1;
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001077
Andy Whitcroft84afd992008-07-23 21:27:32 -07001078 } else {
Roel Kluine2f17d92009-03-31 15:23:15 -07001079 long err;
Andi Kleena5516432008-07-23 21:27:41 -07001080 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
Andy Whitcroft84afd992008-07-23 21:27:32 -07001081 struct resv_map *reservations = vma_resv_map(vma);
1082
1083 err = region_chg(&reservations->regions, idx, idx + 1);
1084 if (err < 0)
1085 return err;
1086 return 0;
1087 }
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001088}
Andi Kleena5516432008-07-23 21:27:41 -07001089static void vma_commit_reservation(struct hstate *h,
1090 struct vm_area_struct *vma, unsigned long addr)
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001091{
1092 struct address_space *mapping = vma->vm_file->f_mapping;
1093 struct inode *inode = mapping->host;
1094
Mel Gormanf83a2752009-05-28 14:34:40 -07001095 if (vma->vm_flags & VM_MAYSHARE) {
Andi Kleena5516432008-07-23 21:27:41 -07001096 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001097 region_add(&inode->i_mapping->private_list, idx, idx + 1);
Andy Whitcroft84afd992008-07-23 21:27:32 -07001098
1099 } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
Andi Kleena5516432008-07-23 21:27:41 -07001100 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
Andy Whitcroft84afd992008-07-23 21:27:32 -07001101 struct resv_map *reservations = vma_resv_map(vma);
1102
1103 /* Mark this page used in the map. */
1104 region_add(&reservations->regions, idx, idx + 1);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001105 }
1106}
1107
David Gibson27a85ef2006-03-22 00:08:56 -08001108static struct page *alloc_huge_page(struct vm_area_struct *vma,
Mel Gorman04f2cbe2008-07-23 21:27:25 -07001109 unsigned long addr, int avoid_reserve)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110{
David Gibson90481622012-03-21 16:34:12 -07001111 struct hugepage_subpool *spool = subpool_vma(vma);
Andi Kleena5516432008-07-23 21:27:41 -07001112 struct hstate *h = hstate_vma(vma);
Adam Litke348ea202007-11-14 16:59:37 -08001113 struct page *page;
Roel Kluine2f17d92009-03-31 15:23:15 -07001114 long chg;
Adam Litke2fc39ce2007-11-14 16:59:39 -08001115
Mel Gormana1e78772008-07-23 21:27:23 -07001116 /*
David Gibson90481622012-03-21 16:34:12 -07001117 * Processes that did not create the mapping will have no
1118 * reserves and will not have accounted against subpool
1119 * limit. Check that the subpool limit can be made before
1120 * satisfying the allocation MAP_NORESERVE mappings may also
1121 * need pages and subpool limit allocated allocated if no reserve
1122 * mapping overlaps.
Mel Gormana1e78772008-07-23 21:27:23 -07001123 */
Andi Kleena5516432008-07-23 21:27:41 -07001124 chg = vma_needs_reservation(h, vma, addr);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001125 if (chg < 0)
Hugh Dickinse0dcd8a2011-06-05 22:03:13 -07001126 return ERR_PTR(-VM_FAULT_OOM);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001127 if (chg)
David Gibson90481622012-03-21 16:34:12 -07001128 if (hugepage_subpool_get_pages(spool, chg))
Hugh Dickinse0dcd8a2011-06-05 22:03:13 -07001129 return ERR_PTR(-VM_FAULT_SIGBUS);
Mel Gormana1e78772008-07-23 21:27:23 -07001130
1131 spin_lock(&hugetlb_lock);
Andi Kleena5516432008-07-23 21:27:41 -07001132 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
Mel Gormana1e78772008-07-23 21:27:23 -07001133 spin_unlock(&hugetlb_lock);
1134
1135 if (!page) {
Naoya Horiguchibf50bab2010-09-08 10:19:33 +09001136 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
Mel Gormana1e78772008-07-23 21:27:23 -07001137 if (!page) {
David Gibson90481622012-03-21 16:34:12 -07001138 hugepage_subpool_put_pages(spool, chg);
Mel Gorman4a6018f2010-05-11 14:06:53 -07001139 return ERR_PTR(-VM_FAULT_SIGBUS);
Mel Gormana1e78772008-07-23 21:27:23 -07001140 }
1141 }
1142
David Gibson90481622012-03-21 16:34:12 -07001143 set_page_private(page, (unsigned long)spool);
Mel Gormana1e78772008-07-23 21:27:23 -07001144
Andi Kleena5516432008-07-23 21:27:41 -07001145 vma_commit_reservation(h, vma, addr);
Andy Whitcroftc37f9fb2008-07-23 21:27:30 -07001146
Adam Litke90d8b7e2007-11-14 16:59:42 -08001147 return page;
David Gibsonb45b5bd2006-03-22 00:08:55 -08001148}
1149
Cyrill Gorcunov91f47662009-01-06 14:40:33 -08001150int __weak alloc_bootmem_huge_page(struct hstate *h)
Andi Kleenaa888a72008-07-23 21:27:47 -07001151{
1152 struct huge_bootmem_page *m;
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001153 int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
Andi Kleenaa888a72008-07-23 21:27:47 -07001154
1155 while (nr_nodes) {
1156 void *addr;
1157
1158 addr = __alloc_bootmem_node_nopanic(
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001159 NODE_DATA(hstate_next_node_to_alloc(h,
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001160 &node_states[N_HIGH_MEMORY])),
Andi Kleenaa888a72008-07-23 21:27:47 -07001161 huge_page_size(h), huge_page_size(h), 0);
1162
1163 if (addr) {
1164 /*
1165 * Use the beginning of the huge page to store the
1166 * huge_bootmem_page struct (until gather_bootmem
1167 * puts them into the mem_map).
1168 */
1169 m = addr;
Cyrill Gorcunov91f47662009-01-06 14:40:33 -08001170 goto found;
Andi Kleenaa888a72008-07-23 21:27:47 -07001171 }
Andi Kleenaa888a72008-07-23 21:27:47 -07001172 nr_nodes--;
1173 }
1174 return 0;
1175
1176found:
1177 BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1178 /* Put them into a private list first because mem_map is not up yet */
1179 list_add(&m->list, &huge_boot_pages);
1180 m->hstate = h;
1181 return 1;
1182}
1183
Andy Whitcroft18229df2008-11-06 12:53:27 -08001184static void prep_compound_huge_page(struct page *page, int order)
1185{
1186 if (unlikely(order > (MAX_ORDER - 1)))
1187 prep_compound_gigantic_page(page, order);
1188 else
1189 prep_compound_page(page, order);
1190}
1191
Andi Kleenaa888a72008-07-23 21:27:47 -07001192/* Put bootmem huge pages into the standard lists after mem_map is up */
1193static void __init gather_bootmem_prealloc(void)
1194{
1195 struct huge_bootmem_page *m;
1196
1197 list_for_each_entry(m, &huge_boot_pages, list) {
Andi Kleenaa888a72008-07-23 21:27:47 -07001198 struct hstate *h = m->hstate;
Becky Bruceee8f2482011-07-25 17:11:50 -07001199 struct page *page;
1200
1201#ifdef CONFIG_HIGHMEM
1202 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1203 free_bootmem_late((unsigned long)m,
1204 sizeof(struct huge_bootmem_page));
1205#else
1206 page = virt_to_page(m);
1207#endif
Andi Kleenaa888a72008-07-23 21:27:47 -07001208 __ClearPageReserved(page);
1209 WARN_ON(page_count(page) != 1);
Andy Whitcroft18229df2008-11-06 12:53:27 -08001210 prep_compound_huge_page(page, h->order);
Andi Kleenaa888a72008-07-23 21:27:47 -07001211 prep_new_huge_page(h, page, page_to_nid(page));
Rafael Aquinib0320c72011-06-15 15:08:39 -07001212 /*
1213 * If we had gigantic hugepages allocated at boot time, we need
1214 * to restore the 'stolen' pages to totalram_pages in order to
1215 * fix confusing memory reports from free(1) and another
1216 * side-effects, like CommitLimit going negative.
1217 */
1218 if (h->order > (MAX_ORDER - 1))
1219 totalram_pages += 1 << h->order;
Andi Kleenaa888a72008-07-23 21:27:47 -07001220 }
1221}
1222
Andi Kleen8faa8b02008-07-23 21:27:48 -07001223static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224{
1225 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226
Andi Kleene5ff2152008-07-23 21:27:42 -07001227 for (i = 0; i < h->max_huge_pages; ++i) {
Andi Kleenaa888a72008-07-23 21:27:47 -07001228 if (h->order >= MAX_ORDER) {
1229 if (!alloc_bootmem_huge_page(h))
1230 break;
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001231 } else if (!alloc_fresh_huge_page(h,
1232 &node_states[N_HIGH_MEMORY]))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 }
Andi Kleen8faa8b02008-07-23 21:27:48 -07001235 h->max_huge_pages = i;
Andi Kleene5ff2152008-07-23 21:27:42 -07001236}
1237
1238static void __init hugetlb_init_hstates(void)
1239{
1240 struct hstate *h;
1241
1242 for_each_hstate(h) {
Andi Kleen8faa8b02008-07-23 21:27:48 -07001243 /* oversize hugepages were init'ed in early boot */
1244 if (h->order < MAX_ORDER)
1245 hugetlb_hstate_alloc_pages(h);
Andi Kleene5ff2152008-07-23 21:27:42 -07001246 }
1247}
1248
Andi Kleen4abd32d2008-07-23 21:27:49 -07001249static char * __init memfmt(char *buf, unsigned long n)
1250{
1251 if (n >= (1UL << 30))
1252 sprintf(buf, "%lu GB", n >> 30);
1253 else if (n >= (1UL << 20))
1254 sprintf(buf, "%lu MB", n >> 20);
1255 else
1256 sprintf(buf, "%lu KB", n >> 10);
1257 return buf;
1258}
1259
Andi Kleene5ff2152008-07-23 21:27:42 -07001260static void __init report_hugepages(void)
1261{
1262 struct hstate *h;
1263
1264 for_each_hstate(h) {
Andi Kleen4abd32d2008-07-23 21:27:49 -07001265 char buf[32];
1266 printk(KERN_INFO "HugeTLB registered %s page size, "
1267 "pre-allocated %ld pages\n",
1268 memfmt(buf, huge_page_size(h)),
1269 h->free_huge_pages);
Andi Kleene5ff2152008-07-23 21:27:42 -07001270 }
1271}
1272
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273#ifdef CONFIG_HIGHMEM
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001274static void try_to_free_low(struct hstate *h, unsigned long count,
1275 nodemask_t *nodes_allowed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276{
Christoph Lameter4415cc82006-09-25 23:31:55 -07001277 int i;
1278
Andi Kleenaa888a72008-07-23 21:27:47 -07001279 if (h->order >= MAX_ORDER)
1280 return;
1281
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001282 for_each_node_mask(i, *nodes_allowed) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 struct page *page, *next;
Andi Kleena5516432008-07-23 21:27:41 -07001284 struct list_head *freel = &h->hugepage_freelists[i];
1285 list_for_each_entry_safe(page, next, freel, lru) {
1286 if (count >= h->nr_huge_pages)
Adam Litke6b0c8802007-10-16 01:26:23 -07001287 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 if (PageHighMem(page))
1289 continue;
1290 list_del(&page->lru);
Andi Kleene5ff2152008-07-23 21:27:42 -07001291 update_and_free_page(h, page);
Andi Kleena5516432008-07-23 21:27:41 -07001292 h->free_huge_pages--;
1293 h->free_huge_pages_node[page_to_nid(page)]--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294 }
1295 }
1296}
1297#else
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001298static inline void try_to_free_low(struct hstate *h, unsigned long count,
1299 nodemask_t *nodes_allowed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300{
1301}
1302#endif
1303
Wu Fengguang20a03072009-06-16 15:32:22 -07001304/*
1305 * Increment or decrement surplus_huge_pages. Keep node-specific counters
1306 * balanced by operating on them in a round-robin fashion.
1307 * Returns 1 if an adjustment was made.
1308 */
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001309static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1310 int delta)
Wu Fengguang20a03072009-06-16 15:32:22 -07001311{
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001312 int start_nid, next_nid;
Wu Fengguang20a03072009-06-16 15:32:22 -07001313 int ret = 0;
1314
1315 VM_BUG_ON(delta != -1 && delta != 1);
Wu Fengguang20a03072009-06-16 15:32:22 -07001316
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001317 if (delta < 0)
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001318 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001319 else
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001320 start_nid = hstate_next_node_to_free(h, nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001321 next_nid = start_nid;
1322
1323 do {
1324 int nid = next_nid;
1325 if (delta < 0) {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001326 /*
1327 * To shrink on this node, there must be a surplus page
1328 */
Lee Schermerhorn9a76db02009-12-14 17:58:15 -08001329 if (!h->surplus_huge_pages_node[nid]) {
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001330 next_nid = hstate_next_node_to_alloc(h,
1331 nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001332 continue;
Lee Schermerhorn9a76db02009-12-14 17:58:15 -08001333 }
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001334 }
1335 if (delta > 0) {
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001336 /*
1337 * Surplus cannot exceed the total number of pages
1338 */
1339 if (h->surplus_huge_pages_node[nid] >=
Lee Schermerhorn9a76db02009-12-14 17:58:15 -08001340 h->nr_huge_pages_node[nid]) {
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001341 next_nid = hstate_next_node_to_free(h,
1342 nodes_allowed);
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001343 continue;
Lee Schermerhorn9a76db02009-12-14 17:58:15 -08001344 }
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001345 }
Wu Fengguang20a03072009-06-16 15:32:22 -07001346
1347 h->surplus_huge_pages += delta;
1348 h->surplus_huge_pages_node[nid] += delta;
1349 ret = 1;
1350 break;
Lee Schermerhorne8c5c822009-09-21 17:01:22 -07001351 } while (next_nid != start_nid);
Wu Fengguang20a03072009-06-16 15:32:22 -07001352
Wu Fengguang20a03072009-06-16 15:32:22 -07001353 return ret;
1354}
1355
Andi Kleena5516432008-07-23 21:27:41 -07001356#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001357static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1358 nodemask_t *nodes_allowed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359{
Adam Litke7893d1d2007-10-16 01:26:18 -07001360 unsigned long min_count, ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
Andi Kleenaa888a72008-07-23 21:27:47 -07001362 if (h->order >= MAX_ORDER)
1363 return h->max_huge_pages;
1364
Adam Litke7893d1d2007-10-16 01:26:18 -07001365 /*
1366 * Increase the pool size
1367 * First take pages out of surplus state. Then make up the
1368 * remaining difference by allocating fresh huge pages.
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -08001369 *
1370 * We might race with alloc_buddy_huge_page() here and be unable
1371 * to convert a surplus huge page to a normal huge page. That is
1372 * not critical, though, it just means the overall size of the
1373 * pool might be one hugepage larger than it needs to be, but
1374 * within all the constraints specified by the sysctls.
Adam Litke7893d1d2007-10-16 01:26:18 -07001375 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 spin_lock(&hugetlb_lock);
Andi Kleena5516432008-07-23 21:27:41 -07001377 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001378 if (!adjust_pool_surplus(h, nodes_allowed, -1))
Adam Litke7893d1d2007-10-16 01:26:18 -07001379 break;
1380 }
1381
Andi Kleena5516432008-07-23 21:27:41 -07001382 while (count > persistent_huge_pages(h)) {
Adam Litke7893d1d2007-10-16 01:26:18 -07001383 /*
1384 * If this allocation races such that we no longer need the
1385 * page, free_huge_page will handle it by freeing the page
1386 * and reducing the surplus.
1387 */
1388 spin_unlock(&hugetlb_lock);
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001389 ret = alloc_fresh_huge_page(h, nodes_allowed);
Adam Litke7893d1d2007-10-16 01:26:18 -07001390 spin_lock(&hugetlb_lock);
1391 if (!ret)
1392 goto out;
1393
Mel Gorman536240f2009-12-14 17:59:56 -08001394 /* Bail for signals. Probably ctrl-c from user */
1395 if (signal_pending(current))
1396 goto out;
Adam Litke7893d1d2007-10-16 01:26:18 -07001397 }
Adam Litke7893d1d2007-10-16 01:26:18 -07001398
1399 /*
1400 * Decrease the pool size
1401 * First return free pages to the buddy allocator (being careful
1402 * to keep enough around to satisfy reservations). Then place
1403 * pages into surplus state as needed so the pool will shrink
1404 * to the desired size as pages become free.
Nishanth Aravamudand1c3fb12007-12-17 16:20:12 -08001405 *
1406 * By placing pages into the surplus state independent of the
1407 * overcommit value, we are allowing the surplus pool size to
1408 * exceed overcommit. There are few sane options here. Since
1409 * alloc_buddy_huge_page() is checking the global counter,
1410 * though, we'll note that we're not allowed to exceed surplus
1411 * and won't grow the pool anywhere else. Not until one of the
1412 * sysctls are changed, or the surplus pages go out of use.
Adam Litke7893d1d2007-10-16 01:26:18 -07001413 */
Andi Kleena5516432008-07-23 21:27:41 -07001414 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
Adam Litke6b0c8802007-10-16 01:26:23 -07001415 min_count = max(count, min_count);
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001416 try_to_free_low(h, min_count, nodes_allowed);
Andi Kleena5516432008-07-23 21:27:41 -07001417 while (min_count < persistent_huge_pages(h)) {
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001418 if (!free_pool_huge_page(h, nodes_allowed, 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420 }
Andi Kleena5516432008-07-23 21:27:41 -07001421 while (count < persistent_huge_pages(h)) {
Lee Schermerhorn6ae11b22009-12-14 17:58:16 -08001422 if (!adjust_pool_surplus(h, nodes_allowed, 1))
Adam Litke7893d1d2007-10-16 01:26:18 -07001423 break;
1424 }
1425out:
Andi Kleena5516432008-07-23 21:27:41 -07001426 ret = persistent_huge_pages(h);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427 spin_unlock(&hugetlb_lock);
Adam Litke7893d1d2007-10-16 01:26:18 -07001428 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429}
1430
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001431#define HSTATE_ATTR_RO(_name) \
1432 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1433
1434#define HSTATE_ATTR(_name) \
1435 static struct kobj_attribute _name##_attr = \
1436 __ATTR(_name, 0644, _name##_show, _name##_store)
1437
1438static struct kobject *hugepages_kobj;
1439static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1440
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001441static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1442
1443static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001444{
1445 int i;
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001446
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001447 for (i = 0; i < HUGE_MAX_HSTATE; i++)
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001448 if (hstate_kobjs[i] == kobj) {
1449 if (nidp)
1450 *nidp = NUMA_NO_NODE;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001451 return &hstates[i];
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001452 }
1453
1454 return kobj_to_node_hstate(kobj, nidp);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001455}
1456
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001457static ssize_t nr_hugepages_show_common(struct kobject *kobj,
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001458 struct kobj_attribute *attr, char *buf)
1459{
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001460 struct hstate *h;
1461 unsigned long nr_huge_pages;
1462 int nid;
1463
1464 h = kobj_to_hstate(kobj, &nid);
1465 if (nid == NUMA_NO_NODE)
1466 nr_huge_pages = h->nr_huge_pages;
1467 else
1468 nr_huge_pages = h->nr_huge_pages_node[nid];
1469
1470 return sprintf(buf, "%lu\n", nr_huge_pages);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001471}
Eric B Munsonadbe8722011-01-13 15:47:27 -08001472
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001473static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1474 struct kobject *kobj, struct kobj_attribute *attr,
1475 const char *buf, size_t len)
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001476{
1477 int err;
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001478 int nid;
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001479 unsigned long count;
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001480 struct hstate *h;
David Rientjesbad44b52009-12-14 17:58:38 -08001481 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001482
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001483 err = strict_strtoul(buf, 10, &count);
Eric B Munson73ae31e2011-01-13 15:47:28 -08001484 if (err)
Eric B Munsonadbe8722011-01-13 15:47:27 -08001485 goto out;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001486
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001487 h = kobj_to_hstate(kobj, &nid);
Eric B Munsonadbe8722011-01-13 15:47:27 -08001488 if (h->order >= MAX_ORDER) {
1489 err = -EINVAL;
1490 goto out;
1491 }
1492
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001493 if (nid == NUMA_NO_NODE) {
1494 /*
1495 * global hstate attribute
1496 */
1497 if (!(obey_mempolicy &&
1498 init_nodemask_of_mempolicy(nodes_allowed))) {
1499 NODEMASK_FREE(nodes_allowed);
1500 nodes_allowed = &node_states[N_HIGH_MEMORY];
1501 }
1502 } else if (nodes_allowed) {
1503 /*
1504 * per node hstate attribute: adjust count to global,
1505 * but restrict alloc/free to the specified node.
1506 */
1507 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1508 init_nodemask_of_node(nodes_allowed, nid);
1509 } else
1510 nodes_allowed = &node_states[N_HIGH_MEMORY];
1511
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001512 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001513
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001514 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001515 NODEMASK_FREE(nodes_allowed);
1516
1517 return len;
Eric B Munsonadbe8722011-01-13 15:47:27 -08001518out:
1519 NODEMASK_FREE(nodes_allowed);
1520 return err;
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001521}
1522
1523static ssize_t nr_hugepages_show(struct kobject *kobj,
1524 struct kobj_attribute *attr, char *buf)
1525{
1526 return nr_hugepages_show_common(kobj, attr, buf);
1527}
1528
1529static ssize_t nr_hugepages_store(struct kobject *kobj,
1530 struct kobj_attribute *attr, const char *buf, size_t len)
1531{
1532 return nr_hugepages_store_common(false, kobj, attr, buf, len);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001533}
1534HSTATE_ATTR(nr_hugepages);
1535
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001536#ifdef CONFIG_NUMA
1537
1538/*
1539 * hstate attribute for optionally mempolicy-based constraint on persistent
1540 * huge page alloc/free.
1541 */
1542static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1543 struct kobj_attribute *attr, char *buf)
1544{
1545 return nr_hugepages_show_common(kobj, attr, buf);
1546}
1547
1548static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1549 struct kobj_attribute *attr, const char *buf, size_t len)
1550{
1551 return nr_hugepages_store_common(true, kobj, attr, buf, len);
1552}
1553HSTATE_ATTR(nr_hugepages_mempolicy);
1554#endif
1555
1556
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001557static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1558 struct kobj_attribute *attr, char *buf)
1559{
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001560 struct hstate *h = kobj_to_hstate(kobj, NULL);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001561 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1562}
Eric B Munsonadbe8722011-01-13 15:47:27 -08001563
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001564static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1565 struct kobj_attribute *attr, const char *buf, size_t count)
1566{
1567 int err;
1568 unsigned long input;
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001569 struct hstate *h = kobj_to_hstate(kobj, NULL);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001570
Eric B Munsonadbe8722011-01-13 15:47:27 -08001571 if (h->order >= MAX_ORDER)
1572 return -EINVAL;
1573
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001574 err = strict_strtoul(buf, 10, &input);
1575 if (err)
Eric B Munson73ae31e2011-01-13 15:47:28 -08001576 return err;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001577
1578 spin_lock(&hugetlb_lock);
1579 h->nr_overcommit_huge_pages = input;
1580 spin_unlock(&hugetlb_lock);
1581
1582 return count;
1583}
1584HSTATE_ATTR(nr_overcommit_hugepages);
1585
1586static ssize_t free_hugepages_show(struct kobject *kobj,
1587 struct kobj_attribute *attr, char *buf)
1588{
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001589 struct hstate *h;
1590 unsigned long free_huge_pages;
1591 int nid;
1592
1593 h = kobj_to_hstate(kobj, &nid);
1594 if (nid == NUMA_NO_NODE)
1595 free_huge_pages = h->free_huge_pages;
1596 else
1597 free_huge_pages = h->free_huge_pages_node[nid];
1598
1599 return sprintf(buf, "%lu\n", free_huge_pages);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001600}
1601HSTATE_ATTR_RO(free_hugepages);
1602
1603static ssize_t resv_hugepages_show(struct kobject *kobj,
1604 struct kobj_attribute *attr, char *buf)
1605{
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001606 struct hstate *h = kobj_to_hstate(kobj, NULL);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001607 return sprintf(buf, "%lu\n", h->resv_huge_pages);
1608}
1609HSTATE_ATTR_RO(resv_hugepages);
1610
1611static ssize_t surplus_hugepages_show(struct kobject *kobj,
1612 struct kobj_attribute *attr, char *buf)
1613{
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001614 struct hstate *h;
1615 unsigned long surplus_huge_pages;
1616 int nid;
1617
1618 h = kobj_to_hstate(kobj, &nid);
1619 if (nid == NUMA_NO_NODE)
1620 surplus_huge_pages = h->surplus_huge_pages;
1621 else
1622 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1623
1624 return sprintf(buf, "%lu\n", surplus_huge_pages);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001625}
1626HSTATE_ATTR_RO(surplus_hugepages);
1627
1628static struct attribute *hstate_attrs[] = {
1629 &nr_hugepages_attr.attr,
1630 &nr_overcommit_hugepages_attr.attr,
1631 &free_hugepages_attr.attr,
1632 &resv_hugepages_attr.attr,
1633 &surplus_hugepages_attr.attr,
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001634#ifdef CONFIG_NUMA
1635 &nr_hugepages_mempolicy_attr.attr,
1636#endif
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001637 NULL,
1638};
1639
1640static struct attribute_group hstate_attr_group = {
1641 .attrs = hstate_attrs,
1642};
1643
Jeff Mahoney094e9532010-02-02 13:44:14 -08001644static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1645 struct kobject **hstate_kobjs,
1646 struct attribute_group *hstate_attr_group)
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001647{
1648 int retval;
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001649 int hi = h - hstates;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001650
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001651 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1652 if (!hstate_kobjs[hi])
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001653 return -ENOMEM;
1654
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001655 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001656 if (retval)
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001657 kobject_put(hstate_kobjs[hi]);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001658
1659 return retval;
1660}
1661
1662static void __init hugetlb_sysfs_init(void)
1663{
1664 struct hstate *h;
1665 int err;
1666
1667 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1668 if (!hugepages_kobj)
1669 return;
1670
1671 for_each_hstate(h) {
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001672 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1673 hstate_kobjs, &hstate_attr_group);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001674 if (err)
1675 printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1676 h->name);
1677 }
1678}
1679
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001680#ifdef CONFIG_NUMA
1681
1682/*
1683 * node_hstate/s - associate per node hstate attributes, via their kobjects,
Kay Sievers10fbcf42011-12-21 14:48:43 -08001684 * with node devices in node_devices[] using a parallel array. The array
1685 * index of a node device or _hstate == node id.
1686 * This is here to avoid any static dependency of the node device driver, in
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001687 * the base kernel, on the hugetlb module.
1688 */
1689struct node_hstate {
1690 struct kobject *hugepages_kobj;
1691 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1692};
1693struct node_hstate node_hstates[MAX_NUMNODES];
1694
1695/*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001696 * A subset of global hstate attributes for node devices
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001697 */
1698static struct attribute *per_node_hstate_attrs[] = {
1699 &nr_hugepages_attr.attr,
1700 &free_hugepages_attr.attr,
1701 &surplus_hugepages_attr.attr,
1702 NULL,
1703};
1704
1705static struct attribute_group per_node_hstate_attr_group = {
1706 .attrs = per_node_hstate_attrs,
1707};
1708
1709/*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001710 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001711 * Returns node id via non-NULL nidp.
1712 */
1713static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1714{
1715 int nid;
1716
1717 for (nid = 0; nid < nr_node_ids; nid++) {
1718 struct node_hstate *nhs = &node_hstates[nid];
1719 int i;
1720 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1721 if (nhs->hstate_kobjs[i] == kobj) {
1722 if (nidp)
1723 *nidp = nid;
1724 return &hstates[i];
1725 }
1726 }
1727
1728 BUG();
1729 return NULL;
1730}
1731
1732/*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001733 * Unregister hstate attributes from a single node device.
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001734 * No-op if no hstate attributes attached.
1735 */
1736void hugetlb_unregister_node(struct node *node)
1737{
1738 struct hstate *h;
Kay Sievers10fbcf42011-12-21 14:48:43 -08001739 struct node_hstate *nhs = &node_hstates[node->dev.id];
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001740
1741 if (!nhs->hugepages_kobj)
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001742 return; /* no hstate attributes */
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001743
1744 for_each_hstate(h)
1745 if (nhs->hstate_kobjs[h - hstates]) {
1746 kobject_put(nhs->hstate_kobjs[h - hstates]);
1747 nhs->hstate_kobjs[h - hstates] = NULL;
1748 }
1749
1750 kobject_put(nhs->hugepages_kobj);
1751 nhs->hugepages_kobj = NULL;
1752}
1753
1754/*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001755 * hugetlb module exit: unregister hstate attributes from node devices
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001756 * that have them.
1757 */
1758static void hugetlb_unregister_all_nodes(void)
1759{
1760 int nid;
1761
1762 /*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001763 * disable node device registrations.
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001764 */
1765 register_hugetlbfs_with_node(NULL, NULL);
1766
1767 /*
1768 * remove hstate attributes from any nodes that have them.
1769 */
1770 for (nid = 0; nid < nr_node_ids; nid++)
1771 hugetlb_unregister_node(&node_devices[nid]);
1772}
1773
1774/*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001775 * Register hstate attributes for a single node device.
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001776 * No-op if attributes already registered.
1777 */
1778void hugetlb_register_node(struct node *node)
1779{
1780 struct hstate *h;
Kay Sievers10fbcf42011-12-21 14:48:43 -08001781 struct node_hstate *nhs = &node_hstates[node->dev.id];
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001782 int err;
1783
1784 if (nhs->hugepages_kobj)
1785 return; /* already allocated */
1786
1787 nhs->hugepages_kobj = kobject_create_and_add("hugepages",
Kay Sievers10fbcf42011-12-21 14:48:43 -08001788 &node->dev.kobj);
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001789 if (!nhs->hugepages_kobj)
1790 return;
1791
1792 for_each_hstate(h) {
1793 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1794 nhs->hstate_kobjs,
1795 &per_node_hstate_attr_group);
1796 if (err) {
1797 printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1798 " for node %d\n",
Kay Sievers10fbcf42011-12-21 14:48:43 -08001799 h->name, node->dev.id);
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001800 hugetlb_unregister_node(node);
1801 break;
1802 }
1803 }
1804}
1805
1806/*
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001807 * hugetlb init time: register hstate attributes for all registered node
Kay Sievers10fbcf42011-12-21 14:48:43 -08001808 * devices of nodes that have memory. All on-line nodes should have
1809 * registered their associated device by this time.
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001810 */
1811static void hugetlb_register_all_nodes(void)
1812{
1813 int nid;
1814
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001815 for_each_node_state(nid, N_HIGH_MEMORY) {
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001816 struct node *node = &node_devices[nid];
Kay Sievers10fbcf42011-12-21 14:48:43 -08001817 if (node->dev.id == nid)
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001818 hugetlb_register_node(node);
1819 }
1820
1821 /*
Kay Sievers10fbcf42011-12-21 14:48:43 -08001822 * Let the node device driver know we're here so it can
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001823 * [un]register hstate attributes on node hotplug.
1824 */
1825 register_hugetlbfs_with_node(hugetlb_register_node,
1826 hugetlb_unregister_node);
1827}
1828#else /* !CONFIG_NUMA */
1829
1830static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831{
1832 BUG();
1833 if (nidp)
1834 *nidp = -1;
1835 return NULL;
1836}
1837
1838static void hugetlb_unregister_all_nodes(void) { }
1839
1840static void hugetlb_register_all_nodes(void) { }
1841
1842#endif
1843
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001844static void __exit hugetlb_exit(void)
1845{
1846 struct hstate *h;
1847
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001848 hugetlb_unregister_all_nodes();
1849
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001850 for_each_hstate(h) {
1851 kobject_put(hstate_kobjs[h - hstates]);
1852 }
1853
1854 kobject_put(hugepages_kobj);
1855}
1856module_exit(hugetlb_exit);
1857
1858static int __init hugetlb_init(void)
1859{
Benjamin Herrenschmidt0ef89d22008-07-31 00:07:30 -07001860 /* Some platform decide whether they support huge pages at boot
1861 * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1862 * there is no such support
1863 */
1864 if (HPAGE_SHIFT == 0)
1865 return 0;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001866
Nick Piggine11bfbf2008-07-23 21:27:52 -07001867 if (!size_to_hstate(default_hstate_size)) {
1868 default_hstate_size = HPAGE_SIZE;
1869 if (!size_to_hstate(default_hstate_size))
1870 hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001871 }
Nick Piggine11bfbf2008-07-23 21:27:52 -07001872 default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1873 if (default_hstate_max_huge_pages)
1874 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001875
1876 hugetlb_init_hstates();
1877
Andi Kleenaa888a72008-07-23 21:27:47 -07001878 gather_bootmem_prealloc();
1879
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001880 report_hugepages();
1881
1882 hugetlb_sysfs_init();
1883
Lee Schermerhorn9a305232009-12-14 17:58:25 -08001884 hugetlb_register_all_nodes();
1885
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001886 return 0;
1887}
1888module_init(hugetlb_init);
1889
1890/* Should be called on processing a hugepagesz=... option */
1891void __init hugetlb_add_hstate(unsigned order)
1892{
1893 struct hstate *h;
Andi Kleen8faa8b02008-07-23 21:27:48 -07001894 unsigned long i;
1895
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001896 if (size_to_hstate(PAGE_SIZE << order)) {
1897 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1898 return;
1899 }
1900 BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1901 BUG_ON(order == 0);
1902 h = &hstates[max_hstate++];
1903 h->order = order;
1904 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
Andi Kleen8faa8b02008-07-23 21:27:48 -07001905 h->nr_huge_pages = 0;
1906 h->free_huge_pages = 0;
1907 for (i = 0; i < MAX_NUMNODES; ++i)
1908 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
Lee Schermerhorn9b5e5d02009-12-14 17:58:32 -08001909 h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1910 h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001911 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1912 huge_page_size(h)/1024);
Andi Kleen8faa8b02008-07-23 21:27:48 -07001913
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001914 parsed_hstate = h;
1915}
1916
Nick Piggine11bfbf2008-07-23 21:27:52 -07001917static int __init hugetlb_nrpages_setup(char *s)
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001918{
1919 unsigned long *mhp;
Andi Kleen8faa8b02008-07-23 21:27:48 -07001920 static unsigned long *last_mhp;
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001921
1922 /*
1923 * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1924 * so this hugepages= parameter goes to the "default hstate".
1925 */
1926 if (!max_hstate)
1927 mhp = &default_hstate_max_huge_pages;
1928 else
1929 mhp = &parsed_hstate->max_huge_pages;
1930
Andi Kleen8faa8b02008-07-23 21:27:48 -07001931 if (mhp == last_mhp) {
1932 printk(KERN_WARNING "hugepages= specified twice without "
1933 "interleaving hugepagesz=, ignoring\n");
1934 return 1;
1935 }
1936
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001937 if (sscanf(s, "%lu", mhp) <= 0)
1938 *mhp = 0;
1939
Andi Kleen8faa8b02008-07-23 21:27:48 -07001940 /*
1941 * Global state is always initialized later in hugetlb_init.
1942 * But we need to allocate >= MAX_ORDER hstates here early to still
1943 * use the bootmem allocator.
1944 */
1945 if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1946 hugetlb_hstate_alloc_pages(parsed_hstate);
1947
1948 last_mhp = mhp;
1949
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001950 return 1;
1951}
Nick Piggine11bfbf2008-07-23 21:27:52 -07001952__setup("hugepages=", hugetlb_nrpages_setup);
1953
1954static int __init hugetlb_default_setup(char *s)
1955{
1956 default_hstate_size = memparse(s, &s);
1957 return 1;
1958}
1959__setup("default_hugepagesz=", hugetlb_default_setup);
Nishanth Aravamudana3437872008-07-23 21:27:44 -07001960
Nishanth Aravamudan8a213462008-07-25 19:44:37 -07001961static unsigned int cpuset_mems_nr(unsigned int *array)
1962{
1963 int node;
1964 unsigned int nr = 0;
1965
1966 for_each_node_mask(node, cpuset_current_mems_allowed)
1967 nr += array[node];
1968
1969 return nr;
1970}
1971
1972#ifdef CONFIG_SYSCTL
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001973static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1974 struct ctl_table *table, int write,
1975 void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976{
Andi Kleene5ff2152008-07-23 21:27:42 -07001977 struct hstate *h = &default_hstate;
1978 unsigned long tmp;
Michal Hocko08d4a242011-01-13 15:47:26 -08001979 int ret;
Andi Kleene5ff2152008-07-23 21:27:42 -07001980
Petr Holasekc033a932011-03-22 16:33:05 -07001981 tmp = h->max_huge_pages;
Andi Kleene5ff2152008-07-23 21:27:42 -07001982
Eric B Munsonadbe8722011-01-13 15:47:27 -08001983 if (write && h->order >= MAX_ORDER)
1984 return -EINVAL;
1985
Andi Kleene5ff2152008-07-23 21:27:42 -07001986 table->data = &tmp;
1987 table->maxlen = sizeof(unsigned long);
Michal Hocko08d4a242011-01-13 15:47:26 -08001988 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
1989 if (ret)
1990 goto out;
Andi Kleene5ff2152008-07-23 21:27:42 -07001991
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001992 if (write) {
David Rientjesbad44b52009-12-14 17:58:38 -08001993 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1994 GFP_KERNEL | __GFP_NORETRY);
Lee Schermerhorn06808b02009-12-14 17:58:21 -08001995 if (!(obey_mempolicy &&
1996 init_nodemask_of_mempolicy(nodes_allowed))) {
1997 NODEMASK_FREE(nodes_allowed);
1998 nodes_allowed = &node_states[N_HIGH_MEMORY];
1999 }
2000 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2001
2002 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
2003 NODEMASK_FREE(nodes_allowed);
2004 }
Michal Hocko08d4a242011-01-13 15:47:26 -08002005out:
2006 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002007}
Mel Gorman396faf02007-07-17 04:03:13 -07002008
Lee Schermerhorn06808b02009-12-14 17:58:21 -08002009int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2010 void __user *buffer, size_t *length, loff_t *ppos)
2011{
2012
2013 return hugetlb_sysctl_handler_common(false, table, write,
2014 buffer, length, ppos);
2015}
2016
2017#ifdef CONFIG_NUMA
2018int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2019 void __user *buffer, size_t *length, loff_t *ppos)
2020{
2021 return hugetlb_sysctl_handler_common(true, table, write,
2022 buffer, length, ppos);
2023}
2024#endif /* CONFIG_NUMA */
2025
Mel Gorman396faf02007-07-17 04:03:13 -07002026int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002027 void __user *buffer,
Mel Gorman396faf02007-07-17 04:03:13 -07002028 size_t *length, loff_t *ppos)
2029{
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002030 proc_dointvec(table, write, buffer, length, ppos);
Mel Gorman396faf02007-07-17 04:03:13 -07002031 if (hugepages_treat_as_movable)
2032 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
2033 else
2034 htlb_alloc_mask = GFP_HIGHUSER;
2035 return 0;
2036}
2037
Nishanth Aravamudana3d0c6a2008-02-08 04:18:18 -08002038int hugetlb_overcommit_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002039 void __user *buffer,
Nishanth Aravamudana3d0c6a2008-02-08 04:18:18 -08002040 size_t *length, loff_t *ppos)
2041{
Andi Kleena5516432008-07-23 21:27:41 -07002042 struct hstate *h = &default_hstate;
Andi Kleene5ff2152008-07-23 21:27:42 -07002043 unsigned long tmp;
Michal Hocko08d4a242011-01-13 15:47:26 -08002044 int ret;
Andi Kleene5ff2152008-07-23 21:27:42 -07002045
Petr Holasekc033a932011-03-22 16:33:05 -07002046 tmp = h->nr_overcommit_huge_pages;
Andi Kleene5ff2152008-07-23 21:27:42 -07002047
Eric B Munsonadbe8722011-01-13 15:47:27 -08002048 if (write && h->order >= MAX_ORDER)
2049 return -EINVAL;
2050
Andi Kleene5ff2152008-07-23 21:27:42 -07002051 table->data = &tmp;
2052 table->maxlen = sizeof(unsigned long);
Michal Hocko08d4a242011-01-13 15:47:26 -08002053 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2054 if (ret)
2055 goto out;
Andi Kleene5ff2152008-07-23 21:27:42 -07002056
2057 if (write) {
2058 spin_lock(&hugetlb_lock);
2059 h->nr_overcommit_huge_pages = tmp;
2060 spin_unlock(&hugetlb_lock);
2061 }
Michal Hocko08d4a242011-01-13 15:47:26 -08002062out:
2063 return ret;
Nishanth Aravamudana3d0c6a2008-02-08 04:18:18 -08002064}
2065
Linus Torvalds1da177e2005-04-16 15:20:36 -07002066#endif /* CONFIG_SYSCTL */
2067
Alexey Dobriyane1759c22008-10-15 23:50:22 +04002068void hugetlb_report_meminfo(struct seq_file *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069{
Andi Kleena5516432008-07-23 21:27:41 -07002070 struct hstate *h = &default_hstate;
Alexey Dobriyane1759c22008-10-15 23:50:22 +04002071 seq_printf(m,
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002072 "HugePages_Total: %5lu\n"
2073 "HugePages_Free: %5lu\n"
2074 "HugePages_Rsvd: %5lu\n"
2075 "HugePages_Surp: %5lu\n"
2076 "Hugepagesize: %8lu kB\n",
Andi Kleena5516432008-07-23 21:27:41 -07002077 h->nr_huge_pages,
2078 h->free_huge_pages,
2079 h->resv_huge_pages,
2080 h->surplus_huge_pages,
2081 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082}
2083
2084int hugetlb_report_node_meminfo(int nid, char *buf)
2085{
Andi Kleena5516432008-07-23 21:27:41 -07002086 struct hstate *h = &default_hstate;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087 return sprintf(buf,
2088 "Node %d HugePages_Total: %5u\n"
Nishanth Aravamudana1de0912008-03-26 14:37:53 -07002089 "Node %d HugePages_Free: %5u\n"
2090 "Node %d HugePages_Surp: %5u\n",
Andi Kleena5516432008-07-23 21:27:41 -07002091 nid, h->nr_huge_pages_node[nid],
2092 nid, h->free_huge_pages_node[nid],
2093 nid, h->surplus_huge_pages_node[nid]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094}
2095
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2097unsigned long hugetlb_total_pages(void)
2098{
Andi Kleena5516432008-07-23 21:27:41 -07002099 struct hstate *h = &default_hstate;
2100 return h->nr_huge_pages * pages_per_huge_page(h);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002101}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102
Andi Kleena5516432008-07-23 21:27:41 -07002103static int hugetlb_acct_memory(struct hstate *h, long delta)
Mel Gormanfc1b8a72008-07-23 21:27:22 -07002104{
2105 int ret = -ENOMEM;
2106
2107 spin_lock(&hugetlb_lock);
2108 /*
2109 * When cpuset is configured, it breaks the strict hugetlb page
2110 * reservation as the accounting is done on a global variable. Such
2111 * reservation is completely rubbish in the presence of cpuset because
2112 * the reservation is not checked against page availability for the
2113 * current cpuset. Application can still potentially OOM'ed by kernel
2114 * with lack of free htlb page in cpuset that the task is in.
2115 * Attempt to enforce strict accounting with cpuset is almost
2116 * impossible (or too ugly) because cpuset is too fluid that
2117 * task or memory node can be dynamically moved between cpusets.
2118 *
2119 * The change of semantics for shared hugetlb mapping with cpuset is
2120 * undesirable. However, in order to preserve some of the semantics,
2121 * we fall back to check against current free page availability as
2122 * a best attempt and hopefully to minimize the impact of changing
2123 * semantics that cpuset has.
2124 */
2125 if (delta > 0) {
Andi Kleena5516432008-07-23 21:27:41 -07002126 if (gather_surplus_pages(h, delta) < 0)
Mel Gormanfc1b8a72008-07-23 21:27:22 -07002127 goto out;
2128
Andi Kleena5516432008-07-23 21:27:41 -07002129 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2130 return_unused_surplus_pages(h, delta);
Mel Gormanfc1b8a72008-07-23 21:27:22 -07002131 goto out;
2132 }
2133 }
2134
2135 ret = 0;
2136 if (delta < 0)
Andi Kleena5516432008-07-23 21:27:41 -07002137 return_unused_surplus_pages(h, (unsigned long) -delta);
Mel Gormanfc1b8a72008-07-23 21:27:22 -07002138
2139out:
2140 spin_unlock(&hugetlb_lock);
2141 return ret;
2142}
2143
Andy Whitcroft84afd992008-07-23 21:27:32 -07002144static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2145{
2146 struct resv_map *reservations = vma_resv_map(vma);
2147
2148 /*
2149 * This new VMA should share its siblings reservation map if present.
2150 * The VMA will only ever have a valid reservation map pointer where
2151 * it is being copied for another still existing VMA. As that VMA
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002152 * has a reference to the reservation map it cannot disappear until
Andy Whitcroft84afd992008-07-23 21:27:32 -07002153 * after this open call completes. It is therefore safe to take a
2154 * new reference here without additional locking.
2155 */
2156 if (reservations)
2157 kref_get(&reservations->refs);
2158}
2159
Mel Gormana1e78772008-07-23 21:27:23 -07002160static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2161{
Andi Kleena5516432008-07-23 21:27:41 -07002162 struct hstate *h = hstate_vma(vma);
Andy Whitcroft84afd992008-07-23 21:27:32 -07002163 struct resv_map *reservations = vma_resv_map(vma);
David Gibson90481622012-03-21 16:34:12 -07002164 struct hugepage_subpool *spool = subpool_vma(vma);
Andy Whitcroft84afd992008-07-23 21:27:32 -07002165 unsigned long reserve;
2166 unsigned long start;
2167 unsigned long end;
2168
2169 if (reservations) {
Andi Kleena5516432008-07-23 21:27:41 -07002170 start = vma_hugecache_offset(h, vma, vma->vm_start);
2171 end = vma_hugecache_offset(h, vma, vma->vm_end);
Andy Whitcroft84afd992008-07-23 21:27:32 -07002172
2173 reserve = (end - start) -
2174 region_count(&reservations->regions, start, end);
2175
2176 kref_put(&reservations->refs, resv_map_release);
2177
Adam Litke7251ff782008-07-23 21:27:59 -07002178 if (reserve) {
Andi Kleena5516432008-07-23 21:27:41 -07002179 hugetlb_acct_memory(h, -reserve);
David Gibson90481622012-03-21 16:34:12 -07002180 hugepage_subpool_put_pages(spool, reserve);
Adam Litke7251ff782008-07-23 21:27:59 -07002181 }
Andy Whitcroft84afd992008-07-23 21:27:32 -07002182 }
Mel Gormana1e78772008-07-23 21:27:23 -07002183}
2184
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185/*
2186 * We cannot handle pagefaults against hugetlb pages at all. They cause
2187 * handle_mm_fault() to try to instantiate regular-sized pages in the
2188 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
2189 * this far.
2190 */
Nick Piggind0217ac2007-07-19 01:47:03 -07002191static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192{
2193 BUG();
Nick Piggind0217ac2007-07-19 01:47:03 -07002194 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002195}
2196
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04002197const struct vm_operations_struct hugetlb_vm_ops = {
Nick Piggind0217ac2007-07-19 01:47:03 -07002198 .fault = hugetlb_vm_op_fault,
Andy Whitcroft84afd992008-07-23 21:27:32 -07002199 .open = hugetlb_vm_op_open,
Mel Gormana1e78772008-07-23 21:27:23 -07002200 .close = hugetlb_vm_op_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002201};
2202
David Gibson1e8f8892006-01-06 00:10:44 -08002203static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2204 int writable)
David Gibson63551ae2005-06-21 17:14:44 -07002205{
2206 pte_t entry;
2207
David Gibson1e8f8892006-01-06 00:10:44 -08002208 if (writable) {
David Gibson63551ae2005-06-21 17:14:44 -07002209 entry =
2210 pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2211 } else {
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002212 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
David Gibson63551ae2005-06-21 17:14:44 -07002213 }
2214 entry = pte_mkyoung(entry);
2215 entry = pte_mkhuge(entry);
2216
2217 return entry;
2218}
2219
David Gibson1e8f8892006-01-06 00:10:44 -08002220static void set_huge_ptep_writable(struct vm_area_struct *vma,
2221 unsigned long address, pte_t *ptep)
2222{
2223 pte_t entry;
2224
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002225 entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
Chris Forbes32f84522011-07-25 17:12:14 -07002226 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
Russell King4b3073e2009-12-18 16:40:18 +00002227 update_mmu_cache(vma, address, ptep);
David Gibson1e8f8892006-01-06 00:10:44 -08002228}
2229
2230
David Gibson63551ae2005-06-21 17:14:44 -07002231int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2232 struct vm_area_struct *vma)
2233{
2234 pte_t *src_pte, *dst_pte, entry;
2235 struct page *ptepage;
Hugh Dickins1c598272005-10-19 21:23:43 -07002236 unsigned long addr;
David Gibson1e8f8892006-01-06 00:10:44 -08002237 int cow;
Andi Kleena5516432008-07-23 21:27:41 -07002238 struct hstate *h = hstate_vma(vma);
2239 unsigned long sz = huge_page_size(h);
David Gibson1e8f8892006-01-06 00:10:44 -08002240
2241 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
David Gibson63551ae2005-06-21 17:14:44 -07002242
Andi Kleena5516432008-07-23 21:27:41 -07002243 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
Hugh Dickinsc74df322005-10-29 18:16:23 -07002244 src_pte = huge_pte_offset(src, addr);
2245 if (!src_pte)
2246 continue;
Andi Kleena5516432008-07-23 21:27:41 -07002247 dst_pte = huge_pte_alloc(dst, addr, sz);
David Gibson63551ae2005-06-21 17:14:44 -07002248 if (!dst_pte)
2249 goto nomem;
Larry Woodmanc5c99422008-01-24 05:49:25 -08002250
2251 /* If the pagetables are shared don't copy or take references */
2252 if (dst_pte == src_pte)
2253 continue;
2254
Hugh Dickinsc74df322005-10-29 18:16:23 -07002255 spin_lock(&dst->page_table_lock);
Nick Piggin46478752008-06-05 22:45:57 -07002256 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002257 if (!huge_pte_none(huge_ptep_get(src_pte))) {
David Gibson1e8f8892006-01-06 00:10:44 -08002258 if (cow)
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002259 huge_ptep_set_wrprotect(src, addr, src_pte);
2260 entry = huge_ptep_get(src_pte);
Hugh Dickins1c598272005-10-19 21:23:43 -07002261 ptepage = pte_page(entry);
2262 get_page(ptepage);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002263 page_dup_rmap(ptepage);
Hugh Dickins1c598272005-10-19 21:23:43 -07002264 set_huge_pte_at(dst, addr, dst_pte, entry);
2265 }
2266 spin_unlock(&src->page_table_lock);
Hugh Dickinsc74df322005-10-29 18:16:23 -07002267 spin_unlock(&dst->page_table_lock);
David Gibson63551ae2005-06-21 17:14:44 -07002268 }
2269 return 0;
2270
2271nomem:
2272 return -ENOMEM;
2273}
2274
Naoya Horiguchi290408d2010-09-08 10:19:35 +09002275static int is_hugetlb_entry_migration(pte_t pte)
2276{
2277 swp_entry_t swp;
2278
2279 if (huge_pte_none(pte) || pte_present(pte))
2280 return 0;
2281 swp = pte_to_swp_entry(pte);
Chris Forbes32f84522011-07-25 17:12:14 -07002282 if (non_swap_entry(swp) && is_migration_entry(swp))
Naoya Horiguchi290408d2010-09-08 10:19:35 +09002283 return 1;
Chris Forbes32f84522011-07-25 17:12:14 -07002284 else
Naoya Horiguchi290408d2010-09-08 10:19:35 +09002285 return 0;
2286}
2287
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002288static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2289{
2290 swp_entry_t swp;
2291
2292 if (huge_pte_none(pte) || pte_present(pte))
2293 return 0;
2294 swp = pte_to_swp_entry(pte);
Chris Forbes32f84522011-07-25 17:12:14 -07002295 if (non_swap_entry(swp) && is_hwpoison_entry(swp))
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002296 return 1;
Chris Forbes32f84522011-07-25 17:12:14 -07002297 else
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002298 return 0;
2299}
2300
Chen, Kenneth W502717f2006-10-11 01:20:46 -07002301void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002302 unsigned long end, struct page *ref_page)
David Gibson63551ae2005-06-21 17:14:44 -07002303{
2304 struct mm_struct *mm = vma->vm_mm;
2305 unsigned long address;
David Gibsonc7546f82005-08-05 11:59:35 -07002306 pte_t *ptep;
David Gibson63551ae2005-06-21 17:14:44 -07002307 pte_t pte;
2308 struct page *page;
Chen, Kenneth Wfe1668a2006-10-04 02:15:24 -07002309 struct page *tmp;
Andi Kleena5516432008-07-23 21:27:41 -07002310 struct hstate *h = hstate_vma(vma);
2311 unsigned long sz = huge_page_size(h);
2312
Chen, Kenneth Wc0a499c2006-12-06 20:31:39 -08002313 /*
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002314 * A page gathering list, protected by per file i_mmap_mutex. The
Chen, Kenneth Wc0a499c2006-12-06 20:31:39 -08002315 * lock is used to avoid list corruption from multiple unmapping
2316 * of the same page since we are using page->lru.
2317 */
Chen, Kenneth Wfe1668a2006-10-04 02:15:24 -07002318 LIST_HEAD(page_list);
David Gibson63551ae2005-06-21 17:14:44 -07002319
2320 WARN_ON(!is_vm_hugetlb_page(vma));
Andi Kleena5516432008-07-23 21:27:41 -07002321 BUG_ON(start & ~huge_page_mask(h));
2322 BUG_ON(end & ~huge_page_mask(h));
David Gibson63551ae2005-06-21 17:14:44 -07002323
Andrea Arcangelicddb8a52008-07-28 15:46:29 -07002324 mmu_notifier_invalidate_range_start(mm, start, end);
Hugh Dickins508034a2005-10-29 18:16:30 -07002325 spin_lock(&mm->page_table_lock);
Andi Kleena5516432008-07-23 21:27:41 -07002326 for (address = start; address < end; address += sz) {
David Gibsonc7546f82005-08-05 11:59:35 -07002327 ptep = huge_pte_offset(mm, address);
Adam Litke4c887262005-10-29 18:16:46 -07002328 if (!ptep)
David Gibsonc7546f82005-08-05 11:59:35 -07002329 continue;
2330
Chen, Kenneth W39dde652006-12-06 20:32:03 -08002331 if (huge_pmd_unshare(mm, &address, ptep))
2332 continue;
2333
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002334 /*
2335 * If a reference page is supplied, it is because a specific
2336 * page is being unmapped, not a range. Ensure the page we
2337 * are about to unmap is the actual page of interest.
2338 */
2339 if (ref_page) {
2340 pte = huge_ptep_get(ptep);
2341 if (huge_pte_none(pte))
2342 continue;
2343 page = pte_page(pte);
2344 if (page != ref_page)
2345 continue;
2346
2347 /*
2348 * Mark the VMA as having unmapped its page so that
2349 * future faults in this VMA will fail rather than
2350 * looking like data was lost
2351 */
2352 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2353 }
2354
David Gibsonc7546f82005-08-05 11:59:35 -07002355 pte = huge_ptep_get_and_clear(mm, address, ptep);
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002356 if (huge_pte_none(pte))
David Gibson63551ae2005-06-21 17:14:44 -07002357 continue;
David Gibsonc7546f82005-08-05 11:59:35 -07002358
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002359 /*
2360 * HWPoisoned hugepage is already unmapped and dropped reference
2361 */
2362 if (unlikely(is_hugetlb_entry_hwpoisoned(pte)))
2363 continue;
2364
David Gibson63551ae2005-06-21 17:14:44 -07002365 page = pte_page(pte);
Ken Chen6649a382007-02-08 14:20:27 -08002366 if (pte_dirty(pte))
2367 set_page_dirty(page);
Chen, Kenneth Wfe1668a2006-10-04 02:15:24 -07002368 list_add(&page->lru, &page_list);
Hillf Danton9e811302012-03-21 16:34:03 -07002369
2370 /* Bail out after unmapping reference page if supplied */
2371 if (ref_page)
2372 break;
David Gibson63551ae2005-06-21 17:14:44 -07002373 }
Hugh Dickins508034a2005-10-29 18:16:30 -07002374 flush_tlb_range(vma, start, end);
Al Virocd2934a2012-03-05 06:40:29 +00002375 spin_unlock(&mm->page_table_lock);
Andrea Arcangelicddb8a52008-07-28 15:46:29 -07002376 mmu_notifier_invalidate_range_end(mm, start, end);
Chen, Kenneth Wfe1668a2006-10-04 02:15:24 -07002377 list_for_each_entry_safe(page, tmp, &page_list, lru) {
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002378 page_remove_rmap(page);
Chen, Kenneth Wfe1668a2006-10-04 02:15:24 -07002379 list_del(&page->lru);
2380 put_page(page);
2381 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002382}
David Gibson63551ae2005-06-21 17:14:44 -07002383
Chen, Kenneth W502717f2006-10-11 01:20:46 -07002384void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002385 unsigned long end, struct page *ref_page)
Chen, Kenneth W502717f2006-10-11 01:20:46 -07002386{
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002387 mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
Andi Kleena137e1c2008-07-23 21:27:43 -07002388 __unmap_hugepage_range(vma, start, end, ref_page);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002389 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
Chen, Kenneth W502717f2006-10-11 01:20:46 -07002390}
2391
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002392/*
2393 * This is called when the original mapper is failing to COW a MAP_PRIVATE
2394 * mappping it owns the reserve page for. The intention is to unmap the page
2395 * from other VMAs and let the children be SIGKILLed if they are faulting the
2396 * same region.
2397 */
Harvey Harrison2a4b3de2008-10-18 20:27:06 -07002398static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2399 struct page *page, unsigned long address)
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002400{
Adam Litke75266742008-11-12 13:24:56 -08002401 struct hstate *h = hstate_vma(vma);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002402 struct vm_area_struct *iter_vma;
2403 struct address_space *mapping;
2404 struct prio_tree_iter iter;
2405 pgoff_t pgoff;
2406
2407 /*
2408 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2409 * from page cache lookup which is in HPAGE_SIZE units.
2410 */
Adam Litke75266742008-11-12 13:24:56 -08002411 address = address & huge_page_mask(h);
Hillf Danton0c176d52012-01-10 15:08:19 -08002412 pgoff = vma_hugecache_offset(h, vma, address);
David Gibson90481622012-03-21 16:34:12 -07002413 mapping = vma->vm_file->f_dentry->d_inode->i_mapping;
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002414
Mel Gorman4eb2b1d2009-12-14 17:59:53 -08002415 /*
2416 * Take the mapping lock for the duration of the table walk. As
2417 * this mapping should be shared between all the VMAs,
2418 * __unmap_hugepage_range() is called as the lock is already held
2419 */
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002420 mutex_lock(&mapping->i_mmap_mutex);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002421 vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2422 /* Do not unmap the current VMA */
2423 if (iter_vma == vma)
2424 continue;
2425
2426 /*
2427 * Unmap the page from other VMAs without their own reserves.
2428 * They get marked to be SIGKILLed if they fault in these
2429 * areas. This is because a future no-page fault on this VMA
2430 * could insert a zeroed page instead of the data existing
2431 * from the time of fork. This would look like data corruption
2432 */
2433 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
Mel Gorman4eb2b1d2009-12-14 17:59:53 -08002434 __unmap_hugepage_range(iter_vma,
Adam Litke75266742008-11-12 13:24:56 -08002435 address, address + huge_page_size(h),
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002436 page);
2437 }
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002438 mutex_unlock(&mapping->i_mmap_mutex);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002439
2440 return 1;
2441}
2442
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002443/*
2444 * Hugetlb_cow() should be called with page lock of the original hugepage held.
Michal Hockoef009b22012-01-10 15:07:21 -08002445 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2446 * cannot race with other handlers or page migration.
2447 * Keep the pte_same checks anyway to make transition from the mutex easier.
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002448 */
David Gibson1e8f8892006-01-06 00:10:44 -08002449static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002450 unsigned long address, pte_t *ptep, pte_t pte,
2451 struct page *pagecache_page)
David Gibson1e8f8892006-01-06 00:10:44 -08002452{
Andi Kleena5516432008-07-23 21:27:41 -07002453 struct hstate *h = hstate_vma(vma);
David Gibson1e8f8892006-01-06 00:10:44 -08002454 struct page *old_page, *new_page;
David Gibson79ac6ba2006-03-22 00:08:51 -08002455 int avoidcopy;
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002456 int outside_reserve = 0;
David Gibson1e8f8892006-01-06 00:10:44 -08002457
2458 old_page = pte_page(pte);
2459
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002460retry_avoidcopy:
David Gibson1e8f8892006-01-06 00:10:44 -08002461 /* If no-one else is actually using this page, avoid the copy
2462 * and just make the page writable */
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002463 avoidcopy = (page_mapcount(old_page) == 1);
David Gibson1e8f8892006-01-06 00:10:44 -08002464 if (avoidcopy) {
Naoya Horiguchi56c9cfb2010-09-10 13:23:04 +09002465 if (PageAnon(old_page))
2466 page_move_anon_rmap(old_page, vma, address);
David Gibson1e8f8892006-01-06 00:10:44 -08002467 set_huge_ptep_writable(vma, address, ptep);
Nick Piggin83c54072007-07-19 01:47:05 -07002468 return 0;
David Gibson1e8f8892006-01-06 00:10:44 -08002469 }
2470
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002471 /*
2472 * If the process that created a MAP_PRIVATE mapping is about to
2473 * perform a COW due to a shared page count, attempt to satisfy
2474 * the allocation without using the existing reserves. The pagecache
2475 * page is used to determine if the reserve at this address was
2476 * consumed or not. If reserves were used, a partial faulted mapping
2477 * at the time of fork() could consume its reserves on COW instead
2478 * of the full address range.
2479 */
Mel Gormanf83a2752009-05-28 14:34:40 -07002480 if (!(vma->vm_flags & VM_MAYSHARE) &&
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002481 is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2482 old_page != pagecache_page)
2483 outside_reserve = 1;
2484
David Gibson1e8f8892006-01-06 00:10:44 -08002485 page_cache_get(old_page);
Larry Woodmanb76c8cf2009-12-14 17:59:37 -08002486
2487 /* Drop page_table_lock as buddy allocator may be called */
2488 spin_unlock(&mm->page_table_lock);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002489 new_page = alloc_huge_page(vma, address, outside_reserve);
David Gibson1e8f8892006-01-06 00:10:44 -08002490
Adam Litke2fc39ce2007-11-14 16:59:39 -08002491 if (IS_ERR(new_page)) {
David Gibson1e8f8892006-01-06 00:10:44 -08002492 page_cache_release(old_page);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002493
2494 /*
2495 * If a process owning a MAP_PRIVATE mapping fails to COW,
2496 * it is due to references held by a child and an insufficient
2497 * huge page pool. To guarantee the original mappers
2498 * reliability, unmap the page from child processes. The child
2499 * may get SIGKILLed if it later faults.
2500 */
2501 if (outside_reserve) {
2502 BUG_ON(huge_pte_none(pte));
2503 if (unmap_ref_private(mm, vma, old_page, address)) {
2504 BUG_ON(page_count(old_page) != 1);
2505 BUG_ON(huge_pte_none(pte));
Larry Woodmanb76c8cf2009-12-14 17:59:37 -08002506 spin_lock(&mm->page_table_lock);
Hillf Dantona734bcc2012-01-10 15:07:20 -08002507 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2508 if (likely(pte_same(huge_ptep_get(ptep), pte)))
2509 goto retry_avoidcopy;
2510 /*
2511 * race occurs while re-acquiring page_table_lock, and
2512 * our job is done.
2513 */
2514 return 0;
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002515 }
2516 WARN_ON_ONCE(1);
2517 }
2518
Larry Woodmanb76c8cf2009-12-14 17:59:37 -08002519 /* Caller expects lock to be held */
2520 spin_lock(&mm->page_table_lock);
Adam Litke2fc39ce2007-11-14 16:59:39 -08002521 return -PTR_ERR(new_page);
David Gibson1e8f8892006-01-06 00:10:44 -08002522 }
2523
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002524 /*
2525 * When the original hugepage is shared one, it does not have
2526 * anon_vma prepared.
2527 */
Dean Nelson44e2aa92010-10-26 14:22:08 -07002528 if (unlikely(anon_vma_prepare(vma))) {
Hillf Dantonea4039a2011-11-15 14:36:12 -08002529 page_cache_release(new_page);
2530 page_cache_release(old_page);
Dean Nelson44e2aa92010-10-26 14:22:08 -07002531 /* Caller expects lock to be held */
2532 spin_lock(&mm->page_table_lock);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002533 return VM_FAULT_OOM;
Dean Nelson44e2aa92010-10-26 14:22:08 -07002534 }
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002535
Andrea Arcangeli47ad8472011-01-13 15:46:47 -08002536 copy_user_huge_page(new_page, old_page, address, vma,
2537 pages_per_huge_page(h));
Nick Piggin0ed361d2008-02-04 22:29:34 -08002538 __SetPageUptodate(new_page);
David Gibson1e8f8892006-01-06 00:10:44 -08002539
Larry Woodmanb76c8cf2009-12-14 17:59:37 -08002540 /*
2541 * Retake the page_table_lock to check for racing updates
2542 * before the page tables are altered
2543 */
2544 spin_lock(&mm->page_table_lock);
Andi Kleena5516432008-07-23 21:27:41 -07002545 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002546 if (likely(pte_same(huge_ptep_get(ptep), pte))) {
David Gibson1e8f8892006-01-06 00:10:44 -08002547 /* Break COW */
Doug Doan3edd4fc2010-08-09 17:18:30 -07002548 mmu_notifier_invalidate_range_start(mm,
2549 address & huge_page_mask(h),
2550 (address & huge_page_mask(h)) + huge_page_size(h));
Gerald Schaefer8fe627e2008-04-28 02:13:28 -07002551 huge_ptep_clear_flush(vma, address, ptep);
David Gibson1e8f8892006-01-06 00:10:44 -08002552 set_huge_pte_at(mm, address, ptep,
2553 make_huge_pte(vma, new_page, 1));
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002554 page_remove_rmap(old_page);
Naoya Horiguchicd67f0d2010-09-10 13:23:04 +09002555 hugepage_add_new_anon_rmap(new_page, vma, address);
David Gibson1e8f8892006-01-06 00:10:44 -08002556 /* Make the old page be freed below */
2557 new_page = old_page;
Doug Doan3edd4fc2010-08-09 17:18:30 -07002558 mmu_notifier_invalidate_range_end(mm,
2559 address & huge_page_mask(h),
2560 (address & huge_page_mask(h)) + huge_page_size(h));
David Gibson1e8f8892006-01-06 00:10:44 -08002561 }
2562 page_cache_release(new_page);
2563 page_cache_release(old_page);
Nick Piggin83c54072007-07-19 01:47:05 -07002564 return 0;
David Gibson1e8f8892006-01-06 00:10:44 -08002565}
2566
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002567/* Return the pagecache page at a given address within a VMA */
Andi Kleena5516432008-07-23 21:27:41 -07002568static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2569 struct vm_area_struct *vma, unsigned long address)
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002570{
2571 struct address_space *mapping;
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -07002572 pgoff_t idx;
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002573
2574 mapping = vma->vm_file->f_mapping;
Andi Kleena5516432008-07-23 21:27:41 -07002575 idx = vma_hugecache_offset(h, vma, address);
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002576
2577 return find_lock_page(mapping, idx);
2578}
2579
Hugh Dickins3ae77f42009-09-21 17:03:33 -07002580/*
2581 * Return whether there is a pagecache page to back given address within VMA.
2582 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2583 */
2584static bool hugetlbfs_pagecache_present(struct hstate *h,
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002585 struct vm_area_struct *vma, unsigned long address)
2586{
2587 struct address_space *mapping;
2588 pgoff_t idx;
2589 struct page *page;
2590
2591 mapping = vma->vm_file->f_mapping;
2592 idx = vma_hugecache_offset(h, vma, address);
2593
2594 page = find_get_page(mapping, idx);
2595 if (page)
2596 put_page(page);
2597 return page != NULL;
2598}
2599
Robert P. J. Daya1ed3dd2007-07-17 04:03:33 -07002600static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +01002601 unsigned long address, pte_t *ptep, unsigned int flags)
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002602{
Andi Kleena5516432008-07-23 21:27:41 -07002603 struct hstate *h = hstate_vma(vma);
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002604 int ret = VM_FAULT_SIGBUS;
Hillf Danton409eb8c2012-01-20 14:34:13 -08002605 int anon_rmap = 0;
Andy Whitcrofte7c4b0b2008-07-23 21:27:26 -07002606 pgoff_t idx;
Adam Litke4c887262005-10-29 18:16:46 -07002607 unsigned long size;
Adam Litke4c887262005-10-29 18:16:46 -07002608 struct page *page;
2609 struct address_space *mapping;
David Gibson1e8f8892006-01-06 00:10:44 -08002610 pte_t new_pte;
Adam Litke4c887262005-10-29 18:16:46 -07002611
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002612 /*
2613 * Currently, we are forced to kill the process in the event the
2614 * original mapper has unmapped pages from the child due to a failed
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002615 * COW. Warn that such a situation has occurred as it may not be obvious
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002616 */
2617 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2618 printk(KERN_WARNING
2619 "PID %d killed due to inadequate hugepage pool\n",
2620 current->pid);
2621 return ret;
2622 }
2623
Adam Litke4c887262005-10-29 18:16:46 -07002624 mapping = vma->vm_file->f_mapping;
Andi Kleena5516432008-07-23 21:27:41 -07002625 idx = vma_hugecache_offset(h, vma, address);
Adam Litke4c887262005-10-29 18:16:46 -07002626
2627 /*
2628 * Use page lock to guard against racing truncation
2629 * before we get page_table_lock.
2630 */
Christoph Lameter6bda6662006-01-06 00:10:49 -08002631retry:
2632 page = find_lock_page(mapping, idx);
2633 if (!page) {
Andi Kleena5516432008-07-23 21:27:41 -07002634 size = i_size_read(mapping->host) >> huge_page_shift(h);
Hugh Dickinsebed4bf2006-10-28 10:38:43 -07002635 if (idx >= size)
2636 goto out;
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002637 page = alloc_huge_page(vma, address, 0);
Adam Litke2fc39ce2007-11-14 16:59:39 -08002638 if (IS_ERR(page)) {
2639 ret = -PTR_ERR(page);
Christoph Lameter6bda6662006-01-06 00:10:49 -08002640 goto out;
2641 }
Andrea Arcangeli47ad8472011-01-13 15:46:47 -08002642 clear_huge_page(page, address, pages_per_huge_page(h));
Nick Piggin0ed361d2008-02-04 22:29:34 -08002643 __SetPageUptodate(page);
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002644
Mel Gormanf83a2752009-05-28 14:34:40 -07002645 if (vma->vm_flags & VM_MAYSHARE) {
Christoph Lameter6bda6662006-01-06 00:10:49 -08002646 int err;
Ken Chen45c682a2007-11-14 16:59:44 -08002647 struct inode *inode = mapping->host;
Christoph Lameter6bda6662006-01-06 00:10:49 -08002648
2649 err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2650 if (err) {
2651 put_page(page);
Christoph Lameter6bda6662006-01-06 00:10:49 -08002652 if (err == -EEXIST)
2653 goto retry;
2654 goto out;
2655 }
Ken Chen45c682a2007-11-14 16:59:44 -08002656
2657 spin_lock(&inode->i_lock);
Andi Kleena5516432008-07-23 21:27:41 -07002658 inode->i_blocks += blocks_per_huge_page(h);
Ken Chen45c682a2007-11-14 16:59:44 -08002659 spin_unlock(&inode->i_lock);
Mel Gorman23be7462010-04-23 13:17:56 -04002660 } else {
Christoph Lameter6bda6662006-01-06 00:10:49 -08002661 lock_page(page);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002662 if (unlikely(anon_vma_prepare(vma))) {
2663 ret = VM_FAULT_OOM;
2664 goto backout_unlocked;
2665 }
Hillf Danton409eb8c2012-01-20 14:34:13 -08002666 anon_rmap = 1;
Mel Gorman23be7462010-04-23 13:17:56 -04002667 }
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002668 } else {
Naoya Horiguchi998b4382010-09-08 10:19:32 +09002669 /*
2670 * If memory error occurs between mmap() and fault, some process
2671 * don't have hwpoisoned swap entry for errored virtual address.
2672 * So we need to block hugepage fault by PG_hwpoison bit check.
2673 */
2674 if (unlikely(PageHWPoison(page))) {
Chris Forbes32f84522011-07-25 17:12:14 -07002675 ret = VM_FAULT_HWPOISON |
Andi Kleenaa50d3a2010-10-06 21:45:00 +02002676 VM_FAULT_SET_HINDEX(h - hstates);
Naoya Horiguchi998b4382010-09-08 10:19:32 +09002677 goto backout_unlocked;
2678 }
Christoph Lameter6bda6662006-01-06 00:10:49 -08002679 }
David Gibson1e8f8892006-01-06 00:10:44 -08002680
Andy Whitcroft57303d82008-08-12 15:08:47 -07002681 /*
2682 * If we are going to COW a private mapping later, we examine the
2683 * pending reservations for this page now. This will ensure that
2684 * any allocations necessary to record that reservation occur outside
2685 * the spinlock.
2686 */
Hugh Dickins788c7df2009-06-23 13:49:05 +01002687 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
Andy Whitcroft2b267362008-08-12 15:08:49 -07002688 if (vma_needs_reservation(h, vma, address) < 0) {
2689 ret = VM_FAULT_OOM;
2690 goto backout_unlocked;
2691 }
Andy Whitcroft57303d82008-08-12 15:08:47 -07002692
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002693 spin_lock(&mm->page_table_lock);
Andi Kleena5516432008-07-23 21:27:41 -07002694 size = i_size_read(mapping->host) >> huge_page_shift(h);
Adam Litke4c887262005-10-29 18:16:46 -07002695 if (idx >= size)
2696 goto backout;
2697
Nick Piggin83c54072007-07-19 01:47:05 -07002698 ret = 0;
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002699 if (!huge_pte_none(huge_ptep_get(ptep)))
Adam Litke4c887262005-10-29 18:16:46 -07002700 goto backout;
2701
Hillf Danton409eb8c2012-01-20 14:34:13 -08002702 if (anon_rmap)
2703 hugepage_add_new_anon_rmap(page, vma, address);
2704 else
2705 page_dup_rmap(page);
David Gibson1e8f8892006-01-06 00:10:44 -08002706 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2707 && (vma->vm_flags & VM_SHARED)));
2708 set_huge_pte_at(mm, address, ptep, new_pte);
2709
Hugh Dickins788c7df2009-06-23 13:49:05 +01002710 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
David Gibson1e8f8892006-01-06 00:10:44 -08002711 /* Optimization, do the COW without a second fault */
Mel Gorman04f2cbe2008-07-23 21:27:25 -07002712 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
David Gibson1e8f8892006-01-06 00:10:44 -08002713 }
2714
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002715 spin_unlock(&mm->page_table_lock);
Adam Litke4c887262005-10-29 18:16:46 -07002716 unlock_page(page);
2717out:
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002718 return ret;
Adam Litke4c887262005-10-29 18:16:46 -07002719
2720backout:
2721 spin_unlock(&mm->page_table_lock);
Andy Whitcroft2b267362008-08-12 15:08:49 -07002722backout_unlocked:
Adam Litke4c887262005-10-29 18:16:46 -07002723 unlock_page(page);
2724 put_page(page);
2725 goto out;
Hugh Dickinsac9b9c62005-10-20 16:24:28 +01002726}
2727
Adam Litke86e52162006-01-06 00:10:43 -08002728int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Hugh Dickins788c7df2009-06-23 13:49:05 +01002729 unsigned long address, unsigned int flags)
Adam Litke86e52162006-01-06 00:10:43 -08002730{
2731 pte_t *ptep;
2732 pte_t entry;
David Gibson1e8f8892006-01-06 00:10:44 -08002733 int ret;
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002734 struct page *page = NULL;
Andy Whitcroft57303d82008-08-12 15:08:47 -07002735 struct page *pagecache_page = NULL;
David Gibson3935baa2006-03-22 00:08:53 -08002736 static DEFINE_MUTEX(hugetlb_instantiation_mutex);
Andi Kleena5516432008-07-23 21:27:41 -07002737 struct hstate *h = hstate_vma(vma);
Adam Litke86e52162006-01-06 00:10:43 -08002738
KAMEZAWA Hiroyuki1e16a532012-01-10 15:07:22 -08002739 address &= huge_page_mask(h);
2740
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002741 ptep = huge_pte_offset(mm, address);
2742 if (ptep) {
2743 entry = huge_ptep_get(ptep);
Naoya Horiguchi290408d2010-09-08 10:19:35 +09002744 if (unlikely(is_hugetlb_entry_migration(entry))) {
2745 migration_entry_wait(mm, (pmd_t *)ptep, address);
2746 return 0;
2747 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
Chris Forbes32f84522011-07-25 17:12:14 -07002748 return VM_FAULT_HWPOISON_LARGE |
Andi Kleenaa50d3a2010-10-06 21:45:00 +02002749 VM_FAULT_SET_HINDEX(h - hstates);
Naoya Horiguchifd6a03e2010-05-28 09:29:21 +09002750 }
2751
Andi Kleena5516432008-07-23 21:27:41 -07002752 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
Adam Litke86e52162006-01-06 00:10:43 -08002753 if (!ptep)
2754 return VM_FAULT_OOM;
2755
David Gibson3935baa2006-03-22 00:08:53 -08002756 /*
2757 * Serialize hugepage allocation and instantiation, so that we don't
2758 * get spurious allocation failures if two CPUs race to instantiate
2759 * the same page in the page cache.
2760 */
2761 mutex_lock(&hugetlb_instantiation_mutex);
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002762 entry = huge_ptep_get(ptep);
2763 if (huge_pte_none(entry)) {
Hugh Dickins788c7df2009-06-23 13:49:05 +01002764 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
David Gibsonb4d1d992008-10-15 22:01:11 -07002765 goto out_mutex;
David Gibson3935baa2006-03-22 00:08:53 -08002766 }
Adam Litke86e52162006-01-06 00:10:43 -08002767
Nick Piggin83c54072007-07-19 01:47:05 -07002768 ret = 0;
David Gibson1e8f8892006-01-06 00:10:44 -08002769
Andy Whitcroft57303d82008-08-12 15:08:47 -07002770 /*
2771 * If we are going to COW the mapping later, we examine the pending
2772 * reservations for this page now. This will ensure that any
2773 * allocations necessary to record that reservation occur outside the
2774 * spinlock. For private mappings, we also lookup the pagecache
2775 * page now as it is used to determine if a reservation has been
2776 * consumed.
2777 */
Hugh Dickins788c7df2009-06-23 13:49:05 +01002778 if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
Andy Whitcroft2b267362008-08-12 15:08:49 -07002779 if (vma_needs_reservation(h, vma, address) < 0) {
2780 ret = VM_FAULT_OOM;
David Gibsonb4d1d992008-10-15 22:01:11 -07002781 goto out_mutex;
Andy Whitcroft2b267362008-08-12 15:08:49 -07002782 }
Andy Whitcroft57303d82008-08-12 15:08:47 -07002783
Mel Gormanf83a2752009-05-28 14:34:40 -07002784 if (!(vma->vm_flags & VM_MAYSHARE))
Andy Whitcroft57303d82008-08-12 15:08:47 -07002785 pagecache_page = hugetlbfs_pagecache_page(h,
2786 vma, address);
2787 }
2788
Naoya Horiguchi56c9cfb2010-09-10 13:23:04 +09002789 /*
2790 * hugetlb_cow() requires page locks of pte_page(entry) and
2791 * pagecache_page, so here we need take the former one
2792 * when page != pagecache_page or !pagecache_page.
2793 * Note that locking order is always pagecache_page -> page,
2794 * so no worry about deadlock.
2795 */
2796 page = pte_page(entry);
2797 if (page != pagecache_page)
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002798 lock_page(page);
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +09002799
David Gibson1e8f8892006-01-06 00:10:44 -08002800 spin_lock(&mm->page_table_lock);
2801 /* Check for a racing update before calling hugetlb_cow */
David Gibsonb4d1d992008-10-15 22:01:11 -07002802 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2803 goto out_page_table_lock;
2804
2805
Hugh Dickins788c7df2009-06-23 13:49:05 +01002806 if (flags & FAULT_FLAG_WRITE) {
David Gibsonb4d1d992008-10-15 22:01:11 -07002807 if (!pte_write(entry)) {
Andy Whitcroft57303d82008-08-12 15:08:47 -07002808 ret = hugetlb_cow(mm, vma, address, ptep, entry,
2809 pagecache_page);
David Gibsonb4d1d992008-10-15 22:01:11 -07002810 goto out_page_table_lock;
2811 }
2812 entry = pte_mkdirty(entry);
2813 }
2814 entry = pte_mkyoung(entry);
Hugh Dickins788c7df2009-06-23 13:49:05 +01002815 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2816 flags & FAULT_FLAG_WRITE))
Russell King4b3073e2009-12-18 16:40:18 +00002817 update_mmu_cache(vma, address, ptep);
David Gibsonb4d1d992008-10-15 22:01:11 -07002818
2819out_page_table_lock:
David Gibson1e8f8892006-01-06 00:10:44 -08002820 spin_unlock(&mm->page_table_lock);
Andy Whitcroft57303d82008-08-12 15:08:47 -07002821
2822 if (pagecache_page) {
2823 unlock_page(pagecache_page);
2824 put_page(pagecache_page);
2825 }
Dean Nelson1f64d692010-12-02 14:31:12 -08002826 if (page != pagecache_page)
2827 unlock_page(page);
Andy Whitcroft57303d82008-08-12 15:08:47 -07002828
David Gibsonb4d1d992008-10-15 22:01:11 -07002829out_mutex:
David Gibson3935baa2006-03-22 00:08:53 -08002830 mutex_unlock(&hugetlb_instantiation_mutex);
David Gibson1e8f8892006-01-06 00:10:44 -08002831
2832 return ret;
Adam Litke86e52162006-01-06 00:10:43 -08002833}
2834
Andi Kleenceb86872008-07-23 21:27:50 -07002835/* Can be overriden by architectures */
2836__attribute__((weak)) struct page *
2837follow_huge_pud(struct mm_struct *mm, unsigned long address,
2838 pud_t *pud, int write)
2839{
2840 BUG();
2841 return NULL;
2842}
2843
David Gibson63551ae2005-06-21 17:14:44 -07002844int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2845 struct page **pages, struct vm_area_struct **vmas,
Adam Litke5b23dbe2007-11-14 16:59:33 -08002846 unsigned long *position, int *length, int i,
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002847 unsigned int flags)
David Gibson63551ae2005-06-21 17:14:44 -07002848{
Chen, Kenneth Wd5d4b0a2006-03-22 00:09:03 -08002849 unsigned long pfn_offset;
2850 unsigned long vaddr = *position;
David Gibson63551ae2005-06-21 17:14:44 -07002851 int remainder = *length;
Andi Kleena5516432008-07-23 21:27:41 -07002852 struct hstate *h = hstate_vma(vma);
David Gibson63551ae2005-06-21 17:14:44 -07002853
Hugh Dickins1c598272005-10-19 21:23:43 -07002854 spin_lock(&mm->page_table_lock);
David Gibson63551ae2005-06-21 17:14:44 -07002855 while (vaddr < vma->vm_end && remainder) {
Adam Litke4c887262005-10-29 18:16:46 -07002856 pte_t *pte;
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002857 int absent;
Adam Litke4c887262005-10-29 18:16:46 -07002858 struct page *page;
2859
2860 /*
2861 * Some archs (sparc64, sh*) have multiple pte_ts to
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002862 * each hugepage. We have to make sure we get the
Adam Litke4c887262005-10-29 18:16:46 -07002863 * first, for the page indexing below to work.
2864 */
Andi Kleena5516432008-07-23 21:27:41 -07002865 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002866 absent = !pte || huge_pte_none(huge_ptep_get(pte));
Adam Litke4c887262005-10-29 18:16:46 -07002867
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002868 /*
2869 * When coredumping, it suits get_dump_page if we just return
Hugh Dickins3ae77f42009-09-21 17:03:33 -07002870 * an error where there's an empty slot with no huge pagecache
2871 * to back it. This way, we avoid allocating a hugepage, and
2872 * the sparse dumpfile avoids allocating disk blocks, but its
2873 * huge holes still show up with zeroes where they need to be.
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002874 */
Hugh Dickins3ae77f42009-09-21 17:03:33 -07002875 if (absent && (flags & FOLL_DUMP) &&
2876 !hugetlbfs_pagecache_present(h, vma, vaddr)) {
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002877 remainder = 0;
2878 break;
2879 }
2880
2881 if (absent ||
2882 ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
Adam Litke4c887262005-10-29 18:16:46 -07002883 int ret;
2884
2885 spin_unlock(&mm->page_table_lock);
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002886 ret = hugetlb_fault(mm, vma, vaddr,
2887 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
Adam Litke4c887262005-10-29 18:16:46 -07002888 spin_lock(&mm->page_table_lock);
Adam Litkea89182c2007-08-22 14:01:51 -07002889 if (!(ret & VM_FAULT_ERROR))
Adam Litke4c887262005-10-29 18:16:46 -07002890 continue;
2891
2892 remainder = 0;
Adam Litke4c887262005-10-29 18:16:46 -07002893 break;
2894 }
David Gibson63551ae2005-06-21 17:14:44 -07002895
Andi Kleena5516432008-07-23 21:27:41 -07002896 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002897 page = pte_page(huge_ptep_get(pte));
Chen, Kenneth Wd5d4b0a2006-03-22 00:09:03 -08002898same_page:
Chen, Kenneth Wd6692182006-03-31 02:29:57 -08002899 if (pages) {
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002900 pages[i] = mem_map_offset(page, pfn_offset);
KOSAKI Motohiro4b2e38a2008-10-18 20:27:10 -07002901 get_page(pages[i]);
Chen, Kenneth Wd6692182006-03-31 02:29:57 -08002902 }
David Gibson63551ae2005-06-21 17:14:44 -07002903
2904 if (vmas)
2905 vmas[i] = vma;
2906
2907 vaddr += PAGE_SIZE;
Chen, Kenneth Wd5d4b0a2006-03-22 00:09:03 -08002908 ++pfn_offset;
David Gibson63551ae2005-06-21 17:14:44 -07002909 --remainder;
2910 ++i;
Chen, Kenneth Wd5d4b0a2006-03-22 00:09:03 -08002911 if (vaddr < vma->vm_end && remainder &&
Andi Kleena5516432008-07-23 21:27:41 -07002912 pfn_offset < pages_per_huge_page(h)) {
Chen, Kenneth Wd5d4b0a2006-03-22 00:09:03 -08002913 /*
2914 * We use pfn_offset to avoid touching the pageframes
2915 * of this compound page.
2916 */
2917 goto same_page;
2918 }
David Gibson63551ae2005-06-21 17:14:44 -07002919 }
Hugh Dickins1c598272005-10-19 21:23:43 -07002920 spin_unlock(&mm->page_table_lock);
David Gibson63551ae2005-06-21 17:14:44 -07002921 *length = remainder;
2922 *position = vaddr;
2923
Hugh Dickins2a15efc2009-09-21 17:03:27 -07002924 return i ? i : -EFAULT;
David Gibson63551ae2005-06-21 17:14:44 -07002925}
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002926
2927void hugetlb_change_protection(struct vm_area_struct *vma,
2928 unsigned long address, unsigned long end, pgprot_t newprot)
2929{
2930 struct mm_struct *mm = vma->vm_mm;
2931 unsigned long start = address;
2932 pte_t *ptep;
2933 pte_t pte;
Andi Kleena5516432008-07-23 21:27:41 -07002934 struct hstate *h = hstate_vma(vma);
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002935
2936 BUG_ON(address >= end);
2937 flush_cache_range(vma, address, end);
2938
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002939 mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002940 spin_lock(&mm->page_table_lock);
Andi Kleena5516432008-07-23 21:27:41 -07002941 for (; address < end; address += huge_page_size(h)) {
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002942 ptep = huge_pte_offset(mm, address);
2943 if (!ptep)
2944 continue;
Chen, Kenneth W39dde652006-12-06 20:32:03 -08002945 if (huge_pmd_unshare(mm, &address, ptep))
2946 continue;
Gerald Schaefer7f2e9522008-04-28 02:13:29 -07002947 if (!huge_pte_none(huge_ptep_get(ptep))) {
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002948 pte = huge_ptep_get_and_clear(mm, address, ptep);
2949 pte = pte_mkhuge(pte_modify(pte, newprot));
2950 set_huge_pte_at(mm, address, ptep, pte);
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002951 }
2952 }
2953 spin_unlock(&mm->page_table_lock);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -07002954 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
Zhang, Yanmin8f860592006-03-22 00:08:50 -08002955
2956 flush_tlb_range(vma, start, end);
2957}
2958
Mel Gormana1e78772008-07-23 21:27:23 -07002959int hugetlb_reserve_pages(struct inode *inode,
2960 long from, long to,
Mel Gorman5a6fe122009-02-10 14:02:27 +00002961 struct vm_area_struct *vma,
KOSAKI Motohiroca16d142011-05-26 19:16:19 +09002962 vm_flags_t vm_flags)
Adam Litkee4e574b2007-10-16 01:26:19 -07002963{
Mel Gorman17c9d122009-02-11 16:34:16 +00002964 long ret, chg;
Andi Kleena5516432008-07-23 21:27:41 -07002965 struct hstate *h = hstate_inode(inode);
David Gibson90481622012-03-21 16:34:12 -07002966 struct hugepage_subpool *spool = subpool_inode(inode);
Adam Litkee4e574b2007-10-16 01:26:19 -07002967
Mel Gormana1e78772008-07-23 21:27:23 -07002968 /*
Mel Gorman17c9d122009-02-11 16:34:16 +00002969 * Only apply hugepage reservation if asked. At fault time, an
2970 * attempt will be made for VM_NORESERVE to allocate a page
David Gibson90481622012-03-21 16:34:12 -07002971 * without using reserves
Mel Gorman17c9d122009-02-11 16:34:16 +00002972 */
KOSAKI Motohiroca16d142011-05-26 19:16:19 +09002973 if (vm_flags & VM_NORESERVE)
Mel Gorman17c9d122009-02-11 16:34:16 +00002974 return 0;
2975
2976 /*
Mel Gormana1e78772008-07-23 21:27:23 -07002977 * Shared mappings base their reservation on the number of pages that
2978 * are already allocated on behalf of the file. Private mappings need
2979 * to reserve the full area even if read-only as mprotect() may be
2980 * called to make the mapping read-write. Assume !vma is a shm mapping
2981 */
Mel Gormanf83a2752009-05-28 14:34:40 -07002982 if (!vma || vma->vm_flags & VM_MAYSHARE)
Mel Gormana1e78772008-07-23 21:27:23 -07002983 chg = region_chg(&inode->i_mapping->private_list, from, to);
Mel Gorman5a6fe122009-02-10 14:02:27 +00002984 else {
2985 struct resv_map *resv_map = resv_map_alloc();
Mel Gorman5a6fe122009-02-10 14:02:27 +00002986 if (!resv_map)
2987 return -ENOMEM;
2988
Mel Gorman17c9d122009-02-11 16:34:16 +00002989 chg = to - from;
2990
Mel Gorman5a6fe122009-02-10 14:02:27 +00002991 set_vma_resv_map(vma, resv_map);
2992 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
2993 }
2994
Mel Gorman17c9d122009-02-11 16:34:16 +00002995 if (chg < 0)
2996 return chg;
2997
David Gibson90481622012-03-21 16:34:12 -07002998 /* There must be enough pages in the subpool for the mapping */
2999 if (hugepage_subpool_get_pages(spool, chg))
Mel Gorman17c9d122009-02-11 16:34:16 +00003000 return -ENOSPC;
3001
3002 /*
3003 * Check enough hugepages are available for the reservation.
David Gibson90481622012-03-21 16:34:12 -07003004 * Hand the pages back to the subpool if there are not
Mel Gorman17c9d122009-02-11 16:34:16 +00003005 */
3006 ret = hugetlb_acct_memory(h, chg);
3007 if (ret < 0) {
David Gibson90481622012-03-21 16:34:12 -07003008 hugepage_subpool_put_pages(spool, chg);
Mel Gorman17c9d122009-02-11 16:34:16 +00003009 return ret;
3010 }
3011
3012 /*
3013 * Account for the reservations made. Shared mappings record regions
3014 * that have reservations as they are shared by multiple VMAs.
3015 * When the last VMA disappears, the region map says how much
3016 * the reservation was and the page cache tells how much of
3017 * the reservation was consumed. Private mappings are per-VMA and
3018 * only the consumed reservations are tracked. When the VMA
3019 * disappears, the original reservation is the VMA size and the
3020 * consumed reservations are stored in the map. Hence, nothing
3021 * else has to be done for private mappings here
3022 */
Mel Gormanf83a2752009-05-28 14:34:40 -07003023 if (!vma || vma->vm_flags & VM_MAYSHARE)
Mel Gorman17c9d122009-02-11 16:34:16 +00003024 region_add(&inode->i_mapping->private_list, from, to);
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -07003025 return 0;
3026}
3027
3028void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3029{
Andi Kleena5516432008-07-23 21:27:41 -07003030 struct hstate *h = hstate_inode(inode);
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -07003031 long chg = region_truncate(&inode->i_mapping->private_list, offset);
David Gibson90481622012-03-21 16:34:12 -07003032 struct hugepage_subpool *spool = subpool_inode(inode);
Ken Chen45c682a2007-11-14 16:59:44 -08003033
3034 spin_lock(&inode->i_lock);
Eric Sandeene4c6f8b2009-07-29 15:02:16 -07003035 inode->i_blocks -= (blocks_per_huge_page(h) * freed);
Ken Chen45c682a2007-11-14 16:59:44 -08003036 spin_unlock(&inode->i_lock);
3037
David Gibson90481622012-03-21 16:34:12 -07003038 hugepage_subpool_put_pages(spool, (chg - freed));
Andi Kleena5516432008-07-23 21:27:41 -07003039 hugetlb_acct_memory(h, -(chg - freed));
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -07003040}
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003041
Andi Kleend5bd9102010-09-27 09:00:12 +02003042#ifdef CONFIG_MEMORY_FAILURE
3043
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003044/* Should be called in hugetlb_lock */
3045static int is_hugepage_on_freelist(struct page *hpage)
3046{
3047 struct page *page;
3048 struct page *tmp;
3049 struct hstate *h = page_hstate(hpage);
3050 int nid = page_to_nid(hpage);
3051
3052 list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3053 if (page == hpage)
3054 return 1;
3055 return 0;
3056}
3057
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003058/*
3059 * This function is called from memory failure code.
3060 * Assume the caller holds page lock of the head page.
3061 */
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003062int dequeue_hwpoisoned_huge_page(struct page *hpage)
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003063{
3064 struct hstate *h = page_hstate(hpage);
3065 int nid = page_to_nid(hpage);
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003066 int ret = -EBUSY;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003067
3068 spin_lock(&hugetlb_lock);
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003069 if (is_hugepage_on_freelist(hpage)) {
3070 list_del(&hpage->lru);
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09003071 set_page_refcounted(hpage);
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003072 h->free_huge_pages--;
3073 h->free_huge_pages_node[nid]--;
3074 ret = 0;
3075 }
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003076 spin_unlock(&hugetlb_lock);
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003077 return ret;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +09003078}
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +09003079#endif