blob: 1e2872b25343f106df7a96d0b667e194dd070c54 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * hugetlbpage-backed filesystem. Based on ramfs.
3 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01004 * Nadia Yvette Chambers, 2002
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Copyright (C) 2002 Linus Torvalds.
7 */
8
Andrew Morton9b857d22014-06-04 16:07:21 -07009#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/module.h>
12#include <linux/thread_info.h>
13#include <asm/current.h>
14#include <linux/sched.h> /* remove ASAP */
15#include <linux/fs.h>
16#include <linux/mount.h>
17#include <linux/file.h>
Randy Dunlape73a75f2007-07-15 23:40:52 -070018#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/writeback.h>
20#include <linux/pagemap.h>
21#include <linux/highmem.h>
22#include <linux/init.h>
23#include <linux/string.h>
Randy Dunlap16f7e0f2006-01-11 12:17:46 -080024#include <linux/capability.h>
Randy Dunlape73a75f2007-07-15 23:40:52 -070025#include <linux/ctype.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/backing-dev.h>
27#include <linux/hugetlb.h>
28#include <linux/pagevec.h>
Randy Dunlape73a75f2007-07-15 23:40:52 -070029#include <linux/parser.h>
Benjamin Herrenschmidt036e0852007-05-06 14:50:12 -070030#include <linux/mman.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <linux/slab.h>
32#include <linux/dnotify.h>
33#include <linux/statfs.h>
34#include <linux/security.h>
Nick Black1fd7317d2009-09-22 16:43:33 -070035#include <linux/magic.h>
Naoya Horiguchi290408d2010-09-08 10:19:35 +090036#include <linux/migrate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
38#include <asm/uaccess.h>
39
Josef 'Jeff' Sipekee9b6d62007-02-12 00:55:41 -080040static const struct super_operations hugetlbfs_ops;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -070041static const struct address_space_operations hugetlbfs_aops;
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080042const struct file_operations hugetlbfs_file_operations;
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -080043static const struct inode_operations hugetlbfs_dir_inode_operations;
44static const struct inode_operations hugetlbfs_inode_operations;
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
David Gibsona1d776e2012-03-21 16:34:12 -070046struct hugetlbfs_config {
Eric W. Biedermana0eb3a02012-02-07 16:19:25 -080047 kuid_t uid;
48 kgid_t gid;
David Gibsona1d776e2012-03-21 16:34:12 -070049 umode_t mode;
50 long nr_blocks;
51 long nr_inodes;
52 struct hstate *hstate;
53};
54
55struct hugetlbfs_inode_info {
56 struct shared_policy policy;
57 struct inode vfs_inode;
58};
59
60static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
61{
62 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
63}
64
Linus Torvalds1da177e2005-04-16 15:20:36 -070065static struct backing_dev_info hugetlbfs_backing_dev_info = {
Jens Axboed9938312009-06-12 14:45:52 +020066 .name = "hugetlbfs",
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 .ra_pages = 0, /* No readahead */
Miklos Szeredie4ad08f2008-04-30 00:54:37 -070068 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
Linus Torvalds1da177e2005-04-16 15:20:36 -070069};
70
71int sysctl_hugetlb_shm_group;
72
Randy Dunlape73a75f2007-07-15 23:40:52 -070073enum {
74 Opt_size, Opt_nr_inodes,
75 Opt_mode, Opt_uid, Opt_gid,
Andi Kleena137e1c2008-07-23 21:27:43 -070076 Opt_pagesize,
Randy Dunlape73a75f2007-07-15 23:40:52 -070077 Opt_err,
78};
79
Steven Whitehousea447c092008-10-13 10:46:57 +010080static const match_table_t tokens = {
Randy Dunlape73a75f2007-07-15 23:40:52 -070081 {Opt_size, "size=%s"},
82 {Opt_nr_inodes, "nr_inodes=%s"},
83 {Opt_mode, "mode=%o"},
84 {Opt_uid, "uid=%u"},
85 {Opt_gid, "gid=%u"},
Andi Kleena137e1c2008-07-23 21:27:43 -070086 {Opt_pagesize, "pagesize=%s"},
Randy Dunlape73a75f2007-07-15 23:40:52 -070087 {Opt_err, NULL},
88};
89
Adam Litke2e9b3672005-10-29 18:16:47 -070090static void huge_pagevec_release(struct pagevec *pvec)
91{
92 int i;
93
94 for (i = 0; i < pagevec_count(pvec); ++i)
95 put_page(pvec->pages[i]);
96
97 pagevec_reinit(pvec);
98}
99
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
101{
Al Viro496ad9a2013-01-23 17:07:38 -0500102 struct inode *inode = file_inode(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 loff_t len, vma_len;
104 int ret;
Andi Kleena5516432008-07-23 21:27:41 -0700105 struct hstate *h = hstate_file(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Hugh Dickins68589bc2006-11-14 02:03:32 -0800107 /*
David Gibsondec4ad82007-08-30 23:56:40 -0700108 * vma address alignment (but not the pgoff alignment) has
109 * already been checked by prepare_hugepage_range. If you add
110 * any error returns here, do so after setting VM_HUGETLB, so
111 * is_vm_hugetlb_page tests below unmap_region go the right
112 * way when do_mmap_pgoff unwinds (may be important on powerpc
113 * and ia64).
Hugh Dickins68589bc2006-11-14 02:03:32 -0800114 */
Naoya Horiguchia2fce912013-04-17 15:58:27 -0700115 vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
Hugh Dickins68589bc2006-11-14 02:03:32 -0800116 vma->vm_ops = &hugetlb_vm_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117
Becky Bruce2b37c352011-07-25 17:11:49 -0700118 if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
David Gibsondec4ad82007-08-30 23:56:40 -0700119 return -EINVAL;
120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 vma_len = (loff_t)(vma->vm_end - vma->vm_start);
122
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800123 mutex_lock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124 file_accessed(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126 ret = -ENOMEM;
127 len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Mel Gormana1e78772008-07-23 21:27:23 -0700129 if (hugetlb_reserve_pages(inode,
Andi Kleena5516432008-07-23 21:27:41 -0700130 vma->vm_pgoff >> huge_page_order(h),
Mel Gorman5a6fe122009-02-10 14:02:27 +0000131 len >> huge_page_shift(h), vma,
132 vma->vm_flags))
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -0700133 goto out;
David Gibsonb45b5bd2006-03-22 00:08:55 -0800134
Adam Litke4c887262005-10-29 18:16:46 -0700135 ret = 0;
136 hugetlb_prefault_arch_hook(vma->vm_mm);
Zhang, Yanminb6174df2006-07-10 04:44:49 -0700137 if (vma->vm_flags & VM_WRITE && inode->i_size < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 inode->i_size = len;
139out:
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800140 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
142 return ret;
143}
144
145/*
Hugh Dickins508034a2005-10-29 18:16:30 -0700146 * Called under down_write(mmap_sem).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 */
148
Adrian Bunkd2ba27e82007-05-06 14:49:00 -0700149#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150static unsigned long
151hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
152 unsigned long len, unsigned long pgoff, unsigned long flags)
153{
154 struct mm_struct *mm = current->mm;
155 struct vm_area_struct *vma;
Andi Kleena5516432008-07-23 21:27:41 -0700156 struct hstate *h = hstate_file(file);
Michel Lespinasse08659352012-12-11 16:02:00 -0800157 struct vm_unmapped_area_info info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158
Andi Kleena5516432008-07-23 21:27:41 -0700159 if (len & ~huge_page_mask(h))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 return -EINVAL;
161 if (len > TASK_SIZE)
162 return -ENOMEM;
163
Benjamin Herrenschmidt036e0852007-05-06 14:50:12 -0700164 if (flags & MAP_FIXED) {
Andi Kleena5516432008-07-23 21:27:41 -0700165 if (prepare_hugepage_range(file, addr, len))
Benjamin Herrenschmidt036e0852007-05-06 14:50:12 -0700166 return -EINVAL;
167 return addr;
168 }
169
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 if (addr) {
Andi Kleena5516432008-07-23 21:27:41 -0700171 addr = ALIGN(addr, huge_page_size(h));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 vma = find_vma(mm, addr);
173 if (TASK_SIZE - len >= addr &&
174 (!vma || addr + len <= vma->vm_start))
175 return addr;
176 }
177
Michel Lespinasse08659352012-12-11 16:02:00 -0800178 info.flags = 0;
179 info.length = len;
180 info.low_limit = TASK_UNMAPPED_BASE;
181 info.high_limit = TASK_SIZE;
182 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
183 info.align_offset = 0;
184 return vm_unmapped_area(&info);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185}
186#endif
187
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700188static int
189hugetlbfs_read_actor(struct page *page, unsigned long offset,
190 char __user *buf, unsigned long count,
191 unsigned long size)
192{
193 char *kaddr;
194 unsigned long left, copied = 0;
195 int i, chunksize;
196
197 if (size > count)
198 size = count;
199
200 /* Find which 4k chunk and offset with in that chunk */
201 i = offset >> PAGE_CACHE_SHIFT;
202 offset = offset & ~PAGE_CACHE_MASK;
203
204 while (size) {
205 chunksize = PAGE_CACHE_SIZE;
206 if (offset)
207 chunksize -= offset;
208 if (chunksize > size)
209 chunksize = size;
210 kaddr = kmap(&page[i]);
211 left = __copy_to_user(buf, kaddr + offset, chunksize);
212 kunmap(&page[i]);
213 if (left) {
214 copied += (chunksize - left);
215 break;
216 }
217 offset = 0;
218 size -= chunksize;
219 buf += chunksize;
220 copied += chunksize;
221 i++;
222 }
223 return copied ? copied : -EFAULT;
224}
225
226/*
227 * Support for read() - Find the page attached to f_mapping and copy out the
228 * data. Its *very* similar to do_generic_mapping_read(), we can't use that
229 * since it has PAGE_CACHE_SIZE assumptions.
230 */
231static ssize_t hugetlbfs_read(struct file *filp, char __user *buf,
232 size_t len, loff_t *ppos)
233{
Andi Kleena5516432008-07-23 21:27:41 -0700234 struct hstate *h = hstate_file(filp);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700235 struct address_space *mapping = filp->f_mapping;
236 struct inode *inode = mapping->host;
Andi Kleena5516432008-07-23 21:27:41 -0700237 unsigned long index = *ppos >> huge_page_shift(h);
238 unsigned long offset = *ppos & ~huge_page_mask(h);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700239 unsigned long end_index;
240 loff_t isize;
241 ssize_t retval = 0;
242
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700243 /* validate length */
244 if (len == 0)
245 goto out;
246
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700247 for (;;) {
248 struct page *page;
Andi Kleena5516432008-07-23 21:27:41 -0700249 unsigned long nr, ret;
Roel Kluin91bf189c2009-01-06 14:40:14 -0800250 int ra;
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700251
252 /* nr is the maximum number of bytes to copy from this page */
Andi Kleena5516432008-07-23 21:27:41 -0700253 nr = huge_page_size(h);
Aneesh Kumar K.Va05b0852012-03-21 16:34:08 -0700254 isize = i_size_read(inode);
255 if (!isize)
256 goto out;
257 end_index = (isize - 1) >> huge_page_shift(h);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700258 if (index >= end_index) {
259 if (index > end_index)
260 goto out;
Andi Kleena5516432008-07-23 21:27:41 -0700261 nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
Aneesh Kumar K.Va05b0852012-03-21 16:34:08 -0700262 if (nr <= offset)
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700263 goto out;
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700264 }
265 nr = nr - offset;
266
267 /* Find the page */
Aneesh Kumar K.Va05b0852012-03-21 16:34:08 -0700268 page = find_lock_page(mapping, index);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700269 if (unlikely(page == NULL)) {
270 /*
271 * We have a HOLE, zero out the user-buffer for the
272 * length of the hole or request.
273 */
274 ret = len < nr ? len : nr;
275 if (clear_user(buf, ret))
Roel Kluin91bf189c2009-01-06 14:40:14 -0800276 ra = -EFAULT;
277 else
278 ra = 0;
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700279 } else {
Aneesh Kumar K.Va05b0852012-03-21 16:34:08 -0700280 unlock_page(page);
281
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700282 /*
283 * We have the page, copy it to user space buffer.
284 */
Roel Kluin91bf189c2009-01-06 14:40:14 -0800285 ra = hugetlbfs_read_actor(page, offset, buf, len, nr);
286 ret = ra;
Aneesh Kumar K.Va05b0852012-03-21 16:34:08 -0700287 page_cache_release(page);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700288 }
Roel Kluin91bf189c2009-01-06 14:40:14 -0800289 if (ra < 0) {
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700290 if (retval == 0)
Roel Kluin91bf189c2009-01-06 14:40:14 -0800291 retval = ra;
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700292 goto out;
293 }
294
295 offset += ret;
296 retval += ret;
297 len -= ret;
Andi Kleena5516432008-07-23 21:27:41 -0700298 index += offset >> huge_page_shift(h);
299 offset &= ~huge_page_mask(h);
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700300
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700301 /* short read or no more work */
302 if ((ret != nr) || (len == 0))
303 break;
304 }
305out:
Andi Kleena5516432008-07-23 21:27:41 -0700306 *ppos = ((loff_t)index << huge_page_shift(h)) + offset;
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700307 return retval;
308}
309
Nick Piggin800d15a2007-10-16 01:25:03 -0700310static int hugetlbfs_write_begin(struct file *file,
311 struct address_space *mapping,
312 loff_t pos, unsigned len, unsigned flags,
313 struct page **pagep, void **fsdata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314{
315 return -EINVAL;
316}
317
Nick Piggin800d15a2007-10-16 01:25:03 -0700318static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
319 loff_t pos, unsigned len, unsigned copied,
320 struct page *page, void *fsdata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
Nick Piggin800d15a2007-10-16 01:25:03 -0700322 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 return -EINVAL;
324}
325
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326static void truncate_huge_page(struct page *page)
327{
Linus Torvaldsfba25912006-12-20 13:46:42 -0800328 cancel_dirty_page(page, /* No IO accounting for huge pages? */0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 ClearPageUptodate(page);
Minchan Kimbd65cb82011-03-22 16:30:54 -0700330 delete_from_page_cache(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
David Gibsonb45b5bd2006-03-22 00:08:55 -0800333static void truncate_hugepages(struct inode *inode, loff_t lstart)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334{
Andi Kleena5516432008-07-23 21:27:41 -0700335 struct hstate *h = hstate_inode(inode);
David Gibsonb45b5bd2006-03-22 00:08:55 -0800336 struct address_space *mapping = &inode->i_data;
Andi Kleena5516432008-07-23 21:27:41 -0700337 const pgoff_t start = lstart >> huge_page_shift(h);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 struct pagevec pvec;
339 pgoff_t next;
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -0700340 int i, freed = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341
342 pagevec_init(&pvec, 0);
343 next = start;
344 while (1) {
345 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
346 if (next == start)
347 break;
348 next = start;
349 continue;
350 }
351
352 for (i = 0; i < pagevec_count(&pvec); ++i) {
353 struct page *page = pvec.pages[i];
354
355 lock_page(page);
356 if (page->index > next)
357 next = page->index;
358 ++next;
359 truncate_huge_page(page);
360 unlock_page(page);
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -0700361 freed++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 }
363 huge_pagevec_release(&pvec);
364 }
365 BUG_ON(!lstart && mapping->nrpages);
Chen, Kenneth Wa43a8c32006-06-23 02:03:15 -0700366 hugetlb_unreserve_pages(inode, start, freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367}
368
Al Viro2bbbda32010-06-04 19:52:12 -0400369static void hugetlbfs_evict_inode(struct inode *inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370{
Joonsoo Kim9119a412014-04-03 14:47:25 -0700371 struct resv_map *resv_map;
372
David Gibsonb45b5bd2006-03-22 00:08:55 -0800373 truncate_hugepages(inode, 0);
Joonsoo Kim9119a412014-04-03 14:47:25 -0700374 resv_map = (struct resv_map *)inode->i_mapping->private_data;
375 /* root inode doesn't have the resv_map, so we should check it */
376 if (resv_map)
377 resv_map_release(&resv_map->refs);
Jan Karadbd57682012-05-03 14:48:02 +0200378 clear_inode(inode);
Christoph Hellwig149f4212005-10-29 18:16:43 -0700379}
380
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381static inline void
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700382hugetlb_vmtruncate_list(struct rb_root *root, pgoff_t pgoff)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383{
384 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700386 vma_interval_tree_foreach(vma, root, pgoff, ULONG_MAX) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 unsigned long v_offset;
388
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 /*
Hugh Dickins856fc292006-10-28 10:38:43 -0700390 * Can the expression below overflow on 32-bit arches?
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700391 * No, because the interval tree returns us only those vmas
Hugh Dickins856fc292006-10-28 10:38:43 -0700392 * which overlap the truncated area starting at pgoff,
393 * and no vma on a 32-bit arch can span beyond the 4GB.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 */
Hugh Dickins856fc292006-10-28 10:38:43 -0700395 if (vma->vm_pgoff < pgoff)
396 v_offset = (pgoff - vma->vm_pgoff) << PAGE_SHIFT;
397 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 v_offset = 0;
399
Aneesh Kumar K.V24669e52012-07-31 16:42:03 -0700400 unmap_hugepage_range(vma, vma->vm_start + v_offset,
401 vma->vm_end, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402 }
403}
404
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
406{
Hugh Dickins856fc292006-10-28 10:38:43 -0700407 pgoff_t pgoff;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 struct address_space *mapping = inode->i_mapping;
Andi Kleena5516432008-07-23 21:27:41 -0700409 struct hstate *h = hstate_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Andi Kleena5516432008-07-23 21:27:41 -0700411 BUG_ON(offset & ~huge_page_mask(h));
Hugh Dickins856fc292006-10-28 10:38:43 -0700412 pgoff = offset >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Ken Chen7aa91e12007-10-16 01:26:21 -0700414 i_size_write(inode, offset);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700415 mutex_lock(&mapping->i_mmap_mutex);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700416 if (!RB_EMPTY_ROOT(&mapping->i_mmap))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700418 mutex_unlock(&mapping->i_mmap_mutex);
David Gibsonb45b5bd2006-03-22 00:08:55 -0800419 truncate_hugepages(inode, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 return 0;
421}
422
423static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
424{
425 struct inode *inode = dentry->d_inode;
Andi Kleena5516432008-07-23 21:27:41 -0700426 struct hstate *h = hstate_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 int error;
428 unsigned int ia_valid = attr->ia_valid;
429
430 BUG_ON(!inode);
431
432 error = inode_change_ok(inode, attr);
433 if (error)
Christoph Hellwig10257742010-06-04 11:30:02 +0200434 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
436 if (ia_valid & ATTR_SIZE) {
437 error = -EINVAL;
Christoph Hellwig10257742010-06-04 11:30:02 +0200438 if (attr->ia_size & ~huge_page_mask(h))
439 return -EINVAL;
440 error = hugetlb_vmtruncate(inode, attr->ia_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 if (error)
Christoph Hellwig10257742010-06-04 11:30:02 +0200442 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 }
Christoph Hellwig10257742010-06-04 11:30:02 +0200444
445 setattr_copy(inode, attr);
446 mark_inode_dirty(inode);
447 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448}
449
Al Viro7d54fa62011-07-24 20:20:48 -0400450static struct inode *hugetlbfs_get_root(struct super_block *sb,
451 struct hugetlbfs_config *config)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452{
453 struct inode *inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454
455 inode = new_inode(sb);
456 if (inode) {
457 struct hugetlbfs_inode_info *info;
Christoph Hellwig85fe4022010-10-23 11:19:54 -0400458 inode->i_ino = get_next_ino();
Al Viro7d54fa62011-07-24 20:20:48 -0400459 inode->i_mode = S_IFDIR | config->mode;
460 inode->i_uid = config->uid;
461 inode->i_gid = config->gid;
462 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
463 info = HUGETLBFS_I(inode);
464 mpol_shared_policy_init(&info->policy, NULL);
465 inode->i_op = &hugetlbfs_dir_inode_operations;
466 inode->i_fop = &simple_dir_operations;
467 /* directory inodes start off with i_nlink == 2 (for "." entry) */
468 inc_nlink(inode);
Aneesh Kumar K.V65ed7602012-04-25 16:01:50 -0700469 lockdep_annotate_inode_mutex_key(inode);
Al Viro7d54fa62011-07-24 20:20:48 -0400470 }
471 return inode;
472}
473
Michal Hockob610ded2013-08-13 16:00:55 -0700474/*
475 * Hugetlbfs is not reclaimable; therefore its i_mmap_mutex will never
476 * be taken from reclaim -- unlike regular filesystems. This needs an
477 * annotation because huge_pmd_share() does an allocation under
478 * i_mmap_mutex.
479 */
Fabian Frederick422b2442014-06-04 16:10:39 -0700480static struct lock_class_key hugetlbfs_i_mmap_mutex_key;
Michal Hockob610ded2013-08-13 16:00:55 -0700481
Al Viro7d54fa62011-07-24 20:20:48 -0400482static struct inode *hugetlbfs_get_inode(struct super_block *sb,
483 struct inode *dir,
Al Viro18df2252011-07-24 23:17:40 -0400484 umode_t mode, dev_t dev)
Al Viro7d54fa62011-07-24 20:20:48 -0400485{
486 struct inode *inode;
Joonsoo Kim9119a412014-04-03 14:47:25 -0700487 struct resv_map *resv_map;
488
489 resv_map = resv_map_alloc();
490 if (!resv_map)
491 return NULL;
Al Viro7d54fa62011-07-24 20:20:48 -0400492
493 inode = new_inode(sb);
494 if (inode) {
495 struct hugetlbfs_inode_info *info;
496 inode->i_ino = get_next_ino();
497 inode_init_owner(inode, dir, mode);
Michal Hockob610ded2013-08-13 16:00:55 -0700498 lockdep_set_class(&inode->i_mapping->i_mmap_mutex,
499 &hugetlbfs_i_mmap_mutex_key);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 inode->i_mapping->a_ops = &hugetlbfs_aops;
501 inode->i_mapping->backing_dev_info =&hugetlbfs_backing_dev_info;
502 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
Joonsoo Kim9119a412014-04-03 14:47:25 -0700503 inode->i_mapping->private_data = resv_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 info = HUGETLBFS_I(inode);
Eric B Munson6bfde052009-09-21 17:03:43 -0700505 /*
506 * The policy is initialized here even if we are creating a
507 * private inode because initialization simply creates an
508 * an empty rb tree and calls spin_lock_init(), later when we
509 * call mpol_free_shared_policy() it will just return because
510 * the rb tree will still be empty.
511 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700512 mpol_shared_policy_init(&info->policy, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 switch (mode & S_IFMT) {
514 default:
515 init_special_inode(inode, mode, dev);
516 break;
517 case S_IFREG:
518 inode->i_op = &hugetlbfs_inode_operations;
519 inode->i_fop = &hugetlbfs_file_operations;
520 break;
521 case S_IFDIR:
522 inode->i_op = &hugetlbfs_dir_inode_operations;
523 inode->i_fop = &simple_dir_operations;
524
525 /* directory inodes start off with i_nlink == 2 (for "." entry) */
Dave Hansend8c76e62006-09-30 23:29:04 -0700526 inc_nlink(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 break;
528 case S_IFLNK:
529 inode->i_op = &page_symlink_inode_operations;
530 break;
531 }
Josh Boyere096d0c2011-08-25 07:48:12 -0400532 lockdep_annotate_inode_mutex_key(inode);
Joonsoo Kim9119a412014-04-03 14:47:25 -0700533 } else
534 kref_put(&resv_map->refs, resv_map_release);
535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 return inode;
537}
538
539/*
540 * File creation. Allocate an inode, and we're done..
541 */
542static int hugetlbfs_mknod(struct inode *dir,
Al Viro1a67aaf2011-07-26 01:52:52 -0400543 struct dentry *dentry, umode_t mode, dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544{
545 struct inode *inode;
546 int error = -ENOSPC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
Al Viro7d54fa62011-07-24 20:20:48 -0400548 inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 if (inode) {
550 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
551 d_instantiate(dentry, inode);
552 dget(dentry); /* Extra count - pin the dentry in core */
553 error = 0;
554 }
555 return error;
556}
557
Al Viro18bb1db2011-07-26 01:41:39 -0400558static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559{
560 int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
561 if (!retval)
Dave Hansend8c76e62006-09-30 23:29:04 -0700562 inc_nlink(dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 return retval;
564}
565
Al Viroebfc3b42012-06-10 18:05:36 -0400566static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567{
568 return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
569}
570
571static int hugetlbfs_symlink(struct inode *dir,
572 struct dentry *dentry, const char *symname)
573{
574 struct inode *inode;
575 int error = -ENOSPC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
Al Viro7d54fa62011-07-24 20:20:48 -0400577 inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 if (inode) {
579 int l = strlen(symname)+1;
580 error = page_symlink(inode, symname, l);
581 if (!error) {
582 d_instantiate(dentry, inode);
583 dget(dentry);
584 } else
585 iput(inode);
586 }
587 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
588
589 return error;
590}
591
592/*
Ken Chen6649a382007-02-08 14:20:27 -0800593 * mark the head page dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 */
595static int hugetlbfs_set_page_dirty(struct page *page)
596{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700597 struct page *head = compound_head(page);
Ken Chen6649a382007-02-08 14:20:27 -0800598
599 SetPageDirty(head);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 return 0;
601}
602
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900603static int hugetlbfs_migrate_page(struct address_space *mapping,
Mel Gormanb969c4a2012-01-12 17:19:34 -0800604 struct page *newpage, struct page *page,
Mel Gormana6bc32b2012-01-12 17:19:43 -0800605 enum migrate_mode mode)
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900606{
607 int rc;
608
609 rc = migrate_huge_page_move_mapping(mapping, newpage, page);
Rafael Aquini78bd5202012-12-11 16:02:31 -0800610 if (rc != MIGRATEPAGE_SUCCESS)
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900611 return rc;
612 migrate_page_copy(newpage, page);
613
Rafael Aquini78bd5202012-12-11 16:02:31 -0800614 return MIGRATEPAGE_SUCCESS;
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900615}
616
David Howells726c3342006-06-23 02:02:58 -0700617static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618{
David Howells726c3342006-06-23 02:02:58 -0700619 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
Andi Kleena5516432008-07-23 21:27:41 -0700620 struct hstate *h = hstate_inode(dentry->d_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
622 buf->f_type = HUGETLBFS_MAGIC;
Andi Kleena5516432008-07-23 21:27:41 -0700623 buf->f_bsize = huge_page_size(h);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 if (sbinfo) {
625 spin_lock(&sbinfo->stat_lock);
David Gibson74a8a652005-11-21 21:32:24 -0800626 /* If no limits set, just report 0 for max/free/used
627 * blocks, like simple_statfs() */
David Gibson90481622012-03-21 16:34:12 -0700628 if (sbinfo->spool) {
629 long free_pages;
630
631 spin_lock(&sbinfo->spool->lock);
632 buf->f_blocks = sbinfo->spool->max_hpages;
633 free_pages = sbinfo->spool->max_hpages
634 - sbinfo->spool->used_hpages;
635 buf->f_bavail = buf->f_bfree = free_pages;
636 spin_unlock(&sbinfo->spool->lock);
David Gibson74a8a652005-11-21 21:32:24 -0800637 buf->f_files = sbinfo->max_inodes;
638 buf->f_ffree = sbinfo->free_inodes;
639 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 spin_unlock(&sbinfo->stat_lock);
641 }
642 buf->f_namelen = NAME_MAX;
643 return 0;
644}
645
646static void hugetlbfs_put_super(struct super_block *sb)
647{
648 struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
649
650 if (sbi) {
651 sb->s_fs_info = NULL;
David Gibson90481622012-03-21 16:34:12 -0700652
653 if (sbi->spool)
654 hugepage_put_subpool(sbi->spool);
655
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 kfree(sbi);
657 }
658}
659
Christoph Hellwig96527982005-10-29 18:16:42 -0700660static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
661{
662 if (sbinfo->free_inodes >= 0) {
663 spin_lock(&sbinfo->stat_lock);
664 if (unlikely(!sbinfo->free_inodes)) {
665 spin_unlock(&sbinfo->stat_lock);
666 return 0;
667 }
668 sbinfo->free_inodes--;
669 spin_unlock(&sbinfo->stat_lock);
670 }
671
672 return 1;
673}
674
675static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
676{
677 if (sbinfo->free_inodes >= 0) {
678 spin_lock(&sbinfo->stat_lock);
679 sbinfo->free_inodes++;
680 spin_unlock(&sbinfo->stat_lock);
681 }
682}
683
684
Christoph Lametere18b8902006-12-06 20:33:20 -0800685static struct kmem_cache *hugetlbfs_inode_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686
687static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
688{
Christoph Hellwig96527982005-10-29 18:16:42 -0700689 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690 struct hugetlbfs_inode_info *p;
691
Christoph Hellwig96527982005-10-29 18:16:42 -0700692 if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 return NULL;
Christoph Lametere94b1762006-12-06 20:33:17 -0800694 p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
Christoph Hellwig96527982005-10-29 18:16:42 -0700695 if (unlikely(!p)) {
696 hugetlbfs_inc_free_inodes(sbinfo);
697 return NULL;
698 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 return &p->vfs_inode;
700}
701
Nick Pigginfa0d7e3d2011-01-07 17:49:49 +1100702static void hugetlbfs_i_callback(struct rcu_head *head)
703{
704 struct inode *inode = container_of(head, struct inode, i_rcu);
Nick Pigginfa0d7e3d2011-01-07 17:49:49 +1100705 kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
706}
707
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708static void hugetlbfs_destroy_inode(struct inode *inode)
709{
Christoph Hellwig96527982005-10-29 18:16:42 -0700710 hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
Nick Pigginfa0d7e3d2011-01-07 17:49:49 +1100712 call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713}
714
Christoph Hellwigf5e54d62006-06-28 04:26:44 -0700715static const struct address_space_operations hugetlbfs_aops = {
Nick Piggin800d15a2007-10-16 01:25:03 -0700716 .write_begin = hugetlbfs_write_begin,
717 .write_end = hugetlbfs_write_end,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 .set_page_dirty = hugetlbfs_set_page_dirty,
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900719 .migratepage = hugetlbfs_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720};
721
Christoph Hellwig96527982005-10-29 18:16:42 -0700722
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700723static void init_once(void *foo)
Christoph Hellwig96527982005-10-29 18:16:42 -0700724{
725 struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
726
Christoph Lametera35afb82007-05-16 22:10:57 -0700727 inode_init_once(&ei->vfs_inode);
Christoph Hellwig96527982005-10-29 18:16:42 -0700728}
729
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -0800730const struct file_operations hugetlbfs_file_operations = {
Badari Pulavartye63e1e52007-10-16 01:26:22 -0700731 .read = hugetlbfs_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 .mmap = hugetlbfs_file_mmap,
Christoph Hellwig1b061d92010-05-26 17:53:41 +0200733 .fsync = noop_fsync,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 .get_unmapped_area = hugetlb_get_unmapped_area,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200735 .llseek = default_llseek,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736};
737
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -0800738static const struct inode_operations hugetlbfs_dir_inode_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 .create = hugetlbfs_create,
740 .lookup = simple_lookup,
741 .link = simple_link,
742 .unlink = simple_unlink,
743 .symlink = hugetlbfs_symlink,
744 .mkdir = hugetlbfs_mkdir,
745 .rmdir = simple_rmdir,
746 .mknod = hugetlbfs_mknod,
747 .rename = simple_rename,
748 .setattr = hugetlbfs_setattr,
749};
750
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -0800751static const struct inode_operations hugetlbfs_inode_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 .setattr = hugetlbfs_setattr,
753};
754
Josef 'Jeff' Sipekee9b6d62007-02-12 00:55:41 -0800755static const struct super_operations hugetlbfs_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 .alloc_inode = hugetlbfs_alloc_inode,
757 .destroy_inode = hugetlbfs_destroy_inode,
Al Viro2bbbda32010-06-04 19:52:12 -0400758 .evict_inode = hugetlbfs_evict_inode,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 .statfs = hugetlbfs_statfs,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 .put_super = hugetlbfs_put_super,
Miklos Szeredi10f19a82008-02-08 04:21:45 -0800761 .show_options = generic_show_options,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762};
763
764static int
765hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
766{
Randy Dunlape73a75f2007-07-15 23:40:52 -0700767 char *p, *rest;
768 substring_t args[MAX_OPT_ARGS];
769 int option;
Andi Kleena137e1c2008-07-23 21:27:43 -0700770 unsigned long long size = 0;
771 enum { NO_SIZE, SIZE_STD, SIZE_PERCENT } setsize = NO_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
773 if (!options)
774 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Randy Dunlape73a75f2007-07-15 23:40:52 -0700776 while ((p = strsep(&options, ",")) != NULL) {
777 int token;
Lee Schermerhornb4c07bc2007-07-15 23:40:54 -0700778 if (!*p)
779 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780
Randy Dunlape73a75f2007-07-15 23:40:52 -0700781 token = match_token(p, tokens, args);
782 switch (token) {
783 case Opt_uid:
784 if (match_int(&args[0], &option))
785 goto bad_val;
Eric W. Biedermana0eb3a02012-02-07 16:19:25 -0800786 pconfig->uid = make_kuid(current_user_ns(), option);
787 if (!uid_valid(pconfig->uid))
788 goto bad_val;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700789 break;
790
791 case Opt_gid:
792 if (match_int(&args[0], &option))
793 goto bad_val;
Eric W. Biedermana0eb3a02012-02-07 16:19:25 -0800794 pconfig->gid = make_kgid(current_user_ns(), option);
795 if (!gid_valid(pconfig->gid))
796 goto bad_val;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700797 break;
798
799 case Opt_mode:
800 if (match_octal(&args[0], &option))
801 goto bad_val;
Ken Chen75897d62008-02-04 22:28:36 -0800802 pconfig->mode = option & 01777U;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700803 break;
804
805 case Opt_size: {
Randy Dunlape73a75f2007-07-15 23:40:52 -0700806 /* memparse() will accept a K/M/G without a digit */
807 if (!isdigit(*args[0].from))
808 goto bad_val;
809 size = memparse(args[0].from, &rest);
Andi Kleena137e1c2008-07-23 21:27:43 -0700810 setsize = SIZE_STD;
811 if (*rest == '%')
812 setsize = SIZE_PERCENT;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700813 break;
814 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
Randy Dunlape73a75f2007-07-15 23:40:52 -0700816 case Opt_nr_inodes:
817 /* memparse() will accept a K/M/G without a digit */
818 if (!isdigit(*args[0].from))
819 goto bad_val;
820 pconfig->nr_inodes = memparse(args[0].from, &rest);
821 break;
822
Andi Kleena137e1c2008-07-23 21:27:43 -0700823 case Opt_pagesize: {
824 unsigned long ps;
825 ps = memparse(args[0].from, &rest);
826 pconfig->hstate = size_to_hstate(ps);
827 if (!pconfig->hstate) {
Andrew Morton9b857d22014-06-04 16:07:21 -0700828 pr_err("Unsupported page size %lu MB\n",
Andi Kleena137e1c2008-07-23 21:27:43 -0700829 ps >> 20);
830 return -EINVAL;
831 }
832 break;
833 }
834
Randy Dunlape73a75f2007-07-15 23:40:52 -0700835 default:
Andrew Morton9b857d22014-06-04 16:07:21 -0700836 pr_err("Bad mount option: \"%s\"\n", p);
Lee Schermerhornb4c07bc2007-07-15 23:40:54 -0700837 return -EINVAL;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700838 break;
839 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 }
Andi Kleena137e1c2008-07-23 21:27:43 -0700841
842 /* Do size after hstate is set up */
843 if (setsize > NO_SIZE) {
844 struct hstate *h = pconfig->hstate;
845 if (setsize == SIZE_PERCENT) {
846 size <<= huge_page_shift(h);
847 size *= h->max_huge_pages;
848 do_div(size, 100);
849 }
850 pconfig->nr_blocks = (size >> huge_page_shift(h));
851 }
852
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 return 0;
Randy Dunlape73a75f2007-07-15 23:40:52 -0700854
855bad_val:
Andrew Morton9b857d22014-06-04 16:07:21 -0700856 pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
Akinobu Mitac12ddba2009-04-21 12:24:05 -0700857 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858}
859
860static int
861hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
862{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 int ret;
864 struct hugetlbfs_config config;
865 struct hugetlbfs_sb_info *sbinfo;
866
Miklos Szeredi10f19a82008-02-08 04:21:45 -0800867 save_mount_options(sb, data);
868
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 config.nr_blocks = -1; /* No limit on size by default */
870 config.nr_inodes = -1; /* No limit on number of inodes by default */
David Howells77c70de2008-11-14 10:38:56 +1100871 config.uid = current_fsuid();
872 config.gid = current_fsgid();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 config.mode = 0755;
Andi Kleena137e1c2008-07-23 21:27:43 -0700874 config.hstate = &default_hstate;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 ret = hugetlbfs_parse_options(data, &config);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 if (ret)
877 return ret;
878
879 sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
880 if (!sbinfo)
881 return -ENOMEM;
882 sb->s_fs_info = sbinfo;
Andi Kleena137e1c2008-07-23 21:27:43 -0700883 sbinfo->hstate = config.hstate;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 spin_lock_init(&sbinfo->stat_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 sbinfo->max_inodes = config.nr_inodes;
886 sbinfo->free_inodes = config.nr_inodes;
David Gibson90481622012-03-21 16:34:12 -0700887 sbinfo->spool = NULL;
888 if (config.nr_blocks != -1) {
889 sbinfo->spool = hugepage_new_subpool(config.nr_blocks);
890 if (!sbinfo->spool)
891 goto out_free;
892 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 sb->s_maxbytes = MAX_LFS_FILESIZE;
Andi Kleena137e1c2008-07-23 21:27:43 -0700894 sb->s_blocksize = huge_page_size(config.hstate);
895 sb->s_blocksize_bits = huge_page_shift(config.hstate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 sb->s_magic = HUGETLBFS_MAGIC;
897 sb->s_op = &hugetlbfs_ops;
898 sb->s_time_gran = 1;
Al Viro48fde702012-01-08 22:15:13 -0500899 sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
900 if (!sb->s_root)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 return 0;
903out_free:
Fabian Frederick6e6870d2014-06-04 16:10:40 -0700904 kfree(sbinfo->spool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 kfree(sbinfo);
906 return -ENOMEM;
907}
908
Al Viro3c26ff62010-07-25 11:46:36 +0400909static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
910 int flags, const char *dev_name, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911{
Al Viro3c26ff62010-07-25 11:46:36 +0400912 return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913}
914
915static struct file_system_type hugetlbfs_fs_type = {
916 .name = "hugetlbfs",
Al Viro3c26ff62010-07-25 11:46:36 +0400917 .mount = hugetlbfs_mount,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 .kill_sb = kill_litter_super,
919};
Eric W. Biederman7f78e032013-03-02 19:39:14 -0800920MODULE_ALIAS_FS("hugetlbfs");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921
Andi Kleen42d73952012-12-11 16:01:34 -0800922static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923
From: Mel Gormanef1ff6b2009-09-23 15:56:05 -0700924static int can_do_hugetlb_shm(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925{
Eric W. Biedermana0eb3a02012-02-07 16:19:25 -0800926 kgid_t shm_group;
927 shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
928 return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929}
930
Andi Kleen42d73952012-12-11 16:01:34 -0800931static int get_hstate_idx(int page_size_log)
932{
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700933 struct hstate *h = hstate_sizelog(page_size_log);
Andi Kleen42d73952012-12-11 16:01:34 -0800934
Andi Kleen42d73952012-12-11 16:01:34 -0800935 if (!h)
936 return -1;
937 return h - hstates;
938}
939
Fabian Frederickbe1d2cf2014-06-04 16:10:39 -0700940static const struct dentry_operations anon_ops = {
Al Viro118b2302013-08-24 12:08:17 -0400941 .d_dname = simple_dname
Al Viro0df4d6e2013-02-14 22:39:53 -0500942};
943
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700944/*
945 * Note that size should be aligned to proper hugepage size in caller side,
946 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
947 */
948struct file *hugetlb_file_setup(const char *name, size_t size,
949 vm_flags_t acctflag, struct user_struct **user,
Andi Kleen42d73952012-12-11 16:01:34 -0800950 int creat_flags, int page_size_log)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951{
Anatol Pomozov39b65252012-09-12 20:11:55 -0700952 struct file *file = ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 struct inode *inode;
Al Viro2c48b9c2009-08-09 00:52:35 +0400954 struct path path;
Al Viro0df4d6e2013-02-14 22:39:53 -0500955 struct super_block *sb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956 struct qstr quick_string;
Andi Kleen42d73952012-12-11 16:01:34 -0800957 int hstate_idx;
958
959 hstate_idx = get_hstate_idx(page_size_log);
960 if (hstate_idx < 0)
961 return ERR_PTR(-ENODEV);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Hugh Dickins353d5c32009-08-24 16:30:28 +0100963 *user = NULL;
Andi Kleen42d73952012-12-11 16:01:34 -0800964 if (!hugetlbfs_vfsmount[hstate_idx])
Akinobu Mita5bc98592007-05-06 14:50:18 -0700965 return ERR_PTR(-ENOENT);
966
From: Mel Gormanef1ff6b2009-09-23 15:56:05 -0700967 if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
Hugh Dickins353d5c32009-08-24 16:30:28 +0100968 *user = current_user();
969 if (user_shm_lock(size, *user)) {
David Rientjes21a3c272012-03-21 16:34:13 -0700970 task_lock(current);
Andrew Morton9b857d22014-06-04 16:07:21 -0700971 pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
David Rientjes21a3c272012-03-21 16:34:13 -0700972 current->comm, current->pid);
973 task_unlock(current);
Hugh Dickins353d5c32009-08-24 16:30:28 +0100974 } else {
975 *user = NULL;
Ravikiran G Thirumalai2584e512009-03-31 15:21:26 -0700976 return ERR_PTR(-EPERM);
Hugh Dickins353d5c32009-08-24 16:30:28 +0100977 }
Ravikiran G Thirumalai2584e512009-03-31 15:21:26 -0700978 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Al Viro0df4d6e2013-02-14 22:39:53 -0500980 sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
Eric W. Biederman9d665862007-06-16 10:16:16 -0700981 quick_string.name = name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 quick_string.len = strlen(quick_string.name);
983 quick_string.hash = 0;
Al Viro0df4d6e2013-02-14 22:39:53 -0500984 path.dentry = d_alloc_pseudo(sb, &quick_string);
Al Viro2c48b9c2009-08-09 00:52:35 +0400985 if (!path.dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 goto out_shm_unlock;
987
Al Viro0df4d6e2013-02-14 22:39:53 -0500988 d_set_d_op(path.dentry, &anon_ops);
Andi Kleen42d73952012-12-11 16:01:34 -0800989 path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
Anatol Pomozov39b65252012-09-12 20:11:55 -0700990 file = ERR_PTR(-ENOSPC);
Al Viro0df4d6e2013-02-14 22:39:53 -0500991 inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 if (!inode)
Dave Hansence8d2cd2007-10-16 23:31:13 -0700993 goto out_dentry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994
Anatol Pomozov39b65252012-09-12 20:11:55 -0700995 file = ERR_PTR(-ENOMEM);
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700996 if (hugetlb_reserve_pages(inode, 0,
997 size >> huge_page_shift(hstate_inode(inode)), NULL,
998 acctflag))
David Gibsonb45b5bd2006-03-22 00:08:55 -0800999 goto out_inode;
1000
Al Viro2c48b9c2009-08-09 00:52:35 +04001001 d_instantiate(path.dentry, inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 inode->i_size = size;
Miklos Szeredi6d6b77f2011-10-28 14:13:28 +02001003 clear_nlink(inode);
Dave Hansence8d2cd2007-10-16 23:31:13 -07001004
Al Viro2c48b9c2009-08-09 00:52:35 +04001005 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
Dave Hansence8d2cd2007-10-16 23:31:13 -07001006 &hugetlbfs_file_operations);
Anatol Pomozov39b65252012-09-12 20:11:55 -07001007 if (IS_ERR(file))
Al Virob4d232e2008-02-23 05:59:19 -05001008 goto out_dentry; /* inode is already attached */
Dave Hansence8d2cd2007-10-16 23:31:13 -07001009
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 return file;
1011
David Gibsonb45b5bd2006-03-22 00:08:55 -08001012out_inode:
1013 iput(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014out_dentry:
Al Viro2c48b9c2009-08-09 00:52:35 +04001015 path_put(&path);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016out_shm_unlock:
Hugh Dickins353d5c32009-08-24 16:30:28 +01001017 if (*user) {
1018 user_shm_unlock(size, *user);
1019 *user = NULL;
1020 }
Anatol Pomozov39b65252012-09-12 20:11:55 -07001021 return file;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022}
1023
1024static int __init init_hugetlbfs_fs(void)
1025{
Andi Kleen42d73952012-12-11 16:01:34 -08001026 struct hstate *h;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 int error;
Andi Kleen42d73952012-12-11 16:01:34 -08001028 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -07001030 if (!hugepages_supported()) {
Andrew Morton9b857d22014-06-04 16:07:21 -07001031 pr_info("disabling because there are no supported hugepage sizes\n");
Nishanth Aravamudan457c1b22014-05-06 12:50:00 -07001032 return -ENOTSUPP;
1033 }
1034
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07001035 error = bdi_init(&hugetlbfs_backing_dev_info);
1036 if (error)
1037 return error;
1038
Hillf Dantond1d5e052012-03-21 16:34:15 -07001039 error = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1041 sizeof(struct hugetlbfs_inode_info),
Paul Mundt20c2df82007-07-20 10:11:58 +09001042 0, 0, init_once);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 if (hugetlbfs_inode_cachep == NULL)
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07001044 goto out2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045
1046 error = register_filesystem(&hugetlbfs_fs_type);
1047 if (error)
1048 goto out;
1049
Andi Kleen42d73952012-12-11 16:01:34 -08001050 i = 0;
1051 for_each_hstate(h) {
1052 char buf[50];
1053 unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054
Andi Kleen42d73952012-12-11 16:01:34 -08001055 snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
1056 hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
1057 buf);
1058
1059 if (IS_ERR(hugetlbfs_vfsmount[i])) {
Andrew Morton9b857d22014-06-04 16:07:21 -07001060 pr_err("Cannot mount internal hugetlbfs for "
Andi Kleen42d73952012-12-11 16:01:34 -08001061 "page size %uK", ps_kb);
1062 error = PTR_ERR(hugetlbfs_vfsmount[i]);
1063 hugetlbfs_vfsmount[i] = NULL;
1064 }
1065 i++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 }
Andi Kleen42d73952012-12-11 16:01:34 -08001067 /* Non default hstates are optional */
1068 if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
1069 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
1071 out:
Hillf Dantond1d5e052012-03-21 16:34:15 -07001072 kmem_cache_destroy(hugetlbfs_inode_cachep);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07001073 out2:
1074 bdi_destroy(&hugetlbfs_backing_dev_info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 return error;
1076}
1077
1078static void __exit exit_hugetlbfs_fs(void)
1079{
Andi Kleen42d73952012-12-11 16:01:34 -08001080 struct hstate *h;
1081 int i;
1082
1083
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +10001084 /*
1085 * Make sure all delayed rcu free inodes are flushed before we
1086 * destroy cache.
1087 */
1088 rcu_barrier();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 kmem_cache_destroy(hugetlbfs_inode_cachep);
Andi Kleen42d73952012-12-11 16:01:34 -08001090 i = 0;
1091 for_each_hstate(h)
1092 kern_unmount(hugetlbfs_vfsmount[i++]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 unregister_filesystem(&hugetlbfs_fs_type);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07001094 bdi_destroy(&hugetlbfs_backing_dev_info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095}
1096
1097module_init(init_hugetlbfs_fs)
1098module_exit(exit_hugetlbfs_fs)
1099
1100MODULE_LICENSE("GPL");