blob: a15a466d0d1d14b21bd82de2bfd3df31654420db [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
Hugh Dickins6922c0c2011-08-03 16:21:25 -07009 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
Hugh Dickins0edd73b2005-06-21 17:15:04 -070011 * Copyright (C) 2002-2005 VERITAS Software Corporation.
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
Matt Mackall853ac432009-01-06 14:40:20 -080018 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070021 * This file is released under the GPL.
22 */
23
Matt Mackall853ac432009-01-06 14:40:20 -080024#include <linux/fs.h>
25#include <linux/init.h>
26#include <linux/vfs.h>
27#include <linux/mount.h>
Hugh Dickinscaefba12009-04-13 14:40:12 -070028#include <linux/pagemap.h>
Matt Mackall853ac432009-01-06 14:40:20 -080029#include <linux/file.h>
30#include <linux/mm.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040031#include <linux/export.h>
Matt Mackall853ac432009-01-06 14:40:20 -080032#include <linux/swap.h>
33
34static struct vfsmount *shm_mnt;
35
36#ifdef CONFIG_SHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -070037/*
38 * This virtual memory filesystem is heavily based on the ramfs. It
39 * extends ramfs by the ability to use swap and honor resource limits
40 * which makes it a completely usable filesystem.
41 */
42
Andreas Gruenbacher39f02472006-09-29 02:01:35 -070043#include <linux/xattr.h>
Christoph Hellwiga5694252007-07-17 04:04:28 -070044#include <linux/exportfs.h>
Christoph Hellwig1c7c4742009-11-03 16:44:44 +010045#include <linux/posix_acl.h>
Andreas Gruenbacher39f02472006-09-29 02:01:35 -070046#include <linux/generic_acl.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <linux/mman.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/string.h>
49#include <linux/slab.h>
50#include <linux/backing-dev.h>
51#include <linux/shmem_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#include <linux/writeback.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <linux/blkdev.h>
Hugh Dickinsbda97ea2011-08-03 16:21:21 -070054#include <linux/pagevec.h>
Hugh Dickins41ffe5d2011-08-03 16:21:21 -070055#include <linux/percpu_counter.h>
Hugh Dickins83e4fa92012-05-29 15:06:40 -070056#include <linux/falloc.h>
Hugh Dickins708e3502011-07-25 17:12:32 -070057#include <linux/splice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/security.h>
59#include <linux/swapops.h>
60#include <linux/mempolicy.h>
61#include <linux/namei.h>
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +000062#include <linux/ctype.h>
Lee Schermerhorn304dbdb2006-04-22 02:35:48 -070063#include <linux/migrate.h>
Christoph Lameterc1f60a52006-09-25 23:31:11 -070064#include <linux/highmem.h>
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -080065#include <linux/seq_file.h>
Mimi Zohar92562922008-10-07 14:00:12 -040066#include <linux/magic.h>
Lee Schermerhorn304dbdb2006-04-22 02:35:48 -070067
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070069#include <asm/pgtable.h>
70
Linus Torvalds1da177e2005-04-16 15:20:36 -070071#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
73
Linus Torvalds1da177e2005-04-16 15:20:36 -070074/* Pretend that each entry is of this size in directory's i_size */
75#define BOGO_DIRENT_SIZE 20
76
Hugh Dickins69f07ec2011-08-03 16:21:26 -070077/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
78#define SHORT_SYMLINK_LEN 128
79
Eric Parisb09e0fa2011-05-24 17:12:39 -070080struct shmem_xattr {
81 struct list_head list; /* anchored by shmem_inode_info->xattr_list */
82 char *name; /* xattr name */
83 size_t size;
84 char value[0];
85};
86
Hugh Dickins1aac1402012-05-29 15:06:42 -070087/*
88 * shmem_fallocate and shmem_writepage communicate via inode->i_private
89 * (with i_mutex making sure that it has only one user at a time):
90 * we would prefer not to enlarge the shmem inode just for that.
91 */
92struct shmem_falloc {
93 pgoff_t start; /* start of range currently being fallocated */
94 pgoff_t next; /* the next page offset to be fallocated */
95 pgoff_t nr_falloced; /* how many new pages have been fallocated */
96 pgoff_t nr_unswapped; /* how often writepage refused to swap out */
97};
98
Hugh Dickins285b2c42011-08-03 16:21:20 -070099/* Flag allocation requirements to shmem_getpage */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100enum sgp_type {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 SGP_READ, /* don't exceed i_size, don't allocate page */
102 SGP_CACHE, /* don't exceed i_size, may allocate page */
Hugh Dickinsa0ee5ec2008-02-04 22:28:51 -0800103 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700104 SGP_WRITE, /* may exceed i_size, may allocate !Uptodate page */
105 SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106};
107
Andrew Mortonb76db732008-02-08 04:21:49 -0800108#ifdef CONFIG_TMPFS
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800109static unsigned long shmem_default_max_blocks(void)
110{
111 return totalram_pages / 2;
112}
113
114static unsigned long shmem_default_max_inodes(void)
115{
116 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
117}
Andrew Mortonb76db732008-02-08 04:21:49 -0800118#endif
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800119
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700120static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
121static int shmem_replace_page(struct page **pagep, gfp_t gfp,
122 struct shmem_inode_info *info, pgoff_t index);
Hugh Dickins68da9f02011-07-25 17:12:34 -0700123static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
124 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
125
126static inline int shmem_getpage(struct inode *inode, pgoff_t index,
127 struct page **pagep, enum sgp_type sgp, int *fault_type)
128{
129 return shmem_getpage_gfp(inode, index, pagep, sgp,
130 mapping_gfp_mask(inode->i_mapping), fault_type);
131}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
134{
135 return sb->s_fs_info;
136}
137
138/*
139 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
140 * for shared memory and for shared anonymous (/dev/zero) mappings
141 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
142 * consistent with the pre-accounting of private mappings ...
143 */
144static inline int shmem_acct_size(unsigned long flags, loff_t size)
145{
Hugh Dickins0b0a0802009-02-24 20:51:52 +0000146 return (flags & VM_NORESERVE) ?
Al Viro191c5422012-02-13 03:58:52 +0000147 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148}
149
150static inline void shmem_unacct_size(unsigned long flags, loff_t size)
151{
Hugh Dickins0b0a0802009-02-24 20:51:52 +0000152 if (!(flags & VM_NORESERVE))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 vm_unacct_memory(VM_ACCT(size));
154}
155
156/*
157 * ... whereas tmpfs objects are accounted incrementally as
158 * pages are allocated, in order to allow huge sparse files.
159 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
160 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
161 */
162static inline int shmem_acct_block(unsigned long flags)
163{
Hugh Dickins0b0a0802009-02-24 20:51:52 +0000164 return (flags & VM_NORESERVE) ?
Al Viro191c5422012-02-13 03:58:52 +0000165 security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166}
167
168static inline void shmem_unacct_blocks(unsigned long flags, long pages)
169{
Hugh Dickins0b0a0802009-02-24 20:51:52 +0000170 if (flags & VM_NORESERVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
172}
173
Hugh Dickins759b9772007-03-05 00:30:28 -0800174static const struct super_operations shmem_ops;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -0700175static const struct address_space_operations shmem_aops;
Helge Deller15ad7cd2006-12-06 20:40:36 -0800176static const struct file_operations shmem_file_operations;
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -0800177static const struct inode_operations shmem_inode_operations;
178static const struct inode_operations shmem_dir_inode_operations;
179static const struct inode_operations shmem_special_inode_operations;
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +0400180static const struct vm_operations_struct shmem_vm_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Ravikiran G Thirumalai6c231b72005-09-06 15:17:45 -0700182static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 .ra_pages = 0, /* No readahead */
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700184 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185};
186
187static LIST_HEAD(shmem_swaplist);
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800188static DEFINE_MUTEX(shmem_swaplist_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189
Pavel Emelyanov5b04c682008-02-04 22:28:47 -0800190static int shmem_reserve_inode(struct super_block *sb)
191{
192 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
193 if (sbinfo->max_inodes) {
194 spin_lock(&sbinfo->stat_lock);
195 if (!sbinfo->free_inodes) {
196 spin_unlock(&sbinfo->stat_lock);
197 return -ENOSPC;
198 }
199 sbinfo->free_inodes--;
200 spin_unlock(&sbinfo->stat_lock);
201 }
202 return 0;
203}
204
205static void shmem_free_inode(struct super_block *sb)
206{
207 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
208 if (sbinfo->max_inodes) {
209 spin_lock(&sbinfo->stat_lock);
210 sbinfo->free_inodes++;
211 spin_unlock(&sbinfo->stat_lock);
212 }
213}
214
Randy Dunlap46711812008-03-19 17:00:41 -0700215/**
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700216 * shmem_recalc_inode - recalculate the block usage of an inode
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 * @inode: inode to recalc
218 *
219 * We have to calculate the free blocks since the mm can drop
220 * undirtied hole pages behind our back.
221 *
222 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
223 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
224 *
225 * It has to be called with the spinlock held.
226 */
227static void shmem_recalc_inode(struct inode *inode)
228{
229 struct shmem_inode_info *info = SHMEM_I(inode);
230 long freed;
231
232 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
233 if (freed > 0) {
Hugh Dickins54af60422011-08-03 16:21:24 -0700234 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
235 if (sbinfo->max_blocks)
236 percpu_counter_add(&sbinfo->used_blocks, -freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 info->alloced -= freed;
Hugh Dickins54af60422011-08-03 16:21:24 -0700238 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 shmem_unacct_blocks(info->flags, freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 }
241}
242
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700243/*
244 * Replace item expected in radix tree by a new item, while holding tree lock.
245 */
246static int shmem_radix_tree_replace(struct address_space *mapping,
247 pgoff_t index, void *expected, void *replacement)
248{
249 void **pslot;
250 void *item = NULL;
251
252 VM_BUG_ON(!expected);
253 pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
254 if (pslot)
255 item = radix_tree_deref_slot_protected(pslot,
256 &mapping->tree_lock);
257 if (item != expected)
258 return -ENOENT;
259 if (replacement)
260 radix_tree_replace_slot(pslot, replacement);
261 else
262 radix_tree_delete(&mapping->page_tree, index);
263 return 0;
264}
265
266/*
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700267 * Like add_to_page_cache_locked, but error if expected item has gone.
268 */
269static int shmem_add_to_page_cache(struct page *page,
270 struct address_space *mapping,
271 pgoff_t index, gfp_t gfp, void *expected)
272{
Hugh Dickinsaa3b1892011-08-03 16:21:24 -0700273 int error = 0;
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700274
275 VM_BUG_ON(!PageLocked(page));
276 VM_BUG_ON(!PageSwapBacked(page));
277
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700278 if (!expected)
279 error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
280 if (!error) {
281 page_cache_get(page);
282 page->mapping = mapping;
283 page->index = index;
284
285 spin_lock_irq(&mapping->tree_lock);
286 if (!expected)
287 error = radix_tree_insert(&mapping->page_tree,
288 index, page);
289 else
290 error = shmem_radix_tree_replace(mapping, index,
291 expected, page);
292 if (!error) {
293 mapping->nrpages++;
294 __inc_zone_page_state(page, NR_FILE_PAGES);
295 __inc_zone_page_state(page, NR_SHMEM);
296 spin_unlock_irq(&mapping->tree_lock);
297 } else {
298 page->mapping = NULL;
299 spin_unlock_irq(&mapping->tree_lock);
300 page_cache_release(page);
301 }
302 if (!expected)
303 radix_tree_preload_end();
304 }
305 if (error)
306 mem_cgroup_uncharge_cache_page(page);
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700307 return error;
308}
309
310/*
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700311 * Like delete_from_page_cache, but substitutes swap for page.
312 */
313static void shmem_delete_from_page_cache(struct page *page, void *radswap)
314{
315 struct address_space *mapping = page->mapping;
316 int error;
317
318 spin_lock_irq(&mapping->tree_lock);
319 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
320 page->mapping = NULL;
321 mapping->nrpages--;
322 __dec_zone_page_state(page, NR_FILE_PAGES);
323 __dec_zone_page_state(page, NR_SHMEM);
324 spin_unlock_irq(&mapping->tree_lock);
325 page_cache_release(page);
326 BUG_ON(error);
327}
328
329/*
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700330 * Like find_get_pages, but collecting swap entries as well as pages.
331 */
332static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
333 pgoff_t start, unsigned int nr_pages,
334 struct page **pages, pgoff_t *indices)
335{
336 unsigned int i;
337 unsigned int ret;
338 unsigned int nr_found;
339
340 rcu_read_lock();
341restart:
342 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
343 (void ***)pages, indices, start, nr_pages);
344 ret = 0;
345 for (i = 0; i < nr_found; i++) {
346 struct page *page;
347repeat:
348 page = radix_tree_deref_slot((void **)pages[i]);
349 if (unlikely(!page))
350 continue;
351 if (radix_tree_exception(page)) {
Hugh Dickins8079b1c2011-08-03 16:21:28 -0700352 if (radix_tree_deref_retry(page))
353 goto restart;
354 /*
355 * Otherwise, we must be storing a swap entry
356 * here as an exceptional entry: so return it
357 * without attempting to raise page count.
358 */
359 goto export;
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700360 }
361 if (!page_cache_get_speculative(page))
362 goto repeat;
363
364 /* Has the page moved? */
365 if (unlikely(page != *((void **)pages[i]))) {
366 page_cache_release(page);
367 goto repeat;
368 }
369export:
370 indices[ret] = indices[i];
371 pages[ret] = page;
372 ret++;
373 }
374 if (unlikely(!ret && nr_found))
375 goto restart;
376 rcu_read_unlock();
377 return ret;
378}
379
380/*
381 * Remove swap entry from radix tree, free the swap and its page cache.
382 */
383static int shmem_free_swap(struct address_space *mapping,
384 pgoff_t index, void *radswap)
385{
386 int error;
387
388 spin_lock_irq(&mapping->tree_lock);
389 error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
390 spin_unlock_irq(&mapping->tree_lock);
391 if (!error)
392 free_swap_and_cache(radix_to_swp_entry(radswap));
393 return error;
394}
395
396/*
397 * Pagevec may contain swap entries, so shuffle up pages before releasing.
398 */
Hugh Dickins24513262012-01-20 14:34:21 -0800399static void shmem_deswap_pagevec(struct pagevec *pvec)
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700400{
401 int i, j;
402
403 for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
404 struct page *page = pvec->pages[i];
405 if (!radix_tree_exceptional_entry(page))
406 pvec->pages[j++] = page;
407 }
408 pvec->nr = j;
Hugh Dickins24513262012-01-20 14:34:21 -0800409}
410
411/*
412 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
413 */
414void shmem_unlock_mapping(struct address_space *mapping)
415{
416 struct pagevec pvec;
417 pgoff_t indices[PAGEVEC_SIZE];
418 pgoff_t index = 0;
419
420 pagevec_init(&pvec, 0);
421 /*
422 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
423 */
424 while (!mapping_unevictable(mapping)) {
425 /*
426 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
427 * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
428 */
429 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
430 PAGEVEC_SIZE, pvec.pages, indices);
431 if (!pvec.nr)
432 break;
433 index = indices[pvec.nr - 1] + 1;
434 shmem_deswap_pagevec(&pvec);
435 check_move_unevictable_pages(pvec.pages, pvec.nr);
436 pagevec_release(&pvec);
437 cond_resched();
438 }
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700439}
440
441/*
442 * Remove range of pages and swap entries from radix tree, and free them.
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700443 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700444 */
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700445static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
446 bool unfalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447{
Hugh Dickins285b2c42011-08-03 16:21:20 -0700448 struct address_space *mapping = inode->i_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 struct shmem_inode_info *info = SHMEM_I(inode);
Hugh Dickins285b2c42011-08-03 16:21:20 -0700450 pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700451 pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
452 unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
453 unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700454 struct pagevec pvec;
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700455 pgoff_t indices[PAGEVEC_SIZE];
456 long nr_swaps_freed = 0;
Hugh Dickins285b2c42011-08-03 16:21:20 -0700457 pgoff_t index;
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700458 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700460 if (lend == -1)
461 end = -1; /* unsigned, so actually very big */
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700462
463 pagevec_init(&pvec, 0);
464 index = start;
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700465 while (index < end) {
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700466 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700467 min(end - index, (pgoff_t)PAGEVEC_SIZE),
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700468 pvec.pages, indices);
469 if (!pvec.nr)
470 break;
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700471 mem_cgroup_uncharge_start();
472 for (i = 0; i < pagevec_count(&pvec); i++) {
473 struct page *page = pvec.pages[i];
474
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700475 index = indices[i];
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700476 if (index >= end)
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700477 break;
478
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700479 if (radix_tree_exceptional_entry(page)) {
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700480 if (unfalloc)
481 continue;
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700482 nr_swaps_freed += !shmem_free_swap(mapping,
483 index, page);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700484 continue;
485 }
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700486
487 if (!trylock_page(page))
488 continue;
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700489 if (!unfalloc || !PageUptodate(page)) {
490 if (page->mapping == mapping) {
491 VM_BUG_ON(PageWriteback(page));
492 truncate_inode_page(mapping, page);
493 }
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700494 }
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700495 unlock_page(page);
496 }
Hugh Dickins24513262012-01-20 14:34:21 -0800497 shmem_deswap_pagevec(&pvec);
498 pagevec_release(&pvec);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700499 mem_cgroup_uncharge_end();
500 cond_resched();
501 index++;
502 }
503
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700504 if (partial_start) {
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700505 struct page *page = NULL;
506 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
507 if (page) {
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700508 unsigned int top = PAGE_CACHE_SIZE;
509 if (start > end) {
510 top = partial_end;
511 partial_end = 0;
512 }
513 zero_user_segment(page, partial_start, top);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700514 set_page_dirty(page);
515 unlock_page(page);
516 page_cache_release(page);
517 }
518 }
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700519 if (partial_end) {
520 struct page *page = NULL;
521 shmem_getpage(inode, end, &page, SGP_READ, NULL);
522 if (page) {
523 zero_user_segment(page, 0, partial_end);
524 set_page_dirty(page);
525 unlock_page(page);
526 page_cache_release(page);
527 }
528 }
529 if (start >= end)
530 return;
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700531
532 index = start;
533 for ( ; ; ) {
534 cond_resched();
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700535 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700536 min(end - index, (pgoff_t)PAGEVEC_SIZE),
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700537 pvec.pages, indices);
538 if (!pvec.nr) {
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700539 if (index == start || unfalloc)
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700540 break;
541 index = start;
542 continue;
543 }
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700544 if ((index == start || unfalloc) && indices[0] >= end) {
Hugh Dickins24513262012-01-20 14:34:21 -0800545 shmem_deswap_pagevec(&pvec);
546 pagevec_release(&pvec);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700547 break;
548 }
549 mem_cgroup_uncharge_start();
550 for (i = 0; i < pagevec_count(&pvec); i++) {
551 struct page *page = pvec.pages[i];
552
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700553 index = indices[i];
Hugh Dickins83e4fa92012-05-29 15:06:40 -0700554 if (index >= end)
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700555 break;
556
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700557 if (radix_tree_exceptional_entry(page)) {
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700558 if (unfalloc)
559 continue;
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700560 nr_swaps_freed += !shmem_free_swap(mapping,
561 index, page);
562 continue;
563 }
564
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700565 lock_page(page);
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700566 if (!unfalloc || !PageUptodate(page)) {
567 if (page->mapping == mapping) {
568 VM_BUG_ON(PageWriteback(page));
569 truncate_inode_page(mapping, page);
570 }
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700571 }
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700572 unlock_page(page);
573 }
Hugh Dickins24513262012-01-20 14:34:21 -0800574 shmem_deswap_pagevec(&pvec);
575 pagevec_release(&pvec);
Hugh Dickinsbda97ea2011-08-03 16:21:21 -0700576 mem_cgroup_uncharge_end();
577 index++;
578 }
Hugh Dickins94c1e622011-06-27 16:18:03 -0700579
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 spin_lock(&info->lock);
Hugh Dickins7a5d0fb2011-08-03 16:21:22 -0700581 info->swapped -= nr_swaps_freed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 shmem_recalc_inode(inode);
583 spin_unlock(&info->lock);
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700584}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700586void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
587{
588 shmem_undo_range(inode, lstart, lend, false);
Hugh Dickins285b2c42011-08-03 16:21:20 -0700589 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590}
Hugh Dickins94c1e622011-06-27 16:18:03 -0700591EXPORT_SYMBOL_GPL(shmem_truncate_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592
Hugh Dickins94c1e622011-06-27 16:18:03 -0700593static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594{
595 struct inode *inode = dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 int error;
597
Christoph Hellwigdb78b872010-06-04 11:30:03 +0200598 error = inode_change_ok(inode, attr);
599 if (error)
600 return error;
601
Hugh Dickins94c1e622011-06-27 16:18:03 -0700602 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
603 loff_t oldsize = inode->i_size;
604 loff_t newsize = attr->ia_size;
npiggin@suse.de3889e6e2010-05-27 01:05:36 +1000605
Hugh Dickins94c1e622011-06-27 16:18:03 -0700606 if (newsize != oldsize) {
607 i_size_write(inode, newsize);
608 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
609 }
610 if (newsize < oldsize) {
611 loff_t holebegin = round_up(newsize, PAGE_SIZE);
612 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
613 shmem_truncate_range(inode, newsize, (loff_t)-1);
614 /* unmap again to remove racily COWed private pages */
615 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
616 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 }
618
Christoph Hellwigdb78b872010-06-04 11:30:03 +0200619 setattr_copy(inode, attr);
Andreas Gruenbacher39f02472006-09-29 02:01:35 -0700620#ifdef CONFIG_TMPFS_POSIX_ACL
Christoph Hellwigdb78b872010-06-04 11:30:03 +0200621 if (attr->ia_valid & ATTR_MODE)
Christoph Hellwig1c7c4742009-11-03 16:44:44 +0100622 error = generic_acl_chmod(inode);
Andreas Gruenbacher39f02472006-09-29 02:01:35 -0700623#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 return error;
625}
626
Al Viro1f895f72010-06-05 19:10:41 -0400627static void shmem_evict_inode(struct inode *inode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 struct shmem_inode_info *info = SHMEM_I(inode);
Eric Parisb09e0fa2011-05-24 17:12:39 -0700630 struct shmem_xattr *xattr, *nxattr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
npiggin@suse.de3889e6e2010-05-27 01:05:36 +1000632 if (inode->i_mapping->a_ops == &shmem_aops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 shmem_unacct_size(info->flags, inode->i_size);
634 inode->i_size = 0;
npiggin@suse.de3889e6e2010-05-27 01:05:36 +1000635 shmem_truncate_range(inode, 0, (loff_t)-1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 if (!list_empty(&info->swaplist)) {
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800637 mutex_lock(&shmem_swaplist_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 list_del_init(&info->swaplist);
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800639 mutex_unlock(&shmem_swaplist_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 }
Hugh Dickins69f07ec2011-08-03 16:21:26 -0700641 } else
642 kfree(info->symlink);
Eric Parisb09e0fa2011-05-24 17:12:39 -0700643
644 list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
645 kfree(xattr->name);
646 kfree(xattr);
647 }
Hugh Dickins0edd73b2005-06-21 17:15:04 -0700648 BUG_ON(inode->i_blocks);
Pavel Emelyanov5b04c682008-02-04 22:28:47 -0800649 shmem_free_inode(inode->i_sb);
Jan Karadbd57682012-05-03 14:48:02 +0200650 clear_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651}
652
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700653/*
654 * If swap found in inode, free it and move page from swapcache to filecache.
655 */
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700656static int shmem_unuse_inode(struct shmem_inode_info *info,
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700657 swp_entry_t swap, struct page **pagep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658{
Hugh Dickins285b2c42011-08-03 16:21:20 -0700659 struct address_space *mapping = info->vfs_inode.i_mapping;
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700660 void *radswap;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700661 pgoff_t index;
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700662 gfp_t gfp;
663 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700665 radswap = swp_to_radix_entry(swap);
Hugh Dickinse504f3f2011-08-03 16:21:27 -0700666 index = radix_tree_locate_item(&mapping->page_tree, radswap);
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700667 if (index == -1)
Hugh Dickins285b2c42011-08-03 16:21:20 -0700668 return 0;
Hugh Dickins2e0e26c2008-02-04 22:28:53 -0800669
Hugh Dickins1b1b32f2008-02-04 22:28:55 -0800670 /*
671 * Move _head_ to start search for next from here.
Al Viro1f895f72010-06-05 19:10:41 -0400672 * But be careful: shmem_evict_inode checks list_empty without taking
Hugh Dickins1b1b32f2008-02-04 22:28:55 -0800673 * mutex, and there's an instant in list_move_tail when info->swaplist
Hugh Dickins285b2c42011-08-03 16:21:20 -0700674 * would appear empty, if it were the only one on shmem_swaplist.
Hugh Dickins1b1b32f2008-02-04 22:28:55 -0800675 */
676 if (shmem_swaplist.next != &info->swaplist)
677 list_move_tail(&shmem_swaplist, &info->swaplist);
Hugh Dickins2e0e26c2008-02-04 22:28:53 -0800678
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700679 gfp = mapping_gfp_mask(mapping);
680 if (shmem_should_replace_page(*pagep, gfp)) {
681 mutex_unlock(&shmem_swaplist_mutex);
682 error = shmem_replace_page(pagep, gfp, info, index);
683 mutex_lock(&shmem_swaplist_mutex);
684 /*
685 * We needed to drop mutex to make that restrictive page
Hugh Dickins0142ef62012-06-07 14:21:09 -0700686 * allocation, but the inode might have been freed while we
687 * dropped it: although a racing shmem_evict_inode() cannot
688 * complete without emptying the radix_tree, our page lock
689 * on this swapcache page is not enough to prevent that -
690 * free_swap_and_cache() of our swap entry will only
691 * trylock_page(), removing swap from radix_tree whatever.
692 *
693 * We must not proceed to shmem_add_to_page_cache() if the
694 * inode has been freed, but of course we cannot rely on
695 * inode or mapping or info to check that. However, we can
696 * safely check if our swap entry is still in use (and here
697 * it can't have got reused for another page): if it's still
698 * in use, then the inode cannot have been freed yet, and we
699 * can safely proceed (if it's no longer in use, that tells
700 * nothing about the inode, but we don't need to unuse swap).
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700701 */
702 if (!page_swapcount(*pagep))
703 error = -ENOENT;
704 }
705
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800706 /*
Hugh Dickins778dd892011-05-11 15:13:37 -0700707 * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
708 * but also to hold up shmem_evict_inode(): so inode cannot be freed
709 * beneath us (pagelock doesn't help until the page is in pagecache).
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800710 */
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700711 if (!error)
712 error = shmem_add_to_page_cache(*pagep, mapping, index,
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700713 GFP_NOWAIT, radswap);
Hugh Dickins48f170f2011-07-25 17:12:37 -0700714 if (error != -ENOMEM) {
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700715 /*
716 * Truncation and eviction use free_swap_and_cache(), which
717 * only does trylock page: if we raced, best clean up here.
718 */
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700719 delete_from_swap_cache(*pagep);
720 set_page_dirty(*pagep);
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700721 if (!error) {
722 spin_lock(&info->lock);
723 info->swapped--;
724 spin_unlock(&info->lock);
725 swap_free(swap);
726 }
Hugh Dickins2e0e26c2008-02-04 22:28:53 -0800727 error = 1; /* not an error, but entry was found */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 }
Hugh Dickins2e0e26c2008-02-04 22:28:53 -0800729 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730}
731
732/*
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700733 * Search through swapped inodes to find and replace swap by page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 */
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700735int shmem_unuse(swp_entry_t swap, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700737 struct list_head *this, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 struct shmem_inode_info *info;
739 int found = 0;
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700740 int error = 0;
741
742 /*
743 * There's a faint possibility that swap page was replaced before
Hugh Dickins0142ef62012-06-07 14:21:09 -0700744 * caller locked it: caller will come back later with the right page.
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700745 */
Hugh Dickins0142ef62012-06-07 14:21:09 -0700746 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700747 goto out;
Hugh Dickins778dd892011-05-11 15:13:37 -0700748
749 /*
750 * Charge page using GFP_KERNEL while we can wait, before taking
751 * the shmem_swaplist_mutex which might hold up shmem_writepage().
752 * Charged back to the user (not to caller) when swap account is used.
Hugh Dickins778dd892011-05-11 15:13:37 -0700753 */
754 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
755 if (error)
756 goto out;
Hugh Dickins46f65ec2011-08-03 16:21:23 -0700757 /* No radix_tree_preload: swap entry keeps a place for page in tree */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800759 mutex_lock(&shmem_swaplist_mutex);
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700760 list_for_each_safe(this, next, &shmem_swaplist) {
761 info = list_entry(this, struct shmem_inode_info, swaplist);
Hugh Dickins285b2c42011-08-03 16:21:20 -0700762 if (info->swapped)
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700763 found = shmem_unuse_inode(info, swap, &page);
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700764 else
765 list_del_init(&info->swaplist);
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800766 cond_resched();
Hugh Dickins2e0e26c2008-02-04 22:28:53 -0800767 if (found)
Hugh Dickins778dd892011-05-11 15:13:37 -0700768 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 }
Hugh Dickinscb5f7b92008-02-04 22:28:52 -0800770 mutex_unlock(&shmem_swaplist_mutex);
Hugh Dickins778dd892011-05-11 15:13:37 -0700771
Hugh Dickins778dd892011-05-11 15:13:37 -0700772 if (found < 0)
773 error = found;
774out:
Hugh Dickinsaaa46862009-12-14 17:58:47 -0800775 unlock_page(page);
776 page_cache_release(page);
Hugh Dickins778dd892011-05-11 15:13:37 -0700777 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778}
779
780/*
781 * Move the page from the page cache to the swap cache.
782 */
783static int shmem_writepage(struct page *page, struct writeback_control *wbc)
784{
785 struct shmem_inode_info *info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 struct address_space *mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 struct inode *inode;
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700788 swp_entry_t swap;
789 pgoff_t index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790
791 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 mapping = page->mapping;
793 index = page->index;
794 inode = mapping->host;
795 info = SHMEM_I(inode);
796 if (info->flags & VM_LOCKED)
797 goto redirty;
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800798 if (!total_swap_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 goto redirty;
800
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800801 /*
802 * shmem_backing_dev_info's capabilities prevent regular writeback or
803 * sync from ever calling shmem_writepage; but a stacking filesystem
Hugh Dickins48f170f2011-07-25 17:12:37 -0700804 * might use ->writepage of its underlying filesystem, in which case
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800805 * tmpfs should write out to swap only in response to memory pressure,
Hugh Dickins48f170f2011-07-25 17:12:37 -0700806 * and not for the writeback threads or sync.
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800807 */
Hugh Dickins48f170f2011-07-25 17:12:37 -0700808 if (!wbc->for_reclaim) {
809 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
810 goto redirty;
811 }
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700812
813 /*
814 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
815 * value into swapfile.c, the only way we can correctly account for a
816 * fallocated page arriving here is now to initialize it and write it.
Hugh Dickins1aac1402012-05-29 15:06:42 -0700817 *
818 * That's okay for a page already fallocated earlier, but if we have
819 * not yet completed the fallocation, then (a) we want to keep track
820 * of this page in case we have to undo it, and (b) it may not be a
821 * good idea to continue anyway, once we're pushing into swap. So
822 * reactivate the page, and let shmem_fallocate() quit when too many.
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700823 */
824 if (!PageUptodate(page)) {
Hugh Dickins1aac1402012-05-29 15:06:42 -0700825 if (inode->i_private) {
826 struct shmem_falloc *shmem_falloc;
827 spin_lock(&inode->i_lock);
828 shmem_falloc = inode->i_private;
829 if (shmem_falloc &&
830 index >= shmem_falloc->start &&
831 index < shmem_falloc->next)
832 shmem_falloc->nr_unswapped++;
833 else
834 shmem_falloc = NULL;
835 spin_unlock(&inode->i_lock);
836 if (shmem_falloc)
837 goto redirty;
838 }
Hugh Dickins1635f6a2012-05-29 15:06:42 -0700839 clear_highpage(page);
840 flush_dcache_page(page);
841 SetPageUptodate(page);
842 }
843
Hugh Dickins48f170f2011-07-25 17:12:37 -0700844 swap = get_swap_page();
845 if (!swap.val)
846 goto redirty;
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800847
Hugh Dickinsb1dea802011-05-11 15:13:36 -0700848 /*
849 * Add inode to shmem_unuse()'s list of swapped-out inodes,
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700850 * if it's not already there. Do it now before the page is
851 * moved to swap cache, when its pagelock no longer protects
Hugh Dickinsb1dea802011-05-11 15:13:36 -0700852 * the inode from eviction. But don't unlock the mutex until
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700853 * we've incremented swapped, because shmem_unuse_inode() will
854 * prune a !swapped inode from the swaplist under this mutex.
Hugh Dickinsb1dea802011-05-11 15:13:36 -0700855 */
Hugh Dickins48f170f2011-07-25 17:12:37 -0700856 mutex_lock(&shmem_swaplist_mutex);
857 if (list_empty(&info->swaplist))
858 list_add_tail(&info->swaplist, &shmem_swaplist);
Hugh Dickinsb1dea802011-05-11 15:13:36 -0700859
Hugh Dickins48f170f2011-07-25 17:12:37 -0700860 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
Hugh Dickinsaaa46862009-12-14 17:58:47 -0800861 swap_shmem_alloc(swap);
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700862 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
863
864 spin_lock(&info->lock);
865 info->swapped++;
866 shmem_recalc_inode(inode);
Hugh Dickins826267c2011-05-28 13:14:09 -0700867 spin_unlock(&info->lock);
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700868
869 mutex_unlock(&shmem_swaplist_mutex);
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800870 BUG_ON(page_mapped(page));
Hugh Dickins9fab5612009-03-31 15:23:33 -0700871 swap_writepage(page, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872 return 0;
873 }
874
Hugh Dickins6922c0c2011-08-03 16:21:25 -0700875 mutex_unlock(&shmem_swaplist_mutex);
KAMEZAWA Hiroyukicb4b86b2009-06-16 15:32:52 -0700876 swapcache_free(swap, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877redirty:
878 set_page_dirty(page);
Hugh Dickinsd9fe5262008-02-04 22:28:51 -0800879 if (wbc->for_reclaim)
880 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
881 unlock_page(page);
882 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883}
884
885#ifdef CONFIG_NUMA
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800886#ifdef CONFIG_TMPFS
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700887static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800888{
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -0700889 char buffer[64];
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800890
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700891 if (!mpol || mpol->mode == MPOL_DEFAULT)
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -0700892 return; /* show nothing */
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800893
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700894 mpol_to_str(buffer, sizeof(buffer), mpol, 1);
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800895
Lee Schermerhorn095f1fc2008-04-28 02:13:23 -0700896 seq_printf(seq, ",mpol=%s", buffer);
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800897}
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700898
899static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
900{
901 struct mempolicy *mpol = NULL;
902 if (sbinfo->mpol) {
903 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
904 mpol = sbinfo->mpol;
905 mpol_get(mpol);
906 spin_unlock(&sbinfo->stat_lock);
907 }
908 return mpol;
909}
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800910#endif /* CONFIG_TMPFS */
911
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700912static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
913 struct shmem_inode_info *info, pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914{
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700915 struct mempolicy mpol, *spol;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 struct vm_area_struct pvma;
917
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700918 spol = mpol_cond_copy(&mpol,
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700919 mpol_shared_policy_lookup(&info->policy, index));
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700920
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 /* Create a pseudo vma that just contains the policy */
Hugh Dickinsc4cc6d02008-02-04 22:28:40 -0800922 pvma.vm_start = 0;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700923 pvma.vm_pgoff = index;
Hugh Dickinsc4cc6d02008-02-04 22:28:40 -0800924 pvma.vm_ops = NULL;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700925 pvma.vm_policy = spol;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700926 return swapin_readahead(swap, gfp, &pvma, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927}
928
Hugh Dickins02098fe2008-02-04 22:28:42 -0800929static struct page *shmem_alloc_page(gfp_t gfp,
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700930 struct shmem_inode_info *info, pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931{
932 struct vm_area_struct pvma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933
Hugh Dickinsc4cc6d02008-02-04 22:28:40 -0800934 /* Create a pseudo vma that just contains the policy */
935 pvma.vm_start = 0;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700936 pvma.vm_pgoff = index;
Hugh Dickinsc4cc6d02008-02-04 22:28:40 -0800937 pvma.vm_ops = NULL;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700938 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700939
940 /*
941 * alloc_page_vma() will drop the shared policy reference
942 */
943 return alloc_page_vma(gfp, &pvma, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944}
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800945#else /* !CONFIG_NUMA */
946#ifdef CONFIG_TMPFS
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700947static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800948{
949}
950#endif /* CONFIG_TMPFS */
951
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700952static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
953 struct shmem_inode_info *info, pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700955 return swapin_readahead(swap, gfp, NULL, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956}
957
Hugh Dickins02098fe2008-02-04 22:28:42 -0800958static inline struct page *shmem_alloc_page(gfp_t gfp,
Hugh Dickins41ffe5d2011-08-03 16:21:21 -0700959 struct shmem_inode_info *info, pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960{
Hugh Dickinse84e2e12007-11-28 18:55:10 +0000961 return alloc_page(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962}
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -0800963#endif /* CONFIG_NUMA */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964
Lee Schermerhorn71fe8042008-04-28 02:13:26 -0700965#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
966static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
967{
968 return NULL;
969}
970#endif
971
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972/*
Hugh Dickinsbde05d12012-05-29 15:06:38 -0700973 * When a page is moved from swapcache to shmem filecache (either by the
974 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
975 * shmem_unuse_inode()), it may have been read in earlier from swap, in
976 * ignorance of the mapping it belongs to. If that mapping has special
977 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
978 * we may need to copy to a suitable page before moving to filecache.
979 *
980 * In a future release, this may well be extended to respect cpuset and
981 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
982 * but for now it is a simple matter of zone.
983 */
984static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
985{
986 return page_zonenum(page) > gfp_zone(gfp);
987}
988
989static int shmem_replace_page(struct page **pagep, gfp_t gfp,
990 struct shmem_inode_info *info, pgoff_t index)
991{
992 struct page *oldpage, *newpage;
993 struct address_space *swap_mapping;
994 pgoff_t swap_index;
995 int error;
996
997 oldpage = *pagep;
998 swap_index = page_private(oldpage);
999 swap_mapping = page_mapping(oldpage);
1000
1001 /*
1002 * We have arrived here because our zones are constrained, so don't
1003 * limit chance of success by further cpuset and node constraints.
1004 */
1005 gfp &= ~GFP_CONSTRAINT_MASK;
1006 newpage = shmem_alloc_page(gfp, info, index);
1007 if (!newpage)
1008 return -ENOMEM;
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001009
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001010 page_cache_get(newpage);
1011 copy_highpage(newpage, oldpage);
Hugh Dickins0142ef62012-06-07 14:21:09 -07001012 flush_dcache_page(newpage);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001013
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001014 __set_page_locked(newpage);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001015 SetPageUptodate(newpage);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001016 SetPageSwapBacked(newpage);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001017 set_page_private(newpage, swap_index);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001018 SetPageSwapCache(newpage);
1019
1020 /*
1021 * Our caller will very soon move newpage out of swapcache, but it's
1022 * a nice clean interface for us to replace oldpage by newpage there.
1023 */
1024 spin_lock_irq(&swap_mapping->tree_lock);
1025 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1026 newpage);
Hugh Dickins0142ef62012-06-07 14:21:09 -07001027 if (!error) {
1028 __inc_zone_page_state(newpage, NR_FILE_PAGES);
1029 __dec_zone_page_state(oldpage, NR_FILE_PAGES);
1030 }
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001031 spin_unlock_irq(&swap_mapping->tree_lock);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001032
Hugh Dickins0142ef62012-06-07 14:21:09 -07001033 if (unlikely(error)) {
1034 /*
1035 * Is this possible? I think not, now that our callers check
1036 * both PageSwapCache and page_private after getting page lock;
1037 * but be defensive. Reverse old to newpage for clear and free.
1038 */
1039 oldpage = newpage;
1040 } else {
1041 mem_cgroup_replace_page_cache(oldpage, newpage);
1042 lru_cache_add_anon(newpage);
1043 *pagep = newpage;
1044 }
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001045
1046 ClearPageSwapCache(oldpage);
1047 set_page_private(oldpage, 0);
1048
1049 unlock_page(oldpage);
1050 page_cache_release(oldpage);
1051 page_cache_release(oldpage);
Hugh Dickins0142ef62012-06-07 14:21:09 -07001052 return error;
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001053}
1054
1055/*
Hugh Dickins68da9f02011-07-25 17:12:34 -07001056 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 *
1058 * If we allocate a new one we do not mark it dirty. That's up to the
1059 * vm. If we swap it in we mark it dirty since we also free the swap
1060 * entry since a page cannot live in both the swap and page cache
1061 */
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001062static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
Hugh Dickins68da9f02011-07-25 17:12:34 -07001063 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064{
1065 struct address_space *mapping = inode->i_mapping;
Hugh Dickins54af60422011-08-03 16:21:24 -07001066 struct shmem_inode_info *info;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 struct shmem_sb_info *sbinfo;
Hugh Dickins27ab7002011-07-25 17:12:36 -07001068 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 swp_entry_t swap;
1070 int error;
Hugh Dickins54af60422011-08-03 16:21:24 -07001071 int once = 0;
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001072 int alloced = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001074 if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 return -EFBIG;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076repeat:
Hugh Dickins54af60422011-08-03 16:21:24 -07001077 swap.val = 0;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001078 page = find_lock_page(mapping, index);
Hugh Dickins54af60422011-08-03 16:21:24 -07001079 if (radix_tree_exceptional_entry(page)) {
1080 swap = radix_to_swp_entry(page);
1081 page = NULL;
1082 }
1083
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001084 if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
Hugh Dickins54af60422011-08-03 16:21:24 -07001085 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1086 error = -EINVAL;
1087 goto failed;
1088 }
1089
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001090 /* fallocated page? */
1091 if (page && !PageUptodate(page)) {
1092 if (sgp != SGP_READ)
1093 goto clear;
1094 unlock_page(page);
1095 page_cache_release(page);
1096 page = NULL;
1097 }
Hugh Dickins54af60422011-08-03 16:21:24 -07001098 if (page || (sgp == SGP_READ && !swap.val)) {
Hugh Dickins54af60422011-08-03 16:21:24 -07001099 *pagep = page;
1100 return 0;
Hugh Dickins27ab7002011-07-25 17:12:36 -07001101 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102
1103 /*
Hugh Dickins54af60422011-08-03 16:21:24 -07001104 * Fast cache lookup did not find it:
1105 * bring it back from swap or allocate.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106 */
Hugh Dickins54af60422011-08-03 16:21:24 -07001107 info = SHMEM_I(inode);
1108 sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001109
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110 if (swap.val) {
1111 /* Look it up and read it in.. */
Hugh Dickins27ab7002011-07-25 17:12:36 -07001112 page = lookup_swap_cache(swap);
1113 if (!page) {
Ying Han456f9982011-05-26 16:25:38 -07001114 /* here we actually do the io */
Hugh Dickins68da9f02011-07-25 17:12:34 -07001115 if (fault_type)
1116 *fault_type |= VM_FAULT_MAJOR;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001117 page = shmem_swapin(swap, gfp, info, index);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001118 if (!page) {
Hugh Dickins54af60422011-08-03 16:21:24 -07001119 error = -ENOMEM;
1120 goto failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 }
1123
1124 /* We have to do this with page locked to prevent races */
Hugh Dickins54af60422011-08-03 16:21:24 -07001125 lock_page(page);
Hugh Dickins0142ef62012-06-07 14:21:09 -07001126 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1127 page->mapping) {
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001128 error = -EEXIST; /* try again */
1129 goto failed;
1130 }
Hugh Dickins27ab7002011-07-25 17:12:36 -07001131 if (!PageUptodate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132 error = -EIO;
Hugh Dickins54af60422011-08-03 16:21:24 -07001133 goto failed;
1134 }
1135 wait_on_page_writeback(page);
1136
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001137 if (shmem_should_replace_page(page, gfp)) {
1138 error = shmem_replace_page(&page, gfp, info, index);
1139 if (error)
1140 goto failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141 }
1142
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07001143 error = mem_cgroup_cache_charge(page, current->mm,
1144 gfp & GFP_RECLAIM_MASK);
1145 if (!error)
1146 error = shmem_add_to_page_cache(page, mapping, index,
1147 gfp, swp_to_radix_entry(swap));
Hugh Dickins54af60422011-08-03 16:21:24 -07001148 if (error)
1149 goto failed;
1150
1151 spin_lock(&info->lock);
1152 info->swapped--;
1153 shmem_recalc_inode(inode);
1154 spin_unlock(&info->lock);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001155
Hugh Dickins27ab7002011-07-25 17:12:36 -07001156 delete_from_swap_cache(page);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001157 set_page_dirty(page);
1158 swap_free(swap);
1159
Hugh Dickins54af60422011-08-03 16:21:24 -07001160 } else {
1161 if (shmem_acct_block(info->flags)) {
1162 error = -ENOSPC;
1163 goto failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 }
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001165 if (sbinfo->max_blocks) {
Hugh Dickinsfc5da222011-04-14 15:22:07 -07001166 if (percpu_counter_compare(&sbinfo->used_blocks,
Hugh Dickins54af60422011-08-03 16:21:24 -07001167 sbinfo->max_blocks) >= 0) {
1168 error = -ENOSPC;
1169 goto unacct;
1170 }
Tim Chen7e496292010-08-09 17:19:05 -07001171 percpu_counter_inc(&sbinfo->used_blocks);
Hugh Dickins59a16ea2011-05-11 15:13:38 -07001172 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173
Hugh Dickins54af60422011-08-03 16:21:24 -07001174 page = shmem_alloc_page(gfp, info, index);
1175 if (!page) {
1176 error = -ENOMEM;
1177 goto decused;
1178 }
1179
1180 SetPageSwapBacked(page);
1181 __set_page_locked(page);
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07001182 error = mem_cgroup_cache_charge(page, current->mm,
1183 gfp & GFP_RECLAIM_MASK);
1184 if (!error)
1185 error = shmem_add_to_page_cache(page, mapping, index,
1186 gfp, NULL);
Hugh Dickins54af60422011-08-03 16:21:24 -07001187 if (error)
1188 goto decused;
1189 lru_cache_add_anon(page);
1190
1191 spin_lock(&info->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 info->alloced++;
Hugh Dickins54af60422011-08-03 16:21:24 -07001193 inode->i_blocks += BLOCKS_PER_PAGE;
1194 shmem_recalc_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 spin_unlock(&info->lock);
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001196 alloced = true;
Hugh Dickins54af60422011-08-03 16:21:24 -07001197
Hugh Dickinsec9516f2012-05-29 15:06:39 -07001198 /*
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001199 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1200 */
1201 if (sgp == SGP_FALLOC)
1202 sgp = SGP_WRITE;
1203clear:
1204 /*
1205 * Let SGP_WRITE caller clear ends if write does not fill page;
1206 * but SGP_FALLOC on a page fallocated earlier must initialize
1207 * it now, lest undo on failure cancel our earlier guarantee.
Hugh Dickinsec9516f2012-05-29 15:06:39 -07001208 */
1209 if (sgp != SGP_WRITE) {
1210 clear_highpage(page);
1211 flush_dcache_page(page);
1212 SetPageUptodate(page);
1213 }
Hugh Dickinsa0ee5ec2008-02-04 22:28:51 -08001214 if (sgp == SGP_DIRTY)
Hugh Dickins27ab7002011-07-25 17:12:36 -07001215 set_page_dirty(page);
Hugh Dickins59a16ea2011-05-11 15:13:38 -07001216 }
Hugh Dickinsbde05d12012-05-29 15:06:38 -07001217
Hugh Dickins54af60422011-08-03 16:21:24 -07001218 /* Perhaps the file has been truncated since we checked */
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001219 if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
Hugh Dickins54af60422011-08-03 16:21:24 -07001220 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1221 error = -EINVAL;
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001222 if (alloced)
1223 goto trunc;
1224 else
1225 goto failed;
Shaohua Liff36b8012010-08-09 17:19:06 -07001226 }
Hugh Dickins54af60422011-08-03 16:21:24 -07001227 *pagep = page;
1228 return 0;
Nick Piggind00806b2007-07-19 01:46:57 -07001229
Nick Piggind0217ac2007-07-19 01:47:03 -07001230 /*
Hugh Dickins54af60422011-08-03 16:21:24 -07001231 * Error recovery.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 */
Hugh Dickins54af60422011-08-03 16:21:24 -07001233trunc:
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001234 info = SHMEM_I(inode);
Hugh Dickins54af60422011-08-03 16:21:24 -07001235 ClearPageDirty(page);
1236 delete_from_page_cache(page);
1237 spin_lock(&info->lock);
1238 info->alloced--;
1239 inode->i_blocks -= BLOCKS_PER_PAGE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 spin_unlock(&info->lock);
Hugh Dickins54af60422011-08-03 16:21:24 -07001241decused:
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001242 sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins54af60422011-08-03 16:21:24 -07001243 if (sbinfo->max_blocks)
1244 percpu_counter_add(&sbinfo->used_blocks, -1);
1245unacct:
1246 shmem_unacct_blocks(info->flags, 1);
1247failed:
1248 if (swap.val && error != -EINVAL) {
1249 struct page *test = find_get_page(mapping, index);
1250 if (test && !radix_tree_exceptional_entry(test))
1251 page_cache_release(test);
1252 /* Have another try if the entry has changed */
1253 if (test != swp_to_radix_entry(swap))
1254 error = -EEXIST;
1255 }
Hugh Dickins27ab7002011-07-25 17:12:36 -07001256 if (page) {
Hugh Dickins54af60422011-08-03 16:21:24 -07001257 unlock_page(page);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001258 page_cache_release(page);
Hugh Dickins54af60422011-08-03 16:21:24 -07001259 }
1260 if (error == -ENOSPC && !once++) {
1261 info = SHMEM_I(inode);
1262 spin_lock(&info->lock);
1263 shmem_recalc_inode(inode);
1264 spin_unlock(&info->lock);
Hugh Dickins27ab7002011-07-25 17:12:36 -07001265 goto repeat;
Josef "Jeff" Sipekd3ac7f82006-12-08 02:36:44 -08001266 }
Hugh Dickins54af60422011-08-03 16:21:24 -07001267 if (error == -EEXIST)
1268 goto repeat;
1269 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270}
1271
1272static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1273{
1274 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1275 int error;
Hugh Dickins68da9f02011-07-25 17:12:34 -07001276 int ret = VM_FAULT_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277
Linus Torvalds1da177e2005-04-16 15:20:36 -07001278 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1279 if (error)
1280 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
Hugh Dickins68da9f02011-07-25 17:12:34 -07001281
Ying Han456f9982011-05-26 16:25:38 -07001282 if (ret & VM_FAULT_MAJOR) {
1283 count_vm_event(PGMAJFAULT);
1284 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1285 }
Hugh Dickins68da9f02011-07-25 17:12:34 -07001286 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287}
1288
1289#ifdef CONFIG_NUMA
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001290static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001292 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1293 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294}
1295
Adrian Bunkd8dc74f2007-10-16 01:26:26 -07001296static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1297 unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001299 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1300 pgoff_t index;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001302 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1303 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304}
1305#endif
1306
1307int shmem_lock(struct file *file, int lock, struct user_struct *user)
1308{
Josef "Jeff" Sipekd3ac7f82006-12-08 02:36:44 -08001309 struct inode *inode = file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 struct shmem_inode_info *info = SHMEM_I(inode);
1311 int retval = -ENOMEM;
1312
1313 spin_lock(&info->lock);
1314 if (lock && !(info->flags & VM_LOCKED)) {
1315 if (!user_shm_lock(inode->i_size, user))
1316 goto out_nomem;
1317 info->flags |= VM_LOCKED;
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07001318 mapping_set_unevictable(file->f_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 }
1320 if (!lock && (info->flags & VM_LOCKED) && user) {
1321 user_shm_unlock(inode->i_size, user);
1322 info->flags &= ~VM_LOCKED;
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07001323 mapping_clear_unevictable(file->f_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 }
1325 retval = 0;
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07001326
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327out_nomem:
1328 spin_unlock(&info->lock);
1329 return retval;
1330}
1331
Adrian Bunk9b83a6a2007-02-28 20:11:03 -08001332static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333{
1334 file_accessed(file);
1335 vma->vm_ops = &shmem_vm_ops;
Nick Piggind0217ac2007-07-19 01:47:03 -07001336 vma->vm_flags |= VM_CAN_NONLINEAR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 return 0;
1338}
1339
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03001340static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
Al Viro09208d12011-07-26 03:15:03 -04001341 umode_t mode, dev_t dev, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342{
1343 struct inode *inode;
1344 struct shmem_inode_info *info;
1345 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1346
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001347 if (shmem_reserve_inode(sb))
1348 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
1350 inode = new_inode(sb);
1351 if (inode) {
Christoph Hellwig85fe4022010-10-23 11:19:54 -04001352 inode->i_ino = get_next_ino();
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03001353 inode_init_owner(inode, dir, mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 inode->i_blocks = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1356 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
David M. Grimes91828a42006-10-17 00:09:45 -07001357 inode->i_generation = get_seconds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 info = SHMEM_I(inode);
1359 memset(info, 0, (char *)inode - (char *)info);
1360 spin_lock_init(&info->lock);
Hugh Dickins0b0a0802009-02-24 20:51:52 +00001361 info->flags = flags & VM_NORESERVE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362 INIT_LIST_HEAD(&info->swaplist);
Eric Parisb09e0fa2011-05-24 17:12:39 -07001363 INIT_LIST_HEAD(&info->xattr_list);
Al Viro72c04902009-06-24 16:58:48 -04001364 cache_no_acl(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365
1366 switch (mode & S_IFMT) {
1367 default:
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07001368 inode->i_op = &shmem_special_inode_operations;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 init_special_inode(inode, mode, dev);
1370 break;
1371 case S_IFREG:
Hugh Dickins14fcc232008-07-28 15:46:19 -07001372 inode->i_mapping->a_ops = &shmem_aops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 inode->i_op = &shmem_inode_operations;
1374 inode->i_fop = &shmem_file_operations;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001375 mpol_shared_policy_init(&info->policy,
1376 shmem_get_sbmpol(sbinfo));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377 break;
1378 case S_IFDIR:
Dave Hansend8c76e62006-09-30 23:29:04 -07001379 inc_nlink(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 /* Some things misbehave if size == 0 on a directory */
1381 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1382 inode->i_op = &shmem_dir_inode_operations;
1383 inode->i_fop = &simple_dir_operations;
1384 break;
1385 case S_IFLNK:
1386 /*
1387 * Must not load anything in the rbtree,
1388 * mpol_free_shared_policy will not be called.
1389 */
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07001390 mpol_shared_policy_init(&info->policy, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 break;
1392 }
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001393 } else
1394 shmem_free_inode(sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 return inode;
1396}
1397
1398#ifdef CONFIG_TMPFS
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -08001399static const struct inode_operations shmem_symlink_inode_operations;
Hugh Dickins69f07ec2011-08-03 16:21:26 -07001400static const struct inode_operations shmem_short_symlink_operations;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07001402#ifdef CONFIG_TMPFS_XATTR
1403static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
1404#else
1405#define shmem_initxattrs NULL
1406#endif
1407
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408static int
Nick Piggin800d15a2007-10-16 01:25:03 -07001409shmem_write_begin(struct file *file, struct address_space *mapping,
1410 loff_t pos, unsigned len, unsigned flags,
1411 struct page **pagep, void **fsdata)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412{
Nick Piggin800d15a2007-10-16 01:25:03 -07001413 struct inode *inode = mapping->host;
1414 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
Nick Piggin800d15a2007-10-16 01:25:03 -07001415 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1416}
1417
1418static int
1419shmem_write_end(struct file *file, struct address_space *mapping,
1420 loff_t pos, unsigned len, unsigned copied,
1421 struct page *page, void *fsdata)
1422{
1423 struct inode *inode = mapping->host;
1424
Hugh Dickinsd3602442008-02-04 22:28:44 -08001425 if (pos + copied > inode->i_size)
1426 i_size_write(inode, pos + copied);
1427
Hugh Dickinsec9516f2012-05-29 15:06:39 -07001428 if (!PageUptodate(page)) {
1429 if (copied < PAGE_CACHE_SIZE) {
1430 unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1431 zero_user_segments(page, 0, from,
1432 from + copied, PAGE_CACHE_SIZE);
1433 }
1434 SetPageUptodate(page);
1435 }
Nick Piggin800d15a2007-10-16 01:25:03 -07001436 set_page_dirty(page);
Wu Fengguang6746aff2009-09-16 11:50:14 +02001437 unlock_page(page);
Nick Piggin800d15a2007-10-16 01:25:03 -07001438 page_cache_release(page);
1439
Nick Piggin800d15a2007-10-16 01:25:03 -07001440 return copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441}
1442
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1444{
Josef "Jeff" Sipekd3ac7f82006-12-08 02:36:44 -08001445 struct inode *inode = filp->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 struct address_space *mapping = inode->i_mapping;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001447 pgoff_t index;
1448 unsigned long offset;
Hugh Dickinsa0ee5ec2008-02-04 22:28:51 -08001449 enum sgp_type sgp = SGP_READ;
1450
1451 /*
1452 * Might this read be for a stacking filesystem? Then when reading
1453 * holes of a sparse file, we actually need to allocate those pages,
1454 * and even mark them dirty, so it cannot exceed the max_blocks limit.
1455 */
1456 if (segment_eq(get_fs(), KERNEL_DS))
1457 sgp = SGP_DIRTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458
1459 index = *ppos >> PAGE_CACHE_SHIFT;
1460 offset = *ppos & ~PAGE_CACHE_MASK;
1461
1462 for (;;) {
1463 struct page *page = NULL;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001464 pgoff_t end_index;
1465 unsigned long nr, ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 loff_t i_size = i_size_read(inode);
1467
1468 end_index = i_size >> PAGE_CACHE_SHIFT;
1469 if (index > end_index)
1470 break;
1471 if (index == end_index) {
1472 nr = i_size & ~PAGE_CACHE_MASK;
1473 if (nr <= offset)
1474 break;
1475 }
1476
Hugh Dickinsa0ee5ec2008-02-04 22:28:51 -08001477 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478 if (desc->error) {
1479 if (desc->error == -EINVAL)
1480 desc->error = 0;
1481 break;
1482 }
Hugh Dickinsd3602442008-02-04 22:28:44 -08001483 if (page)
1484 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485
1486 /*
1487 * We must evaluate after, since reads (unlike writes)
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001488 * are called without i_mutex protection against truncate
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 */
1490 nr = PAGE_CACHE_SIZE;
1491 i_size = i_size_read(inode);
1492 end_index = i_size >> PAGE_CACHE_SHIFT;
1493 if (index == end_index) {
1494 nr = i_size & ~PAGE_CACHE_MASK;
1495 if (nr <= offset) {
1496 if (page)
1497 page_cache_release(page);
1498 break;
1499 }
1500 }
1501 nr -= offset;
1502
1503 if (page) {
1504 /*
1505 * If users can be writing to this page using arbitrary
1506 * virtual addresses, take care about potential aliasing
1507 * before reading the page on the kernel side.
1508 */
1509 if (mapping_writably_mapped(mapping))
1510 flush_dcache_page(page);
1511 /*
1512 * Mark the page accessed if we read the beginning.
1513 */
1514 if (!offset)
1515 mark_page_accessed(page);
Nick Pigginb5810032005-10-29 18:16:12 -07001516 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517 page = ZERO_PAGE(0);
Nick Pigginb5810032005-10-29 18:16:12 -07001518 page_cache_get(page);
1519 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
1521 /*
1522 * Ok, we have the page, and it's up-to-date, so
1523 * now we can copy it to user space...
1524 *
1525 * The actor routine returns how many bytes were actually used..
1526 * NOTE! This may not be the same as how much of a user buffer
1527 * we filled up (we may be padding etc), so we can only update
1528 * "pos" here (the actor routine has to update the user buffer
1529 * pointers and the remaining count).
1530 */
1531 ret = actor(desc, page, offset, nr);
1532 offset += ret;
1533 index += offset >> PAGE_CACHE_SHIFT;
1534 offset &= ~PAGE_CACHE_MASK;
1535
1536 page_cache_release(page);
1537 if (ret != nr || !desc->count)
1538 break;
1539
1540 cond_resched();
1541 }
1542
1543 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1544 file_accessed(filp);
1545}
1546
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07001547static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1548 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549{
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07001550 struct file *filp = iocb->ki_filp;
1551 ssize_t retval;
1552 unsigned long seg;
1553 size_t count;
1554 loff_t *ppos = &iocb->ki_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07001556 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1557 if (retval)
1558 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07001560 for (seg = 0; seg < nr_segs; seg++) {
1561 read_descriptor_t desc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07001563 desc.written = 0;
1564 desc.arg.buf = iov[seg].iov_base;
1565 desc.count = iov[seg].iov_len;
1566 if (desc.count == 0)
1567 continue;
1568 desc.error = 0;
1569 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1570 retval += desc.written;
1571 if (desc.error) {
1572 retval = retval ?: desc.error;
1573 break;
1574 }
1575 if (desc.count > 0)
1576 break;
1577 }
1578 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001579}
1580
Hugh Dickins708e3502011-07-25 17:12:32 -07001581static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1582 struct pipe_inode_info *pipe, size_t len,
1583 unsigned int flags)
1584{
1585 struct address_space *mapping = in->f_mapping;
Hugh Dickins71f0e072011-07-25 17:12:33 -07001586 struct inode *inode = mapping->host;
Hugh Dickins708e3502011-07-25 17:12:32 -07001587 unsigned int loff, nr_pages, req_pages;
1588 struct page *pages[PIPE_DEF_BUFFERS];
1589 struct partial_page partial[PIPE_DEF_BUFFERS];
1590 struct page *page;
1591 pgoff_t index, end_index;
1592 loff_t isize, left;
1593 int error, page_nr;
1594 struct splice_pipe_desc spd = {
1595 .pages = pages,
1596 .partial = partial,
1597 .flags = flags,
1598 .ops = &page_cache_pipe_buf_ops,
1599 .spd_release = spd_release_page,
1600 };
1601
Hugh Dickins71f0e072011-07-25 17:12:33 -07001602 isize = i_size_read(inode);
Hugh Dickins708e3502011-07-25 17:12:32 -07001603 if (unlikely(*ppos >= isize))
1604 return 0;
1605
1606 left = isize - *ppos;
1607 if (unlikely(left < len))
1608 len = left;
1609
1610 if (splice_grow_spd(pipe, &spd))
1611 return -ENOMEM;
1612
1613 index = *ppos >> PAGE_CACHE_SHIFT;
1614 loff = *ppos & ~PAGE_CACHE_MASK;
1615 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1616 nr_pages = min(req_pages, pipe->buffers);
1617
Hugh Dickins708e3502011-07-25 17:12:32 -07001618 spd.nr_pages = find_get_pages_contig(mapping, index,
1619 nr_pages, spd.pages);
1620 index += spd.nr_pages;
Hugh Dickins708e3502011-07-25 17:12:32 -07001621 error = 0;
Hugh Dickins71f0e072011-07-25 17:12:33 -07001622
Hugh Dickins708e3502011-07-25 17:12:32 -07001623 while (spd.nr_pages < nr_pages) {
Hugh Dickins71f0e072011-07-25 17:12:33 -07001624 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1625 if (error)
1626 break;
1627 unlock_page(page);
Hugh Dickins708e3502011-07-25 17:12:32 -07001628 spd.pages[spd.nr_pages++] = page;
1629 index++;
1630 }
1631
Hugh Dickins708e3502011-07-25 17:12:32 -07001632 index = *ppos >> PAGE_CACHE_SHIFT;
1633 nr_pages = spd.nr_pages;
1634 spd.nr_pages = 0;
Hugh Dickins71f0e072011-07-25 17:12:33 -07001635
Hugh Dickins708e3502011-07-25 17:12:32 -07001636 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1637 unsigned int this_len;
1638
1639 if (!len)
1640 break;
1641
Hugh Dickins708e3502011-07-25 17:12:32 -07001642 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1643 page = spd.pages[page_nr];
1644
Hugh Dickins71f0e072011-07-25 17:12:33 -07001645 if (!PageUptodate(page) || page->mapping != mapping) {
Hugh Dickins71f0e072011-07-25 17:12:33 -07001646 error = shmem_getpage(inode, index, &page,
1647 SGP_CACHE, NULL);
1648 if (error)
Hugh Dickins708e3502011-07-25 17:12:32 -07001649 break;
Hugh Dickins71f0e072011-07-25 17:12:33 -07001650 unlock_page(page);
1651 page_cache_release(spd.pages[page_nr]);
1652 spd.pages[page_nr] = page;
Hugh Dickins708e3502011-07-25 17:12:32 -07001653 }
Hugh Dickins71f0e072011-07-25 17:12:33 -07001654
1655 isize = i_size_read(inode);
Hugh Dickins708e3502011-07-25 17:12:32 -07001656 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1657 if (unlikely(!isize || index > end_index))
1658 break;
1659
Hugh Dickins708e3502011-07-25 17:12:32 -07001660 if (end_index == index) {
1661 unsigned int plen;
1662
Hugh Dickins708e3502011-07-25 17:12:32 -07001663 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1664 if (plen <= loff)
1665 break;
1666
Hugh Dickins708e3502011-07-25 17:12:32 -07001667 this_len = min(this_len, plen - loff);
1668 len = this_len;
1669 }
1670
1671 spd.partial[page_nr].offset = loff;
1672 spd.partial[page_nr].len = this_len;
1673 len -= this_len;
1674 loff = 0;
1675 spd.nr_pages++;
1676 index++;
1677 }
1678
Hugh Dickins708e3502011-07-25 17:12:32 -07001679 while (page_nr < nr_pages)
1680 page_cache_release(spd.pages[page_nr++]);
Hugh Dickins708e3502011-07-25 17:12:32 -07001681
1682 if (spd.nr_pages)
1683 error = splice_to_pipe(pipe, &spd);
1684
1685 splice_shrink_spd(pipe, &spd);
1686
1687 if (error > 0) {
1688 *ppos += error;
1689 file_accessed(in);
1690 }
1691 return error;
1692}
1693
Hugh Dickins4fb5ef02012-05-29 15:06:43 -07001694/*
1695 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
1696 */
1697static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
1698 pgoff_t index, pgoff_t end, int origin)
1699{
1700 struct page *page;
1701 struct pagevec pvec;
1702 pgoff_t indices[PAGEVEC_SIZE];
1703 bool done = false;
1704 int i;
1705
1706 pagevec_init(&pvec, 0);
1707 pvec.nr = 1; /* start small: we may be there already */
1708 while (!done) {
1709 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
1710 pvec.nr, pvec.pages, indices);
1711 if (!pvec.nr) {
1712 if (origin == SEEK_DATA)
1713 index = end;
1714 break;
1715 }
1716 for (i = 0; i < pvec.nr; i++, index++) {
1717 if (index < indices[i]) {
1718 if (origin == SEEK_HOLE) {
1719 done = true;
1720 break;
1721 }
1722 index = indices[i];
1723 }
1724 page = pvec.pages[i];
1725 if (page && !radix_tree_exceptional_entry(page)) {
1726 if (!PageUptodate(page))
1727 page = NULL;
1728 }
1729 if (index >= end ||
1730 (page && origin == SEEK_DATA) ||
1731 (!page && origin == SEEK_HOLE)) {
1732 done = true;
1733 break;
1734 }
1735 }
1736 shmem_deswap_pagevec(&pvec);
1737 pagevec_release(&pvec);
1738 pvec.nr = PAGEVEC_SIZE;
1739 cond_resched();
1740 }
1741 return index;
1742}
1743
1744static loff_t shmem_file_llseek(struct file *file, loff_t offset, int origin)
1745{
1746 struct address_space *mapping;
1747 struct inode *inode;
1748 pgoff_t start, end;
1749 loff_t new_offset;
1750
1751 if (origin != SEEK_DATA && origin != SEEK_HOLE)
1752 return generic_file_llseek_size(file, offset, origin,
1753 MAX_LFS_FILESIZE);
1754 mapping = file->f_mapping;
1755 inode = mapping->host;
1756 mutex_lock(&inode->i_mutex);
1757 /* We're holding i_mutex so we can access i_size directly */
1758
1759 if (offset < 0)
1760 offset = -EINVAL;
1761 else if (offset >= inode->i_size)
1762 offset = -ENXIO;
1763 else {
1764 start = offset >> PAGE_CACHE_SHIFT;
1765 end = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1766 new_offset = shmem_seek_hole_data(mapping, start, end, origin);
1767 new_offset <<= PAGE_CACHE_SHIFT;
1768 if (new_offset > offset) {
1769 if (new_offset < inode->i_size)
1770 offset = new_offset;
1771 else if (origin == SEEK_DATA)
1772 offset = -ENXIO;
1773 else
1774 offset = inode->i_size;
1775 }
1776 }
1777
1778 if (offset >= 0 && offset != file->f_pos) {
1779 file->f_pos = offset;
1780 file->f_version = 0;
1781 }
1782 mutex_unlock(&inode->i_mutex);
1783 return offset;
1784}
1785
Hugh Dickins83e4fa92012-05-29 15:06:40 -07001786static long shmem_fallocate(struct file *file, int mode, loff_t offset,
1787 loff_t len)
1788{
1789 struct inode *inode = file->f_path.dentry->d_inode;
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001790 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins1aac1402012-05-29 15:06:42 -07001791 struct shmem_falloc shmem_falloc;
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001792 pgoff_t start, index, end;
1793 int error;
Hugh Dickins83e4fa92012-05-29 15:06:40 -07001794
1795 mutex_lock(&inode->i_mutex);
1796
1797 if (mode & FALLOC_FL_PUNCH_HOLE) {
1798 struct address_space *mapping = file->f_mapping;
1799 loff_t unmap_start = round_up(offset, PAGE_SIZE);
1800 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
1801
1802 if ((u64)unmap_end > (u64)unmap_start)
1803 unmap_mapping_range(mapping, unmap_start,
1804 1 + unmap_end - unmap_start, 0);
1805 shmem_truncate_range(inode, offset, offset + len - 1);
1806 /* No need to unmap again: hole-punching leaves COWed pages */
1807 error = 0;
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001808 goto out;
Hugh Dickins83e4fa92012-05-29 15:06:40 -07001809 }
1810
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001811 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
1812 error = inode_newsize_ok(inode, offset + len);
1813 if (error)
1814 goto out;
1815
1816 start = offset >> PAGE_CACHE_SHIFT;
1817 end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1818 /* Try to avoid a swapstorm if len is impossible to satisfy */
1819 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
1820 error = -ENOSPC;
1821 goto out;
1822 }
1823
Hugh Dickins1aac1402012-05-29 15:06:42 -07001824 shmem_falloc.start = start;
1825 shmem_falloc.next = start;
1826 shmem_falloc.nr_falloced = 0;
1827 shmem_falloc.nr_unswapped = 0;
1828 spin_lock(&inode->i_lock);
1829 inode->i_private = &shmem_falloc;
1830 spin_unlock(&inode->i_lock);
1831
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001832 for (index = start; index < end; index++) {
1833 struct page *page;
1834
1835 /*
1836 * Good, the fallocate(2) manpage permits EINTR: we may have
1837 * been interrupted because we are using up too much memory.
1838 */
1839 if (signal_pending(current))
1840 error = -EINTR;
Hugh Dickins1aac1402012-05-29 15:06:42 -07001841 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
1842 error = -ENOMEM;
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001843 else
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001844 error = shmem_getpage(inode, index, &page, SGP_FALLOC,
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001845 NULL);
1846 if (error) {
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001847 /* Remove the !PageUptodate pages we added */
1848 shmem_undo_range(inode,
1849 (loff_t)start << PAGE_CACHE_SHIFT,
1850 (loff_t)index << PAGE_CACHE_SHIFT, true);
Hugh Dickins1aac1402012-05-29 15:06:42 -07001851 goto undone;
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001852 }
1853
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001854 /*
Hugh Dickins1aac1402012-05-29 15:06:42 -07001855 * Inform shmem_writepage() how far we have reached.
1856 * No need for lock or barrier: we have the page lock.
1857 */
1858 shmem_falloc.next++;
1859 if (!PageUptodate(page))
1860 shmem_falloc.nr_falloced++;
1861
1862 /*
Hugh Dickins1635f6a2012-05-29 15:06:42 -07001863 * If !PageUptodate, leave it that way so that freeable pages
1864 * can be recognized if we need to rollback on error later.
1865 * But set_page_dirty so that memory pressure will swap rather
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001866 * than free the pages we are allocating (and SGP_CACHE pages
1867 * might still be clean: we now need to mark those dirty too).
1868 */
1869 set_page_dirty(page);
1870 unlock_page(page);
1871 page_cache_release(page);
1872 cond_resched();
1873 }
1874
1875 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
1876 i_size_write(inode, offset + len);
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001877 inode->i_ctime = CURRENT_TIME;
Hugh Dickins1aac1402012-05-29 15:06:42 -07001878undone:
1879 spin_lock(&inode->i_lock);
1880 inode->i_private = NULL;
1881 spin_unlock(&inode->i_lock);
Hugh Dickinse2d12e22012-05-29 15:06:41 -07001882out:
Hugh Dickins83e4fa92012-05-29 15:06:40 -07001883 mutex_unlock(&inode->i_mutex);
1884 return error;
1885}
1886
David Howells726c3342006-06-23 02:02:58 -07001887static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888{
David Howells726c3342006-06-23 02:02:58 -07001889 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890
1891 buf->f_type = TMPFS_MAGIC;
1892 buf->f_bsize = PAGE_CACHE_SIZE;
1893 buf->f_namelen = NAME_MAX;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001894 if (sbinfo->max_blocks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895 buf->f_blocks = sbinfo->max_blocks;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07001896 buf->f_bavail =
1897 buf->f_bfree = sbinfo->max_blocks -
1898 percpu_counter_sum(&sbinfo->used_blocks);
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001899 }
1900 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901 buf->f_files = sbinfo->max_inodes;
1902 buf->f_ffree = sbinfo->free_inodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903 }
1904 /* else leave those fields 0 like simple_statfs */
1905 return 0;
1906}
1907
1908/*
1909 * File creation. Allocate an inode, and we're done..
1910 */
1911static int
Al Viro1a67aaf2011-07-26 01:52:52 -04001912shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913{
Hugh Dickins0b0a0802009-02-24 20:51:52 +00001914 struct inode *inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915 int error = -ENOSPC;
1916
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03001917 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 if (inode) {
Eric Paris2a7dba32011-02-01 11:05:39 -05001919 error = security_inode_init_security(inode, dir,
Mimi Zohar9d8f13b2011-06-06 15:29:25 -04001920 &dentry->d_name,
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07001921 shmem_initxattrs, NULL);
Stephen Smalley570bc1c2005-09-09 13:01:43 -07001922 if (error) {
1923 if (error != -EOPNOTSUPP) {
1924 iput(inode);
1925 return error;
1926 }
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07001927 }
Christoph Hellwig1c7c4742009-11-03 16:44:44 +01001928#ifdef CONFIG_TMPFS_POSIX_ACL
1929 error = generic_acl_init(inode, dir);
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07001930 if (error) {
1931 iput(inode);
1932 return error;
Stephen Smalley570bc1c2005-09-09 13:01:43 -07001933 }
Al Viro718deb62009-12-16 19:35:36 -05001934#else
1935 error = 0;
Christoph Hellwig1c7c4742009-11-03 16:44:44 +01001936#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 dir->i_size += BOGO_DIRENT_SIZE;
1938 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1939 d_instantiate(dentry, inode);
1940 dget(dentry); /* Extra count - pin the dentry in core */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001941 }
1942 return error;
1943}
1944
Al Viro18bb1db2011-07-26 01:41:39 -04001945static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001946{
1947 int error;
1948
1949 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1950 return error;
Dave Hansend8c76e62006-09-30 23:29:04 -07001951 inc_nlink(dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952 return 0;
1953}
1954
Al Viro4acdaf22011-07-26 01:42:34 -04001955static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956 struct nameidata *nd)
1957{
1958 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1959}
1960
1961/*
1962 * Link a file..
1963 */
1964static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1965{
1966 struct inode *inode = old_dentry->d_inode;
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001967 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968
1969 /*
1970 * No ordinary (disk based) filesystem counts links as inodes;
1971 * but each new link needs a new dentry, pinning lowmem, and
1972 * tmpfs dentries cannot be pruned until they are unlinked.
1973 */
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001974 ret = shmem_reserve_inode(inode->i_sb);
1975 if (ret)
1976 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977
1978 dir->i_size += BOGO_DIRENT_SIZE;
1979 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
Dave Hansend8c76e62006-09-30 23:29:04 -07001980 inc_nlink(inode);
Al Viro7de9c6ee2010-10-23 11:11:40 -04001981 ihold(inode); /* New dentry reference */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001982 dget(dentry); /* Extra pinning count for the created dentry */
1983 d_instantiate(dentry, inode);
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001984out:
1985 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001986}
1987
1988static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1989{
1990 struct inode *inode = dentry->d_inode;
1991
Pavel Emelyanov5b04c682008-02-04 22:28:47 -08001992 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1993 shmem_free_inode(inode->i_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001994
1995 dir->i_size -= BOGO_DIRENT_SIZE;
1996 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
Dave Hansen9a53c3a2006-09-30 23:29:03 -07001997 drop_nlink(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998 dput(dentry); /* Undo the count from "create" - this does all the work */
1999 return 0;
2000}
2001
2002static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2003{
2004 if (!simple_empty(dentry))
2005 return -ENOTEMPTY;
2006
Dave Hansen9a53c3a2006-09-30 23:29:03 -07002007 drop_nlink(dentry->d_inode);
2008 drop_nlink(dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009 return shmem_unlink(dir, dentry);
2010}
2011
2012/*
2013 * The VFS layer already does all the dentry stuff for rename,
2014 * we just have to decrement the usage count for the target if
2015 * it exists so that the VFS layer correctly free's it when it
2016 * gets overwritten.
2017 */
2018static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2019{
2020 struct inode *inode = old_dentry->d_inode;
2021 int they_are_dirs = S_ISDIR(inode->i_mode);
2022
2023 if (!simple_empty(new_dentry))
2024 return -ENOTEMPTY;
2025
2026 if (new_dentry->d_inode) {
2027 (void) shmem_unlink(new_dir, new_dentry);
2028 if (they_are_dirs)
Dave Hansen9a53c3a2006-09-30 23:29:03 -07002029 drop_nlink(old_dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002030 } else if (they_are_dirs) {
Dave Hansen9a53c3a2006-09-30 23:29:03 -07002031 drop_nlink(old_dir);
Dave Hansend8c76e62006-09-30 23:29:04 -07002032 inc_nlink(new_dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002033 }
2034
2035 old_dir->i_size -= BOGO_DIRENT_SIZE;
2036 new_dir->i_size += BOGO_DIRENT_SIZE;
2037 old_dir->i_ctime = old_dir->i_mtime =
2038 new_dir->i_ctime = new_dir->i_mtime =
2039 inode->i_ctime = CURRENT_TIME;
2040 return 0;
2041}
2042
2043static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
2044{
2045 int error;
2046 int len;
2047 struct inode *inode;
Hugh Dickins9276aad2011-07-25 17:12:34 -07002048 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 char *kaddr;
2050 struct shmem_inode_info *info;
2051
2052 len = strlen(symname) + 1;
2053 if (len > PAGE_CACHE_SIZE)
2054 return -ENAMETOOLONG;
2055
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03002056 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057 if (!inode)
2058 return -ENOSPC;
2059
Mimi Zohar9d8f13b2011-06-06 15:29:25 -04002060 error = security_inode_init_security(inode, dir, &dentry->d_name,
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002061 shmem_initxattrs, NULL);
Stephen Smalley570bc1c2005-09-09 13:01:43 -07002062 if (error) {
2063 if (error != -EOPNOTSUPP) {
2064 iput(inode);
2065 return error;
2066 }
2067 error = 0;
2068 }
2069
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070 info = SHMEM_I(inode);
2071 inode->i_size = len-1;
Hugh Dickins69f07ec2011-08-03 16:21:26 -07002072 if (len <= SHORT_SYMLINK_LEN) {
2073 info->symlink = kmemdup(symname, len, GFP_KERNEL);
2074 if (!info->symlink) {
2075 iput(inode);
2076 return -ENOMEM;
2077 }
2078 inode->i_op = &shmem_short_symlink_operations;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079 } else {
2080 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2081 if (error) {
2082 iput(inode);
2083 return error;
2084 }
Hugh Dickins14fcc232008-07-28 15:46:19 -07002085 inode->i_mapping->a_ops = &shmem_aops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 inode->i_op = &shmem_symlink_inode_operations;
Cong Wang9b04c5f2011-11-25 23:14:39 +08002087 kaddr = kmap_atomic(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088 memcpy(kaddr, symname, len);
Cong Wang9b04c5f2011-11-25 23:14:39 +08002089 kunmap_atomic(kaddr);
Hugh Dickinsec9516f2012-05-29 15:06:39 -07002090 SetPageUptodate(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 set_page_dirty(page);
Wu Fengguang6746aff2009-09-16 11:50:14 +02002092 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093 page_cache_release(page);
2094 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 dir->i_size += BOGO_DIRENT_SIZE;
2096 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2097 d_instantiate(dentry, inode);
2098 dget(dentry);
2099 return 0;
2100}
2101
Hugh Dickins69f07ec2011-08-03 16:21:26 -07002102static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002103{
Hugh Dickins69f07ec2011-08-03 16:21:26 -07002104 nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
Linus Torvaldscc314ee2005-08-19 18:02:56 -07002105 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106}
2107
Linus Torvaldscc314ee2005-08-19 18:02:56 -07002108static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002109{
2110 struct page *page = NULL;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002111 int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2112 nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
Hugh Dickinsd3602442008-02-04 22:28:44 -08002113 if (page)
2114 unlock_page(page);
Linus Torvaldscc314ee2005-08-19 18:02:56 -07002115 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116}
2117
Linus Torvaldscc314ee2005-08-19 18:02:56 -07002118static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119{
2120 if (!IS_ERR(nd_get_link(nd))) {
Linus Torvaldscc314ee2005-08-19 18:02:56 -07002121 struct page *page = cookie;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002122 kunmap(page);
2123 mark_page_accessed(page);
2124 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002125 }
2126}
2127
Eric Parisb09e0fa2011-05-24 17:12:39 -07002128#ifdef CONFIG_TMPFS_XATTR
2129/*
2130 * Superblocks without xattr inode operations may get some security.* xattr
2131 * support from the LSM "for free". As soon as we have any other xattrs
2132 * like ACLs, we also need to implement the security.* handlers at
2133 * filesystem level, though.
2134 */
2135
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002136/*
2137 * Allocate new xattr and copy in the value; but leave the name to callers.
2138 */
2139static struct shmem_xattr *shmem_xattr_alloc(const void *value, size_t size)
2140{
2141 struct shmem_xattr *new_xattr;
2142 size_t len;
2143
2144 /* wrap around? */
2145 len = sizeof(*new_xattr) + size;
2146 if (len <= sizeof(*new_xattr))
2147 return NULL;
2148
2149 new_xattr = kmalloc(len, GFP_KERNEL);
2150 if (!new_xattr)
2151 return NULL;
2152
2153 new_xattr->size = size;
2154 memcpy(new_xattr->value, value, size);
2155 return new_xattr;
2156}
2157
2158/*
2159 * Callback for security_inode_init_security() for acquiring xattrs.
2160 */
2161static int shmem_initxattrs(struct inode *inode,
2162 const struct xattr *xattr_array,
2163 void *fs_info)
2164{
2165 struct shmem_inode_info *info = SHMEM_I(inode);
2166 const struct xattr *xattr;
2167 struct shmem_xattr *new_xattr;
2168 size_t len;
2169
2170 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
2171 new_xattr = shmem_xattr_alloc(xattr->value, xattr->value_len);
2172 if (!new_xattr)
2173 return -ENOMEM;
2174
2175 len = strlen(xattr->name) + 1;
2176 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
2177 GFP_KERNEL);
2178 if (!new_xattr->name) {
2179 kfree(new_xattr);
2180 return -ENOMEM;
2181 }
2182
2183 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
2184 XATTR_SECURITY_PREFIX_LEN);
2185 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
2186 xattr->name, len);
2187
2188 spin_lock(&info->lock);
2189 list_add(&new_xattr->list, &info->xattr_list);
2190 spin_unlock(&info->lock);
2191 }
2192
2193 return 0;
2194}
2195
Eric Parisb09e0fa2011-05-24 17:12:39 -07002196static int shmem_xattr_get(struct dentry *dentry, const char *name,
2197 void *buffer, size_t size)
2198{
2199 struct shmem_inode_info *info;
2200 struct shmem_xattr *xattr;
2201 int ret = -ENODATA;
2202
2203 info = SHMEM_I(dentry->d_inode);
2204
2205 spin_lock(&info->lock);
2206 list_for_each_entry(xattr, &info->xattr_list, list) {
2207 if (strcmp(name, xattr->name))
2208 continue;
2209
2210 ret = xattr->size;
2211 if (buffer) {
2212 if (size < xattr->size)
2213 ret = -ERANGE;
2214 else
2215 memcpy(buffer, xattr->value, xattr->size);
2216 }
2217 break;
2218 }
2219 spin_unlock(&info->lock);
2220 return ret;
2221}
2222
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002223static int shmem_xattr_set(struct inode *inode, const char *name,
Eric Parisb09e0fa2011-05-24 17:12:39 -07002224 const void *value, size_t size, int flags)
2225{
Eric Parisb09e0fa2011-05-24 17:12:39 -07002226 struct shmem_inode_info *info = SHMEM_I(inode);
2227 struct shmem_xattr *xattr;
2228 struct shmem_xattr *new_xattr = NULL;
Eric Parisb09e0fa2011-05-24 17:12:39 -07002229 int err = 0;
2230
2231 /* value == NULL means remove */
2232 if (value) {
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002233 new_xattr = shmem_xattr_alloc(value, size);
Eric Parisb09e0fa2011-05-24 17:12:39 -07002234 if (!new_xattr)
2235 return -ENOMEM;
2236
2237 new_xattr->name = kstrdup(name, GFP_KERNEL);
2238 if (!new_xattr->name) {
2239 kfree(new_xattr);
2240 return -ENOMEM;
2241 }
Eric Parisb09e0fa2011-05-24 17:12:39 -07002242 }
2243
2244 spin_lock(&info->lock);
2245 list_for_each_entry(xattr, &info->xattr_list, list) {
2246 if (!strcmp(name, xattr->name)) {
2247 if (flags & XATTR_CREATE) {
2248 xattr = new_xattr;
2249 err = -EEXIST;
2250 } else if (new_xattr) {
2251 list_replace(&xattr->list, &new_xattr->list);
2252 } else {
2253 list_del(&xattr->list);
2254 }
2255 goto out;
2256 }
2257 }
2258 if (flags & XATTR_REPLACE) {
2259 xattr = new_xattr;
2260 err = -ENODATA;
2261 } else {
2262 list_add(&new_xattr->list, &info->xattr_list);
2263 xattr = NULL;
2264 }
2265out:
2266 spin_unlock(&info->lock);
2267 if (xattr)
2268 kfree(xattr->name);
2269 kfree(xattr);
2270 return err;
2271}
2272
Eric Parisb09e0fa2011-05-24 17:12:39 -07002273static const struct xattr_handler *shmem_xattr_handlers[] = {
2274#ifdef CONFIG_TMPFS_POSIX_ACL
2275 &generic_acl_access_handler,
2276 &generic_acl_default_handler,
2277#endif
2278 NULL
2279};
2280
2281static int shmem_xattr_validate(const char *name)
2282{
2283 struct { const char *prefix; size_t len; } arr[] = {
2284 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
2285 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
2286 };
2287 int i;
2288
2289 for (i = 0; i < ARRAY_SIZE(arr); i++) {
2290 size_t preflen = arr[i].len;
2291 if (strncmp(name, arr[i].prefix, preflen) == 0) {
2292 if (!name[preflen])
2293 return -EINVAL;
2294 return 0;
2295 }
2296 }
2297 return -EOPNOTSUPP;
2298}
2299
2300static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
2301 void *buffer, size_t size)
2302{
2303 int err;
2304
2305 /*
2306 * If this is a request for a synthetic attribute in the system.*
2307 * namespace use the generic infrastructure to resolve a handler
2308 * for it via sb->s_xattr.
2309 */
2310 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2311 return generic_getxattr(dentry, name, buffer, size);
2312
2313 err = shmem_xattr_validate(name);
2314 if (err)
2315 return err;
2316
2317 return shmem_xattr_get(dentry, name, buffer, size);
2318}
2319
2320static int shmem_setxattr(struct dentry *dentry, const char *name,
2321 const void *value, size_t size, int flags)
2322{
2323 int err;
2324
2325 /*
2326 * If this is a request for a synthetic attribute in the system.*
2327 * namespace use the generic infrastructure to resolve a handler
2328 * for it via sb->s_xattr.
2329 */
2330 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2331 return generic_setxattr(dentry, name, value, size, flags);
2332
2333 err = shmem_xattr_validate(name);
2334 if (err)
2335 return err;
2336
2337 if (size == 0)
2338 value = ""; /* empty EA, do not remove */
2339
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002340 return shmem_xattr_set(dentry->d_inode, name, value, size, flags);
Eric Parisb09e0fa2011-05-24 17:12:39 -07002341
2342}
2343
2344static int shmem_removexattr(struct dentry *dentry, const char *name)
2345{
2346 int err;
2347
2348 /*
2349 * If this is a request for a synthetic attribute in the system.*
2350 * namespace use the generic infrastructure to resolve a handler
2351 * for it via sb->s_xattr.
2352 */
2353 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2354 return generic_removexattr(dentry, name);
2355
2356 err = shmem_xattr_validate(name);
2357 if (err)
2358 return err;
2359
Jarkko Sakkinen6d9d88d2012-03-21 16:34:05 -07002360 return shmem_xattr_set(dentry->d_inode, name, NULL, 0, XATTR_REPLACE);
Eric Parisb09e0fa2011-05-24 17:12:39 -07002361}
2362
2363static bool xattr_is_trusted(const char *name)
2364{
2365 return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
2366}
2367
2368static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
2369{
2370 bool trusted = capable(CAP_SYS_ADMIN);
2371 struct shmem_xattr *xattr;
2372 struct shmem_inode_info *info;
2373 size_t used = 0;
2374
2375 info = SHMEM_I(dentry->d_inode);
2376
2377 spin_lock(&info->lock);
2378 list_for_each_entry(xattr, &info->xattr_list, list) {
2379 size_t len;
2380
2381 /* skip "trusted." attributes for unprivileged callers */
2382 if (!trusted && xattr_is_trusted(xattr->name))
2383 continue;
2384
2385 len = strlen(xattr->name) + 1;
2386 used += len;
2387 if (buffer) {
2388 if (size < used) {
2389 used = -ERANGE;
2390 break;
2391 }
2392 memcpy(buffer, xattr->name, len);
2393 buffer += len;
2394 }
2395 }
2396 spin_unlock(&info->lock);
2397
2398 return used;
2399}
2400#endif /* CONFIG_TMPFS_XATTR */
2401
Hugh Dickins69f07ec2011-08-03 16:21:26 -07002402static const struct inode_operations shmem_short_symlink_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403 .readlink = generic_readlink,
Hugh Dickins69f07ec2011-08-03 16:21:26 -07002404 .follow_link = shmem_follow_short_symlink,
Eric Parisb09e0fa2011-05-24 17:12:39 -07002405#ifdef CONFIG_TMPFS_XATTR
2406 .setxattr = shmem_setxattr,
2407 .getxattr = shmem_getxattr,
2408 .listxattr = shmem_listxattr,
2409 .removexattr = shmem_removexattr,
2410#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002411};
2412
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -08002413static const struct inode_operations shmem_symlink_inode_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414 .readlink = generic_readlink,
2415 .follow_link = shmem_follow_link,
2416 .put_link = shmem_put_link,
Eric Parisb09e0fa2011-05-24 17:12:39 -07002417#ifdef CONFIG_TMPFS_XATTR
2418 .setxattr = shmem_setxattr,
2419 .getxattr = shmem_getxattr,
2420 .listxattr = shmem_listxattr,
2421 .removexattr = shmem_removexattr,
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002422#endif
Eric Parisb09e0fa2011-05-24 17:12:39 -07002423};
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002424
David M. Grimes91828a42006-10-17 00:09:45 -07002425static struct dentry *shmem_get_parent(struct dentry *child)
2426{
2427 return ERR_PTR(-ESTALE);
2428}
2429
2430static int shmem_match(struct inode *ino, void *vfh)
2431{
2432 __u32 *fh = vfh;
2433 __u64 inum = fh[2];
2434 inum = (inum << 32) | fh[1];
2435 return ino->i_ino == inum && fh[0] == ino->i_generation;
2436}
2437
Christoph Hellwig480b1162007-10-21 16:42:13 -07002438static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2439 struct fid *fid, int fh_len, int fh_type)
David M. Grimes91828a42006-10-17 00:09:45 -07002440{
David M. Grimes91828a42006-10-17 00:09:45 -07002441 struct inode *inode;
Christoph Hellwig480b1162007-10-21 16:42:13 -07002442 struct dentry *dentry = NULL;
2443 u64 inum = fid->raw[2];
2444 inum = (inum << 32) | fid->raw[1];
David M. Grimes91828a42006-10-17 00:09:45 -07002445
Christoph Hellwig480b1162007-10-21 16:42:13 -07002446 if (fh_len < 3)
2447 return NULL;
2448
2449 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2450 shmem_match, fid->raw);
David M. Grimes91828a42006-10-17 00:09:45 -07002451 if (inode) {
Christoph Hellwig480b1162007-10-21 16:42:13 -07002452 dentry = d_find_alias(inode);
David M. Grimes91828a42006-10-17 00:09:45 -07002453 iput(inode);
2454 }
2455
Christoph Hellwig480b1162007-10-21 16:42:13 -07002456 return dentry;
David M. Grimes91828a42006-10-17 00:09:45 -07002457}
2458
Al Virob0b03822012-04-02 14:34:06 -04002459static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
2460 struct inode *parent)
David M. Grimes91828a42006-10-17 00:09:45 -07002461{
Aneesh Kumar K.V5fe0c232011-01-29 18:43:25 +05302462 if (*len < 3) {
2463 *len = 3;
David M. Grimes91828a42006-10-17 00:09:45 -07002464 return 255;
Aneesh Kumar K.V5fe0c232011-01-29 18:43:25 +05302465 }
David M. Grimes91828a42006-10-17 00:09:45 -07002466
Al Viro1d3382cb2010-10-23 15:19:20 -04002467 if (inode_unhashed(inode)) {
David M. Grimes91828a42006-10-17 00:09:45 -07002468 /* Unfortunately insert_inode_hash is not idempotent,
2469 * so as we hash inodes here rather than at creation
2470 * time, we need a lock to ensure we only try
2471 * to do it once
2472 */
2473 static DEFINE_SPINLOCK(lock);
2474 spin_lock(&lock);
Al Viro1d3382cb2010-10-23 15:19:20 -04002475 if (inode_unhashed(inode))
David M. Grimes91828a42006-10-17 00:09:45 -07002476 __insert_inode_hash(inode,
2477 inode->i_ino + inode->i_generation);
2478 spin_unlock(&lock);
2479 }
2480
2481 fh[0] = inode->i_generation;
2482 fh[1] = inode->i_ino;
2483 fh[2] = ((__u64)inode->i_ino) >> 32;
2484
2485 *len = 3;
2486 return 1;
2487}
2488
Christoph Hellwig39655162007-10-21 16:42:17 -07002489static const struct export_operations shmem_export_ops = {
David M. Grimes91828a42006-10-17 00:09:45 -07002490 .get_parent = shmem_get_parent,
David M. Grimes91828a42006-10-17 00:09:45 -07002491 .encode_fh = shmem_encode_fh,
Christoph Hellwig480b1162007-10-21 16:42:13 -07002492 .fh_to_dentry = shmem_fh_to_dentry,
David M. Grimes91828a42006-10-17 00:09:45 -07002493};
2494
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002495static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2496 bool remount)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497{
2498 char *this_char, *value, *rest;
Eric W. Biederman8751e032012-02-07 16:46:12 -08002499 uid_t uid;
2500 gid_t gid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +00002502 while (options != NULL) {
2503 this_char = options;
2504 for (;;) {
2505 /*
2506 * NUL-terminate this option: unfortunately,
2507 * mount options form a comma-separated list,
2508 * but mpol's nodelist may also contain commas.
2509 */
2510 options = strchr(options, ',');
2511 if (options == NULL)
2512 break;
2513 options++;
2514 if (!isdigit(*options)) {
2515 options[-1] = '\0';
2516 break;
2517 }
2518 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002519 if (!*this_char)
2520 continue;
2521 if ((value = strchr(this_char,'=')) != NULL) {
2522 *value++ = 0;
2523 } else {
2524 printk(KERN_ERR
2525 "tmpfs: No value for mount option '%s'\n",
2526 this_char);
2527 return 1;
2528 }
2529
2530 if (!strcmp(this_char,"size")) {
2531 unsigned long long size;
2532 size = memparse(value,&rest);
2533 if (*rest == '%') {
2534 size <<= PAGE_SHIFT;
2535 size *= totalram_pages;
2536 do_div(size, 100);
2537 rest++;
2538 }
2539 if (*rest)
2540 goto bad_val;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002541 sbinfo->max_blocks =
2542 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543 } else if (!strcmp(this_char,"nr_blocks")) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002544 sbinfo->max_blocks = memparse(value, &rest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002545 if (*rest)
2546 goto bad_val;
2547 } else if (!strcmp(this_char,"nr_inodes")) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002548 sbinfo->max_inodes = memparse(value, &rest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002549 if (*rest)
2550 goto bad_val;
2551 } else if (!strcmp(this_char,"mode")) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002552 if (remount)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002553 continue;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002554 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555 if (*rest)
2556 goto bad_val;
2557 } else if (!strcmp(this_char,"uid")) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002558 if (remount)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 continue;
Eric W. Biederman8751e032012-02-07 16:46:12 -08002560 uid = simple_strtoul(value, &rest, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561 if (*rest)
2562 goto bad_val;
Eric W. Biederman8751e032012-02-07 16:46:12 -08002563 sbinfo->uid = make_kuid(current_user_ns(), uid);
2564 if (!uid_valid(sbinfo->uid))
2565 goto bad_val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566 } else if (!strcmp(this_char,"gid")) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002567 if (remount)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002568 continue;
Eric W. Biederman8751e032012-02-07 16:46:12 -08002569 gid = simple_strtoul(value, &rest, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002570 if (*rest)
2571 goto bad_val;
Eric W. Biederman8751e032012-02-07 16:46:12 -08002572 sbinfo->gid = make_kgid(current_user_ns(), gid);
2573 if (!gid_valid(sbinfo->gid))
2574 goto bad_val;
Robin Holt7339ff82006-01-14 13:20:48 -08002575 } else if (!strcmp(this_char,"mpol")) {
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002576 if (mpol_parse_str(value, &sbinfo->mpol, 1))
Robin Holt7339ff82006-01-14 13:20:48 -08002577 goto bad_val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002578 } else {
2579 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2580 this_char);
2581 return 1;
2582 }
2583 }
2584 return 0;
2585
2586bad_val:
2587 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2588 value, this_char);
2589 return 1;
2590
2591}
2592
2593static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2594{
2595 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002596 struct shmem_sb_info config = *sbinfo;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002597 unsigned long inodes;
2598 int error = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002599
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002600 if (shmem_parse_options(data, &config, true))
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002601 return error;
2602
2603 spin_lock(&sbinfo->stat_lock);
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002604 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
Tim Chen7e496292010-08-09 17:19:05 -07002605 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002606 goto out;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002607 if (config.max_inodes < inodes)
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002608 goto out;
2609 /*
Hugh Dickins54af60422011-08-03 16:21:24 -07002610 * Those tests disallow limited->unlimited while any are in use;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002611 * but we must separately disallow unlimited->limited, because
2612 * in that case we have no record of how much is already in use.
2613 */
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002614 if (config.max_blocks && !sbinfo->max_blocks)
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002615 goto out;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002616 if (config.max_inodes && !sbinfo->max_inodes)
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002617 goto out;
2618
2619 error = 0;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002620 sbinfo->max_blocks = config.max_blocks;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002621 sbinfo->max_inodes = config.max_inodes;
2622 sbinfo->free_inodes = config.max_inodes - inodes;
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002623
2624 mpol_put(sbinfo->mpol);
2625 sbinfo->mpol = config.mpol; /* transfers initial ref */
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002626out:
2627 spin_unlock(&sbinfo->stat_lock);
2628 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002629}
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002630
Al Viro34c80b12011-12-08 21:32:45 -05002631static int shmem_show_options(struct seq_file *seq, struct dentry *root)
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002632{
Al Viro34c80b12011-12-08 21:32:45 -05002633 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002634
2635 if (sbinfo->max_blocks != shmem_default_max_blocks())
2636 seq_printf(seq, ",size=%luk",
2637 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2638 if (sbinfo->max_inodes != shmem_default_max_inodes())
2639 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2640 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
Al Viro09208d12011-07-26 03:15:03 -04002641 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
Eric W. Biederman8751e032012-02-07 16:46:12 -08002642 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2643 seq_printf(seq, ",uid=%u",
2644 from_kuid_munged(&init_user_ns, sbinfo->uid));
2645 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2646 seq_printf(seq, ",gid=%u",
2647 from_kgid_munged(&init_user_ns, sbinfo->gid));
Lee Schermerhorn71fe8042008-04-28 02:13:26 -07002648 shmem_show_mpol(seq, sbinfo->mpol);
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002649 return 0;
2650}
2651#endif /* CONFIG_TMPFS */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002652
2653static void shmem_put_super(struct super_block *sb)
2654{
Hugh Dickins602586a2010-08-17 15:23:56 -07002655 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2656
2657 percpu_counter_destroy(&sbinfo->used_blocks);
2658 kfree(sbinfo);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002659 sb->s_fs_info = NULL;
2660}
2661
Kay Sievers2b2af542009-04-30 15:23:42 +02002662int shmem_fill_super(struct super_block *sb, void *data, int silent)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663{
2664 struct inode *inode;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002665 struct shmem_sb_info *sbinfo;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002666 int err = -ENOMEM;
2667
2668 /* Round up to L1_CACHE_BYTES to resist false sharing */
Pekka Enberg425fbf02009-09-21 17:03:50 -07002669 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002670 L1_CACHE_BYTES), GFP_KERNEL);
2671 if (!sbinfo)
2672 return -ENOMEM;
2673
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002674 sbinfo->mode = S_IRWXUGO | S_ISVTX;
David Howells76aac0e2008-11-14 10:39:12 +11002675 sbinfo->uid = current_fsuid();
2676 sbinfo->gid = current_fsgid();
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002677 sb->s_fs_info = sbinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002678
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002679#ifdef CONFIG_TMPFS
Linus Torvalds1da177e2005-04-16 15:20:36 -07002680 /*
2681 * Per default we only allow half of the physical ram per
2682 * tmpfs instance, limiting inodes to one per page of lowmem;
2683 * but the internal instance is left unlimited.
2684 */
2685 if (!(sb->s_flags & MS_NOUSER)) {
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002686 sbinfo->max_blocks = shmem_default_max_blocks();
2687 sbinfo->max_inodes = shmem_default_max_inodes();
2688 if (shmem_parse_options(data, sbinfo, false)) {
2689 err = -EINVAL;
2690 goto failed;
2691 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002692 }
David M. Grimes91828a42006-10-17 00:09:45 -07002693 sb->s_export_op = &shmem_export_ops;
Hugh Dickins2f6e38f2012-05-29 15:06:38 -07002694 sb->s_flags |= MS_NOSEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002695#else
2696 sb->s_flags |= MS_NOUSER;
2697#endif
2698
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002699 spin_lock_init(&sbinfo->stat_lock);
Hugh Dickins602586a2010-08-17 15:23:56 -07002700 if (percpu_counter_init(&sbinfo->used_blocks, 0))
2701 goto failed;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002702 sbinfo->free_inodes = sbinfo->max_inodes;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002703
Hugh Dickins285b2c42011-08-03 16:21:20 -07002704 sb->s_maxbytes = MAX_LFS_FILESIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705 sb->s_blocksize = PAGE_CACHE_SIZE;
2706 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2707 sb->s_magic = TMPFS_MAGIC;
2708 sb->s_op = &shmem_ops;
Robin H. Johnsoncfd95a92006-06-12 21:50:25 +01002709 sb->s_time_gran = 1;
Eric Parisb09e0fa2011-05-24 17:12:39 -07002710#ifdef CONFIG_TMPFS_XATTR
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002711 sb->s_xattr = shmem_xattr_handlers;
Eric Parisb09e0fa2011-05-24 17:12:39 -07002712#endif
2713#ifdef CONFIG_TMPFS_POSIX_ACL
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002714 sb->s_flags |= MS_POSIXACL;
2715#endif
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002716
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03002717 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718 if (!inode)
2719 goto failed;
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002720 inode->i_uid = sbinfo->uid;
2721 inode->i_gid = sbinfo->gid;
Al Viro318ceed2012-02-12 22:08:01 -05002722 sb->s_root = d_make_root(inode);
2723 if (!sb->s_root)
Al Viro48fde702012-01-08 22:15:13 -05002724 goto failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002725 return 0;
2726
Linus Torvalds1da177e2005-04-16 15:20:36 -07002727failed:
2728 shmem_put_super(sb);
2729 return err;
2730}
2731
Pekka Enbergfcc234f2006-03-22 00:08:13 -08002732static struct kmem_cache *shmem_inode_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002733
2734static struct inode *shmem_alloc_inode(struct super_block *sb)
2735{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002736 struct shmem_inode_info *info;
2737 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2738 if (!info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739 return NULL;
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002740 return &info->vfs_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741}
2742
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002743static void shmem_destroy_callback(struct rcu_head *head)
Nick Pigginfa0d7e3d2011-01-07 17:49:49 +11002744{
2745 struct inode *inode = container_of(head, struct inode, i_rcu);
Nick Pigginfa0d7e3d2011-01-07 17:49:49 +11002746 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2747}
2748
Linus Torvalds1da177e2005-04-16 15:20:36 -07002749static void shmem_destroy_inode(struct inode *inode)
2750{
Al Viro09208d12011-07-26 03:15:03 -04002751 if (S_ISREG(inode->i_mode))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002753 call_rcu(&inode->i_rcu, shmem_destroy_callback);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002754}
2755
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002756static void shmem_init_inode(void *foo)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002757{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002758 struct shmem_inode_info *info = foo;
2759 inode_init_once(&info->vfs_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002760}
2761
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002762static int shmem_init_inodecache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002763{
2764 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2765 sizeof(struct shmem_inode_info),
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002766 0, SLAB_PANIC, shmem_init_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767 return 0;
2768}
2769
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002770static void shmem_destroy_inodecache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002771{
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07002772 kmem_cache_destroy(shmem_inode_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002773}
2774
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07002775static const struct address_space_operations shmem_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002776 .writepage = shmem_writepage,
Ken Chen76719322007-02-10 01:43:15 -08002777 .set_page_dirty = __set_page_dirty_no_writeback,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002778#ifdef CONFIG_TMPFS
Nick Piggin800d15a2007-10-16 01:25:03 -07002779 .write_begin = shmem_write_begin,
2780 .write_end = shmem_write_end,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781#endif
Lee Schermerhorn304dbdb2006-04-22 02:35:48 -07002782 .migratepage = migrate_page,
Andi Kleenaa261f52009-09-16 11:50:16 +02002783 .error_remove_page = generic_error_remove_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002784};
2785
Helge Deller15ad7cd2006-12-06 20:40:36 -08002786static const struct file_operations shmem_file_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787 .mmap = shmem_mmap,
2788#ifdef CONFIG_TMPFS
Hugh Dickins4fb5ef02012-05-29 15:06:43 -07002789 .llseek = shmem_file_llseek,
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07002790 .read = do_sync_read,
Hugh Dickins5402b972008-02-04 22:28:44 -08002791 .write = do_sync_write,
Hugh Dickinsbcd78e42008-07-23 21:27:35 -07002792 .aio_read = shmem_file_aio_read,
Hugh Dickins5402b972008-02-04 22:28:44 -08002793 .aio_write = generic_file_aio_write,
Christoph Hellwig1b061d92010-05-26 17:53:41 +02002794 .fsync = noop_fsync,
Hugh Dickins708e3502011-07-25 17:12:32 -07002795 .splice_read = shmem_file_splice_read,
Hugh Dickinsae9764162007-06-04 10:00:39 +02002796 .splice_write = generic_file_splice_write,
Hugh Dickins83e4fa92012-05-29 15:06:40 -07002797 .fallocate = shmem_fallocate,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002798#endif
2799};
2800
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -08002801static const struct inode_operations shmem_inode_operations = {
Hugh Dickins94c1e622011-06-27 16:18:03 -07002802 .setattr = shmem_setattr,
Eric Parisb09e0fa2011-05-24 17:12:39 -07002803#ifdef CONFIG_TMPFS_XATTR
2804 .setxattr = shmem_setxattr,
2805 .getxattr = shmem_getxattr,
2806 .listxattr = shmem_listxattr,
2807 .removexattr = shmem_removexattr,
2808#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809};
2810
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -08002811static const struct inode_operations shmem_dir_inode_operations = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002812#ifdef CONFIG_TMPFS
2813 .create = shmem_create,
2814 .lookup = simple_lookup,
2815 .link = shmem_link,
2816 .unlink = shmem_unlink,
2817 .symlink = shmem_symlink,
2818 .mkdir = shmem_mkdir,
2819 .rmdir = shmem_rmdir,
2820 .mknod = shmem_mknod,
2821 .rename = shmem_rename,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002822#endif
Eric Parisb09e0fa2011-05-24 17:12:39 -07002823#ifdef CONFIG_TMPFS_XATTR
2824 .setxattr = shmem_setxattr,
2825 .getxattr = shmem_getxattr,
2826 .listxattr = shmem_listxattr,
2827 .removexattr = shmem_removexattr,
2828#endif
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002829#ifdef CONFIG_TMPFS_POSIX_ACL
Hugh Dickins94c1e622011-06-27 16:18:03 -07002830 .setattr = shmem_setattr,
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002831#endif
2832};
2833
Arjan van de Ven92e1d5b2007-02-12 00:55:39 -08002834static const struct inode_operations shmem_special_inode_operations = {
Eric Parisb09e0fa2011-05-24 17:12:39 -07002835#ifdef CONFIG_TMPFS_XATTR
2836 .setxattr = shmem_setxattr,
2837 .getxattr = shmem_getxattr,
2838 .listxattr = shmem_listxattr,
2839 .removexattr = shmem_removexattr,
2840#endif
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002841#ifdef CONFIG_TMPFS_POSIX_ACL
Hugh Dickins94c1e622011-06-27 16:18:03 -07002842 .setattr = shmem_setattr,
Andreas Gruenbacher39f02472006-09-29 02:01:35 -07002843#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844};
2845
Hugh Dickins759b9772007-03-05 00:30:28 -08002846static const struct super_operations shmem_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847 .alloc_inode = shmem_alloc_inode,
2848 .destroy_inode = shmem_destroy_inode,
2849#ifdef CONFIG_TMPFS
2850 .statfs = shmem_statfs,
2851 .remount_fs = shmem_remount_fs,
akpm@linux-foundation.org680d7942008-02-08 04:21:48 -08002852 .show_options = shmem_show_options,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853#endif
Al Viro1f895f72010-06-05 19:10:41 -04002854 .evict_inode = shmem_evict_inode,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855 .drop_inode = generic_delete_inode,
2856 .put_super = shmem_put_super,
2857};
2858
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04002859static const struct vm_operations_struct shmem_vm_ops = {
Nick Piggin54cb8822007-07-19 01:46:59 -07002860 .fault = shmem_fault,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002861#ifdef CONFIG_NUMA
2862 .set_policy = shmem_set_policy,
2863 .get_policy = shmem_get_policy,
2864#endif
2865};
2866
Al Viro3c26ff62010-07-25 11:46:36 +04002867static struct dentry *shmem_mount(struct file_system_type *fs_type,
2868 int flags, const char *dev_name, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002869{
Al Viro3c26ff62010-07-25 11:46:36 +04002870 return mount_nodev(fs_type, flags, data, shmem_fill_super);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871}
2872
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002873static struct file_system_type shmem_fs_type = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874 .owner = THIS_MODULE,
2875 .name = "tmpfs",
Al Viro3c26ff62010-07-25 11:46:36 +04002876 .mount = shmem_mount,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877 .kill_sb = kill_litter_super,
2878};
Linus Torvalds1da177e2005-04-16 15:20:36 -07002879
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002880int __init shmem_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002881{
2882 int error;
2883
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07002884 error = bdi_init(&shmem_backing_dev_info);
2885 if (error)
2886 goto out4;
2887
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002888 error = shmem_init_inodecache();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002889 if (error)
2890 goto out3;
2891
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002892 error = register_filesystem(&shmem_fs_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002893 if (error) {
2894 printk(KERN_ERR "Could not register tmpfs\n");
2895 goto out2;
2896 }
Greg Kroah-Hartman95dc1122005-06-20 21:15:16 -07002897
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002898 shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
2899 shmem_fs_type.name, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002900 if (IS_ERR(shm_mnt)) {
2901 error = PTR_ERR(shm_mnt);
2902 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2903 goto out1;
2904 }
2905 return 0;
2906
2907out1:
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002908 unregister_filesystem(&shmem_fs_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002909out2:
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002910 shmem_destroy_inodecache();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002911out3:
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -07002912 bdi_destroy(&shmem_backing_dev_info);
2913out4:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002914 shm_mnt = ERR_PTR(error);
2915 return error;
2916}
Matt Mackall853ac432009-01-06 14:40:20 -08002917
2918#else /* !CONFIG_SHMEM */
2919
2920/*
2921 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2922 *
2923 * This is intended for small system where the benefits of the full
2924 * shmem code (swap-backed and resource-limited) are outweighed by
2925 * their complexity. On systems without swap this code should be
2926 * effectively equivalent, but much lighter weight.
2927 */
2928
2929#include <linux/ramfs.h>
2930
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002931static struct file_system_type shmem_fs_type = {
Matt Mackall853ac432009-01-06 14:40:20 -08002932 .name = "tmpfs",
Al Viro3c26ff62010-07-25 11:46:36 +04002933 .mount = ramfs_mount,
Matt Mackall853ac432009-01-06 14:40:20 -08002934 .kill_sb = kill_litter_super,
2935};
2936
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002937int __init shmem_init(void)
Matt Mackall853ac432009-01-06 14:40:20 -08002938{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002939 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
Matt Mackall853ac432009-01-06 14:40:20 -08002940
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002941 shm_mnt = kern_mount(&shmem_fs_type);
Matt Mackall853ac432009-01-06 14:40:20 -08002942 BUG_ON(IS_ERR(shm_mnt));
2943
2944 return 0;
2945}
2946
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002947int shmem_unuse(swp_entry_t swap, struct page *page)
Matt Mackall853ac432009-01-06 14:40:20 -08002948{
2949 return 0;
2950}
2951
Hugh Dickins3f96b792009-09-21 17:03:37 -07002952int shmem_lock(struct file *file, int lock, struct user_struct *user)
2953{
2954 return 0;
2955}
2956
Hugh Dickins24513262012-01-20 14:34:21 -08002957void shmem_unlock_mapping(struct address_space *mapping)
2958{
2959}
2960
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002961void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
Hugh Dickins94c1e622011-06-27 16:18:03 -07002962{
Hugh Dickins41ffe5d2011-08-03 16:21:21 -07002963 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
Hugh Dickins94c1e622011-06-27 16:18:03 -07002964}
2965EXPORT_SYMBOL_GPL(shmem_truncate_range);
2966
Hugh Dickins0b0a0802009-02-24 20:51:52 +00002967#define shmem_vm_ops generic_file_vm_ops
2968#define shmem_file_operations ramfs_file_operations
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03002969#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
Hugh Dickins0b0a0802009-02-24 20:51:52 +00002970#define shmem_acct_size(flags, size) 0
2971#define shmem_unacct_size(flags, size) do {} while (0)
Matt Mackall853ac432009-01-06 14:40:20 -08002972
2973#endif /* CONFIG_SHMEM */
2974
2975/* common code */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976
Randy Dunlap46711812008-03-19 17:00:41 -07002977/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002978 * shmem_file_setup - get an unlinked file living in tmpfs
Linus Torvalds1da177e2005-04-16 15:20:36 -07002979 * @name: name for dentry (to be seen in /proc/<pid>/maps
2980 * @size: size to be set for the file
Hugh Dickins0b0a0802009-02-24 20:51:52 +00002981 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
Linus Torvalds1da177e2005-04-16 15:20:36 -07002982 */
Sergei Trofimovich168f5ac2009-06-16 15:33:02 -07002983struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002984{
2985 int error;
2986 struct file *file;
2987 struct inode *inode;
Al Viro2c48b9c2009-08-09 00:52:35 +04002988 struct path path;
2989 struct dentry *root;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002990 struct qstr this;
2991
2992 if (IS_ERR(shm_mnt))
2993 return (void *)shm_mnt;
2994
Hugh Dickins285b2c42011-08-03 16:21:20 -07002995 if (size < 0 || size > MAX_LFS_FILESIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002996 return ERR_PTR(-EINVAL);
2997
2998 if (shmem_acct_size(flags, size))
2999 return ERR_PTR(-ENOMEM);
3000
3001 error = -ENOMEM;
3002 this.name = name;
3003 this.len = strlen(name);
3004 this.hash = 0; /* will go */
3005 root = shm_mnt->mnt_root;
Al Viro2c48b9c2009-08-09 00:52:35 +04003006 path.dentry = d_alloc(root, &this);
3007 if (!path.dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008 goto put_memory;
Al Viro2c48b9c2009-08-09 00:52:35 +04003009 path.mnt = mntget(shm_mnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011 error = -ENOSPC;
Dmitry Monakhov454abaf2010-03-04 17:32:18 +03003012 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003013 if (!inode)
Al Viro4b42af82009-08-05 18:25:56 +04003014 goto put_dentry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003015
Al Viro2c48b9c2009-08-09 00:52:35 +04003016 d_instantiate(path.dentry, inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017 inode->i_size = size;
Miklos Szeredi6d6b77f2011-10-28 14:13:28 +02003018 clear_nlink(inode); /* It is unlinked */
Matt Mackall853ac432009-01-06 14:40:20 -08003019#ifndef CONFIG_MMU
3020 error = ramfs_nommu_expand_for_mapping(inode, size);
3021 if (error)
Al Viro4b42af82009-08-05 18:25:56 +04003022 goto put_dentry;
Matt Mackall853ac432009-01-06 14:40:20 -08003023#endif
Al Viro4b42af82009-08-05 18:25:56 +04003024
3025 error = -ENFILE;
Al Viro2c48b9c2009-08-09 00:52:35 +04003026 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
Al Viro4b42af82009-08-05 18:25:56 +04003027 &shmem_file_operations);
3028 if (!file)
3029 goto put_dentry;
3030
Linus Torvalds1da177e2005-04-16 15:20:36 -07003031 return file;
3032
Linus Torvalds1da177e2005-04-16 15:20:36 -07003033put_dentry:
Al Viro2c48b9c2009-08-09 00:52:35 +04003034 path_put(&path);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003035put_memory:
3036 shmem_unacct_size(flags, size);
3037 return ERR_PTR(error);
3038}
Keith Packard395e0dd2008-06-20 00:08:06 -07003039EXPORT_SYMBOL_GPL(shmem_file_setup);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003040
Randy Dunlap46711812008-03-19 17:00:41 -07003041/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003042 * shmem_zero_setup - setup a shared anonymous mapping
Linus Torvalds1da177e2005-04-16 15:20:36 -07003043 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
3044 */
3045int shmem_zero_setup(struct vm_area_struct *vma)
3046{
3047 struct file *file;
3048 loff_t size = vma->vm_end - vma->vm_start;
3049
3050 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
3051 if (IS_ERR(file))
3052 return PTR_ERR(file);
3053
3054 if (vma->vm_file)
3055 fput(vma->vm_file);
3056 vma->vm_file = file;
3057 vma->vm_ops = &shmem_vm_ops;
Hugh Dickinsbee4c36a2011-03-22 16:33:43 -07003058 vma->vm_flags |= VM_CAN_NONLINEAR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003059 return 0;
3060}
Hugh Dickinsd9d90e52011-06-27 16:18:04 -07003061
3062/**
3063 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
3064 * @mapping: the page's address_space
3065 * @index: the page index
3066 * @gfp: the page allocator flags to use if allocating
3067 *
3068 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
3069 * with any new page allocations done using the specified allocation flags.
3070 * But read_cache_page_gfp() uses the ->readpage() method: which does not
3071 * suit tmpfs, since it may have pages in swapcache, and needs to find those
3072 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
3073 *
Hugh Dickins68da9f02011-07-25 17:12:34 -07003074 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
3075 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
Hugh Dickinsd9d90e52011-06-27 16:18:04 -07003076 */
3077struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
3078 pgoff_t index, gfp_t gfp)
3079{
Hugh Dickins68da9f02011-07-25 17:12:34 -07003080#ifdef CONFIG_SHMEM
3081 struct inode *inode = mapping->host;
Hugh Dickins9276aad2011-07-25 17:12:34 -07003082 struct page *page;
Hugh Dickins68da9f02011-07-25 17:12:34 -07003083 int error;
3084
3085 BUG_ON(mapping->a_ops != &shmem_aops);
3086 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
3087 if (error)
3088 page = ERR_PTR(error);
3089 else
3090 unlock_page(page);
3091 return page;
3092#else
3093 /*
3094 * The tiny !SHMEM case uses ramfs without swap
3095 */
Hugh Dickinsd9d90e52011-06-27 16:18:04 -07003096 return read_cache_page_gfp(mapping, index, gfp);
Hugh Dickins68da9f02011-07-25 17:12:34 -07003097#endif
Hugh Dickinsd9d90e52011-06-27 16:18:04 -07003098}
3099EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);