blob: 38bc3334f2633a4bfe9ed0e40361d075ec35d8ec [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
Hugh Dickins0edd73b2005-06-21 17:15:04 -07009 * Copyright (C) 2002-2005 Hugh Dickins.
10 * Copyright (C) 2002-2005 VERITAS Software Corporation.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Copyright (C) 2004 Andi Kleen, SuSE Labs
12 *
13 * Extended attribute support for tmpfs:
14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16 *
17 * This file is released under the GPL.
18 */
19
20/*
21 * This virtual memory filesystem is heavily based on the ramfs. It
22 * extends ramfs by the ability to use swap and honor resource limits
23 * which makes it a completely usable filesystem.
24 */
25
26#include <linux/config.h>
27#include <linux/module.h>
28#include <linux/init.h>
29#include <linux/devfs_fs_kernel.h>
30#include <linux/fs.h>
31#include <linux/mm.h>
32#include <linux/mman.h>
33#include <linux/file.h>
34#include <linux/swap.h>
35#include <linux/pagemap.h>
36#include <linux/string.h>
37#include <linux/slab.h>
38#include <linux/backing-dev.h>
39#include <linux/shmem_fs.h>
40#include <linux/mount.h>
41#include <linux/writeback.h>
42#include <linux/vfs.h>
43#include <linux/blkdev.h>
44#include <linux/security.h>
45#include <linux/swapops.h>
46#include <linux/mempolicy.h>
47#include <linux/namei.h>
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +000048#include <linux/ctype.h>
Lee Schermerhorn304dbdb2006-04-22 02:35:48 -070049#include <linux/migrate.h>
50
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#include <asm/uaccess.h>
52#include <asm/div64.h>
53#include <asm/pgtable.h>
54
55/* This magic number is used in glibc for posix shared memory */
56#define TMPFS_MAGIC 0x01021994
57
58#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
59#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
60#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
61
62#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
63#define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
64
65#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
66
67/* info->flags needs VM_flags to handle pagein/truncate races efficiently */
68#define SHMEM_PAGEIN VM_READ
69#define SHMEM_TRUNCATE VM_WRITE
70
71/* Definition to limit shmem_truncate's steps between cond_rescheds */
72#define LATENCY_LIMIT 64
73
74/* Pretend that each entry is of this size in directory's i_size */
75#define BOGO_DIRENT_SIZE 20
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
78enum sgp_type {
79 SGP_QUICK, /* don't try more than file page cache lookup */
80 SGP_READ, /* don't exceed i_size, don't allocate page */
81 SGP_CACHE, /* don't exceed i_size, may allocate page */
82 SGP_WRITE, /* may exceed i_size, may allocate page */
83};
84
85static int shmem_getpage(struct inode *inode, unsigned long idx,
86 struct page **pagep, enum sgp_type sgp, int *type);
87
Al Viro6daa0e22005-10-21 03:18:50 -040088static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089{
90 /*
91 * The above definition of ENTRIES_PER_PAGE, and the use of
92 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
93 * might be reconsidered if it ever diverges from PAGE_SIZE.
94 */
95 return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT);
96}
97
98static inline void shmem_dir_free(struct page *page)
99{
100 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
101}
102
103static struct page **shmem_dir_map(struct page *page)
104{
105 return (struct page **)kmap_atomic(page, KM_USER0);
106}
107
108static inline void shmem_dir_unmap(struct page **dir)
109{
110 kunmap_atomic(dir, KM_USER0);
111}
112
113static swp_entry_t *shmem_swp_map(struct page *page)
114{
115 return (swp_entry_t *)kmap_atomic(page, KM_USER1);
116}
117
118static inline void shmem_swp_balance_unmap(void)
119{
120 /*
121 * When passing a pointer to an i_direct entry, to code which
122 * also handles indirect entries and so will shmem_swp_unmap,
123 * we must arrange for the preempt count to remain in balance.
124 * What kmap_atomic of a lowmem page does depends on config
125 * and architecture, so pretend to kmap_atomic some lowmem page.
126 */
127 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
128}
129
130static inline void shmem_swp_unmap(swp_entry_t *entry)
131{
132 kunmap_atomic(entry, KM_USER1);
133}
134
135static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
136{
137 return sb->s_fs_info;
138}
139
140/*
141 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
142 * for shared memory and for shared anonymous (/dev/zero) mappings
143 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
144 * consistent with the pre-accounting of private mappings ...
145 */
146static inline int shmem_acct_size(unsigned long flags, loff_t size)
147{
148 return (flags & VM_ACCOUNT)?
149 security_vm_enough_memory(VM_ACCT(size)): 0;
150}
151
152static inline void shmem_unacct_size(unsigned long flags, loff_t size)
153{
154 if (flags & VM_ACCOUNT)
155 vm_unacct_memory(VM_ACCT(size));
156}
157
158/*
159 * ... whereas tmpfs objects are accounted incrementally as
160 * pages are allocated, in order to allow huge sparse files.
161 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
162 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
163 */
164static inline int shmem_acct_block(unsigned long flags)
165{
166 return (flags & VM_ACCOUNT)?
167 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE));
168}
169
170static inline void shmem_unacct_blocks(unsigned long flags, long pages)
171{
172 if (!(flags & VM_ACCOUNT))
173 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
174}
175
176static struct super_operations shmem_ops;
177static struct address_space_operations shmem_aops;
178static struct file_operations shmem_file_operations;
179static struct inode_operations shmem_inode_operations;
180static struct inode_operations shmem_dir_inode_operations;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181static struct vm_operations_struct shmem_vm_ops;
182
Ravikiran G Thirumalai6c231b72005-09-06 15:17:45 -0700183static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 .ra_pages = 0, /* No readahead */
185 .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
186 .unplug_io_fn = default_unplug_io_fn,
187};
188
189static LIST_HEAD(shmem_swaplist);
190static DEFINE_SPINLOCK(shmem_swaplist_lock);
191
192static void shmem_free_blocks(struct inode *inode, long pages)
193{
194 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins0edd73b2005-06-21 17:15:04 -0700195 if (sbinfo->max_blocks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 spin_lock(&sbinfo->stat_lock);
197 sbinfo->free_blocks += pages;
198 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
199 spin_unlock(&sbinfo->stat_lock);
200 }
201}
202
203/*
204 * shmem_recalc_inode - recalculate the size of an inode
205 *
206 * @inode: inode to recalc
207 *
208 * We have to calculate the free blocks since the mm can drop
209 * undirtied hole pages behind our back.
210 *
211 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
212 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
213 *
214 * It has to be called with the spinlock held.
215 */
216static void shmem_recalc_inode(struct inode *inode)
217{
218 struct shmem_inode_info *info = SHMEM_I(inode);
219 long freed;
220
221 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
222 if (freed > 0) {
223 info->alloced -= freed;
224 shmem_unacct_blocks(info->flags, freed);
225 shmem_free_blocks(inode, freed);
226 }
227}
228
229/*
230 * shmem_swp_entry - find the swap vector position in the info structure
231 *
232 * @info: info structure for the inode
233 * @index: index of the page to find
234 * @page: optional page to add to the structure. Has to be preset to
235 * all zeros
236 *
237 * If there is no space allocated yet it will return NULL when
238 * page is NULL, else it will use the page for the needed block,
239 * setting it to NULL on return to indicate that it has been used.
240 *
241 * The swap vector is organized the following way:
242 *
243 * There are SHMEM_NR_DIRECT entries directly stored in the
244 * shmem_inode_info structure. So small files do not need an addional
245 * allocation.
246 *
247 * For pages with index > SHMEM_NR_DIRECT there is the pointer
248 * i_indirect which points to a page which holds in the first half
249 * doubly indirect blocks, in the second half triple indirect blocks:
250 *
251 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
252 * following layout (for SHMEM_NR_DIRECT == 16):
253 *
254 * i_indirect -> dir --> 16-19
255 * | +-> 20-23
256 * |
257 * +-->dir2 --> 24-27
258 * | +-> 28-31
259 * | +-> 32-35
260 * | +-> 36-39
261 * |
262 * +-->dir3 --> 40-43
263 * +-> 44-47
264 * +-> 48-51
265 * +-> 52-55
266 */
267static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
268{
269 unsigned long offset;
270 struct page **dir;
271 struct page *subdir;
272
273 if (index < SHMEM_NR_DIRECT) {
274 shmem_swp_balance_unmap();
275 return info->i_direct+index;
276 }
277 if (!info->i_indirect) {
278 if (page) {
279 info->i_indirect = *page;
280 *page = NULL;
281 }
282 return NULL; /* need another page */
283 }
284
285 index -= SHMEM_NR_DIRECT;
286 offset = index % ENTRIES_PER_PAGE;
287 index /= ENTRIES_PER_PAGE;
288 dir = shmem_dir_map(info->i_indirect);
289
290 if (index >= ENTRIES_PER_PAGE/2) {
291 index -= ENTRIES_PER_PAGE/2;
292 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
293 index %= ENTRIES_PER_PAGE;
294 subdir = *dir;
295 if (!subdir) {
296 if (page) {
297 *dir = *page;
298 *page = NULL;
299 }
300 shmem_dir_unmap(dir);
301 return NULL; /* need another page */
302 }
303 shmem_dir_unmap(dir);
304 dir = shmem_dir_map(subdir);
305 }
306
307 dir += index;
308 subdir = *dir;
309 if (!subdir) {
310 if (!page || !(subdir = *page)) {
311 shmem_dir_unmap(dir);
312 return NULL; /* need a page */
313 }
314 *dir = subdir;
315 *page = NULL;
316 }
317 shmem_dir_unmap(dir);
318 return shmem_swp_map(subdir) + offset;
319}
320
321static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
322{
323 long incdec = value? 1: -1;
324
325 entry->val = value;
326 info->swapped += incdec;
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700327 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
328 struct page *page = kmap_atomic_to_page(entry);
329 set_page_private(page, page_private(page) + incdec);
330 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
333/*
334 * shmem_swp_alloc - get the position of the swap entry for the page.
335 * If it does not exist allocate the entry.
336 *
337 * @info: info structure for the inode
338 * @index: index of the page to find
339 * @sgp: check and recheck i_size? skip allocation?
340 */
341static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
342{
343 struct inode *inode = &info->vfs_inode;
344 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
345 struct page *page = NULL;
346 swp_entry_t *entry;
347
348 if (sgp != SGP_WRITE &&
349 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
350 return ERR_PTR(-EINVAL);
351
352 while (!(entry = shmem_swp_entry(info, index, &page))) {
353 if (sgp == SGP_READ)
354 return shmem_swp_map(ZERO_PAGE(0));
355 /*
356 * Test free_blocks against 1 not 0, since we have 1 data
357 * page (and perhaps indirect index pages) yet to allocate:
358 * a waste to allocate index if we cannot allocate data.
359 */
Hugh Dickins0edd73b2005-06-21 17:15:04 -0700360 if (sbinfo->max_blocks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 spin_lock(&sbinfo->stat_lock);
362 if (sbinfo->free_blocks <= 1) {
363 spin_unlock(&sbinfo->stat_lock);
364 return ERR_PTR(-ENOSPC);
365 }
366 sbinfo->free_blocks--;
367 inode->i_blocks += BLOCKS_PER_PAGE;
368 spin_unlock(&sbinfo->stat_lock);
369 }
370
371 spin_unlock(&info->lock);
372 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700373 if (page)
374 set_page_private(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 spin_lock(&info->lock);
376
377 if (!page) {
378 shmem_free_blocks(inode, 1);
379 return ERR_PTR(-ENOMEM);
380 }
381 if (sgp != SGP_WRITE &&
382 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
383 entry = ERR_PTR(-EINVAL);
384 break;
385 }
386 if (info->next_index <= index)
387 info->next_index = index + 1;
388 }
389 if (page) {
390 /* another task gave its page, or truncated the file */
391 shmem_free_blocks(inode, 1);
392 shmem_dir_free(page);
393 }
394 if (info->next_index <= index && !IS_ERR(entry))
395 info->next_index = index + 1;
396 return entry;
397}
398
399/*
400 * shmem_free_swp - free some swap entries in a directory
401 *
402 * @dir: pointer to the directory
403 * @edir: pointer after last entry of the directory
404 */
405static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir)
406{
407 swp_entry_t *ptr;
408 int freed = 0;
409
410 for (ptr = dir; ptr < edir; ptr++) {
411 if (ptr->val) {
412 free_swap_and_cache(*ptr);
413 *ptr = (swp_entry_t){0};
414 freed++;
415 }
416 }
417 return freed;
418}
419
420static int shmem_map_and_free_swp(struct page *subdir,
421 int offset, int limit, struct page ***dir)
422{
423 swp_entry_t *ptr;
424 int freed = 0;
425
426 ptr = shmem_swp_map(subdir);
427 for (; offset < limit; offset += LATENCY_LIMIT) {
428 int size = limit - offset;
429 if (size > LATENCY_LIMIT)
430 size = LATENCY_LIMIT;
431 freed += shmem_free_swp(ptr+offset, ptr+offset+size);
432 if (need_resched()) {
433 shmem_swp_unmap(ptr);
434 if (*dir) {
435 shmem_dir_unmap(*dir);
436 *dir = NULL;
437 }
438 cond_resched();
439 ptr = shmem_swp_map(subdir);
440 }
441 }
442 shmem_swp_unmap(ptr);
443 return freed;
444}
445
446static void shmem_free_pages(struct list_head *next)
447{
448 struct page *page;
449 int freed = 0;
450
451 do {
452 page = container_of(next, struct page, lru);
453 next = next->next;
454 shmem_dir_free(page);
455 freed++;
456 if (freed >= LATENCY_LIMIT) {
457 cond_resched();
458 freed = 0;
459 }
460 } while (next);
461}
462
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800463static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464{
465 struct shmem_inode_info *info = SHMEM_I(inode);
466 unsigned long idx;
467 unsigned long size;
468 unsigned long limit;
469 unsigned long stage;
470 unsigned long diroff;
471 struct page **dir;
472 struct page *topdir;
473 struct page *middir;
474 struct page *subdir;
475 swp_entry_t *ptr;
476 LIST_HEAD(pages_to_free);
477 long nr_pages_to_free = 0;
478 long nr_swaps_freed = 0;
479 int offset;
480 int freed;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800481 int punch_hole = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482
483 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800484 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 if (idx >= info->next_index)
486 return;
487
488 spin_lock(&info->lock);
489 info->flags |= SHMEM_TRUNCATE;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800490 if (likely(end == (loff_t) -1)) {
491 limit = info->next_index;
492 info->next_index = idx;
493 } else {
494 limit = (end + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
495 if (limit > info->next_index)
496 limit = info->next_index;
497 punch_hole = 1;
498 }
499
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 topdir = info->i_indirect;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800501 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 info->i_indirect = NULL;
503 nr_pages_to_free++;
504 list_add(&topdir->lru, &pages_to_free);
505 }
506 spin_unlock(&info->lock);
507
508 if (info->swapped && idx < SHMEM_NR_DIRECT) {
509 ptr = info->i_direct;
510 size = limit;
511 if (size > SHMEM_NR_DIRECT)
512 size = SHMEM_NR_DIRECT;
513 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size);
514 }
515 if (!topdir)
516 goto done2;
517
518 BUG_ON(limit <= SHMEM_NR_DIRECT);
519 limit -= SHMEM_NR_DIRECT;
520 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
521 offset = idx % ENTRIES_PER_PAGE;
522 idx -= offset;
523
524 dir = shmem_dir_map(topdir);
525 stage = ENTRIES_PER_PAGEPAGE/2;
526 if (idx < ENTRIES_PER_PAGEPAGE/2) {
527 middir = topdir;
528 diroff = idx/ENTRIES_PER_PAGE;
529 } else {
530 dir += ENTRIES_PER_PAGE/2;
531 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
532 while (stage <= idx)
533 stage += ENTRIES_PER_PAGEPAGE;
534 middir = *dir;
535 if (*dir) {
536 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
537 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
538 if (!diroff && !offset) {
539 *dir = NULL;
540 nr_pages_to_free++;
541 list_add(&middir->lru, &pages_to_free);
542 }
543 shmem_dir_unmap(dir);
544 dir = shmem_dir_map(middir);
545 } else {
546 diroff = 0;
547 offset = 0;
548 idx = stage;
549 }
550 }
551
552 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
553 if (unlikely(idx == stage)) {
554 shmem_dir_unmap(dir);
555 dir = shmem_dir_map(topdir) +
556 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
557 while (!*dir) {
558 dir++;
559 idx += ENTRIES_PER_PAGEPAGE;
560 if (idx >= limit)
561 goto done1;
562 }
563 stage = idx + ENTRIES_PER_PAGEPAGE;
564 middir = *dir;
565 *dir = NULL;
566 nr_pages_to_free++;
567 list_add(&middir->lru, &pages_to_free);
568 shmem_dir_unmap(dir);
569 cond_resched();
570 dir = shmem_dir_map(middir);
571 diroff = 0;
572 }
573 subdir = dir[diroff];
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700574 if (subdir && page_private(subdir)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 size = limit - idx;
576 if (size > ENTRIES_PER_PAGE)
577 size = ENTRIES_PER_PAGE;
578 freed = shmem_map_and_free_swp(subdir,
579 offset, size, &dir);
580 if (!dir)
581 dir = shmem_dir_map(middir);
582 nr_swaps_freed += freed;
583 if (offset)
584 spin_lock(&info->lock);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700585 set_page_private(subdir, page_private(subdir) - freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 if (offset)
587 spin_unlock(&info->lock);
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800588 if (!punch_hole)
589 BUG_ON(page_private(subdir) > offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 }
591 if (offset)
592 offset = 0;
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800593 else if (subdir && !page_private(subdir)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 dir[diroff] = NULL;
595 nr_pages_to_free++;
596 list_add(&subdir->lru, &pages_to_free);
597 }
598 }
599done1:
600 shmem_dir_unmap(dir);
601done2:
602 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
603 /*
604 * Call truncate_inode_pages again: racing shmem_unuse_inode
605 * may have swizzled a page in from swap since vmtruncate or
606 * generic_delete_inode did it, before we lowered next_index.
607 * Also, though shmem_getpage checks i_size before adding to
608 * cache, no recheck after: so fix the narrow window there too.
609 */
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800610 truncate_inode_pages_range(inode->i_mapping, start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 }
612
613 spin_lock(&info->lock);
614 info->flags &= ~SHMEM_TRUNCATE;
615 info->swapped -= nr_swaps_freed;
616 if (nr_pages_to_free)
617 shmem_free_blocks(inode, nr_pages_to_free);
618 shmem_recalc_inode(inode);
619 spin_unlock(&info->lock);
620
621 /*
622 * Empty swap vector directory pages to be freed?
623 */
624 if (!list_empty(&pages_to_free)) {
625 pages_to_free.prev->next = NULL;
626 shmem_free_pages(pages_to_free.next);
627 }
628}
629
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -0800630static void shmem_truncate(struct inode *inode)
631{
632 shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
633}
634
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
636{
637 struct inode *inode = dentry->d_inode;
638 struct page *page = NULL;
639 int error;
640
641 if (attr->ia_valid & ATTR_SIZE) {
642 if (attr->ia_size < inode->i_size) {
643 /*
644 * If truncating down to a partial page, then
645 * if that page is already allocated, hold it
646 * in memory until the truncation is over, so
647 * truncate_partial_page cannnot miss it were
648 * it assigned to swap.
649 */
650 if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
651 (void) shmem_getpage(inode,
652 attr->ia_size>>PAGE_CACHE_SHIFT,
653 &page, SGP_READ, NULL);
654 }
655 /*
656 * Reset SHMEM_PAGEIN flag so that shmem_truncate can
657 * detect if any pages might have been added to cache
658 * after truncate_inode_pages. But we needn't bother
659 * if it's being fully truncated to zero-length: the
660 * nrpages check is efficient enough in that case.
661 */
662 if (attr->ia_size) {
663 struct shmem_inode_info *info = SHMEM_I(inode);
664 spin_lock(&info->lock);
665 info->flags &= ~SHMEM_PAGEIN;
666 spin_unlock(&info->lock);
667 }
668 }
669 }
670
671 error = inode_change_ok(inode, attr);
672 if (!error)
673 error = inode_setattr(inode, attr);
674 if (page)
675 page_cache_release(page);
676 return error;
677}
678
679static void shmem_delete_inode(struct inode *inode)
680{
681 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
682 struct shmem_inode_info *info = SHMEM_I(inode);
683
684 if (inode->i_op->truncate == shmem_truncate) {
Mark Fashehfef26652005-09-09 13:01:31 -0700685 truncate_inode_pages(inode->i_mapping, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 shmem_unacct_size(info->flags, inode->i_size);
687 inode->i_size = 0;
688 shmem_truncate(inode);
689 if (!list_empty(&info->swaplist)) {
690 spin_lock(&shmem_swaplist_lock);
691 list_del_init(&info->swaplist);
692 spin_unlock(&shmem_swaplist_lock);
693 }
694 }
Hugh Dickins0edd73b2005-06-21 17:15:04 -0700695 BUG_ON(inode->i_blocks);
696 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 spin_lock(&sbinfo->stat_lock);
698 sbinfo->free_inodes++;
699 spin_unlock(&sbinfo->stat_lock);
700 }
701 clear_inode(inode);
702}
703
704static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
705{
706 swp_entry_t *ptr;
707
708 for (ptr = dir; ptr < edir; ptr++) {
709 if (ptr->val == entry.val)
710 return ptr - dir;
711 }
712 return -1;
713}
714
715static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
716{
717 struct inode *inode;
718 unsigned long idx;
719 unsigned long size;
720 unsigned long limit;
721 unsigned long stage;
722 struct page **dir;
723 struct page *subdir;
724 swp_entry_t *ptr;
725 int offset;
726
727 idx = 0;
728 ptr = info->i_direct;
729 spin_lock(&info->lock);
730 limit = info->next_index;
731 size = limit;
732 if (size > SHMEM_NR_DIRECT)
733 size = SHMEM_NR_DIRECT;
734 offset = shmem_find_swp(entry, ptr, ptr+size);
735 if (offset >= 0) {
736 shmem_swp_balance_unmap();
737 goto found;
738 }
739 if (!info->i_indirect)
740 goto lost2;
741
742 dir = shmem_dir_map(info->i_indirect);
743 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
744
745 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
746 if (unlikely(idx == stage)) {
747 shmem_dir_unmap(dir-1);
748 dir = shmem_dir_map(info->i_indirect) +
749 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
750 while (!*dir) {
751 dir++;
752 idx += ENTRIES_PER_PAGEPAGE;
753 if (idx >= limit)
754 goto lost1;
755 }
756 stage = idx + ENTRIES_PER_PAGEPAGE;
757 subdir = *dir;
758 shmem_dir_unmap(dir);
759 dir = shmem_dir_map(subdir);
760 }
761 subdir = *dir;
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700762 if (subdir && page_private(subdir)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 ptr = shmem_swp_map(subdir);
764 size = limit - idx;
765 if (size > ENTRIES_PER_PAGE)
766 size = ENTRIES_PER_PAGE;
767 offset = shmem_find_swp(entry, ptr, ptr+size);
768 if (offset >= 0) {
769 shmem_dir_unmap(dir);
770 goto found;
771 }
772 shmem_swp_unmap(ptr);
773 }
774 }
775lost1:
776 shmem_dir_unmap(dir-1);
777lost2:
778 spin_unlock(&info->lock);
779 return 0;
780found:
781 idx += offset;
782 inode = &info->vfs_inode;
783 if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) {
784 info->flags |= SHMEM_PAGEIN;
785 shmem_swp_set(info, ptr + offset, 0);
786 }
787 shmem_swp_unmap(ptr);
788 spin_unlock(&info->lock);
789 /*
790 * Decrement swap count even when the entry is left behind:
791 * try_to_unuse will skip over mms, then reincrement count.
792 */
793 swap_free(entry);
794 return 1;
795}
796
797/*
798 * shmem_unuse() search for an eventually swapped out shmem page.
799 */
800int shmem_unuse(swp_entry_t entry, struct page *page)
801{
802 struct list_head *p, *next;
803 struct shmem_inode_info *info;
804 int found = 0;
805
806 spin_lock(&shmem_swaplist_lock);
807 list_for_each_safe(p, next, &shmem_swaplist) {
808 info = list_entry(p, struct shmem_inode_info, swaplist);
809 if (!info->swapped)
810 list_del_init(&info->swaplist);
811 else if (shmem_unuse_inode(info, entry, page)) {
812 /* move head to start search for next from here */
813 list_move_tail(&shmem_swaplist, &info->swaplist);
814 found = 1;
815 break;
816 }
817 }
818 spin_unlock(&shmem_swaplist_lock);
819 return found;
820}
821
822/*
823 * Move the page from the page cache to the swap cache.
824 */
825static int shmem_writepage(struct page *page, struct writeback_control *wbc)
826{
827 struct shmem_inode_info *info;
828 swp_entry_t *entry, swap;
829 struct address_space *mapping;
830 unsigned long index;
831 struct inode *inode;
832
833 BUG_ON(!PageLocked(page));
834 BUG_ON(page_mapped(page));
835
836 mapping = page->mapping;
837 index = page->index;
838 inode = mapping->host;
839 info = SHMEM_I(inode);
840 if (info->flags & VM_LOCKED)
841 goto redirty;
842 swap = get_swap_page();
843 if (!swap.val)
844 goto redirty;
845
846 spin_lock(&info->lock);
847 shmem_recalc_inode(inode);
848 if (index >= info->next_index) {
849 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
850 goto unlock;
851 }
852 entry = shmem_swp_entry(info, index, NULL);
853 BUG_ON(!entry);
854 BUG_ON(entry->val);
855
856 if (move_to_swap_cache(page, swap) == 0) {
857 shmem_swp_set(info, entry, swap.val);
858 shmem_swp_unmap(entry);
859 spin_unlock(&info->lock);
860 if (list_empty(&info->swaplist)) {
861 spin_lock(&shmem_swaplist_lock);
862 /* move instead of add in case we're racing */
863 list_move_tail(&info->swaplist, &shmem_swaplist);
864 spin_unlock(&shmem_swaplist_lock);
865 }
866 unlock_page(page);
867 return 0;
868 }
869
870 shmem_swp_unmap(entry);
871unlock:
872 spin_unlock(&info->lock);
873 swap_free(swap);
874redirty:
875 set_page_dirty(page);
Zach Brown994fc28c2005-12-15 14:28:17 -0800876 return AOP_WRITEPAGE_ACTIVATE; /* Return with the page locked */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877}
878
879#ifdef CONFIG_NUMA
Hugh Dickinsd15c0232006-03-22 00:08:46 -0800880static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +0000881{
882 char *nodelist = strchr(value, ':');
883 int err = 1;
884
885 if (nodelist) {
886 /* NUL-terminate policy string */
887 *nodelist++ = '\0';
888 if (nodelist_parse(nodelist, *policy_nodes))
889 goto out;
890 }
891 if (!strcmp(value, "default")) {
892 *policy = MPOL_DEFAULT;
893 /* Don't allow a nodelist */
894 if (!nodelist)
895 err = 0;
896 } else if (!strcmp(value, "prefer")) {
897 *policy = MPOL_PREFERRED;
898 /* Insist on a nodelist of one node only */
899 if (nodelist) {
900 char *rest = nodelist;
901 while (isdigit(*rest))
902 rest++;
903 if (!*rest)
904 err = 0;
905 }
906 } else if (!strcmp(value, "bind")) {
907 *policy = MPOL_BIND;
908 /* Insist on a nodelist */
909 if (nodelist)
910 err = 0;
911 } else if (!strcmp(value, "interleave")) {
912 *policy = MPOL_INTERLEAVE;
913 /* Default to nodes online if no nodelist */
914 if (!nodelist)
915 *policy_nodes = node_online_map;
916 err = 0;
917 }
918out:
919 /* Restore string for error message */
920 if (nodelist)
921 *--nodelist = ':';
922 return err;
923}
924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925static struct page *shmem_swapin_async(struct shared_policy *p,
926 swp_entry_t entry, unsigned long idx)
927{
928 struct page *page;
929 struct vm_area_struct pvma;
930
931 /* Create a pseudo vma that just contains the policy */
932 memset(&pvma, 0, sizeof(struct vm_area_struct));
933 pvma.vm_end = PAGE_SIZE;
934 pvma.vm_pgoff = idx;
935 pvma.vm_policy = mpol_shared_policy_lookup(p, idx);
936 page = read_swap_cache_async(entry, &pvma, 0);
937 mpol_free(pvma.vm_policy);
938 return page;
939}
940
941struct page *shmem_swapin(struct shmem_inode_info *info, swp_entry_t entry,
942 unsigned long idx)
943{
944 struct shared_policy *p = &info->policy;
945 int i, num;
946 struct page *page;
947 unsigned long offset;
948
949 num = valid_swaphandles(entry, &offset);
950 for (i = 0; i < num; offset++, i++) {
951 page = shmem_swapin_async(p,
952 swp_entry(swp_type(entry), offset), idx);
953 if (!page)
954 break;
955 page_cache_release(page);
956 }
957 lru_add_drain(); /* Push any new pages onto the LRU now */
958 return shmem_swapin_async(p, entry, idx);
959}
960
961static struct page *
Al Viro6daa0e22005-10-21 03:18:50 -0400962shmem_alloc_page(gfp_t gfp, struct shmem_inode_info *info,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 unsigned long idx)
964{
965 struct vm_area_struct pvma;
966 struct page *page;
967
968 memset(&pvma, 0, sizeof(struct vm_area_struct));
969 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
970 pvma.vm_pgoff = idx;
971 pvma.vm_end = PAGE_SIZE;
972 page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0);
973 mpol_free(pvma.vm_policy);
974 return page;
975}
976#else
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +0000977static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
978{
979 return 1;
980}
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982static inline struct page *
983shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx)
984{
985 swapin_readahead(entry, 0, NULL);
986 return read_swap_cache_async(entry, NULL, 0);
987}
988
989static inline struct page *
Al Virodd0fc662005-10-07 07:46:04 +0100990shmem_alloc_page(gfp_t gfp,struct shmem_inode_info *info, unsigned long idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
992 return alloc_page(gfp | __GFP_ZERO);
993}
994#endif
995
996/*
997 * shmem_getpage - either get the page from swap or allocate a new one
998 *
999 * If we allocate a new one we do not mark it dirty. That's up to the
1000 * vm. If we swap it in we mark it dirty since we also free the swap
1001 * entry since a page cannot live in both the swap and page cache
1002 */
1003static int shmem_getpage(struct inode *inode, unsigned long idx,
1004 struct page **pagep, enum sgp_type sgp, int *type)
1005{
1006 struct address_space *mapping = inode->i_mapping;
1007 struct shmem_inode_info *info = SHMEM_I(inode);
1008 struct shmem_sb_info *sbinfo;
1009 struct page *filepage = *pagep;
1010 struct page *swappage;
1011 swp_entry_t *entry;
1012 swp_entry_t swap;
1013 int error;
1014
1015 if (idx >= SHMEM_MAX_INDEX)
1016 return -EFBIG;
1017 /*
1018 * Normally, filepage is NULL on entry, and either found
1019 * uptodate immediately, or allocated and zeroed, or read
1020 * in under swappage, which is then assigned to filepage.
1021 * But shmem_prepare_write passes in a locked filepage,
1022 * which may be found not uptodate by other callers too,
1023 * and may need to be copied from the swappage read in.
1024 */
1025repeat:
1026 if (!filepage)
1027 filepage = find_lock_page(mapping, idx);
1028 if (filepage && PageUptodate(filepage))
1029 goto done;
1030 error = 0;
1031 if (sgp == SGP_QUICK)
1032 goto failed;
1033
1034 spin_lock(&info->lock);
1035 shmem_recalc_inode(inode);
1036 entry = shmem_swp_alloc(info, idx, sgp);
1037 if (IS_ERR(entry)) {
1038 spin_unlock(&info->lock);
1039 error = PTR_ERR(entry);
1040 goto failed;
1041 }
1042 swap = *entry;
1043
1044 if (swap.val) {
1045 /* Look it up and read it in.. */
1046 swappage = lookup_swap_cache(swap);
1047 if (!swappage) {
1048 shmem_swp_unmap(entry);
1049 spin_unlock(&info->lock);
1050 /* here we actually do the io */
1051 if (type && *type == VM_FAULT_MINOR) {
1052 inc_page_state(pgmajfault);
1053 *type = VM_FAULT_MAJOR;
1054 }
1055 swappage = shmem_swapin(info, swap, idx);
1056 if (!swappage) {
1057 spin_lock(&info->lock);
1058 entry = shmem_swp_alloc(info, idx, sgp);
1059 if (IS_ERR(entry))
1060 error = PTR_ERR(entry);
1061 else {
1062 if (entry->val == swap.val)
1063 error = -ENOMEM;
1064 shmem_swp_unmap(entry);
1065 }
1066 spin_unlock(&info->lock);
1067 if (error)
1068 goto failed;
1069 goto repeat;
1070 }
1071 wait_on_page_locked(swappage);
1072 page_cache_release(swappage);
1073 goto repeat;
1074 }
1075
1076 /* We have to do this with page locked to prevent races */
1077 if (TestSetPageLocked(swappage)) {
1078 shmem_swp_unmap(entry);
1079 spin_unlock(&info->lock);
1080 wait_on_page_locked(swappage);
1081 page_cache_release(swappage);
1082 goto repeat;
1083 }
1084 if (PageWriteback(swappage)) {
1085 shmem_swp_unmap(entry);
1086 spin_unlock(&info->lock);
1087 wait_on_page_writeback(swappage);
1088 unlock_page(swappage);
1089 page_cache_release(swappage);
1090 goto repeat;
1091 }
1092 if (!PageUptodate(swappage)) {
1093 shmem_swp_unmap(entry);
1094 spin_unlock(&info->lock);
1095 unlock_page(swappage);
1096 page_cache_release(swappage);
1097 error = -EIO;
1098 goto failed;
1099 }
1100
1101 if (filepage) {
1102 shmem_swp_set(info, entry, 0);
1103 shmem_swp_unmap(entry);
1104 delete_from_swap_cache(swappage);
1105 spin_unlock(&info->lock);
1106 copy_highpage(filepage, swappage);
1107 unlock_page(swappage);
1108 page_cache_release(swappage);
1109 flush_dcache_page(filepage);
1110 SetPageUptodate(filepage);
1111 set_page_dirty(filepage);
1112 swap_free(swap);
1113 } else if (!(error = move_from_swap_cache(
1114 swappage, idx, mapping))) {
1115 info->flags |= SHMEM_PAGEIN;
1116 shmem_swp_set(info, entry, 0);
1117 shmem_swp_unmap(entry);
1118 spin_unlock(&info->lock);
1119 filepage = swappage;
1120 swap_free(swap);
1121 } else {
1122 shmem_swp_unmap(entry);
1123 spin_unlock(&info->lock);
1124 unlock_page(swappage);
1125 page_cache_release(swappage);
1126 if (error == -ENOMEM) {
1127 /* let kswapd refresh zone for GFP_ATOMICs */
1128 blk_congestion_wait(WRITE, HZ/50);
1129 }
1130 goto repeat;
1131 }
1132 } else if (sgp == SGP_READ && !filepage) {
1133 shmem_swp_unmap(entry);
1134 filepage = find_get_page(mapping, idx);
1135 if (filepage &&
1136 (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
1137 spin_unlock(&info->lock);
1138 wait_on_page_locked(filepage);
1139 page_cache_release(filepage);
1140 filepage = NULL;
1141 goto repeat;
1142 }
1143 spin_unlock(&info->lock);
1144 } else {
1145 shmem_swp_unmap(entry);
1146 sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001147 if (sbinfo->max_blocks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 spin_lock(&sbinfo->stat_lock);
1149 if (sbinfo->free_blocks == 0 ||
1150 shmem_acct_block(info->flags)) {
1151 spin_unlock(&sbinfo->stat_lock);
1152 spin_unlock(&info->lock);
1153 error = -ENOSPC;
1154 goto failed;
1155 }
1156 sbinfo->free_blocks--;
1157 inode->i_blocks += BLOCKS_PER_PAGE;
1158 spin_unlock(&sbinfo->stat_lock);
1159 } else if (shmem_acct_block(info->flags)) {
1160 spin_unlock(&info->lock);
1161 error = -ENOSPC;
1162 goto failed;
1163 }
1164
1165 if (!filepage) {
1166 spin_unlock(&info->lock);
1167 filepage = shmem_alloc_page(mapping_gfp_mask(mapping),
1168 info,
1169 idx);
1170 if (!filepage) {
1171 shmem_unacct_blocks(info->flags, 1);
1172 shmem_free_blocks(inode, 1);
1173 error = -ENOMEM;
1174 goto failed;
1175 }
1176
1177 spin_lock(&info->lock);
1178 entry = shmem_swp_alloc(info, idx, sgp);
1179 if (IS_ERR(entry))
1180 error = PTR_ERR(entry);
1181 else {
1182 swap = *entry;
1183 shmem_swp_unmap(entry);
1184 }
1185 if (error || swap.val || 0 != add_to_page_cache_lru(
1186 filepage, mapping, idx, GFP_ATOMIC)) {
1187 spin_unlock(&info->lock);
1188 page_cache_release(filepage);
1189 shmem_unacct_blocks(info->flags, 1);
1190 shmem_free_blocks(inode, 1);
1191 filepage = NULL;
1192 if (error)
1193 goto failed;
1194 goto repeat;
1195 }
1196 info->flags |= SHMEM_PAGEIN;
1197 }
1198
1199 info->alloced++;
1200 spin_unlock(&info->lock);
1201 flush_dcache_page(filepage);
1202 SetPageUptodate(filepage);
1203 }
1204done:
1205 if (*pagep != filepage) {
1206 unlock_page(filepage);
1207 *pagep = filepage;
1208 }
1209 return 0;
1210
1211failed:
1212 if (*pagep != filepage) {
1213 unlock_page(filepage);
1214 page_cache_release(filepage);
1215 }
1216 return error;
1217}
1218
1219struct page *shmem_nopage(struct vm_area_struct *vma, unsigned long address, int *type)
1220{
1221 struct inode *inode = vma->vm_file->f_dentry->d_inode;
1222 struct page *page = NULL;
1223 unsigned long idx;
1224 int error;
1225
1226 idx = (address - vma->vm_start) >> PAGE_SHIFT;
1227 idx += vma->vm_pgoff;
1228 idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
1229 if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1230 return NOPAGE_SIGBUS;
1231
1232 error = shmem_getpage(inode, idx, &page, SGP_CACHE, type);
1233 if (error)
1234 return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS;
1235
1236 mark_page_accessed(page);
1237 return page;
1238}
1239
1240static int shmem_populate(struct vm_area_struct *vma,
1241 unsigned long addr, unsigned long len,
1242 pgprot_t prot, unsigned long pgoff, int nonblock)
1243{
1244 struct inode *inode = vma->vm_file->f_dentry->d_inode;
1245 struct mm_struct *mm = vma->vm_mm;
1246 enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE;
1247 unsigned long size;
1248
1249 size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1250 if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size)
1251 return -EINVAL;
1252
1253 while ((long) len > 0) {
1254 struct page *page = NULL;
1255 int err;
1256 /*
1257 * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
1258 */
1259 err = shmem_getpage(inode, pgoff, &page, sgp, NULL);
1260 if (err)
1261 return err;
Paolo 'Blaisorblade' Giarrussod44ed4f2005-09-03 15:54:55 -07001262 /* Page may still be null, but only if nonblock was set. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 if (page) {
1264 mark_page_accessed(page);
1265 err = install_page(mm, vma, addr, page, prot);
1266 if (err) {
1267 page_cache_release(page);
1268 return err;
1269 }
Hugh Dickins65500d22005-10-29 18:15:59 -07001270 } else if (vma->vm_flags & VM_NONLINEAR) {
Paolo 'Blaisorblade' Giarrussod44ed4f2005-09-03 15:54:55 -07001271 /* No page was found just because we can't read it in
1272 * now (being here implies nonblock != 0), but the page
1273 * may exist, so set the PTE to fault it in later. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 err = install_file_pte(mm, vma, addr, pgoff, prot);
1275 if (err)
1276 return err;
1277 }
1278
1279 len -= PAGE_SIZE;
1280 addr += PAGE_SIZE;
1281 pgoff++;
1282 }
1283 return 0;
1284}
1285
1286#ifdef CONFIG_NUMA
1287int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1288{
1289 struct inode *i = vma->vm_file->f_dentry->d_inode;
1290 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1291}
1292
1293struct mempolicy *
1294shmem_get_policy(struct vm_area_struct *vma, unsigned long addr)
1295{
1296 struct inode *i = vma->vm_file->f_dentry->d_inode;
1297 unsigned long idx;
1298
1299 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1300 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1301}
1302#endif
1303
1304int shmem_lock(struct file *file, int lock, struct user_struct *user)
1305{
1306 struct inode *inode = file->f_dentry->d_inode;
1307 struct shmem_inode_info *info = SHMEM_I(inode);
1308 int retval = -ENOMEM;
1309
1310 spin_lock(&info->lock);
1311 if (lock && !(info->flags & VM_LOCKED)) {
1312 if (!user_shm_lock(inode->i_size, user))
1313 goto out_nomem;
1314 info->flags |= VM_LOCKED;
1315 }
1316 if (!lock && (info->flags & VM_LOCKED) && user) {
1317 user_shm_unlock(inode->i_size, user);
1318 info->flags &= ~VM_LOCKED;
1319 }
1320 retval = 0;
1321out_nomem:
1322 spin_unlock(&info->lock);
1323 return retval;
1324}
1325
David Howellsb0e15192006-01-06 00:11:42 -08001326int shmem_mmap(struct file *file, struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327{
1328 file_accessed(file);
1329 vma->vm_ops = &shmem_vm_ops;
1330 return 0;
1331}
1332
1333static struct inode *
1334shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
1335{
1336 struct inode *inode;
1337 struct shmem_inode_info *info;
1338 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1339
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001340 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 spin_lock(&sbinfo->stat_lock);
1342 if (!sbinfo->free_inodes) {
1343 spin_unlock(&sbinfo->stat_lock);
1344 return NULL;
1345 }
1346 sbinfo->free_inodes--;
1347 spin_unlock(&sbinfo->stat_lock);
1348 }
1349
1350 inode = new_inode(sb);
1351 if (inode) {
1352 inode->i_mode = mode;
1353 inode->i_uid = current->fsuid;
1354 inode->i_gid = current->fsgid;
1355 inode->i_blksize = PAGE_CACHE_SIZE;
1356 inode->i_blocks = 0;
1357 inode->i_mapping->a_ops = &shmem_aops;
1358 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1359 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1360 info = SHMEM_I(inode);
1361 memset(info, 0, (char *)inode - (char *)info);
1362 spin_lock_init(&info->lock);
1363 INIT_LIST_HEAD(&info->swaplist);
1364
1365 switch (mode & S_IFMT) {
1366 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 init_special_inode(inode, mode, dev);
1368 break;
1369 case S_IFREG:
1370 inode->i_op = &shmem_inode_operations;
1371 inode->i_fop = &shmem_file_operations;
Robin Holt7339ff82006-01-14 13:20:48 -08001372 mpol_shared_policy_init(&info->policy, sbinfo->policy,
1373 &sbinfo->policy_nodes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 break;
1375 case S_IFDIR:
1376 inode->i_nlink++;
1377 /* Some things misbehave if size == 0 on a directory */
1378 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1379 inode->i_op = &shmem_dir_inode_operations;
1380 inode->i_fop = &simple_dir_operations;
1381 break;
1382 case S_IFLNK:
1383 /*
1384 * Must not load anything in the rbtree,
1385 * mpol_free_shared_policy will not be called.
1386 */
Robin Holt7339ff82006-01-14 13:20:48 -08001387 mpol_shared_policy_init(&info->policy, MPOL_DEFAULT,
1388 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 break;
1390 }
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001391 } else if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392 spin_lock(&sbinfo->stat_lock);
1393 sbinfo->free_inodes++;
1394 spin_unlock(&sbinfo->stat_lock);
1395 }
1396 return inode;
1397}
1398
1399#ifdef CONFIG_TMPFS
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400static struct inode_operations shmem_symlink_inode_operations;
1401static struct inode_operations shmem_symlink_inline_operations;
1402
1403/*
1404 * Normally tmpfs makes no use of shmem_prepare_write, but it
1405 * lets a tmpfs file be used read-write below the loop driver.
1406 */
1407static int
1408shmem_prepare_write(struct file *file, struct page *page, unsigned offset, unsigned to)
1409{
1410 struct inode *inode = page->mapping->host;
1411 return shmem_getpage(inode, page->index, &page, SGP_WRITE, NULL);
1412}
1413
1414static ssize_t
1415shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
1416{
1417 struct inode *inode = file->f_dentry->d_inode;
1418 loff_t pos;
1419 unsigned long written;
1420 ssize_t err;
1421
1422 if ((ssize_t) count < 0)
1423 return -EINVAL;
1424
1425 if (!access_ok(VERIFY_READ, buf, count))
1426 return -EFAULT;
1427
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001428 mutex_lock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429
1430 pos = *ppos;
1431 written = 0;
1432
1433 err = generic_write_checks(file, &pos, &count, 0);
1434 if (err || !count)
1435 goto out;
1436
1437 err = remove_suid(file->f_dentry);
1438 if (err)
1439 goto out;
1440
1441 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
1442
1443 do {
1444 struct page *page = NULL;
1445 unsigned long bytes, index, offset;
1446 char *kaddr;
1447 int left;
1448
1449 offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
1450 index = pos >> PAGE_CACHE_SHIFT;
1451 bytes = PAGE_CACHE_SIZE - offset;
1452 if (bytes > count)
1453 bytes = count;
1454
1455 /*
1456 * We don't hold page lock across copy from user -
1457 * what would it guard against? - so no deadlock here.
1458 * But it still may be a good idea to prefault below.
1459 */
1460
1461 err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL);
1462 if (err)
1463 break;
1464
1465 left = bytes;
1466 if (PageHighMem(page)) {
1467 volatile unsigned char dummy;
1468 __get_user(dummy, buf);
1469 __get_user(dummy, buf + bytes - 1);
1470
1471 kaddr = kmap_atomic(page, KM_USER0);
1472 left = __copy_from_user_inatomic(kaddr + offset,
1473 buf, bytes);
1474 kunmap_atomic(kaddr, KM_USER0);
1475 }
1476 if (left) {
1477 kaddr = kmap(page);
1478 left = __copy_from_user(kaddr + offset, buf, bytes);
1479 kunmap(page);
1480 }
1481
1482 written += bytes;
1483 count -= bytes;
1484 pos += bytes;
1485 buf += bytes;
1486 if (pos > inode->i_size)
1487 i_size_write(inode, pos);
1488
1489 flush_dcache_page(page);
1490 set_page_dirty(page);
1491 mark_page_accessed(page);
1492 page_cache_release(page);
1493
1494 if (left) {
1495 pos -= left;
1496 written -= left;
1497 err = -EFAULT;
1498 break;
1499 }
1500
1501 /*
1502 * Our dirty pages are not counted in nr_dirty,
1503 * and we do not attempt to balance dirty pages.
1504 */
1505
1506 cond_resched();
1507 } while (count);
1508
1509 *ppos = pos;
1510 if (written)
1511 err = written;
1512out:
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001513 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 return err;
1515}
1516
1517static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1518{
1519 struct inode *inode = filp->f_dentry->d_inode;
1520 struct address_space *mapping = inode->i_mapping;
1521 unsigned long index, offset;
1522
1523 index = *ppos >> PAGE_CACHE_SHIFT;
1524 offset = *ppos & ~PAGE_CACHE_MASK;
1525
1526 for (;;) {
1527 struct page *page = NULL;
1528 unsigned long end_index, nr, ret;
1529 loff_t i_size = i_size_read(inode);
1530
1531 end_index = i_size >> PAGE_CACHE_SHIFT;
1532 if (index > end_index)
1533 break;
1534 if (index == end_index) {
1535 nr = i_size & ~PAGE_CACHE_MASK;
1536 if (nr <= offset)
1537 break;
1538 }
1539
1540 desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL);
1541 if (desc->error) {
1542 if (desc->error == -EINVAL)
1543 desc->error = 0;
1544 break;
1545 }
1546
1547 /*
1548 * We must evaluate after, since reads (unlike writes)
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001549 * are called without i_mutex protection against truncate
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550 */
1551 nr = PAGE_CACHE_SIZE;
1552 i_size = i_size_read(inode);
1553 end_index = i_size >> PAGE_CACHE_SHIFT;
1554 if (index == end_index) {
1555 nr = i_size & ~PAGE_CACHE_MASK;
1556 if (nr <= offset) {
1557 if (page)
1558 page_cache_release(page);
1559 break;
1560 }
1561 }
1562 nr -= offset;
1563
1564 if (page) {
1565 /*
1566 * If users can be writing to this page using arbitrary
1567 * virtual addresses, take care about potential aliasing
1568 * before reading the page on the kernel side.
1569 */
1570 if (mapping_writably_mapped(mapping))
1571 flush_dcache_page(page);
1572 /*
1573 * Mark the page accessed if we read the beginning.
1574 */
1575 if (!offset)
1576 mark_page_accessed(page);
Nick Pigginb5810032005-10-29 18:16:12 -07001577 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 page = ZERO_PAGE(0);
Nick Pigginb5810032005-10-29 18:16:12 -07001579 page_cache_get(page);
1580 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581
1582 /*
1583 * Ok, we have the page, and it's up-to-date, so
1584 * now we can copy it to user space...
1585 *
1586 * The actor routine returns how many bytes were actually used..
1587 * NOTE! This may not be the same as how much of a user buffer
1588 * we filled up (we may be padding etc), so we can only update
1589 * "pos" here (the actor routine has to update the user buffer
1590 * pointers and the remaining count).
1591 */
1592 ret = actor(desc, page, offset, nr);
1593 offset += ret;
1594 index += offset >> PAGE_CACHE_SHIFT;
1595 offset &= ~PAGE_CACHE_MASK;
1596
1597 page_cache_release(page);
1598 if (ret != nr || !desc->count)
1599 break;
1600
1601 cond_resched();
1602 }
1603
1604 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1605 file_accessed(filp);
1606}
1607
1608static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
1609{
1610 read_descriptor_t desc;
1611
1612 if ((ssize_t) count < 0)
1613 return -EINVAL;
1614 if (!access_ok(VERIFY_WRITE, buf, count))
1615 return -EFAULT;
1616 if (!count)
1617 return 0;
1618
1619 desc.written = 0;
1620 desc.count = count;
1621 desc.arg.buf = buf;
1622 desc.error = 0;
1623
1624 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1625 if (desc.written)
1626 return desc.written;
1627 return desc.error;
1628}
1629
1630static ssize_t shmem_file_sendfile(struct file *in_file, loff_t *ppos,
1631 size_t count, read_actor_t actor, void *target)
1632{
1633 read_descriptor_t desc;
1634
1635 if (!count)
1636 return 0;
1637
1638 desc.written = 0;
1639 desc.count = count;
1640 desc.arg.data = target;
1641 desc.error = 0;
1642
1643 do_shmem_file_read(in_file, ppos, &desc, actor);
1644 if (desc.written)
1645 return desc.written;
1646 return desc.error;
1647}
1648
David Howells726c3342006-06-23 02:02:58 -07001649static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650{
David Howells726c3342006-06-23 02:02:58 -07001651 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652
1653 buf->f_type = TMPFS_MAGIC;
1654 buf->f_bsize = PAGE_CACHE_SIZE;
1655 buf->f_namelen = NAME_MAX;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001656 spin_lock(&sbinfo->stat_lock);
1657 if (sbinfo->max_blocks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658 buf->f_blocks = sbinfo->max_blocks;
1659 buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001660 }
1661 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662 buf->f_files = sbinfo->max_inodes;
1663 buf->f_ffree = sbinfo->free_inodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664 }
1665 /* else leave those fields 0 like simple_statfs */
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001666 spin_unlock(&sbinfo->stat_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 return 0;
1668}
1669
1670/*
1671 * File creation. Allocate an inode, and we're done..
1672 */
1673static int
1674shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1675{
1676 struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
1677 int error = -ENOSPC;
1678
1679 if (inode) {
Stephen Smalley570bc1c2005-09-09 13:01:43 -07001680 error = security_inode_init_security(inode, dir, NULL, NULL,
1681 NULL);
1682 if (error) {
1683 if (error != -EOPNOTSUPP) {
1684 iput(inode);
1685 return error;
1686 }
1687 error = 0;
1688 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 if (dir->i_mode & S_ISGID) {
1690 inode->i_gid = dir->i_gid;
1691 if (S_ISDIR(mode))
1692 inode->i_mode |= S_ISGID;
1693 }
1694 dir->i_size += BOGO_DIRENT_SIZE;
1695 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1696 d_instantiate(dentry, inode);
1697 dget(dentry); /* Extra count - pin the dentry in core */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 }
1699 return error;
1700}
1701
1702static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1703{
1704 int error;
1705
1706 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1707 return error;
1708 dir->i_nlink++;
1709 return 0;
1710}
1711
1712static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1713 struct nameidata *nd)
1714{
1715 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1716}
1717
1718/*
1719 * Link a file..
1720 */
1721static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1722{
1723 struct inode *inode = old_dentry->d_inode;
1724 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1725
1726 /*
1727 * No ordinary (disk based) filesystem counts links as inodes;
1728 * but each new link needs a new dentry, pinning lowmem, and
1729 * tmpfs dentries cannot be pruned until they are unlinked.
1730 */
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001731 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001732 spin_lock(&sbinfo->stat_lock);
1733 if (!sbinfo->free_inodes) {
1734 spin_unlock(&sbinfo->stat_lock);
1735 return -ENOSPC;
1736 }
1737 sbinfo->free_inodes--;
1738 spin_unlock(&sbinfo->stat_lock);
1739 }
1740
1741 dir->i_size += BOGO_DIRENT_SIZE;
1742 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1743 inode->i_nlink++;
1744 atomic_inc(&inode->i_count); /* New dentry reference */
1745 dget(dentry); /* Extra pinning count for the created dentry */
1746 d_instantiate(dentry, inode);
1747 return 0;
1748}
1749
1750static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1751{
1752 struct inode *inode = dentry->d_inode;
1753
1754 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) {
1755 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001756 if (sbinfo->max_inodes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757 spin_lock(&sbinfo->stat_lock);
1758 sbinfo->free_inodes++;
1759 spin_unlock(&sbinfo->stat_lock);
1760 }
1761 }
1762
1763 dir->i_size -= BOGO_DIRENT_SIZE;
1764 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1765 inode->i_nlink--;
1766 dput(dentry); /* Undo the count from "create" - this does all the work */
1767 return 0;
1768}
1769
1770static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1771{
1772 if (!simple_empty(dentry))
1773 return -ENOTEMPTY;
1774
Sergey Vlasov86bc8432006-06-12 21:53:23 +01001775 dentry->d_inode->i_nlink--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776 dir->i_nlink--;
1777 return shmem_unlink(dir, dentry);
1778}
1779
1780/*
1781 * The VFS layer already does all the dentry stuff for rename,
1782 * we just have to decrement the usage count for the target if
1783 * it exists so that the VFS layer correctly free's it when it
1784 * gets overwritten.
1785 */
1786static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1787{
1788 struct inode *inode = old_dentry->d_inode;
1789 int they_are_dirs = S_ISDIR(inode->i_mode);
1790
1791 if (!simple_empty(new_dentry))
1792 return -ENOTEMPTY;
1793
1794 if (new_dentry->d_inode) {
1795 (void) shmem_unlink(new_dir, new_dentry);
1796 if (they_are_dirs)
1797 old_dir->i_nlink--;
1798 } else if (they_are_dirs) {
1799 old_dir->i_nlink--;
1800 new_dir->i_nlink++;
1801 }
1802
1803 old_dir->i_size -= BOGO_DIRENT_SIZE;
1804 new_dir->i_size += BOGO_DIRENT_SIZE;
1805 old_dir->i_ctime = old_dir->i_mtime =
1806 new_dir->i_ctime = new_dir->i_mtime =
1807 inode->i_ctime = CURRENT_TIME;
1808 return 0;
1809}
1810
1811static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1812{
1813 int error;
1814 int len;
1815 struct inode *inode;
1816 struct page *page = NULL;
1817 char *kaddr;
1818 struct shmem_inode_info *info;
1819
1820 len = strlen(symname) + 1;
1821 if (len > PAGE_CACHE_SIZE)
1822 return -ENAMETOOLONG;
1823
1824 inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
1825 if (!inode)
1826 return -ENOSPC;
1827
Stephen Smalley570bc1c2005-09-09 13:01:43 -07001828 error = security_inode_init_security(inode, dir, NULL, NULL,
1829 NULL);
1830 if (error) {
1831 if (error != -EOPNOTSUPP) {
1832 iput(inode);
1833 return error;
1834 }
1835 error = 0;
1836 }
1837
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838 info = SHMEM_I(inode);
1839 inode->i_size = len-1;
1840 if (len <= (char *)inode - (char *)info) {
1841 /* do it inline */
1842 memcpy(info, symname, len);
1843 inode->i_op = &shmem_symlink_inline_operations;
1844 } else {
1845 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1846 if (error) {
1847 iput(inode);
1848 return error;
1849 }
1850 inode->i_op = &shmem_symlink_inode_operations;
1851 kaddr = kmap_atomic(page, KM_USER0);
1852 memcpy(kaddr, symname, len);
1853 kunmap_atomic(kaddr, KM_USER0);
1854 set_page_dirty(page);
1855 page_cache_release(page);
1856 }
1857 if (dir->i_mode & S_ISGID)
1858 inode->i_gid = dir->i_gid;
1859 dir->i_size += BOGO_DIRENT_SIZE;
1860 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1861 d_instantiate(dentry, inode);
1862 dget(dentry);
1863 return 0;
1864}
1865
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001866static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867{
1868 nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001869 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870}
1871
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001872static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873{
1874 struct page *page = NULL;
1875 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1876 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001877 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878}
1879
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001880static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881{
1882 if (!IS_ERR(nd_get_link(nd))) {
Linus Torvaldscc314ee2005-08-19 18:02:56 -07001883 struct page *page = cookie;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884 kunmap(page);
1885 mark_page_accessed(page);
1886 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 }
1888}
1889
1890static struct inode_operations shmem_symlink_inline_operations = {
1891 .readlink = generic_readlink,
1892 .follow_link = shmem_follow_link_inline,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893};
1894
1895static struct inode_operations shmem_symlink_inode_operations = {
1896 .truncate = shmem_truncate,
1897 .readlink = generic_readlink,
1898 .follow_link = shmem_follow_link,
1899 .put_link = shmem_put_link,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900};
1901
Robin Holt7339ff82006-01-14 13:20:48 -08001902static int shmem_parse_options(char *options, int *mode, uid_t *uid,
1903 gid_t *gid, unsigned long *blocks, unsigned long *inodes,
1904 int *policy, nodemask_t *policy_nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905{
1906 char *this_char, *value, *rest;
1907
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +00001908 while (options != NULL) {
1909 this_char = options;
1910 for (;;) {
1911 /*
1912 * NUL-terminate this option: unfortunately,
1913 * mount options form a comma-separated list,
1914 * but mpol's nodelist may also contain commas.
1915 */
1916 options = strchr(options, ',');
1917 if (options == NULL)
1918 break;
1919 options++;
1920 if (!isdigit(*options)) {
1921 options[-1] = '\0';
1922 break;
1923 }
1924 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925 if (!*this_char)
1926 continue;
1927 if ((value = strchr(this_char,'=')) != NULL) {
1928 *value++ = 0;
1929 } else {
1930 printk(KERN_ERR
1931 "tmpfs: No value for mount option '%s'\n",
1932 this_char);
1933 return 1;
1934 }
1935
1936 if (!strcmp(this_char,"size")) {
1937 unsigned long long size;
1938 size = memparse(value,&rest);
1939 if (*rest == '%') {
1940 size <<= PAGE_SHIFT;
1941 size *= totalram_pages;
1942 do_div(size, 100);
1943 rest++;
1944 }
1945 if (*rest)
1946 goto bad_val;
1947 *blocks = size >> PAGE_CACHE_SHIFT;
1948 } else if (!strcmp(this_char,"nr_blocks")) {
1949 *blocks = memparse(value,&rest);
1950 if (*rest)
1951 goto bad_val;
1952 } else if (!strcmp(this_char,"nr_inodes")) {
1953 *inodes = memparse(value,&rest);
1954 if (*rest)
1955 goto bad_val;
1956 } else if (!strcmp(this_char,"mode")) {
1957 if (!mode)
1958 continue;
1959 *mode = simple_strtoul(value,&rest,8);
1960 if (*rest)
1961 goto bad_val;
1962 } else if (!strcmp(this_char,"uid")) {
1963 if (!uid)
1964 continue;
1965 *uid = simple_strtoul(value,&rest,0);
1966 if (*rest)
1967 goto bad_val;
1968 } else if (!strcmp(this_char,"gid")) {
1969 if (!gid)
1970 continue;
1971 *gid = simple_strtoul(value,&rest,0);
1972 if (*rest)
1973 goto bad_val;
Robin Holt7339ff82006-01-14 13:20:48 -08001974 } else if (!strcmp(this_char,"mpol")) {
Hugh Dickinsb00dc3a2006-02-21 23:49:47 +00001975 if (shmem_parse_mpol(value,policy,policy_nodes))
Robin Holt7339ff82006-01-14 13:20:48 -08001976 goto bad_val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 } else {
1978 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
1979 this_char);
1980 return 1;
1981 }
1982 }
1983 return 0;
1984
1985bad_val:
1986 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
1987 value, this_char);
1988 return 1;
1989
1990}
1991
1992static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
1993{
1994 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001995 unsigned long max_blocks = sbinfo->max_blocks;
1996 unsigned long max_inodes = sbinfo->max_inodes;
Robin Holt7339ff82006-01-14 13:20:48 -08001997 int policy = sbinfo->policy;
1998 nodemask_t policy_nodes = sbinfo->policy_nodes;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07001999 unsigned long blocks;
2000 unsigned long inodes;
2001 int error = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002
Robin Holt7339ff82006-01-14 13:20:48 -08002003 if (shmem_parse_options(data, NULL, NULL, NULL, &max_blocks,
2004 &max_inodes, &policy, &policy_nodes))
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002005 return error;
2006
2007 spin_lock(&sbinfo->stat_lock);
2008 blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2009 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2010 if (max_blocks < blocks)
2011 goto out;
2012 if (max_inodes < inodes)
2013 goto out;
2014 /*
2015 * Those tests also disallow limited->unlimited while any are in
2016 * use, so i_blocks will always be zero when max_blocks is zero;
2017 * but we must separately disallow unlimited->limited, because
2018 * in that case we have no record of how much is already in use.
2019 */
2020 if (max_blocks && !sbinfo->max_blocks)
2021 goto out;
2022 if (max_inodes && !sbinfo->max_inodes)
2023 goto out;
2024
2025 error = 0;
2026 sbinfo->max_blocks = max_blocks;
2027 sbinfo->free_blocks = max_blocks - blocks;
2028 sbinfo->max_inodes = max_inodes;
2029 sbinfo->free_inodes = max_inodes - inodes;
Robin Holt7339ff82006-01-14 13:20:48 -08002030 sbinfo->policy = policy;
2031 sbinfo->policy_nodes = policy_nodes;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002032out:
2033 spin_unlock(&sbinfo->stat_lock);
2034 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035}
2036#endif
2037
2038static void shmem_put_super(struct super_block *sb)
2039{
2040 kfree(sb->s_fs_info);
2041 sb->s_fs_info = NULL;
2042}
2043
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044static int shmem_fill_super(struct super_block *sb,
2045 void *data, int silent)
2046{
2047 struct inode *inode;
2048 struct dentry *root;
2049 int mode = S_IRWXUGO | S_ISVTX;
2050 uid_t uid = current->fsuid;
2051 gid_t gid = current->fsgid;
2052 int err = -ENOMEM;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002053 struct shmem_sb_info *sbinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 unsigned long blocks = 0;
2055 unsigned long inodes = 0;
Robin Holt7339ff82006-01-14 13:20:48 -08002056 int policy = MPOL_DEFAULT;
2057 nodemask_t policy_nodes = node_online_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002059#ifdef CONFIG_TMPFS
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060 /*
2061 * Per default we only allow half of the physical ram per
2062 * tmpfs instance, limiting inodes to one per page of lowmem;
2063 * but the internal instance is left unlimited.
2064 */
2065 if (!(sb->s_flags & MS_NOUSER)) {
2066 blocks = totalram_pages / 2;
2067 inodes = totalram_pages - totalhigh_pages;
2068 if (inodes > blocks)
2069 inodes = blocks;
Robin Holt7339ff82006-01-14 13:20:48 -08002070 if (shmem_parse_options(data, &mode, &uid, &gid, &blocks,
2071 &inodes, &policy, &policy_nodes))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002072 return -EINVAL;
2073 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074#else
2075 sb->s_flags |= MS_NOUSER;
2076#endif
2077
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002078 /* Round up to L1_CACHE_BYTES to resist false sharing */
2079 sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
2080 L1_CACHE_BYTES), GFP_KERNEL);
2081 if (!sbinfo)
2082 return -ENOMEM;
2083
2084 spin_lock_init(&sbinfo->stat_lock);
2085 sbinfo->max_blocks = blocks;
2086 sbinfo->free_blocks = blocks;
2087 sbinfo->max_inodes = inodes;
2088 sbinfo->free_inodes = inodes;
Robin Holt7339ff82006-01-14 13:20:48 -08002089 sbinfo->policy = policy;
2090 sbinfo->policy_nodes = policy_nodes;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002091
2092 sb->s_fs_info = sbinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093 sb->s_maxbytes = SHMEM_MAX_BYTES;
2094 sb->s_blocksize = PAGE_CACHE_SIZE;
2095 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2096 sb->s_magic = TMPFS_MAGIC;
2097 sb->s_op = &shmem_ops;
Robin H. Johnsoncfd95a92006-06-12 21:50:25 +01002098 sb->s_time_gran = 1;
Hugh Dickins0edd73b2005-06-21 17:15:04 -07002099
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 inode = shmem_get_inode(sb, S_IFDIR | mode, 0);
2101 if (!inode)
2102 goto failed;
2103 inode->i_uid = uid;
2104 inode->i_gid = gid;
2105 root = d_alloc_root(inode);
2106 if (!root)
2107 goto failed_iput;
2108 sb->s_root = root;
2109 return 0;
2110
2111failed_iput:
2112 iput(inode);
2113failed:
2114 shmem_put_super(sb);
2115 return err;
2116}
2117
Pekka Enbergfcc234f2006-03-22 00:08:13 -08002118static struct kmem_cache *shmem_inode_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119
2120static struct inode *shmem_alloc_inode(struct super_block *sb)
2121{
2122 struct shmem_inode_info *p;
2123 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, SLAB_KERNEL);
2124 if (!p)
2125 return NULL;
2126 return &p->vfs_inode;
2127}
2128
2129static void shmem_destroy_inode(struct inode *inode)
2130{
2131 if ((inode->i_mode & S_IFMT) == S_IFREG) {
2132 /* only struct inode is valid if it's an inline symlink */
2133 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2134 }
2135 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2136}
2137
Pekka Enbergfcc234f2006-03-22 00:08:13 -08002138static void init_once(void *foo, struct kmem_cache *cachep,
2139 unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140{
2141 struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2142
2143 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
2144 SLAB_CTOR_CONSTRUCTOR) {
2145 inode_init_once(&p->vfs_inode);
2146 }
2147}
2148
2149static int init_inodecache(void)
2150{
2151 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2152 sizeof(struct shmem_inode_info),
2153 0, 0, init_once, NULL);
2154 if (shmem_inode_cachep == NULL)
2155 return -ENOMEM;
2156 return 0;
2157}
2158
2159static void destroy_inodecache(void)
2160{
2161 if (kmem_cache_destroy(shmem_inode_cachep))
2162 printk(KERN_INFO "shmem_inode_cache: not all structures were freed\n");
2163}
2164
2165static struct address_space_operations shmem_aops = {
2166 .writepage = shmem_writepage,
2167 .set_page_dirty = __set_page_dirty_nobuffers,
2168#ifdef CONFIG_TMPFS
2169 .prepare_write = shmem_prepare_write,
2170 .commit_write = simple_commit_write,
2171#endif
Lee Schermerhorn304dbdb2006-04-22 02:35:48 -07002172 .migratepage = migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002173};
2174
2175static struct file_operations shmem_file_operations = {
2176 .mmap = shmem_mmap,
2177#ifdef CONFIG_TMPFS
2178 .llseek = generic_file_llseek,
2179 .read = shmem_file_read,
2180 .write = shmem_file_write,
2181 .fsync = simple_sync_file,
2182 .sendfile = shmem_file_sendfile,
2183#endif
2184};
2185
2186static struct inode_operations shmem_inode_operations = {
2187 .truncate = shmem_truncate,
2188 .setattr = shmem_notify_change,
Badari Pulavartyf6b3ec22006-01-06 00:10:38 -08002189 .truncate_range = shmem_truncate_range,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190};
2191
2192static struct inode_operations shmem_dir_inode_operations = {
2193#ifdef CONFIG_TMPFS
2194 .create = shmem_create,
2195 .lookup = simple_lookup,
2196 .link = shmem_link,
2197 .unlink = shmem_unlink,
2198 .symlink = shmem_symlink,
2199 .mkdir = shmem_mkdir,
2200 .rmdir = shmem_rmdir,
2201 .mknod = shmem_mknod,
2202 .rename = shmem_rename,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203#endif
2204};
2205
2206static struct super_operations shmem_ops = {
2207 .alloc_inode = shmem_alloc_inode,
2208 .destroy_inode = shmem_destroy_inode,
2209#ifdef CONFIG_TMPFS
2210 .statfs = shmem_statfs,
2211 .remount_fs = shmem_remount_fs,
2212#endif
2213 .delete_inode = shmem_delete_inode,
2214 .drop_inode = generic_delete_inode,
2215 .put_super = shmem_put_super,
2216};
2217
2218static struct vm_operations_struct shmem_vm_ops = {
2219 .nopage = shmem_nopage,
2220 .populate = shmem_populate,
2221#ifdef CONFIG_NUMA
2222 .set_policy = shmem_set_policy,
2223 .get_policy = shmem_get_policy,
2224#endif
2225};
2226
2227
David Howells454e2392006-06-23 02:02:57 -07002228static int shmem_get_sb(struct file_system_type *fs_type,
2229 int flags, const char *dev_name, void *data, struct vfsmount *mnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002230{
David Howells454e2392006-06-23 02:02:57 -07002231 return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002232}
2233
2234static struct file_system_type tmpfs_fs_type = {
2235 .owner = THIS_MODULE,
2236 .name = "tmpfs",
2237 .get_sb = shmem_get_sb,
2238 .kill_sb = kill_litter_super,
2239};
2240static struct vfsmount *shm_mnt;
2241
2242static int __init init_tmpfs(void)
2243{
2244 int error;
2245
2246 error = init_inodecache();
2247 if (error)
2248 goto out3;
2249
2250 error = register_filesystem(&tmpfs_fs_type);
2251 if (error) {
2252 printk(KERN_ERR "Could not register tmpfs\n");
2253 goto out2;
2254 }
2255#ifdef CONFIG_TMPFS
2256 devfs_mk_dir("shm");
2257#endif
Trond Myklebust1f5ce9e2006-06-09 09:34:16 -04002258 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259 tmpfs_fs_type.name, NULL);
2260 if (IS_ERR(shm_mnt)) {
2261 error = PTR_ERR(shm_mnt);
2262 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2263 goto out1;
2264 }
2265 return 0;
2266
2267out1:
2268 unregister_filesystem(&tmpfs_fs_type);
2269out2:
2270 destroy_inodecache();
2271out3:
2272 shm_mnt = ERR_PTR(error);
2273 return error;
2274}
2275module_init(init_tmpfs)
2276
2277/*
2278 * shmem_file_setup - get an unlinked file living in tmpfs
2279 *
2280 * @name: name for dentry (to be seen in /proc/<pid>/maps
2281 * @size: size to be set for the file
2282 *
2283 */
2284struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
2285{
2286 int error;
2287 struct file *file;
2288 struct inode *inode;
2289 struct dentry *dentry, *root;
2290 struct qstr this;
2291
2292 if (IS_ERR(shm_mnt))
2293 return (void *)shm_mnt;
2294
2295 if (size < 0 || size > SHMEM_MAX_BYTES)
2296 return ERR_PTR(-EINVAL);
2297
2298 if (shmem_acct_size(flags, size))
2299 return ERR_PTR(-ENOMEM);
2300
2301 error = -ENOMEM;
2302 this.name = name;
2303 this.len = strlen(name);
2304 this.hash = 0; /* will go */
2305 root = shm_mnt->mnt_root;
2306 dentry = d_alloc(root, &this);
2307 if (!dentry)
2308 goto put_memory;
2309
2310 error = -ENFILE;
2311 file = get_empty_filp();
2312 if (!file)
2313 goto put_dentry;
2314
2315 error = -ENOSPC;
2316 inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
2317 if (!inode)
2318 goto close_file;
2319
2320 SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
2321 d_instantiate(dentry, inode);
2322 inode->i_size = size;
2323 inode->i_nlink = 0; /* It is unlinked */
2324 file->f_vfsmnt = mntget(shm_mnt);
2325 file->f_dentry = dentry;
2326 file->f_mapping = inode->i_mapping;
2327 file->f_op = &shmem_file_operations;
2328 file->f_mode = FMODE_WRITE | FMODE_READ;
2329 return file;
2330
2331close_file:
2332 put_filp(file);
2333put_dentry:
2334 dput(dentry);
2335put_memory:
2336 shmem_unacct_size(flags, size);
2337 return ERR_PTR(error);
2338}
2339
2340/*
2341 * shmem_zero_setup - setup a shared anonymous mapping
2342 *
2343 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2344 */
2345int shmem_zero_setup(struct vm_area_struct *vma)
2346{
2347 struct file *file;
2348 loff_t size = vma->vm_end - vma->vm_start;
2349
2350 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2351 if (IS_ERR(file))
2352 return PTR_ERR(file);
2353
2354 if (vma->vm_file)
2355 fput(vma->vm_file);
2356 vma->vm_file = file;
2357 vma->vm_ops = &shmem_vm_ops;
2358 return 0;
2359}