blob: bcc7372aebbc4375d0763e4f3acd8d096bcb612d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/filemap.c
3 *
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
6
7/*
8 * This file handles the generic file mmap semantics used by
9 * most "normal" filesystems (but you don't /have/ to use this:
10 * the NFS filesystem used to do this differently, for example)
11 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/compiler.h>
15#include <linux/fs.h>
Hiro Yoshiokac22ce142006-06-23 02:04:16 -070016#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/aio.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080018#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/kernel_stat.h>
20#include <linux/mm.h>
21#include <linux/swap.h>
22#include <linux/mman.h>
23#include <linux/pagemap.h>
24#include <linux/file.h>
25#include <linux/uio.h>
26#include <linux/hash.h>
27#include <linux/writeback.h>
Linus Torvalds53253382007-10-18 14:47:32 -070028#include <linux/backing-dev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/pagevec.h>
30#include <linux/blkdev.h>
31#include <linux/security.h>
32#include <linux/syscalls.h>
Paul Jackson44110fe2006-03-24 03:16:04 -080033#include <linux/cpuset.h>
Nick Piggin2f718ff2007-10-16 01:24:59 -070034#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080035#include <linux/memcontrol.h>
Rik van Riel4f98a2f2008-10-18 20:26:32 -070036#include <linux/mm_inline.h> /* for page_is_file_cache() */
Nick Piggin0f8053a2006-03-22 00:08:33 -080037#include "internal.h"
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * FIXME: remove all knowledge of the buffer layer from the core VM
41 */
Jan Kara148f9482009-08-17 19:52:36 +020042#include <linux/buffer_head.h> /* for try_to_free_buffers */
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <asm/mman.h>
45
46/*
47 * Shared mappings implemented 30.11.1994. It's not fully working yet,
48 * though.
49 *
50 * Shared mappings now work. 15.8.1995 Bruno.
51 *
52 * finished 'unifying' the page and buffer cache and SMP-threaded the
53 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
54 *
55 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
56 */
57
58/*
59 * Lock ordering:
60 *
61 * ->i_mmap_lock (vmtruncate)
62 * ->private_lock (__free_pte->__set_page_dirty_buffers)
Hugh Dickins5d337b92005-09-03 15:54:41 -070063 * ->swap_lock (exclusive_swap_page, others)
64 * ->mapping->tree_lock
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -080066 * ->i_mutex
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * ->i_mmap_lock (truncate->unmap_mapping_range)
68 *
69 * ->mmap_sem
70 * ->i_mmap_lock
Hugh Dickinsb8072f02005-10-29 18:16:41 -070071 * ->page_table_lock or pte_lock (various, mainly in memory.c)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
73 *
74 * ->mmap_sem
75 * ->lock_page (access_process_vm)
76 *
Nick Piggin82591e62006-10-19 23:29:10 -070077 * ->i_mutex (generic_file_buffered_write)
78 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -080080 * ->i_mutex
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 * ->i_alloc_sem (various)
82 *
83 * ->inode_lock
84 * ->sb_lock (fs/fs-writeback.c)
85 * ->mapping->tree_lock (__sync_single_inode)
86 *
87 * ->i_mmap_lock
88 * ->anon_vma.lock (vma_adjust)
89 *
90 * ->anon_vma.lock
Hugh Dickinsb8072f02005-10-29 18:16:41 -070091 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 *
Hugh Dickinsb8072f02005-10-29 18:16:41 -070093 * ->page_table_lock or pte_lock
Hugh Dickins5d337b92005-09-03 15:54:41 -070094 * ->swap_lock (try_to_unmap_one)
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 * ->private_lock (try_to_unmap_one)
96 * ->tree_lock (try_to_unmap_one)
97 * ->zone.lru_lock (follow_page->mark_page_accessed)
Nick Piggin053837f2006-01-18 17:42:27 -080098 * ->zone.lru_lock (check_pte_range->isolate_lru_page)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 * ->private_lock (page_remove_rmap->set_page_dirty)
100 * ->tree_lock (page_remove_rmap->set_page_dirty)
101 * ->inode_lock (page_remove_rmap->set_page_dirty)
102 * ->inode_lock (zap_pte_range->set_page_dirty)
103 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
104 *
105 * ->task->proc_lock
106 * ->dcache_lock (proc_pid_lookup)
107 */
108
109/*
110 * Remove a page from the page cache and free it. Caller has to make
111 * sure the page is locked and that nobody else uses it - or that usage
Nick Piggin19fd6232008-07-25 19:45:32 -0700112 * is safe. The caller must hold the mapping's tree_lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 */
114void __remove_from_page_cache(struct page *page)
115{
116 struct address_space *mapping = page->mapping;
117
118 radix_tree_delete(&mapping->page_tree, page->index);
119 page->mapping = NULL;
120 mapping->nrpages--;
Christoph Lameter347ce432006-06-30 01:55:35 -0700121 __dec_zone_page_state(page, NR_FILE_PAGES);
KOSAKI Motohiro4b021082009-09-21 17:01:33 -0700122 if (PageSwapBacked(page))
123 __dec_zone_page_state(page, NR_SHMEM);
Nick Piggin45426812007-07-15 23:38:12 -0700124 BUG_ON(page_mapped(page));
Linus Torvalds3a692792007-12-19 14:05:13 -0800125
126 /*
127 * Some filesystems seem to re-dirty the page even after
128 * the VM has canceled the dirty bit (eg ext3 journaling).
129 *
130 * Fix it up by doing a final dirty accounting check after
131 * having removed the page entirely.
132 */
133 if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
134 dec_zone_page_state(page, NR_FILE_DIRTY);
135 dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
136 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137}
138
139void remove_from_page_cache(struct page *page)
140{
141 struct address_space *mapping = page->mapping;
142
Matt Mackallcd7619d2005-05-01 08:59:01 -0700143 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Nick Piggin19fd6232008-07-25 19:45:32 -0700145 spin_lock_irq(&mapping->tree_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 __remove_from_page_cache(page);
Nick Piggin19fd6232008-07-25 19:45:32 -0700147 spin_unlock_irq(&mapping->tree_lock);
Daisuke Nishimurae767e052009-05-28 14:34:28 -0700148 mem_cgroup_uncharge_cache_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149}
150
151static int sync_page(void *word)
152{
153 struct address_space *mapping;
154 struct page *page;
155
Andi Kleen07808b72005-11-05 17:25:53 +0100156 page = container_of((unsigned long *)word, struct page, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
158 /*
William Lee Irwin IIIdd1d5af2005-05-01 08:58:38 -0700159 * page_mapping() is being called without PG_locked held.
160 * Some knowledge of the state and use of the page is used to
161 * reduce the requirements down to a memory barrier.
162 * The danger here is of a stale page_mapping() return value
163 * indicating a struct address_space different from the one it's
164 * associated with when it is associated with one.
165 * After smp_mb(), it's either the correct page_mapping() for
166 * the page, or an old page_mapping() and the page's own
167 * page_mapping() has gone NULL.
168 * The ->sync_page() address_space operation must tolerate
169 * page_mapping() going NULL. By an amazing coincidence,
170 * this comes about because none of the users of the page
171 * in the ->sync_page() methods make essential use of the
172 * page_mapping(), merely passing the page down to the backing
173 * device's unplug functions when it's non-NULL, which in turn
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700174 * ignore it for all cases but swap, where only page_private(page) is
William Lee Irwin IIIdd1d5af2005-05-01 08:58:38 -0700175 * of interest. When page_mapping() does go NULL, the entire
176 * call stack gracefully ignores the page and returns.
177 * -- wli
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 */
179 smp_mb();
180 mapping = page_mapping(page);
181 if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
182 mapping->a_ops->sync_page(page);
183 io_schedule();
184 return 0;
185}
186
Matthew Wilcox2687a352007-12-06 11:18:49 -0500187static int sync_page_killable(void *word)
188{
189 sync_page(word);
190 return fatal_signal_pending(current) ? -EINTR : 0;
191}
192
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193/**
Randy Dunlap485bb992006-06-23 02:03:49 -0700194 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
Martin Waitz67be2dd2005-05-01 08:59:26 -0700195 * @mapping: address space structure to write
196 * @start: offset in bytes where the range starts
Andrew Morton469eb4d2006-03-24 03:17:45 -0800197 * @end: offset in bytes where the range ends (inclusive)
Martin Waitz67be2dd2005-05-01 08:59:26 -0700198 * @sync_mode: enable synchronous operation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 *
Randy Dunlap485bb992006-06-23 02:03:49 -0700200 * Start writeback against all of a mapping's dirty pages that lie
201 * within the byte offsets <start, end> inclusive.
202 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
Randy Dunlap485bb992006-06-23 02:03:49 -0700204 * opposed to a regular memory cleansing writeback. The difference between
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 * these two operations is that if a dirty page/buffer is encountered, it must
206 * be waited upon, and not just skipped over.
207 */
Andrew Mortonebcf28e2006-03-24 03:18:04 -0800208int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
209 loff_t end, int sync_mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
211 int ret;
212 struct writeback_control wbc = {
213 .sync_mode = sync_mode,
Nick Piggin05fe4782009-01-06 14:39:08 -0800214 .nr_to_write = LONG_MAX,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700215 .range_start = start,
216 .range_end = end,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 };
218
219 if (!mapping_cap_writeback_dirty(mapping))
220 return 0;
221
222 ret = do_writepages(mapping, &wbc);
223 return ret;
224}
225
226static inline int __filemap_fdatawrite(struct address_space *mapping,
227 int sync_mode)
228{
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700229 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230}
231
232int filemap_fdatawrite(struct address_space *mapping)
233{
234 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
235}
236EXPORT_SYMBOL(filemap_fdatawrite);
237
Jan Karaf4c0a0f2008-07-11 19:27:31 -0400238int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
Andrew Mortonebcf28e2006-03-24 03:18:04 -0800239 loff_t end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240{
241 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
242}
Jan Karaf4c0a0f2008-07-11 19:27:31 -0400243EXPORT_SYMBOL(filemap_fdatawrite_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
Randy Dunlap485bb992006-06-23 02:03:49 -0700245/**
246 * filemap_flush - mostly a non-blocking flush
247 * @mapping: target address_space
248 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 * This is a mostly non-blocking flush. Not suitable for data-integrity
250 * purposes - I/O may not be started against all dirty pages.
251 */
252int filemap_flush(struct address_space *mapping)
253{
254 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
255}
256EXPORT_SYMBOL(filemap_flush);
257
Randy Dunlap485bb992006-06-23 02:03:49 -0700258/**
259 * wait_on_page_writeback_range - wait for writeback to complete
260 * @mapping: target address_space
261 * @start: beginning page index
262 * @end: ending page index
263 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 * Wait for writeback to complete against pages indexed by start->end
265 * inclusive
266 */
Andrew Mortonebcf28e2006-03-24 03:18:04 -0800267int wait_on_page_writeback_range(struct address_space *mapping,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 pgoff_t start, pgoff_t end)
269{
270 struct pagevec pvec;
271 int nr_pages;
272 int ret = 0;
273 pgoff_t index;
274
275 if (end < start)
276 return 0;
277
278 pagevec_init(&pvec, 0);
279 index = start;
280 while ((index <= end) &&
281 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
282 PAGECACHE_TAG_WRITEBACK,
283 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
284 unsigned i;
285
286 for (i = 0; i < nr_pages; i++) {
287 struct page *page = pvec.pages[i];
288
289 /* until radix tree lookup accepts end_index */
290 if (page->index > end)
291 continue;
292
293 wait_on_page_writeback(page);
294 if (PageError(page))
295 ret = -EIO;
296 }
297 pagevec_release(&pvec);
298 cond_resched();
299 }
300
301 /* Check for outstanding write errors */
302 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
303 ret = -ENOSPC;
304 if (test_and_clear_bit(AS_EIO, &mapping->flags))
305 ret = -EIO;
306
307 return ret;
308}
309
Randy Dunlap485bb992006-06-23 02:03:49 -0700310/**
Jan Karad3bccb6f2009-08-17 19:30:27 +0200311 * filemap_fdatawait_range - wait for all under-writeback pages to complete in a given range
312 * @mapping: address space structure to wait for
313 * @start: offset in bytes where the range starts
314 * @end: offset in bytes where the range ends (inclusive)
315 *
316 * Walk the list of under-writeback pages of the given address space
317 * in the given range and wait for all of them.
318 *
319 * This is just a simple wrapper so that callers don't have to convert offsets
320 * to page indexes themselves
321 */
322int filemap_fdatawait_range(struct address_space *mapping, loff_t start,
323 loff_t end)
324{
325 return wait_on_page_writeback_range(mapping, start >> PAGE_CACHE_SHIFT,
326 end >> PAGE_CACHE_SHIFT);
327}
328EXPORT_SYMBOL(filemap_fdatawait_range);
329
330/**
Randy Dunlap485bb992006-06-23 02:03:49 -0700331 * filemap_fdatawait - wait for all under-writeback pages to complete
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 * @mapping: address space structure to wait for
Randy Dunlap485bb992006-06-23 02:03:49 -0700333 *
334 * Walk the list of under-writeback pages of the given address space
335 * and wait for all of them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 */
337int filemap_fdatawait(struct address_space *mapping)
338{
339 loff_t i_size = i_size_read(mapping->host);
340
341 if (i_size == 0)
342 return 0;
343
344 return wait_on_page_writeback_range(mapping, 0,
345 (i_size - 1) >> PAGE_CACHE_SHIFT);
346}
347EXPORT_SYMBOL(filemap_fdatawait);
348
349int filemap_write_and_wait(struct address_space *mapping)
350{
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800351 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352
353 if (mapping->nrpages) {
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800354 err = filemap_fdatawrite(mapping);
355 /*
356 * Even if the above returned error, the pages may be
357 * written partially (e.g. -ENOSPC), so we wait for it.
358 * But the -EIO is special case, it may indicate the worst
359 * thing (e.g. bug) happened, so we avoid waiting for it.
360 */
361 if (err != -EIO) {
362 int err2 = filemap_fdatawait(mapping);
363 if (!err)
364 err = err2;
365 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 }
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800367 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368}
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800369EXPORT_SYMBOL(filemap_write_and_wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
Randy Dunlap485bb992006-06-23 02:03:49 -0700371/**
372 * filemap_write_and_wait_range - write out & wait on a file range
373 * @mapping: the address_space for the pages
374 * @lstart: offset in bytes where the range starts
375 * @lend: offset in bytes where the range ends (inclusive)
376 *
Andrew Morton469eb4d2006-03-24 03:17:45 -0800377 * Write out and wait upon file offsets lstart->lend, inclusive.
378 *
379 * Note that `lend' is inclusive (describes the last byte to be written) so
380 * that this function can be used to write to the very end-of-file (end = -1).
381 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382int filemap_write_and_wait_range(struct address_space *mapping,
383 loff_t lstart, loff_t lend)
384{
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800385 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
387 if (mapping->nrpages) {
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800388 err = __filemap_fdatawrite_range(mapping, lstart, lend,
389 WB_SYNC_ALL);
390 /* See comment of filemap_write_and_wait() */
391 if (err != -EIO) {
392 int err2 = wait_on_page_writeback_range(mapping,
393 lstart >> PAGE_CACHE_SHIFT,
394 lend >> PAGE_CACHE_SHIFT);
395 if (!err)
396 err = err2;
397 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 }
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800399 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400}
Chris Masonf6995582009-04-15 13:22:37 -0400401EXPORT_SYMBOL(filemap_write_and_wait_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Randy Dunlap485bb992006-06-23 02:03:49 -0700403/**
Nick Piggine2867812008-07-25 19:45:30 -0700404 * add_to_page_cache_locked - add a locked page to the pagecache
Randy Dunlap485bb992006-06-23 02:03:49 -0700405 * @page: page to add
406 * @mapping: the page's address_space
407 * @offset: page index
408 * @gfp_mask: page allocation mode
409 *
Nick Piggine2867812008-07-25 19:45:30 -0700410 * This function is used to add a page to the pagecache. It must be locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 * This function does not add the page to the LRU. The caller must do that.
412 */
Nick Piggine2867812008-07-25 19:45:30 -0700413int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
Al Viro6daa0e22005-10-21 03:18:50 -0400414 pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415{
Nick Piggine2867812008-07-25 19:45:30 -0700416 int error;
417
418 VM_BUG_ON(!PageLocked(page));
419
420 error = mem_cgroup_cache_charge(page, current->mm,
KAMEZAWA Hiroyuki2c26fdd2009-01-07 18:08:10 -0800421 gfp_mask & GFP_RECLAIM_MASK);
Balbir Singh35c754d2008-02-07 00:14:05 -0800422 if (error)
423 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424
Balbir Singh35c754d2008-02-07 00:14:05 -0800425 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 if (error == 0) {
Nick Piggine2867812008-07-25 19:45:30 -0700427 page_cache_get(page);
428 page->mapping = mapping;
429 page->index = offset;
430
Nick Piggin19fd6232008-07-25 19:45:32 -0700431 spin_lock_irq(&mapping->tree_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 error = radix_tree_insert(&mapping->page_tree, offset, page);
Nick Piggine2867812008-07-25 19:45:30 -0700433 if (likely(!error)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 mapping->nrpages++;
Christoph Lameter347ce432006-06-30 01:55:35 -0700435 __inc_zone_page_state(page, NR_FILE_PAGES);
KOSAKI Motohiro4b021082009-09-21 17:01:33 -0700436 if (PageSwapBacked(page))
437 __inc_zone_page_state(page, NR_SHMEM);
Daisuke Nishimurae767e052009-05-28 14:34:28 -0700438 spin_unlock_irq(&mapping->tree_lock);
Nick Piggine2867812008-07-25 19:45:30 -0700439 } else {
440 page->mapping = NULL;
Daisuke Nishimurae767e052009-05-28 14:34:28 -0700441 spin_unlock_irq(&mapping->tree_lock);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700442 mem_cgroup_uncharge_cache_page(page);
Nick Piggine2867812008-07-25 19:45:30 -0700443 page_cache_release(page);
444 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 radix_tree_preload_end();
Balbir Singh35c754d2008-02-07 00:14:05 -0800446 } else
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700447 mem_cgroup_uncharge_cache_page(page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800448out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 return error;
450}
Nick Piggine2867812008-07-25 19:45:30 -0700451EXPORT_SYMBOL(add_to_page_cache_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
453int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
Al Viro6daa0e22005-10-21 03:18:50 -0400454 pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455{
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700456 int ret;
457
458 /*
459 * Splice_read and readahead add shmem/tmpfs pages into the page cache
460 * before shmem_readpage has a chance to mark them as SwapBacked: they
461 * need to go on the active_anon lru below, and mem_cgroup_cache_charge
462 * (called in add_to_page_cache) needs to know where they're going too.
463 */
464 if (mapping_cap_swap_backed(mapping))
465 SetPageSwapBacked(page);
466
467 ret = add_to_page_cache(page, mapping, offset, gfp_mask);
468 if (ret == 0) {
469 if (page_is_file_cache(page))
470 lru_cache_add_file(page);
471 else
472 lru_cache_add_active_anon(page);
473 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 return ret;
475}
Evgeniy Polyakov18bc0bb2009-02-09 17:02:42 +0300476EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
Paul Jackson44110fe2006-03-24 03:16:04 -0800478#ifdef CONFIG_NUMA
Nick Piggin2ae88142006-10-28 10:38:23 -0700479struct page *__page_cache_alloc(gfp_t gfp)
Paul Jackson44110fe2006-03-24 03:16:04 -0800480{
481 if (cpuset_do_page_mem_spread()) {
482 int n = cpuset_mem_spread_node();
Mel Gorman6484eb32009-06-16 15:31:54 -0700483 return alloc_pages_exact_node(n, gfp, 0);
Paul Jackson44110fe2006-03-24 03:16:04 -0800484 }
Nick Piggin2ae88142006-10-28 10:38:23 -0700485 return alloc_pages(gfp, 0);
Paul Jackson44110fe2006-03-24 03:16:04 -0800486}
Nick Piggin2ae88142006-10-28 10:38:23 -0700487EXPORT_SYMBOL(__page_cache_alloc);
Paul Jackson44110fe2006-03-24 03:16:04 -0800488#endif
489
Nick Piggindb376482006-09-25 23:31:24 -0700490static int __sleep_on_page_lock(void *word)
491{
492 io_schedule();
493 return 0;
494}
495
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496/*
497 * In order to wait for pages to become available there must be
498 * waitqueues associated with pages. By using a hash table of
499 * waitqueues where the bucket discipline is to maintain all
500 * waiters on the same queue and wake all when any of the pages
501 * become available, and for the woken contexts to check to be
502 * sure the appropriate page became available, this saves space
503 * at a cost of "thundering herd" phenomena during rare hash
504 * collisions.
505 */
506static wait_queue_head_t *page_waitqueue(struct page *page)
507{
508 const struct zone *zone = page_zone(page);
509
510 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
511}
512
513static inline void wake_up_page(struct page *page, int bit)
514{
515 __wake_up_bit(page_waitqueue(page), &page->flags, bit);
516}
517
Harvey Harrison920c7a52008-02-04 22:29:26 -0800518void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519{
520 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
521
522 if (test_bit(bit_nr, &page->flags))
523 __wait_on_bit(page_waitqueue(page), &wait, sync_page,
524 TASK_UNINTERRUPTIBLE);
525}
526EXPORT_SYMBOL(wait_on_page_bit);
527
528/**
David Howells385e1ca5f2009-04-03 16:42:39 +0100529 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
Randy Dunlap697f6192009-04-13 14:39:54 -0700530 * @page: Page defining the wait queue of interest
531 * @waiter: Waiter to add to the queue
David Howells385e1ca5f2009-04-03 16:42:39 +0100532 *
533 * Add an arbitrary @waiter to the wait queue for the nominated @page.
534 */
535void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
536{
537 wait_queue_head_t *q = page_waitqueue(page);
538 unsigned long flags;
539
540 spin_lock_irqsave(&q->lock, flags);
541 __add_wait_queue(q, waiter);
542 spin_unlock_irqrestore(&q->lock, flags);
543}
544EXPORT_SYMBOL_GPL(add_page_wait_queue);
545
546/**
Randy Dunlap485bb992006-06-23 02:03:49 -0700547 * unlock_page - unlock a locked page
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 * @page: the page
549 *
550 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
551 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
552 * mechananism between PageLocked pages and PageWriteback pages is shared.
553 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
554 *
Nick Piggin8413ac92008-10-18 20:26:59 -0700555 * The mb is necessary to enforce ordering between the clear_bit and the read
556 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 */
Harvey Harrison920c7a52008-02-04 22:29:26 -0800558void unlock_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559{
Nick Piggin8413ac92008-10-18 20:26:59 -0700560 VM_BUG_ON(!PageLocked(page));
561 clear_bit_unlock(PG_locked, &page->flags);
562 smp_mb__after_clear_bit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 wake_up_page(page, PG_locked);
564}
565EXPORT_SYMBOL(unlock_page);
566
Randy Dunlap485bb992006-06-23 02:03:49 -0700567/**
568 * end_page_writeback - end writeback against a page
569 * @page: the page
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 */
571void end_page_writeback(struct page *page)
572{
Miklos Szerediac6aadb2008-04-28 02:12:38 -0700573 if (TestClearPageReclaim(page))
574 rotate_reclaimable_page(page);
575
576 if (!test_clear_page_writeback(page))
577 BUG();
578
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 smp_mb__after_clear_bit();
580 wake_up_page(page, PG_writeback);
581}
582EXPORT_SYMBOL(end_page_writeback);
583
Randy Dunlap485bb992006-06-23 02:03:49 -0700584/**
585 * __lock_page - get a lock on the page, assuming we need to sleep to get it
586 * @page: the page to lock
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 *
Randy Dunlap485bb992006-06-23 02:03:49 -0700588 * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 * random driver's requestfn sets TASK_RUNNING, we could busywait. However
590 * chances are that on the second loop, the block layer's plug list is empty,
591 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
592 */
Harvey Harrison920c7a52008-02-04 22:29:26 -0800593void __lock_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594{
595 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
596
597 __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page,
598 TASK_UNINTERRUPTIBLE);
599}
600EXPORT_SYMBOL(__lock_page);
601
Harvey Harrisonb5606c22008-02-13 15:03:16 -0800602int __lock_page_killable(struct page *page)
Matthew Wilcox2687a352007-12-06 11:18:49 -0500603{
604 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
605
606 return __wait_on_bit_lock(page_waitqueue(page), &wait,
607 sync_page_killable, TASK_KILLABLE);
608}
Evgeniy Polyakov18bc0bb2009-02-09 17:02:42 +0300609EXPORT_SYMBOL_GPL(__lock_page_killable);
Matthew Wilcox2687a352007-12-06 11:18:49 -0500610
Randy Dunlap76824862008-03-19 17:00:40 -0700611/**
612 * __lock_page_nosync - get a lock on the page, without calling sync_page()
613 * @page: the page to lock
614 *
Nick Piggindb376482006-09-25 23:31:24 -0700615 * Variant of lock_page that does not require the caller to hold a reference
616 * on the page's mapping.
617 */
Harvey Harrison920c7a52008-02-04 22:29:26 -0800618void __lock_page_nosync(struct page *page)
Nick Piggindb376482006-09-25 23:31:24 -0700619{
620 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
621 __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock,
622 TASK_UNINTERRUPTIBLE);
623}
624
Randy Dunlap485bb992006-06-23 02:03:49 -0700625/**
626 * find_get_page - find and get a page reference
627 * @mapping: the address_space to search
628 * @offset: the page index
629 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700630 * Is there a pagecache struct page at the given (mapping, offset) tuple?
631 * If yes, increment its refcount and return it; if no, return NULL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 */
Nick Piggina60637c2008-07-25 19:45:31 -0700633struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
Nick Piggina60637c2008-07-25 19:45:31 -0700635 void **pagep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 struct page *page;
637
Nick Piggina60637c2008-07-25 19:45:31 -0700638 rcu_read_lock();
639repeat:
640 page = NULL;
641 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
642 if (pagep) {
643 page = radix_tree_deref_slot(pagep);
644 if (unlikely(!page || page == RADIX_TREE_RETRY))
645 goto repeat;
646
647 if (!page_cache_get_speculative(page))
648 goto repeat;
649
650 /*
651 * Has the page moved?
652 * This is part of the lockless pagecache protocol. See
653 * include/linux/pagemap.h for details.
654 */
655 if (unlikely(page != *pagep)) {
656 page_cache_release(page);
657 goto repeat;
658 }
659 }
660 rcu_read_unlock();
661
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 return page;
663}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664EXPORT_SYMBOL(find_get_page);
665
Randy Dunlap485bb992006-06-23 02:03:49 -0700666/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667 * find_lock_page - locate, pin and lock a pagecache page
Martin Waitz67be2dd2005-05-01 08:59:26 -0700668 * @mapping: the address_space to search
669 * @offset: the page index
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670 *
671 * Locates the desired pagecache page, locks it, increments its reference
672 * count and returns its address.
673 *
674 * Returns zero if the page was not present. find_lock_page() may sleep.
675 */
Nick Piggina60637c2008-07-25 19:45:31 -0700676struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677{
678 struct page *page;
679
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680repeat:
Nick Piggina60637c2008-07-25 19:45:31 -0700681 page = find_get_page(mapping, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 if (page) {
Nick Piggina60637c2008-07-25 19:45:31 -0700683 lock_page(page);
684 /* Has the page been truncated? */
685 if (unlikely(page->mapping != mapping)) {
686 unlock_page(page);
687 page_cache_release(page);
688 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 }
Nick Piggina60637c2008-07-25 19:45:31 -0700690 VM_BUG_ON(page->index != offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 return page;
693}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694EXPORT_SYMBOL(find_lock_page);
695
696/**
697 * find_or_create_page - locate or add a pagecache page
Martin Waitz67be2dd2005-05-01 08:59:26 -0700698 * @mapping: the page's address_space
699 * @index: the page's index into the mapping
700 * @gfp_mask: page allocation mode
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 *
702 * Locates a page in the pagecache. If the page is not present, a new page
703 * is allocated using @gfp_mask and is added to the pagecache and to the VM's
704 * LRU list. The returned page is locked and has its reference count
705 * incremented.
706 *
707 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
708 * allocation!
709 *
710 * find_or_create_page() returns the desired page's address, or zero on
711 * memory exhaustion.
712 */
713struct page *find_or_create_page(struct address_space *mapping,
Fengguang Wu57f6b962007-10-16 01:24:37 -0700714 pgoff_t index, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715{
Nick Piggineb2be182007-10-16 01:24:57 -0700716 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 int err;
718repeat:
719 page = find_lock_page(mapping, index);
720 if (!page) {
Nick Piggineb2be182007-10-16 01:24:57 -0700721 page = __page_cache_alloc(gfp_mask);
722 if (!page)
723 return NULL;
Nick Piggin67d58ac2009-01-06 14:40:28 -0800724 /*
725 * We want a regular kernel memory (not highmem or DMA etc)
726 * allocation for the radix tree nodes, but we need to honour
727 * the context-specific requirements the caller has asked for.
728 * GFP_RECLAIM_MASK collects those requirements.
729 */
730 err = add_to_page_cache_lru(page, mapping, index,
731 (gfp_mask & GFP_RECLAIM_MASK));
Nick Piggineb2be182007-10-16 01:24:57 -0700732 if (unlikely(err)) {
733 page_cache_release(page);
734 page = NULL;
735 if (err == -EEXIST)
736 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 return page;
740}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741EXPORT_SYMBOL(find_or_create_page);
742
743/**
744 * find_get_pages - gang pagecache lookup
745 * @mapping: The address_space to search
746 * @start: The starting page index
747 * @nr_pages: The maximum number of pages
748 * @pages: Where the resulting pages are placed
749 *
750 * find_get_pages() will search for and return a group of up to
751 * @nr_pages pages in the mapping. The pages are placed at @pages.
752 * find_get_pages() takes a reference against the returned pages.
753 *
754 * The search returns a group of mapping-contiguous pages with ascending
755 * indexes. There may be holes in the indices due to not-present pages.
756 *
757 * find_get_pages() returns the number of pages which were found.
758 */
759unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
760 unsigned int nr_pages, struct page **pages)
761{
762 unsigned int i;
763 unsigned int ret;
Nick Piggina60637c2008-07-25 19:45:31 -0700764 unsigned int nr_found;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765
Nick Piggina60637c2008-07-25 19:45:31 -0700766 rcu_read_lock();
767restart:
768 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
769 (void ***)pages, start, nr_pages);
770 ret = 0;
771 for (i = 0; i < nr_found; i++) {
772 struct page *page;
773repeat:
774 page = radix_tree_deref_slot((void **)pages[i]);
775 if (unlikely(!page))
776 continue;
777 /*
778 * this can only trigger if nr_found == 1, making livelock
779 * a non issue.
780 */
781 if (unlikely(page == RADIX_TREE_RETRY))
782 goto restart;
783
784 if (!page_cache_get_speculative(page))
785 goto repeat;
786
787 /* Has the page moved? */
788 if (unlikely(page != *((void **)pages[i]))) {
789 page_cache_release(page);
790 goto repeat;
791 }
792
793 pages[ret] = page;
794 ret++;
795 }
796 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 return ret;
798}
799
Jens Axboeebf43502006-04-27 08:46:01 +0200800/**
801 * find_get_pages_contig - gang contiguous pagecache lookup
802 * @mapping: The address_space to search
803 * @index: The starting page index
804 * @nr_pages: The maximum number of pages
805 * @pages: Where the resulting pages are placed
806 *
807 * find_get_pages_contig() works exactly like find_get_pages(), except
808 * that the returned number of pages are guaranteed to be contiguous.
809 *
810 * find_get_pages_contig() returns the number of pages which were found.
811 */
812unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
813 unsigned int nr_pages, struct page **pages)
814{
815 unsigned int i;
816 unsigned int ret;
Nick Piggina60637c2008-07-25 19:45:31 -0700817 unsigned int nr_found;
Jens Axboeebf43502006-04-27 08:46:01 +0200818
Nick Piggina60637c2008-07-25 19:45:31 -0700819 rcu_read_lock();
820restart:
821 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
822 (void ***)pages, index, nr_pages);
823 ret = 0;
824 for (i = 0; i < nr_found; i++) {
825 struct page *page;
826repeat:
827 page = radix_tree_deref_slot((void **)pages[i]);
828 if (unlikely(!page))
829 continue;
830 /*
831 * this can only trigger if nr_found == 1, making livelock
832 * a non issue.
833 */
834 if (unlikely(page == RADIX_TREE_RETRY))
835 goto restart;
836
837 if (page->mapping == NULL || page->index != index)
Jens Axboeebf43502006-04-27 08:46:01 +0200838 break;
839
Nick Piggina60637c2008-07-25 19:45:31 -0700840 if (!page_cache_get_speculative(page))
841 goto repeat;
842
843 /* Has the page moved? */
844 if (unlikely(page != *((void **)pages[i]))) {
845 page_cache_release(page);
846 goto repeat;
847 }
848
849 pages[ret] = page;
850 ret++;
Jens Axboeebf43502006-04-27 08:46:01 +0200851 index++;
852 }
Nick Piggina60637c2008-07-25 19:45:31 -0700853 rcu_read_unlock();
854 return ret;
Jens Axboeebf43502006-04-27 08:46:01 +0200855}
David Howellsef71c152007-05-09 02:33:44 -0700856EXPORT_SYMBOL(find_get_pages_contig);
Jens Axboeebf43502006-04-27 08:46:01 +0200857
Randy Dunlap485bb992006-06-23 02:03:49 -0700858/**
859 * find_get_pages_tag - find and return pages that match @tag
860 * @mapping: the address_space to search
861 * @index: the starting page index
862 * @tag: the tag index
863 * @nr_pages: the maximum number of pages
864 * @pages: where the resulting pages are placed
865 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866 * Like find_get_pages, except we only return pages which are tagged with
Randy Dunlap485bb992006-06-23 02:03:49 -0700867 * @tag. We update @index to index the next page for the traversal.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 */
869unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
870 int tag, unsigned int nr_pages, struct page **pages)
871{
872 unsigned int i;
873 unsigned int ret;
Nick Piggina60637c2008-07-25 19:45:31 -0700874 unsigned int nr_found;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875
Nick Piggina60637c2008-07-25 19:45:31 -0700876 rcu_read_lock();
877restart:
878 nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,
879 (void ***)pages, *index, nr_pages, tag);
880 ret = 0;
881 for (i = 0; i < nr_found; i++) {
882 struct page *page;
883repeat:
884 page = radix_tree_deref_slot((void **)pages[i]);
885 if (unlikely(!page))
886 continue;
887 /*
888 * this can only trigger if nr_found == 1, making livelock
889 * a non issue.
890 */
891 if (unlikely(page == RADIX_TREE_RETRY))
892 goto restart;
893
894 if (!page_cache_get_speculative(page))
895 goto repeat;
896
897 /* Has the page moved? */
898 if (unlikely(page != *((void **)pages[i]))) {
899 page_cache_release(page);
900 goto repeat;
901 }
902
903 pages[ret] = page;
904 ret++;
905 }
906 rcu_read_unlock();
907
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 if (ret)
909 *index = pages[ret - 1]->index + 1;
Nick Piggina60637c2008-07-25 19:45:31 -0700910
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 return ret;
912}
David Howellsef71c152007-05-09 02:33:44 -0700913EXPORT_SYMBOL(find_get_pages_tag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Randy Dunlap485bb992006-06-23 02:03:49 -0700915/**
916 * grab_cache_page_nowait - returns locked page at given index in given cache
917 * @mapping: target address_space
918 * @index: the page index
919 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800920 * Same as grab_cache_page(), but do not wait if the page is unavailable.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 * This is intended for speculative data generators, where the data can
922 * be regenerated if the page couldn't be grabbed. This routine should
923 * be safe to call while holding the lock for another page.
924 *
925 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
926 * and deadlock against the caller's locked page.
927 */
928struct page *
Fengguang Wu57f6b962007-10-16 01:24:37 -0700929grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930{
931 struct page *page = find_get_page(mapping, index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
933 if (page) {
Nick Piggin529ae9a2008-08-02 12:01:03 +0200934 if (trylock_page(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935 return page;
936 page_cache_release(page);
937 return NULL;
938 }
Nick Piggin2ae88142006-10-28 10:38:23 -0700939 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
Nick Piggin67d58ac2009-01-06 14:40:28 -0800940 if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 page_cache_release(page);
942 page = NULL;
943 }
944 return page;
945}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946EXPORT_SYMBOL(grab_cache_page_nowait);
947
Wu Fengguang76d42bd2006-06-25 05:48:43 -0700948/*
949 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
950 * a _large_ part of the i/o request. Imagine the worst scenario:
951 *
952 * ---R__________________________________________B__________
953 * ^ reading here ^ bad block(assume 4k)
954 *
955 * read(R) => miss => readahead(R...B) => media error => frustrating retries
956 * => failing the whole request => read(R) => read(R+1) =>
957 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
958 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
959 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
960 *
961 * It is going insane. Fix it by quickly scaling down the readahead size.
962 */
963static void shrink_readahead_size_eio(struct file *filp,
964 struct file_ra_state *ra)
965{
Wu Fengguang76d42bd2006-06-25 05:48:43 -0700966 ra->ra_pages /= 4;
Wu Fengguang76d42bd2006-06-25 05:48:43 -0700967}
968
Randy Dunlap485bb992006-06-23 02:03:49 -0700969/**
Christoph Hellwig36e78912008-02-08 04:21:24 -0800970 * do_generic_file_read - generic file read routine
Randy Dunlap485bb992006-06-23 02:03:49 -0700971 * @filp: the file to read
972 * @ppos: current file position
973 * @desc: read_descriptor
974 * @actor: read method
975 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 * This is a generic file read routine, and uses the
Randy Dunlap485bb992006-06-23 02:03:49 -0700977 * mapping->a_ops->readpage() function for the actual low-level stuff.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 *
979 * This is really ugly. But the goto's actually try to clarify some
980 * of the logic when it comes to error handling etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 */
Christoph Hellwig36e78912008-02-08 04:21:24 -0800982static void do_generic_file_read(struct file *filp, loff_t *ppos,
983 read_descriptor_t *desc, read_actor_t actor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984{
Christoph Hellwig36e78912008-02-08 04:21:24 -0800985 struct address_space *mapping = filp->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 struct inode *inode = mapping->host;
Christoph Hellwig36e78912008-02-08 04:21:24 -0800987 struct file_ra_state *ra = &filp->f_ra;
Fengguang Wu57f6b962007-10-16 01:24:37 -0700988 pgoff_t index;
989 pgoff_t last_index;
990 pgoff_t prev_index;
991 unsigned long offset; /* offset into pagecache page */
Jan Karaec0f1632007-05-06 14:49:25 -0700992 unsigned int prev_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 index = *ppos >> PAGE_CACHE_SHIFT;
Fengguang Wu7ff81072007-10-16 01:24:35 -0700996 prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
997 prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
999 offset = *ppos & ~PAGE_CACHE_MASK;
1000
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 for (;;) {
1002 struct page *page;
Fengguang Wu57f6b962007-10-16 01:24:37 -07001003 pgoff_t end_index;
NeilBrowna32ea1e2007-07-17 04:03:04 -07001004 loff_t isize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 unsigned long nr, ret;
1006
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008find_page:
1009 page = find_get_page(mapping, index);
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001010 if (!page) {
Rusty Russellcf914a72007-07-19 01:48:08 -07001011 page_cache_sync_readahead(mapping,
Fengguang Wu7ff81072007-10-16 01:24:35 -07001012 ra, filp,
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001013 index, last_index - index);
1014 page = find_get_page(mapping, index);
1015 if (unlikely(page == NULL))
1016 goto no_cached_page;
1017 }
1018 if (PageReadahead(page)) {
Rusty Russellcf914a72007-07-19 01:48:08 -07001019 page_cache_async_readahead(mapping,
Fengguang Wu7ff81072007-10-16 01:24:35 -07001020 ra, filp, page,
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001021 index, last_index - index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 }
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07001023 if (!PageUptodate(page)) {
1024 if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
1025 !mapping->a_ops->is_partially_uptodate)
1026 goto page_not_up_to_date;
Nick Piggin529ae9a2008-08-02 12:01:03 +02001027 if (!trylock_page(page))
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07001028 goto page_not_up_to_date;
1029 if (!mapping->a_ops->is_partially_uptodate(page,
1030 desc, offset))
1031 goto page_not_up_to_date_locked;
1032 unlock_page(page);
1033 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034page_ok:
NeilBrowna32ea1e2007-07-17 04:03:04 -07001035 /*
1036 * i_size must be checked after we know the page is Uptodate.
1037 *
1038 * Checking i_size after the check allows us to calculate
1039 * the correct value for "nr", which means the zero-filled
1040 * part of the page is not copied back to userspace (unless
1041 * another truncate extends the file - this is desired though).
1042 */
1043
1044 isize = i_size_read(inode);
1045 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1046 if (unlikely(!isize || index > end_index)) {
1047 page_cache_release(page);
1048 goto out;
1049 }
1050
1051 /* nr is the maximum number of bytes to copy from this page */
1052 nr = PAGE_CACHE_SIZE;
1053 if (index == end_index) {
1054 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1055 if (nr <= offset) {
1056 page_cache_release(page);
1057 goto out;
1058 }
1059 }
1060 nr = nr - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
1062 /* If users can be writing to this page using arbitrary
1063 * virtual addresses, take care about potential aliasing
1064 * before reading the page on the kernel side.
1065 */
1066 if (mapping_writably_mapped(mapping))
1067 flush_dcache_page(page);
1068
1069 /*
Jan Karaec0f1632007-05-06 14:49:25 -07001070 * When a sequential read accesses a page several times,
1071 * only mark it as accessed the first time.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 */
Jan Karaec0f1632007-05-06 14:49:25 -07001073 if (prev_index != index || offset != prev_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 mark_page_accessed(page);
1075 prev_index = index;
1076
1077 /*
1078 * Ok, we have the page, and it's up-to-date, so
1079 * now we can copy it to user space...
1080 *
1081 * The actor routine returns how many bytes were actually used..
1082 * NOTE! This may not be the same as how much of a user buffer
1083 * we filled up (we may be padding etc), so we can only update
1084 * "pos" here (the actor routine has to update the user buffer
1085 * pointers and the remaining count).
1086 */
1087 ret = actor(desc, page, offset, nr);
1088 offset += ret;
1089 index += offset >> PAGE_CACHE_SHIFT;
1090 offset &= ~PAGE_CACHE_MASK;
Jan Kara6ce745e2007-05-06 14:49:26 -07001091 prev_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092
1093 page_cache_release(page);
1094 if (ret == nr && desc->count)
1095 continue;
1096 goto out;
1097
1098page_not_up_to_date:
1099 /* Get exclusive access to the page ... */
Oleg Nesterov85462322008-06-08 21:20:43 +04001100 error = lock_page_killable(page);
1101 if (unlikely(error))
1102 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07001104page_not_up_to_date_locked:
Nick Pigginda6052f2006-09-25 23:31:35 -07001105 /* Did it get truncated before we got the lock? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106 if (!page->mapping) {
1107 unlock_page(page);
1108 page_cache_release(page);
1109 continue;
1110 }
1111
1112 /* Did somebody else fill it already? */
1113 if (PageUptodate(page)) {
1114 unlock_page(page);
1115 goto page_ok;
1116 }
1117
1118readpage:
1119 /* Start the actual read. The read will unlock the page. */
1120 error = mapping->a_ops->readpage(filp, page);
1121
Zach Brown994fc28c2005-12-15 14:28:17 -08001122 if (unlikely(error)) {
1123 if (error == AOP_TRUNCATED_PAGE) {
1124 page_cache_release(page);
1125 goto find_page;
1126 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 goto readpage_error;
Zach Brown994fc28c2005-12-15 14:28:17 -08001128 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129
1130 if (!PageUptodate(page)) {
Oleg Nesterov85462322008-06-08 21:20:43 +04001131 error = lock_page_killable(page);
1132 if (unlikely(error))
1133 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 if (!PageUptodate(page)) {
1135 if (page->mapping == NULL) {
1136 /*
1137 * invalidate_inode_pages got it
1138 */
1139 unlock_page(page);
1140 page_cache_release(page);
1141 goto find_page;
1142 }
1143 unlock_page(page);
Fengguang Wu7ff81072007-10-16 01:24:35 -07001144 shrink_readahead_size_eio(filp, ra);
Oleg Nesterov85462322008-06-08 21:20:43 +04001145 error = -EIO;
1146 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 }
1148 unlock_page(page);
1149 }
1150
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151 goto page_ok;
1152
1153readpage_error:
1154 /* UHHUH! A synchronous read error occurred. Report it */
1155 desc->error = error;
1156 page_cache_release(page);
1157 goto out;
1158
1159no_cached_page:
1160 /*
1161 * Ok, it wasn't cached, so we need to create a new
1162 * page..
1163 */
Nick Piggineb2be182007-10-16 01:24:57 -07001164 page = page_cache_alloc_cold(mapping);
1165 if (!page) {
1166 desc->error = -ENOMEM;
1167 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168 }
Nick Piggineb2be182007-10-16 01:24:57 -07001169 error = add_to_page_cache_lru(page, mapping,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170 index, GFP_KERNEL);
1171 if (error) {
Nick Piggineb2be182007-10-16 01:24:57 -07001172 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 if (error == -EEXIST)
1174 goto find_page;
1175 desc->error = error;
1176 goto out;
1177 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 goto readpage;
1179 }
1180
1181out:
Fengguang Wu7ff81072007-10-16 01:24:35 -07001182 ra->prev_pos = prev_index;
1183 ra->prev_pos <<= PAGE_CACHE_SHIFT;
1184 ra->prev_pos |= prev_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185
Fengguang Wuf4e6b492007-10-16 01:24:33 -07001186 *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
Krishna Kumar0c6aa262008-10-15 22:01:13 -07001187 file_accessed(filp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189
1190int file_read_actor(read_descriptor_t *desc, struct page *page,
1191 unsigned long offset, unsigned long size)
1192{
1193 char *kaddr;
1194 unsigned long left, count = desc->count;
1195
1196 if (size > count)
1197 size = count;
1198
1199 /*
1200 * Faults on the destination of a read are common, so do it before
1201 * taking the kmap.
1202 */
1203 if (!fault_in_pages_writeable(desc->arg.buf, size)) {
1204 kaddr = kmap_atomic(page, KM_USER0);
1205 left = __copy_to_user_inatomic(desc->arg.buf,
1206 kaddr + offset, size);
1207 kunmap_atomic(kaddr, KM_USER0);
1208 if (left == 0)
1209 goto success;
1210 }
1211
1212 /* Do it the slow way */
1213 kaddr = kmap(page);
1214 left = __copy_to_user(desc->arg.buf, kaddr + offset, size);
1215 kunmap(page);
1216
1217 if (left) {
1218 size -= left;
1219 desc->error = -EFAULT;
1220 }
1221success:
1222 desc->count = count - size;
1223 desc->written += size;
1224 desc->arg.buf += size;
1225 return size;
1226}
1227
Dmitriy Monakhov0ceb3312007-05-08 00:23:02 -07001228/*
1229 * Performs necessary checks before doing a write
1230 * @iov: io vector request
1231 * @nr_segs: number of segments in the iovec
1232 * @count: number of bytes to write
1233 * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
1234 *
1235 * Adjust number of segments and amount of bytes to write (nr_segs should be
1236 * properly initialized first). Returns appropriate error code that caller
1237 * should return or zero in case that write should be allowed.
1238 */
1239int generic_segment_checks(const struct iovec *iov,
1240 unsigned long *nr_segs, size_t *count, int access_flags)
1241{
1242 unsigned long seg;
1243 size_t cnt = 0;
1244 for (seg = 0; seg < *nr_segs; seg++) {
1245 const struct iovec *iv = &iov[seg];
1246
1247 /*
1248 * If any segment has a negative length, or the cumulative
1249 * length ever wraps negative then return -EINVAL.
1250 */
1251 cnt += iv->iov_len;
1252 if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
1253 return -EINVAL;
1254 if (access_ok(access_flags, iv->iov_base, iv->iov_len))
1255 continue;
1256 if (seg == 0)
1257 return -EFAULT;
1258 *nr_segs = seg;
1259 cnt -= iv->iov_len; /* This segment is no good */
1260 break;
1261 }
1262 *count = cnt;
1263 return 0;
1264}
1265EXPORT_SYMBOL(generic_segment_checks);
1266
Randy Dunlap485bb992006-06-23 02:03:49 -07001267/**
Henrik Kretzschmarb2abacf2006-10-04 02:15:22 -07001268 * generic_file_aio_read - generic filesystem read routine
Randy Dunlap485bb992006-06-23 02:03:49 -07001269 * @iocb: kernel I/O control block
1270 * @iov: io vector request
1271 * @nr_segs: number of segments in the iovec
Henrik Kretzschmarb2abacf2006-10-04 02:15:22 -07001272 * @pos: current file position
Randy Dunlap485bb992006-06-23 02:03:49 -07001273 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 * This is the "read()" routine for all filesystems
1275 * that can use the page cache directly.
1276 */
1277ssize_t
Badari Pulavarty543ade12006-09-30 23:28:48 -07001278generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
1279 unsigned long nr_segs, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280{
1281 struct file *filp = iocb->ki_filp;
1282 ssize_t retval;
1283 unsigned long seg;
1284 size_t count;
Badari Pulavarty543ade12006-09-30 23:28:48 -07001285 loff_t *ppos = &iocb->ki_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286
1287 count = 0;
Dmitriy Monakhov0ceb3312007-05-08 00:23:02 -07001288 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1289 if (retval)
1290 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291
1292 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1293 if (filp->f_flags & O_DIRECT) {
Badari Pulavarty543ade12006-09-30 23:28:48 -07001294 loff_t size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 struct address_space *mapping;
1296 struct inode *inode;
1297
1298 mapping = filp->f_mapping;
1299 inode = mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 if (!count)
1301 goto out; /* skip atime */
1302 size = i_size_read(inode);
1303 if (pos < size) {
Nick Piggin48b47c52009-01-06 14:40:22 -08001304 retval = filemap_write_and_wait_range(mapping, pos,
1305 pos + iov_length(iov, nr_segs) - 1);
Christoph Hellwiga969e902008-07-23 21:27:04 -07001306 if (!retval) {
1307 retval = mapping->a_ops->direct_IO(READ, iocb,
1308 iov, pos, nr_segs);
1309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 if (retval > 0)
1311 *ppos = pos + retval;
Hugh Dickins11fa9772008-07-23 21:27:34 -07001312 if (retval) {
1313 file_accessed(filp);
1314 goto out;
1315 }
Steven Whitehouse0e0bcae2006-09-27 14:45:07 -04001316 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 }
1318
Hugh Dickins11fa9772008-07-23 21:27:34 -07001319 for (seg = 0; seg < nr_segs; seg++) {
1320 read_descriptor_t desc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321
Hugh Dickins11fa9772008-07-23 21:27:34 -07001322 desc.written = 0;
1323 desc.arg.buf = iov[seg].iov_base;
1324 desc.count = iov[seg].iov_len;
1325 if (desc.count == 0)
1326 continue;
1327 desc.error = 0;
1328 do_generic_file_read(filp, ppos, &desc, file_read_actor);
1329 retval += desc.written;
1330 if (desc.error) {
1331 retval = retval ?: desc.error;
1332 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 }
Hugh Dickins11fa9772008-07-23 21:27:34 -07001334 if (desc.count > 0)
1335 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 }
1337out:
1338 return retval;
1339}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340EXPORT_SYMBOL(generic_file_aio_read);
1341
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342static ssize_t
1343do_readahead(struct address_space *mapping, struct file *filp,
Fengguang Wu57f6b962007-10-16 01:24:37 -07001344 pgoff_t index, unsigned long nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345{
1346 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
1347 return -EINVAL;
1348
Wu Fengguangf7e839d2009-06-16 15:31:20 -07001349 force_page_cache_readahead(mapping, filp, index, nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 return 0;
1351}
1352
Heiko Carstens6673e0c2009-01-14 14:14:02 +01001353SYSCALL_DEFINE(readahead)(int fd, loff_t offset, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354{
1355 ssize_t ret;
1356 struct file *file;
1357
1358 ret = -EBADF;
1359 file = fget(fd);
1360 if (file) {
1361 if (file->f_mode & FMODE_READ) {
1362 struct address_space *mapping = file->f_mapping;
Fengguang Wu57f6b962007-10-16 01:24:37 -07001363 pgoff_t start = offset >> PAGE_CACHE_SHIFT;
1364 pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 unsigned long len = end - start + 1;
1366 ret = do_readahead(mapping, file, start, len);
1367 }
1368 fput(file);
1369 }
1370 return ret;
1371}
Heiko Carstens6673e0c2009-01-14 14:14:02 +01001372#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
1373asmlinkage long SyS_readahead(long fd, loff_t offset, long count)
1374{
1375 return SYSC_readahead((int) fd, offset, (size_t) count);
1376}
1377SYSCALL_ALIAS(sys_readahead, SyS_readahead);
1378#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379
1380#ifdef CONFIG_MMU
Randy Dunlap485bb992006-06-23 02:03:49 -07001381/**
1382 * page_cache_read - adds requested page to the page cache if not already there
1383 * @file: file to read
1384 * @offset: page index
1385 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 * This adds the requested page to the page cache if it isn't already there,
1387 * and schedules an I/O to read in its contents from disk.
1388 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08001389static int page_cache_read(struct file *file, pgoff_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390{
1391 struct address_space *mapping = file->f_mapping;
1392 struct page *page;
Zach Brown994fc28c2005-12-15 14:28:17 -08001393 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Zach Brown994fc28c2005-12-15 14:28:17 -08001395 do {
1396 page = page_cache_alloc_cold(mapping);
1397 if (!page)
1398 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399
Zach Brown994fc28c2005-12-15 14:28:17 -08001400 ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
1401 if (ret == 0)
1402 ret = mapping->a_ops->readpage(file, page);
1403 else if (ret == -EEXIST)
1404 ret = 0; /* losing race to add is OK */
1405
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
Zach Brown994fc28c2005-12-15 14:28:17 -08001408 } while (ret == AOP_TRUNCATED_PAGE);
1409
1410 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411}
1412
1413#define MMAP_LOTSAMISS (100)
1414
Linus Torvaldsef00e082009-06-16 15:31:25 -07001415/*
1416 * Synchronous readahead happens when we don't even find
1417 * a page in the page cache at all.
1418 */
1419static void do_sync_mmap_readahead(struct vm_area_struct *vma,
1420 struct file_ra_state *ra,
1421 struct file *file,
1422 pgoff_t offset)
1423{
1424 unsigned long ra_pages;
1425 struct address_space *mapping = file->f_mapping;
1426
1427 /* If we don't want any read-ahead, don't bother */
1428 if (VM_RandomReadHint(vma))
1429 return;
1430
Wu Fengguang70ac23c2009-06-16 15:31:28 -07001431 if (VM_SequentialReadHint(vma) ||
1432 offset - 1 == (ra->prev_pos >> PAGE_CACHE_SHIFT)) {
Wu Fengguang7ffc59b2009-06-16 15:31:38 -07001433 page_cache_sync_readahead(mapping, ra, file, offset,
1434 ra->ra_pages);
Linus Torvaldsef00e082009-06-16 15:31:25 -07001435 return;
1436 }
1437
1438 if (ra->mmap_miss < INT_MAX)
1439 ra->mmap_miss++;
1440
1441 /*
1442 * Do we miss much more than hit in this file? If so,
1443 * stop bothering with read-ahead. It will only hurt.
1444 */
1445 if (ra->mmap_miss > MMAP_LOTSAMISS)
1446 return;
1447
Wu Fengguangd30a1102009-06-16 15:31:30 -07001448 /*
1449 * mmap read-around
1450 */
Linus Torvaldsef00e082009-06-16 15:31:25 -07001451 ra_pages = max_sane_readahead(ra->ra_pages);
1452 if (ra_pages) {
Wu Fengguangd30a1102009-06-16 15:31:30 -07001453 ra->start = max_t(long, 0, offset - ra_pages/2);
1454 ra->size = ra_pages;
1455 ra->async_size = 0;
1456 ra_submit(ra, mapping, file);
Linus Torvaldsef00e082009-06-16 15:31:25 -07001457 }
1458}
1459
1460/*
1461 * Asynchronous readahead happens when we find the page and PG_readahead,
1462 * so we want to possibly extend the readahead further..
1463 */
1464static void do_async_mmap_readahead(struct vm_area_struct *vma,
1465 struct file_ra_state *ra,
1466 struct file *file,
1467 struct page *page,
1468 pgoff_t offset)
1469{
1470 struct address_space *mapping = file->f_mapping;
1471
1472 /* If we don't want any read-ahead, don't bother */
1473 if (VM_RandomReadHint(vma))
1474 return;
1475 if (ra->mmap_miss > 0)
1476 ra->mmap_miss--;
1477 if (PageReadahead(page))
Wu Fengguang2fad6f52009-06-16 15:31:29 -07001478 page_cache_async_readahead(mapping, ra, file,
1479 page, offset, ra->ra_pages);
Linus Torvaldsef00e082009-06-16 15:31:25 -07001480}
1481
Randy Dunlap485bb992006-06-23 02:03:49 -07001482/**
Nick Piggin54cb8822007-07-19 01:46:59 -07001483 * filemap_fault - read in file data for page fault handling
Nick Piggind0217ac2007-07-19 01:47:03 -07001484 * @vma: vma in which the fault was taken
1485 * @vmf: struct vm_fault containing details of the fault
Randy Dunlap485bb992006-06-23 02:03:49 -07001486 *
Nick Piggin54cb8822007-07-19 01:46:59 -07001487 * filemap_fault() is invoked via the vma operations vector for a
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 * mapped memory region to read in file data during a page fault.
1489 *
1490 * The goto's are kind of ugly, but this streamlines the normal case of having
1491 * it in the page cache, and handles the special cases reasonably without
1492 * having a lot of duplicated code.
1493 */
Nick Piggind0217ac2007-07-19 01:47:03 -07001494int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495{
1496 int error;
Nick Piggin54cb8822007-07-19 01:46:59 -07001497 struct file *file = vma->vm_file;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498 struct address_space *mapping = file->f_mapping;
1499 struct file_ra_state *ra = &file->f_ra;
1500 struct inode *inode = mapping->host;
Linus Torvaldsef00e082009-06-16 15:31:25 -07001501 pgoff_t offset = vmf->pgoff;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 struct page *page;
Jan Kara2004dc82008-02-08 04:20:11 -08001503 pgoff_t size;
Nick Piggin83c54072007-07-19 01:47:05 -07001504 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Linus Torvaldsef00e082009-06-16 15:31:25 -07001507 if (offset >= size)
Linus Torvalds5307cc12007-10-31 09:19:46 -07001508 return VM_FAULT_SIGBUS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 * Do we have something in the page cache already?
1512 */
Linus Torvaldsef00e082009-06-16 15:31:25 -07001513 page = find_get_page(mapping, offset);
1514 if (likely(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 /*
Linus Torvaldsef00e082009-06-16 15:31:25 -07001516 * We found the page, so try async readahead before
1517 * waiting for the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518 */
Linus Torvaldsef00e082009-06-16 15:31:25 -07001519 do_async_mmap_readahead(vma, ra, file, page, offset);
1520 lock_page(page);
1521
1522 /* Did it get truncated? */
1523 if (unlikely(page->mapping != mapping)) {
1524 unlock_page(page);
1525 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 goto no_cached_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 }
Linus Torvaldsef00e082009-06-16 15:31:25 -07001528 } else {
1529 /* No page in the page cache at all */
1530 do_sync_mmap_readahead(vma, ra, file, offset);
1531 count_vm_event(PGMAJFAULT);
1532 ret = VM_FAULT_MAJOR;
1533retry_find:
1534 page = find_lock_page(mapping, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 if (!page)
1536 goto no_cached_page;
1537 }
1538
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 /*
Nick Piggind00806b2007-07-19 01:46:57 -07001540 * We have a locked page in the page cache, now we need to check
1541 * that it's up-to-date. If not, it is going to be due to an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 */
Nick Piggind00806b2007-07-19 01:46:57 -07001543 if (unlikely(!PageUptodate(page)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 goto page_not_uptodate;
1545
Linus Torvaldsef00e082009-06-16 15:31:25 -07001546 /*
1547 * Found the page and have a reference on it.
1548 * We must recheck i_size under page lock.
1549 */
Nick Piggind00806b2007-07-19 01:46:57 -07001550 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Linus Torvaldsef00e082009-06-16 15:31:25 -07001551 if (unlikely(offset >= size)) {
Nick Piggind00806b2007-07-19 01:46:57 -07001552 unlock_page(page);
Yan Zheng745ad482007-10-08 10:08:37 -07001553 page_cache_release(page);
Linus Torvalds5307cc12007-10-31 09:19:46 -07001554 return VM_FAULT_SIGBUS;
Nick Piggind00806b2007-07-19 01:46:57 -07001555 }
1556
Linus Torvaldsef00e082009-06-16 15:31:25 -07001557 ra->prev_pos = (loff_t)offset << PAGE_CACHE_SHIFT;
Nick Piggind0217ac2007-07-19 01:47:03 -07001558 vmf->page = page;
Nick Piggin83c54072007-07-19 01:47:05 -07001559 return ret | VM_FAULT_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561no_cached_page:
1562 /*
1563 * We're only likely to ever get here if MADV_RANDOM is in
1564 * effect.
1565 */
Linus Torvaldsef00e082009-06-16 15:31:25 -07001566 error = page_cache_read(file, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567
1568 /*
1569 * The page we want has now been added to the page cache.
1570 * In the unlikely event that someone removed it in the
1571 * meantime, we'll just come back here and read it again.
1572 */
1573 if (error >= 0)
1574 goto retry_find;
1575
1576 /*
1577 * An error return from page_cache_read can result if the
1578 * system is low on memory, or a problem occurs while trying
1579 * to schedule I/O.
1580 */
1581 if (error == -ENOMEM)
Nick Piggind0217ac2007-07-19 01:47:03 -07001582 return VM_FAULT_OOM;
1583 return VM_FAULT_SIGBUS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584
1585page_not_uptodate:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 /*
1587 * Umm, take care of errors if the page isn't up-to-date.
1588 * Try to re-read it _once_. We do this synchronously,
1589 * because there really aren't any performance issues here
1590 * and we need to check for errors.
1591 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 ClearPageError(page);
Zach Brown994fc28c2005-12-15 14:28:17 -08001593 error = mapping->a_ops->readpage(file, page);
Miklos Szeredi3ef0f722008-05-14 16:05:37 -07001594 if (!error) {
1595 wait_on_page_locked(page);
1596 if (!PageUptodate(page))
1597 error = -EIO;
1598 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 page_cache_release(page);
Nick Piggind00806b2007-07-19 01:46:57 -07001600
1601 if (!error || error == AOP_TRUNCATED_PAGE)
1602 goto retry_find;
1603
1604 /* Things didn't work out. Return zero to tell the mm layer so. */
1605 shrink_readahead_size_eio(file, ra);
Nick Piggind0217ac2007-07-19 01:47:03 -07001606 return VM_FAULT_SIGBUS;
Nick Piggin54cb8822007-07-19 01:46:59 -07001607}
1608EXPORT_SYMBOL(filemap_fault);
1609
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610struct vm_operations_struct generic_file_vm_ops = {
Nick Piggin54cb8822007-07-19 01:46:59 -07001611 .fault = filemap_fault,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612};
1613
1614/* This is used for a general mmap of a disk file */
1615
1616int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1617{
1618 struct address_space *mapping = file->f_mapping;
1619
1620 if (!mapping->a_ops->readpage)
1621 return -ENOEXEC;
1622 file_accessed(file);
1623 vma->vm_ops = &generic_file_vm_ops;
Nick Piggind0217ac2007-07-19 01:47:03 -07001624 vma->vm_flags |= VM_CAN_NONLINEAR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 return 0;
1626}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627
1628/*
1629 * This is for filesystems which do not implement ->writepage.
1630 */
1631int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
1632{
1633 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
1634 return -EINVAL;
1635 return generic_file_mmap(file, vma);
1636}
1637#else
1638int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1639{
1640 return -ENOSYS;
1641}
1642int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
1643{
1644 return -ENOSYS;
1645}
1646#endif /* CONFIG_MMU */
1647
1648EXPORT_SYMBOL(generic_file_mmap);
1649EXPORT_SYMBOL(generic_file_readonly_mmap);
1650
Nick Piggin6fe69002007-05-06 14:49:04 -07001651static struct page *__read_cache_page(struct address_space *mapping,
Fengguang Wu57f6b962007-10-16 01:24:37 -07001652 pgoff_t index,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653 int (*filler)(void *,struct page*),
1654 void *data)
1655{
Nick Piggineb2be182007-10-16 01:24:57 -07001656 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 int err;
1658repeat:
1659 page = find_get_page(mapping, index);
1660 if (!page) {
Nick Piggineb2be182007-10-16 01:24:57 -07001661 page = page_cache_alloc_cold(mapping);
1662 if (!page)
1663 return ERR_PTR(-ENOMEM);
1664 err = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL);
1665 if (unlikely(err)) {
1666 page_cache_release(page);
1667 if (err == -EEXIST)
1668 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 /* Presumably ENOMEM for radix tree node */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 return ERR_PTR(err);
1671 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 err = filler(data, page);
1673 if (err < 0) {
1674 page_cache_release(page);
1675 page = ERR_PTR(err);
1676 }
1677 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678 return page;
1679}
1680
Randy Dunlap76824862008-03-19 17:00:40 -07001681/**
1682 * read_cache_page_async - read into page cache, fill it if needed
1683 * @mapping: the page's address_space
1684 * @index: the page index
1685 * @filler: function to perform the read
1686 * @data: destination for read data
1687 *
Nick Piggin6fe69002007-05-06 14:49:04 -07001688 * Same as read_cache_page, but don't wait for page to become unlocked
1689 * after submitting it to the filler.
Randy Dunlap76824862008-03-19 17:00:40 -07001690 *
1691 * Read into the page cache. If a page already exists, and PageUptodate() is
1692 * not set, try to fill the page but don't wait for it to become unlocked.
1693 *
1694 * If the page does not get brought uptodate, return -EIO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 */
Nick Piggin6fe69002007-05-06 14:49:04 -07001696struct page *read_cache_page_async(struct address_space *mapping,
Fengguang Wu57f6b962007-10-16 01:24:37 -07001697 pgoff_t index,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 int (*filler)(void *,struct page*),
1699 void *data)
1700{
1701 struct page *page;
1702 int err;
1703
1704retry:
1705 page = __read_cache_page(mapping, index, filler, data);
1706 if (IS_ERR(page))
David Howellsc855ff32007-05-09 13:42:20 +01001707 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708 if (PageUptodate(page))
1709 goto out;
1710
1711 lock_page(page);
1712 if (!page->mapping) {
1713 unlock_page(page);
1714 page_cache_release(page);
1715 goto retry;
1716 }
1717 if (PageUptodate(page)) {
1718 unlock_page(page);
1719 goto out;
1720 }
1721 err = filler(data, page);
1722 if (err < 0) {
1723 page_cache_release(page);
David Howellsc855ff32007-05-09 13:42:20 +01001724 return ERR_PTR(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725 }
David Howellsc855ff32007-05-09 13:42:20 +01001726out:
Nick Piggin6fe69002007-05-06 14:49:04 -07001727 mark_page_accessed(page);
1728 return page;
1729}
1730EXPORT_SYMBOL(read_cache_page_async);
1731
1732/**
1733 * read_cache_page - read into page cache, fill it if needed
1734 * @mapping: the page's address_space
1735 * @index: the page index
1736 * @filler: function to perform the read
1737 * @data: destination for read data
1738 *
1739 * Read into the page cache. If a page already exists, and PageUptodate() is
1740 * not set, try to fill the page then wait for it to become unlocked.
1741 *
1742 * If the page does not get brought uptodate, return -EIO.
1743 */
1744struct page *read_cache_page(struct address_space *mapping,
Fengguang Wu57f6b962007-10-16 01:24:37 -07001745 pgoff_t index,
Nick Piggin6fe69002007-05-06 14:49:04 -07001746 int (*filler)(void *,struct page*),
1747 void *data)
1748{
1749 struct page *page;
1750
1751 page = read_cache_page_async(mapping, index, filler, data);
1752 if (IS_ERR(page))
1753 goto out;
1754 wait_on_page_locked(page);
1755 if (!PageUptodate(page)) {
1756 page_cache_release(page);
1757 page = ERR_PTR(-EIO);
1758 }
1759 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760 return page;
1761}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762EXPORT_SYMBOL(read_cache_page);
1763
1764/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 * The logic we want is
1766 *
1767 * if suid or (sgid and xgrp)
1768 * remove privs
1769 */
Jens Axboe01de85e2006-10-17 19:50:36 +02001770int should_remove_suid(struct dentry *dentry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771{
1772 mode_t mode = dentry->d_inode->i_mode;
1773 int kill = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774
1775 /* suid always must be killed */
1776 if (unlikely(mode & S_ISUID))
1777 kill = ATTR_KILL_SUID;
1778
1779 /*
1780 * sgid without any exec bits is just a mandatory locking mark; leave
1781 * it alone. If some exec bits are set, it's a real sgid; kill it.
1782 */
1783 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1784 kill |= ATTR_KILL_SGID;
1785
Dmitri Monakhov7f5ff762008-12-01 14:34:56 -08001786 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
Jens Axboe01de85e2006-10-17 19:50:36 +02001787 return kill;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788
Jens Axboe01de85e2006-10-17 19:50:36 +02001789 return 0;
1790}
Mark Fashehd23a1472006-10-17 17:05:18 -07001791EXPORT_SYMBOL(should_remove_suid);
Jens Axboe01de85e2006-10-17 19:50:36 +02001792
Miklos Szeredi7f3d4ee2008-05-07 09:22:39 +02001793static int __remove_suid(struct dentry *dentry, int kill)
Jens Axboe01de85e2006-10-17 19:50:36 +02001794{
1795 struct iattr newattrs;
1796
1797 newattrs.ia_valid = ATTR_FORCE | kill;
1798 return notify_change(dentry, &newattrs);
1799}
1800
Miklos Szeredi2f1936b2008-06-24 16:50:14 +02001801int file_remove_suid(struct file *file)
Jens Axboe01de85e2006-10-17 19:50:36 +02001802{
Miklos Szeredi2f1936b2008-06-24 16:50:14 +02001803 struct dentry *dentry = file->f_path.dentry;
Serge E. Hallynb5376772007-10-16 23:31:36 -07001804 int killsuid = should_remove_suid(dentry);
1805 int killpriv = security_inode_need_killpriv(dentry);
1806 int error = 0;
Jens Axboe01de85e2006-10-17 19:50:36 +02001807
Serge E. Hallynb5376772007-10-16 23:31:36 -07001808 if (killpriv < 0)
1809 return killpriv;
1810 if (killpriv)
1811 error = security_inode_killpriv(dentry);
1812 if (!error && killsuid)
1813 error = __remove_suid(dentry, killsuid);
Jens Axboe01de85e2006-10-17 19:50:36 +02001814
Serge E. Hallynb5376772007-10-16 23:31:36 -07001815 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001816}
Miklos Szeredi2f1936b2008-06-24 16:50:14 +02001817EXPORT_SYMBOL(file_remove_suid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818
Nick Piggin2f718ff2007-10-16 01:24:59 -07001819static size_t __iovec_copy_from_user_inatomic(char *vaddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820 const struct iovec *iov, size_t base, size_t bytes)
1821{
Ingo Molnarf1800532009-03-02 11:00:57 +01001822 size_t copied = 0, left = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823
1824 while (bytes) {
1825 char __user *buf = iov->iov_base + base;
1826 int copy = min(bytes, iov->iov_len - base);
1827
1828 base = 0;
Ingo Molnarf1800532009-03-02 11:00:57 +01001829 left = __copy_from_user_inatomic(vaddr, buf, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 copied += copy;
1831 bytes -= copy;
1832 vaddr += copy;
1833 iov++;
1834
NeilBrown01408c42006-06-25 05:47:58 -07001835 if (unlikely(left))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001837 }
1838 return copied - left;
1839}
1840
1841/*
Nick Piggin2f718ff2007-10-16 01:24:59 -07001842 * Copy as much as we can into the page and return the number of bytes which
1843 * were sucessfully copied. If a fault is encountered then return the number of
1844 * bytes which were copied.
1845 */
1846size_t iov_iter_copy_from_user_atomic(struct page *page,
1847 struct iov_iter *i, unsigned long offset, size_t bytes)
1848{
1849 char *kaddr;
1850 size_t copied;
1851
1852 BUG_ON(!in_atomic());
1853 kaddr = kmap_atomic(page, KM_USER0);
1854 if (likely(i->nr_segs == 1)) {
1855 int left;
1856 char __user *buf = i->iov->iov_base + i->iov_offset;
Ingo Molnarf1800532009-03-02 11:00:57 +01001857 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001858 copied = bytes - left;
1859 } else {
1860 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
1861 i->iov, i->iov_offset, bytes);
1862 }
1863 kunmap_atomic(kaddr, KM_USER0);
1864
1865 return copied;
1866}
Nick Piggin89e10782007-10-16 01:25:07 -07001867EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001868
1869/*
1870 * This has the same sideeffects and return value as
1871 * iov_iter_copy_from_user_atomic().
1872 * The difference is that it attempts to resolve faults.
1873 * Page must not be locked.
1874 */
1875size_t iov_iter_copy_from_user(struct page *page,
1876 struct iov_iter *i, unsigned long offset, size_t bytes)
1877{
1878 char *kaddr;
1879 size_t copied;
1880
1881 kaddr = kmap(page);
1882 if (likely(i->nr_segs == 1)) {
1883 int left;
1884 char __user *buf = i->iov->iov_base + i->iov_offset;
Ingo Molnarf1800532009-03-02 11:00:57 +01001885 left = __copy_from_user(kaddr + offset, buf, bytes);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001886 copied = bytes - left;
1887 } else {
1888 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
1889 i->iov, i->iov_offset, bytes);
1890 }
1891 kunmap(page);
1892 return copied;
1893}
Nick Piggin89e10782007-10-16 01:25:07 -07001894EXPORT_SYMBOL(iov_iter_copy_from_user);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001895
Nick Pigginf7009262008-03-10 11:43:59 -07001896void iov_iter_advance(struct iov_iter *i, size_t bytes)
Nick Piggin2f718ff2007-10-16 01:24:59 -07001897{
Nick Pigginf7009262008-03-10 11:43:59 -07001898 BUG_ON(i->count < bytes);
1899
Nick Piggin2f718ff2007-10-16 01:24:59 -07001900 if (likely(i->nr_segs == 1)) {
1901 i->iov_offset += bytes;
Nick Pigginf7009262008-03-10 11:43:59 -07001902 i->count -= bytes;
Nick Piggin2f718ff2007-10-16 01:24:59 -07001903 } else {
1904 const struct iovec *iov = i->iov;
1905 size_t base = i->iov_offset;
1906
Nick Piggin124d3b72008-02-02 15:01:17 +01001907 /*
1908 * The !iov->iov_len check ensures we skip over unlikely
Nick Pigginf7009262008-03-10 11:43:59 -07001909 * zero-length segments (without overruning the iovec).
Nick Piggin124d3b72008-02-02 15:01:17 +01001910 */
Linus Torvalds94ad3742008-07-30 14:45:12 -07001911 while (bytes || unlikely(i->count && !iov->iov_len)) {
Nick Pigginf7009262008-03-10 11:43:59 -07001912 int copy;
Nick Piggin2f718ff2007-10-16 01:24:59 -07001913
Nick Pigginf7009262008-03-10 11:43:59 -07001914 copy = min(bytes, iov->iov_len - base);
1915 BUG_ON(!i->count || i->count < copy);
1916 i->count -= copy;
Nick Piggin2f718ff2007-10-16 01:24:59 -07001917 bytes -= copy;
1918 base += copy;
1919 if (iov->iov_len == base) {
1920 iov++;
1921 base = 0;
1922 }
1923 }
1924 i->iov = iov;
1925 i->iov_offset = base;
1926 }
1927}
Nick Piggin89e10782007-10-16 01:25:07 -07001928EXPORT_SYMBOL(iov_iter_advance);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001929
Nick Pigginafddba42007-10-16 01:25:01 -07001930/*
1931 * Fault in the first iovec of the given iov_iter, to a maximum length
1932 * of bytes. Returns 0 on success, or non-zero if the memory could not be
1933 * accessed (ie. because it is an invalid address).
1934 *
1935 * writev-intensive code may want this to prefault several iovecs -- that
1936 * would be possible (callers must not rely on the fact that _only_ the
1937 * first iovec will be faulted with the current implementation).
1938 */
1939int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
Nick Piggin2f718ff2007-10-16 01:24:59 -07001940{
Nick Piggin2f718ff2007-10-16 01:24:59 -07001941 char __user *buf = i->iov->iov_base + i->iov_offset;
Nick Pigginafddba42007-10-16 01:25:01 -07001942 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
1943 return fault_in_pages_readable(buf, bytes);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001944}
Nick Piggin89e10782007-10-16 01:25:07 -07001945EXPORT_SYMBOL(iov_iter_fault_in_readable);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001946
1947/*
1948 * Return the count of just the current iov_iter segment.
1949 */
1950size_t iov_iter_single_seg_count(struct iov_iter *i)
1951{
1952 const struct iovec *iov = i->iov;
1953 if (i->nr_segs == 1)
1954 return i->count;
1955 else
1956 return min(i->count, iov->iov_len - i->iov_offset);
1957}
Nick Piggin89e10782007-10-16 01:25:07 -07001958EXPORT_SYMBOL(iov_iter_single_seg_count);
Nick Piggin2f718ff2007-10-16 01:24:59 -07001959
1960/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961 * Performs necessary checks before doing a write
1962 *
Randy Dunlap485bb992006-06-23 02:03:49 -07001963 * Can adjust writing position or amount of bytes to write.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964 * Returns appropriate error code that caller should return or
1965 * zero in case that write should be allowed.
1966 */
1967inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
1968{
1969 struct inode *inode = file->f_mapping->host;
1970 unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
1971
1972 if (unlikely(*pos < 0))
1973 return -EINVAL;
1974
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975 if (!isblk) {
1976 /* FIXME: this is for backwards compatibility with 2.4 */
1977 if (file->f_flags & O_APPEND)
1978 *pos = i_size_read(inode);
1979
1980 if (limit != RLIM_INFINITY) {
1981 if (*pos >= limit) {
1982 send_sig(SIGXFSZ, current, 0);
1983 return -EFBIG;
1984 }
1985 if (*count > limit - (typeof(limit))*pos) {
1986 *count = limit - (typeof(limit))*pos;
1987 }
1988 }
1989 }
1990
1991 /*
1992 * LFS rule
1993 */
1994 if (unlikely(*pos + *count > MAX_NON_LFS &&
1995 !(file->f_flags & O_LARGEFILE))) {
1996 if (*pos >= MAX_NON_LFS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997 return -EFBIG;
1998 }
1999 if (*count > MAX_NON_LFS - (unsigned long)*pos) {
2000 *count = MAX_NON_LFS - (unsigned long)*pos;
2001 }
2002 }
2003
2004 /*
2005 * Are we about to exceed the fs block limit ?
2006 *
2007 * If we have written data it becomes a short write. If we have
2008 * exceeded without writing data we send a signal and return EFBIG.
2009 * Linus frestrict idea will clean these up nicely..
2010 */
2011 if (likely(!isblk)) {
2012 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
2013 if (*count || *pos > inode->i_sb->s_maxbytes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014 return -EFBIG;
2015 }
2016 /* zero-length writes at ->s_maxbytes are OK */
2017 }
2018
2019 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
2020 *count = inode->i_sb->s_maxbytes - *pos;
2021 } else {
David Howells93614012006-09-30 20:45:40 +02002022#ifdef CONFIG_BLOCK
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023 loff_t isize;
2024 if (bdev_read_only(I_BDEV(inode)))
2025 return -EPERM;
2026 isize = i_size_read(inode);
2027 if (*pos >= isize) {
2028 if (*count || *pos > isize)
2029 return -ENOSPC;
2030 }
2031
2032 if (*pos + *count > isize)
2033 *count = isize - *pos;
David Howells93614012006-09-30 20:45:40 +02002034#else
2035 return -EPERM;
2036#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037 }
2038 return 0;
2039}
2040EXPORT_SYMBOL(generic_write_checks);
2041
Nick Pigginafddba42007-10-16 01:25:01 -07002042int pagecache_write_begin(struct file *file, struct address_space *mapping,
2043 loff_t pos, unsigned len, unsigned flags,
2044 struct page **pagep, void **fsdata)
2045{
2046 const struct address_space_operations *aops = mapping->a_ops;
2047
Nick Piggin4e02ed42008-10-29 14:00:55 -07002048 return aops->write_begin(file, mapping, pos, len, flags,
Nick Pigginafddba42007-10-16 01:25:01 -07002049 pagep, fsdata);
Nick Pigginafddba42007-10-16 01:25:01 -07002050}
2051EXPORT_SYMBOL(pagecache_write_begin);
2052
2053int pagecache_write_end(struct file *file, struct address_space *mapping,
2054 loff_t pos, unsigned len, unsigned copied,
2055 struct page *page, void *fsdata)
2056{
2057 const struct address_space_operations *aops = mapping->a_ops;
Nick Pigginafddba42007-10-16 01:25:01 -07002058
Nick Piggin4e02ed42008-10-29 14:00:55 -07002059 mark_page_accessed(page);
2060 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
Nick Pigginafddba42007-10-16 01:25:01 -07002061}
2062EXPORT_SYMBOL(pagecache_write_end);
2063
Linus Torvalds1da177e2005-04-16 15:20:36 -07002064ssize_t
2065generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
2066 unsigned long *nr_segs, loff_t pos, loff_t *ppos,
2067 size_t count, size_t ocount)
2068{
2069 struct file *file = iocb->ki_filp;
2070 struct address_space *mapping = file->f_mapping;
2071 struct inode *inode = mapping->host;
2072 ssize_t written;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002073 size_t write_len;
2074 pgoff_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075
2076 if (count != ocount)
2077 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
2078
Christoph Hellwiga969e902008-07-23 21:27:04 -07002079 write_len = iov_length(iov, *nr_segs);
2080 end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002081
Nick Piggin48b47c52009-01-06 14:40:22 -08002082 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
Christoph Hellwiga969e902008-07-23 21:27:04 -07002083 if (written)
2084 goto out;
2085
2086 /*
2087 * After a write we want buffered reads to be sure to go to disk to get
2088 * the new data. We invalidate clean cached page from the region we're
2089 * about to write. We do this *before* the write so that we can return
Hisashi Hifumi6ccfa802008-09-02 14:35:40 -07002090 * without clobbering -EIOCBQUEUED from ->direct_IO().
Christoph Hellwiga969e902008-07-23 21:27:04 -07002091 */
2092 if (mapping->nrpages) {
2093 written = invalidate_inode_pages2_range(mapping,
2094 pos >> PAGE_CACHE_SHIFT, end);
Hisashi Hifumi6ccfa802008-09-02 14:35:40 -07002095 /*
2096 * If a page can not be invalidated, return 0 to fall back
2097 * to buffered write.
2098 */
2099 if (written) {
2100 if (written == -EBUSY)
2101 return 0;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002102 goto out;
Hisashi Hifumi6ccfa802008-09-02 14:35:40 -07002103 }
Christoph Hellwiga969e902008-07-23 21:27:04 -07002104 }
2105
2106 written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs);
2107
2108 /*
2109 * Finally, try again to invalidate clean pages which might have been
2110 * cached by non-direct readahead, or faulted in by get_user_pages()
2111 * if the source of the write was an mmap'ed region of the file
2112 * we're writing. Either one is a pretty crazy thing to do,
2113 * so we don't support it 100%. If this invalidation
2114 * fails, tough, the write still worked...
2115 */
2116 if (mapping->nrpages) {
2117 invalidate_inode_pages2_range(mapping,
2118 pos >> PAGE_CACHE_SHIFT, end);
2119 }
2120
Linus Torvalds1da177e2005-04-16 15:20:36 -07002121 if (written > 0) {
2122 loff_t end = pos + written;
2123 if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
2124 i_size_write(inode, end);
2125 mark_inode_dirty(inode);
2126 }
2127 *ppos = end;
2128 }
Christoph Hellwiga969e902008-07-23 21:27:04 -07002129out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130 return written;
2131}
2132EXPORT_SYMBOL(generic_file_direct_write);
2133
Nick Piggineb2be182007-10-16 01:24:57 -07002134/*
2135 * Find or create a page at the given pagecache position. Return the locked
2136 * page. This function is specifically for buffered writes.
2137 */
Nick Piggin54566b22009-01-04 12:00:53 -08002138struct page *grab_cache_page_write_begin(struct address_space *mapping,
2139 pgoff_t index, unsigned flags)
Nick Piggineb2be182007-10-16 01:24:57 -07002140{
2141 int status;
2142 struct page *page;
Nick Piggin54566b22009-01-04 12:00:53 -08002143 gfp_t gfp_notmask = 0;
2144 if (flags & AOP_FLAG_NOFS)
2145 gfp_notmask = __GFP_FS;
Nick Piggineb2be182007-10-16 01:24:57 -07002146repeat:
2147 page = find_lock_page(mapping, index);
2148 if (likely(page))
2149 return page;
2150
Nick Piggin54566b22009-01-04 12:00:53 -08002151 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~gfp_notmask);
Nick Piggineb2be182007-10-16 01:24:57 -07002152 if (!page)
2153 return NULL;
Nick Piggin54566b22009-01-04 12:00:53 -08002154 status = add_to_page_cache_lru(page, mapping, index,
2155 GFP_KERNEL & ~gfp_notmask);
Nick Piggineb2be182007-10-16 01:24:57 -07002156 if (unlikely(status)) {
2157 page_cache_release(page);
2158 if (status == -EEXIST)
2159 goto repeat;
2160 return NULL;
2161 }
2162 return page;
2163}
Nick Piggin54566b22009-01-04 12:00:53 -08002164EXPORT_SYMBOL(grab_cache_page_write_begin);
Nick Piggineb2be182007-10-16 01:24:57 -07002165
Nick Pigginafddba42007-10-16 01:25:01 -07002166static ssize_t generic_perform_write(struct file *file,
2167 struct iov_iter *i, loff_t pos)
2168{
2169 struct address_space *mapping = file->f_mapping;
2170 const struct address_space_operations *a_ops = mapping->a_ops;
2171 long status = 0;
2172 ssize_t written = 0;
Nick Piggin674b8922007-10-16 01:25:03 -07002173 unsigned int flags = 0;
2174
2175 /*
2176 * Copies from kernel address space cannot fail (NFSD is a big user).
2177 */
2178 if (segment_eq(get_fs(), KERNEL_DS))
2179 flags |= AOP_FLAG_UNINTERRUPTIBLE;
Nick Pigginafddba42007-10-16 01:25:01 -07002180
2181 do {
2182 struct page *page;
2183 pgoff_t index; /* Pagecache index for current page */
2184 unsigned long offset; /* Offset into pagecache page */
2185 unsigned long bytes; /* Bytes to write to page */
2186 size_t copied; /* Bytes copied from user */
2187 void *fsdata;
2188
2189 offset = (pos & (PAGE_CACHE_SIZE - 1));
2190 index = pos >> PAGE_CACHE_SHIFT;
2191 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
2192 iov_iter_count(i));
2193
2194again:
2195
2196 /*
2197 * Bring in the user page that we will copy from _first_.
2198 * Otherwise there's a nasty deadlock on copying from the
2199 * same page as we're writing to, without it being marked
2200 * up-to-date.
2201 *
2202 * Not only is this an optimisation, but it is also required
2203 * to check that the address is actually valid, when atomic
2204 * usercopies are used, below.
2205 */
2206 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
2207 status = -EFAULT;
2208 break;
2209 }
2210
Nick Piggin674b8922007-10-16 01:25:03 -07002211 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
Nick Pigginafddba42007-10-16 01:25:01 -07002212 &page, &fsdata);
2213 if (unlikely(status))
2214 break;
2215
2216 pagefault_disable();
2217 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
2218 pagefault_enable();
2219 flush_dcache_page(page);
2220
Josef Bacikc8236db2009-07-05 12:08:18 -07002221 mark_page_accessed(page);
Nick Pigginafddba42007-10-16 01:25:01 -07002222 status = a_ops->write_end(file, mapping, pos, bytes, copied,
2223 page, fsdata);
2224 if (unlikely(status < 0))
2225 break;
2226 copied = status;
2227
2228 cond_resched();
2229
Nick Piggin124d3b72008-02-02 15:01:17 +01002230 iov_iter_advance(i, copied);
Nick Pigginafddba42007-10-16 01:25:01 -07002231 if (unlikely(copied == 0)) {
2232 /*
2233 * If we were unable to copy any data at all, we must
2234 * fall back to a single segment length write.
2235 *
2236 * If we didn't fallback here, we could livelock
2237 * because not all segments in the iov can be copied at
2238 * once without a pagefault.
2239 */
2240 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
2241 iov_iter_single_seg_count(i));
2242 goto again;
2243 }
Nick Pigginafddba42007-10-16 01:25:01 -07002244 pos += copied;
2245 written += copied;
2246
2247 balance_dirty_pages_ratelimited(mapping);
2248
2249 } while (iov_iter_count(i));
2250
2251 return written ? written : status;
2252}
2253
2254ssize_t
2255generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
2256 unsigned long nr_segs, loff_t pos, loff_t *ppos,
2257 size_t count, ssize_t written)
2258{
2259 struct file *file = iocb->ki_filp;
2260 struct address_space *mapping = file->f_mapping;
Nick Pigginafddba42007-10-16 01:25:01 -07002261 ssize_t status;
2262 struct iov_iter i;
2263
2264 iov_iter_init(&i, iov, nr_segs, count, written);
Nick Piggin4e02ed42008-10-29 14:00:55 -07002265 status = generic_perform_write(file, &i, pos);
Nick Pigginafddba42007-10-16 01:25:01 -07002266
Linus Torvalds1da177e2005-04-16 15:20:36 -07002267 if (likely(status >= 0)) {
Nick Pigginafddba42007-10-16 01:25:01 -07002268 written += status;
2269 *ppos = pos + status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270 }
2271
2272 /*
2273 * If we get here for O_DIRECT writes then we must have fallen through
2274 * to buffered writes (block instantiation inside i_size). So we sync
2275 * the file data here, to try to honour O_DIRECT expectations.
2276 */
2277 if (unlikely(file->f_flags & O_DIRECT) && written)
Nick Piggin48b47c52009-01-06 14:40:22 -08002278 status = filemap_write_and_wait_range(mapping,
2279 pos, pos + written - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002280
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 return written ? written : status;
2282}
2283EXPORT_SYMBOL(generic_file_buffered_write);
2284
Jan Karae4dd9de2009-08-17 18:10:06 +02002285/**
2286 * __generic_file_aio_write - write data to a file
2287 * @iocb: IO state structure (file, offset, etc.)
2288 * @iov: vector with data to write
2289 * @nr_segs: number of segments in the vector
2290 * @ppos: position where to write
2291 *
2292 * This function does all the work needed for actually writing data to a
2293 * file. It does all basic checks, removes SUID from the file, updates
2294 * modification times and calls proper subroutines depending on whether we
2295 * do direct IO or a standard buffered write.
2296 *
2297 * It expects i_mutex to be grabbed unless we work on a block device or similar
2298 * object which does not need locking at all.
2299 *
2300 * This function does *not* take care of syncing data in case of O_SYNC write.
2301 * A caller has to handle it. This is mainly due to the fact that we want to
2302 * avoid syncing under i_mutex.
2303 */
2304ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
2305 unsigned long nr_segs, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002306{
2307 struct file *file = iocb->ki_filp;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07002308 struct address_space * mapping = file->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309 size_t ocount; /* original count */
2310 size_t count; /* after file limit checks */
2311 struct inode *inode = mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002312 loff_t pos;
2313 ssize_t written;
2314 ssize_t err;
2315
2316 ocount = 0;
Dmitriy Monakhov0ceb3312007-05-08 00:23:02 -07002317 err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
2318 if (err)
2319 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002320
2321 count = ocount;
2322 pos = *ppos;
2323
2324 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
2325
2326 /* We can write back this queue in page reclaim */
2327 current->backing_dev_info = mapping->backing_dev_info;
2328 written = 0;
2329
2330 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
2331 if (err)
2332 goto out;
2333
2334 if (count == 0)
2335 goto out;
2336
Miklos Szeredi2f1936b2008-06-24 16:50:14 +02002337 err = file_remove_suid(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338 if (err)
2339 goto out;
2340
Christoph Hellwig870f4812006-01-09 20:52:01 -08002341 file_update_time(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342
2343 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
2344 if (unlikely(file->f_flags & O_DIRECT)) {
Jeff Moyerfb5527e2006-10-19 23:28:13 -07002345 loff_t endbyte;
2346 ssize_t written_buffered;
2347
2348 written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
2349 ppos, count, ocount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002350 if (written < 0 || written == count)
2351 goto out;
2352 /*
2353 * direct-io write to a hole: fall through to buffered I/O
2354 * for completing the rest of the request.
2355 */
2356 pos += written;
2357 count -= written;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07002358 written_buffered = generic_file_buffered_write(iocb, iov,
2359 nr_segs, pos, ppos, count,
2360 written);
2361 /*
2362 * If generic_file_buffered_write() retuned a synchronous error
2363 * then we want to return the number of bytes which were
2364 * direct-written, or the error code if that was zero. Note
2365 * that this differs from normal direct-io semantics, which
2366 * will return -EFOO even if some bytes were written.
2367 */
2368 if (written_buffered < 0) {
2369 err = written_buffered;
2370 goto out;
2371 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372
Jeff Moyerfb5527e2006-10-19 23:28:13 -07002373 /*
2374 * We need to ensure that the page cache pages are written to
2375 * disk and invalidated to preserve the expected O_DIRECT
2376 * semantics.
2377 */
2378 endbyte = pos + written_buffered - written - 1;
Mark Fashehef51c972007-05-08 00:27:10 -07002379 err = do_sync_mapping_range(file->f_mapping, pos, endbyte,
2380 SYNC_FILE_RANGE_WAIT_BEFORE|
2381 SYNC_FILE_RANGE_WRITE|
2382 SYNC_FILE_RANGE_WAIT_AFTER);
Jeff Moyerfb5527e2006-10-19 23:28:13 -07002383 if (err == 0) {
2384 written = written_buffered;
2385 invalidate_mapping_pages(mapping,
2386 pos >> PAGE_CACHE_SHIFT,
2387 endbyte >> PAGE_CACHE_SHIFT);
2388 } else {
2389 /*
2390 * We don't know how much we wrote, so just return
2391 * the number of bytes which were direct-written
2392 */
2393 }
2394 } else {
2395 written = generic_file_buffered_write(iocb, iov, nr_segs,
2396 pos, ppos, count, written);
2397 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398out:
2399 current->backing_dev_info = NULL;
2400 return written ? written : err;
2401}
Jan Karae4dd9de2009-08-17 18:10:06 +02002402EXPORT_SYMBOL(__generic_file_aio_write);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403
Jan Karae4dd9de2009-08-17 18:10:06 +02002404/**
2405 * generic_file_aio_write - write data to a file
2406 * @iocb: IO state structure
2407 * @iov: vector with data to write
2408 * @nr_segs: number of segments in the vector
2409 * @pos: position in file where to write
2410 *
2411 * This is a wrapper around __generic_file_aio_write() to be used by most
2412 * filesystems. It takes care of syncing the file in case of O_SYNC file
2413 * and acquires i_mutex as needed.
2414 */
Badari Pulavarty027445c2006-09-30 23:28:46 -07002415ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
2416 unsigned long nr_segs, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002417{
2418 struct file *file = iocb->ki_filp;
Jan Kara148f9482009-08-17 19:52:36 +02002419 struct inode *inode = file->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421
2422 BUG_ON(iocb->ki_pos != pos);
2423
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08002424 mutex_lock(&inode->i_mutex);
Jan Karae4dd9de2009-08-17 18:10:06 +02002425 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08002426 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427
Jan Kara148f9482009-08-17 19:52:36 +02002428 if (ret > 0 || ret == -EIOCBQUEUED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002429 ssize_t err;
2430
Jan Kara148f9482009-08-17 19:52:36 +02002431 err = generic_write_sync(file, pos, ret);
Jan Karac7b50db2009-08-18 16:18:20 +02002432 if (err < 0 && ret > 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433 ret = err;
2434 }
2435 return ret;
2436}
2437EXPORT_SYMBOL(generic_file_aio_write);
2438
David Howellscf9a2ae2006-08-29 19:05:54 +01002439/**
2440 * try_to_release_page() - release old fs-specific metadata on a page
2441 *
2442 * @page: the page which the kernel is trying to free
2443 * @gfp_mask: memory allocation flags (and I/O mode)
2444 *
2445 * The address_space is to try to release any data against the page
2446 * (presumably at page->private). If the release was successful, return `1'.
2447 * Otherwise return zero.
2448 *
David Howells266cf652009-04-03 16:42:36 +01002449 * This may also be called if PG_fscache is set on a page, indicating that the
2450 * page is known to the local caching routines.
2451 *
David Howellscf9a2ae2006-08-29 19:05:54 +01002452 * The @gfp_mask argument specifies whether I/O may be performed to release
Mingming Cao3f31fdd2008-07-25 01:46:22 -07002453 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
David Howellscf9a2ae2006-08-29 19:05:54 +01002454 *
David Howellscf9a2ae2006-08-29 19:05:54 +01002455 */
2456int try_to_release_page(struct page *page, gfp_t gfp_mask)
2457{
2458 struct address_space * const mapping = page->mapping;
2459
2460 BUG_ON(!PageLocked(page));
2461 if (PageWriteback(page))
2462 return 0;
2463
2464 if (mapping && mapping->a_ops->releasepage)
2465 return mapping->a_ops->releasepage(page, gfp_mask);
2466 return try_to_free_buffers(page);
2467}
2468
2469EXPORT_SYMBOL(try_to_release_page);