blob: 5cdfbc1a59fdce4e1a1346ef8c9ddf97f159777d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 akpm@zip.com.au
7 * Initial version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/mm.h>
Nick Piggin0fd0e6b2006-09-27 01:50:02 -070012#include <linux/swap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/module.h>
14#include <linux/pagemap.h>
Nate Diller01f27052007-05-09 02:35:07 -070015#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/pagevec.h>
Andrew Mortone08748c2006-12-10 02:19:31 -080017#include <linux/task_io_accounting_ops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/buffer_head.h> /* grr. try_to_release_page,
Jan Karaaaa40592005-10-30 15:00:16 -080019 do_invalidatepage */
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21
David Howellscf9a2ae2006-08-29 19:05:54 +010022/**
23 * do_invalidatepage - invalidate part of all of a page
24 * @page: the page which is affected
25 * @offset: the index of the truncation point
26 *
27 * do_invalidatepage() is called when all or part of the page has become
28 * invalidated by a truncate operation.
29 *
30 * do_invalidatepage() does not have to release all buffers, but it must
31 * ensure that no dirty buffer is left outside @offset and that no I/O
32 * is underway against any of the blocks which are outside the truncation
33 * point. Because the caller is about to free (and possibly reuse) those
34 * blocks on-disk.
35 */
36void do_invalidatepage(struct page *page, unsigned long offset)
37{
38 void (*invalidatepage)(struct page *, unsigned long);
39 invalidatepage = page->mapping->a_ops->invalidatepage;
David Howells93614012006-09-30 20:45:40 +020040#ifdef CONFIG_BLOCK
David Howellscf9a2ae2006-08-29 19:05:54 +010041 if (!invalidatepage)
42 invalidatepage = block_invalidatepage;
David Howells93614012006-09-30 20:45:40 +020043#endif
David Howellscf9a2ae2006-08-29 19:05:54 +010044 if (invalidatepage)
45 (*invalidatepage)(page, offset);
46}
47
Linus Torvalds1da177e2005-04-16 15:20:36 -070048static inline void truncate_partial_page(struct page *page, unsigned partial)
49{
Nate Diller01f27052007-05-09 02:35:07 -070050 zero_user_page(page, partial, PAGE_CACHE_SIZE - partial, KM_USER0);
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 if (PagePrivate(page))
52 do_invalidatepage(page, partial);
53}
54
Linus Torvaldsecdfc972007-01-26 12:47:06 -080055/*
56 * This cancels just the dirty bit on the kernel page itself, it
57 * does NOT actually remove dirty bits on any mmap's that may be
58 * around. It also leaves the page tagged dirty, so any sync
59 * activity will still find it on the dirty lists, and in particular,
60 * clear_page_dirty_for_io() will still look at the dirty bits in
61 * the VM.
62 *
63 * Doing this should *normally* only ever be done when a page
64 * is truncated, and is not actually mapped anywhere at all. However,
65 * fs/buffer.c does this when it notices that somebody has cleaned
66 * out all the buffers on a page without actually doing it through
67 * the VM. Can you say "ext3 is horribly ugly"? Tought you could.
68 */
Linus Torvaldsfba25912006-12-20 13:46:42 -080069void cancel_dirty_page(struct page *page, unsigned int account_size)
70{
Linus Torvalds8368e322006-12-23 09:25:04 -080071 if (TestClearPageDirty(page)) {
72 struct address_space *mapping = page->mapping;
73 if (mapping && mapping_cap_account_dirty(mapping)) {
74 dec_zone_page_state(page, NR_FILE_DIRTY);
75 if (account_size)
76 task_io_account_cancelled_write(account_size);
77 }
Andrew Morton3e67c092006-12-21 11:00:33 -080078 }
Linus Torvaldsfba25912006-12-20 13:46:42 -080079}
Linus Torvalds8368e322006-12-23 09:25:04 -080080EXPORT_SYMBOL(cancel_dirty_page);
Linus Torvaldsfba25912006-12-20 13:46:42 -080081
Linus Torvalds1da177e2005-04-16 15:20:36 -070082/*
83 * If truncate cannot remove the fs-private metadata from the page, the page
84 * becomes anonymous. It will be left on the LRU and may even be mapped into
Nick Piggin54cb8822007-07-19 01:46:59 -070085 * user pagetables if we're racing with filemap_fault().
Linus Torvalds1da177e2005-04-16 15:20:36 -070086 *
87 * We need to bale out if page->mapping is no longer equal to the original
88 * mapping. This happens a) when the VM reclaimed the page while we waited on
Andrew Mortonfc0ecff2007-02-10 01:45:39 -080089 * its lock, b) when a concurrent invalidate_mapping_pages got there first and
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
91 */
92static void
93truncate_complete_page(struct address_space *mapping, struct page *page)
94{
95 if (page->mapping != mapping)
96 return;
97
Andrew Morton3e67c092006-12-21 11:00:33 -080098 cancel_dirty_page(page, PAGE_CACHE_SIZE);
99
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 if (PagePrivate(page))
101 do_invalidatepage(page, 0);
102
Nick Piggin787d2212007-07-17 04:03:34 -0700103 remove_from_page_cache(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 ClearPageUptodate(page);
105 ClearPageMappedToDisk(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 page_cache_release(page); /* pagecache ref */
107}
108
109/*
Andrew Mortonfc0ecff2007-02-10 01:45:39 -0800110 * This is for invalidate_mapping_pages(). That function can be called at
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 * any time, and is not supposed to throw away dirty pages. But pages can
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700112 * be marked dirty at any time too, so use remove_mapping which safely
113 * discards clean, unused pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 *
115 * Returns non-zero if the page was successfully invalidated.
116 */
117static int
118invalidate_complete_page(struct address_space *mapping, struct page *page)
119{
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700120 int ret;
121
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 if (page->mapping != mapping)
123 return 0;
124
125 if (PagePrivate(page) && !try_to_release_page(page, 0))
126 return 0;
127
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700128 ret = remove_mapping(mapping, page);
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700129
130 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131}
132
133/**
Hans Reiserd7339072006-01-06 00:10:36 -0800134 * truncate_inode_pages - truncate range of pages specified by start and
135 * end byte offsets
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 * @mapping: mapping to truncate
137 * @lstart: offset from which to truncate
Hans Reiserd7339072006-01-06 00:10:36 -0800138 * @lend: offset to which to truncate
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 *
Hans Reiserd7339072006-01-06 00:10:36 -0800140 * Truncate the page cache, removing the pages that are between
141 * specified offsets (and zeroing out partial page
142 * (if lstart is not page aligned)).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143 *
144 * Truncate takes two passes - the first pass is nonblocking. It will not
145 * block on page locks and it will not block on writeback. The second pass
146 * will wait. This is to prevent as much IO as possible in the affected region.
147 * The first pass will remove most pages, so the search cost of the second pass
148 * is low.
149 *
150 * When looking at page->index outside the page lock we need to be careful to
151 * copy it into a local to avoid races (it could change at any time).
152 *
153 * We pass down the cache-hot hint to the page freeing code. Even if the
154 * mapping is large, it is probably the case that the final pages are the most
155 * recently touched, and freeing happens in ascending file offset order.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 */
Hans Reiserd7339072006-01-06 00:10:36 -0800157void truncate_inode_pages_range(struct address_space *mapping,
158 loff_t lstart, loff_t lend)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159{
160 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
Hans Reiserd7339072006-01-06 00:10:36 -0800161 pgoff_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
163 struct pagevec pvec;
164 pgoff_t next;
165 int i;
166
167 if (mapping->nrpages == 0)
168 return;
169
Hans Reiserd7339072006-01-06 00:10:36 -0800170 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
171 end = (lend >> PAGE_CACHE_SHIFT);
172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 pagevec_init(&pvec, 0);
174 next = start;
Hans Reiserd7339072006-01-06 00:10:36 -0800175 while (next <= end &&
176 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 for (i = 0; i < pagevec_count(&pvec); i++) {
178 struct page *page = pvec.pages[i];
179 pgoff_t page_index = page->index;
180
Hans Reiserd7339072006-01-06 00:10:36 -0800181 if (page_index > end) {
182 next = page_index;
183 break;
184 }
185
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 if (page_index > next)
187 next = page_index;
188 next++;
189 if (TestSetPageLocked(page))
190 continue;
191 if (PageWriteback(page)) {
192 unlock_page(page);
193 continue;
194 }
Nick Piggind00806b2007-07-19 01:46:57 -0700195 if (page_mapped(page)) {
196 unmap_mapping_range(mapping,
197 (loff_t)page_index<<PAGE_CACHE_SHIFT,
198 PAGE_CACHE_SIZE, 0);
199 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 truncate_complete_page(mapping, page);
201 unlock_page(page);
202 }
203 pagevec_release(&pvec);
204 cond_resched();
205 }
206
207 if (partial) {
208 struct page *page = find_lock_page(mapping, start - 1);
209 if (page) {
210 wait_on_page_writeback(page);
211 truncate_partial_page(page, partial);
212 unlock_page(page);
213 page_cache_release(page);
214 }
215 }
216
217 next = start;
218 for ( ; ; ) {
219 cond_resched();
220 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
221 if (next == start)
222 break;
223 next = start;
224 continue;
225 }
Hans Reiserd7339072006-01-06 00:10:36 -0800226 if (pvec.pages[0]->index > end) {
227 pagevec_release(&pvec);
228 break;
229 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 for (i = 0; i < pagevec_count(&pvec); i++) {
231 struct page *page = pvec.pages[i];
232
Hans Reiserd7339072006-01-06 00:10:36 -0800233 if (page->index > end)
234 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 lock_page(page);
236 wait_on_page_writeback(page);
Nick Piggind00806b2007-07-19 01:46:57 -0700237 if (page_mapped(page)) {
238 unmap_mapping_range(mapping,
239 (loff_t)page->index<<PAGE_CACHE_SHIFT,
240 PAGE_CACHE_SIZE, 0);
241 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 if (page->index > next)
243 next = page->index;
244 next++;
245 truncate_complete_page(mapping, page);
246 unlock_page(page);
247 }
248 pagevec_release(&pvec);
249 }
250}
Hans Reiserd7339072006-01-06 00:10:36 -0800251EXPORT_SYMBOL(truncate_inode_pages_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252
Hans Reiserd7339072006-01-06 00:10:36 -0800253/**
254 * truncate_inode_pages - truncate *all* the pages from an offset
255 * @mapping: mapping to truncate
256 * @lstart: offset from which to truncate
257 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800258 * Called under (and serialised by) inode->i_mutex.
Hans Reiserd7339072006-01-06 00:10:36 -0800259 */
260void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
261{
262 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
263}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264EXPORT_SYMBOL(truncate_inode_pages);
265
Andrew Mortonfc9a07e2007-07-15 23:38:14 -0700266unsigned long __invalidate_mapping_pages(struct address_space *mapping,
267 pgoff_t start, pgoff_t end, bool be_atomic)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
269 struct pagevec pvec;
270 pgoff_t next = start;
271 unsigned long ret = 0;
272 int i;
273
274 pagevec_init(&pvec, 0);
275 while (next <= end &&
276 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
277 for (i = 0; i < pagevec_count(&pvec); i++) {
278 struct page *page = pvec.pages[i];
NeilBrowne0f23602006-06-23 02:05:48 -0700279 pgoff_t index;
280 int lock_failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
NeilBrowne0f23602006-06-23 02:05:48 -0700282 lock_failed = TestSetPageLocked(page);
283
284 /*
285 * We really shouldn't be looking at the ->index of an
286 * unlocked page. But we're not allowed to lock these
287 * pages. So we rely upon nobody altering the ->index
288 * of this (pinned-by-us) page.
289 */
290 index = page->index;
291 if (index > next)
292 next = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 next++;
NeilBrowne0f23602006-06-23 02:05:48 -0700294 if (lock_failed)
295 continue;
296
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 if (PageDirty(page) || PageWriteback(page))
298 goto unlock;
299 if (page_mapped(page))
300 goto unlock;
301 ret += invalidate_complete_page(mapping, page);
302unlock:
303 unlock_page(page);
304 if (next > end)
305 break;
306 }
307 pagevec_release(&pvec);
Andrew Mortonfc9a07e2007-07-15 23:38:14 -0700308 if (likely(!be_atomic))
309 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 }
311 return ret;
312}
Andrew Mortonfc9a07e2007-07-15 23:38:14 -0700313
314/**
315 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
316 * @mapping: the address_space which holds the pages to invalidate
317 * @start: the offset 'from' which to invalidate
318 * @end: the offset 'to' which to invalidate (inclusive)
319 *
320 * This function only removes the unlocked pages, if you want to
321 * remove all the pages of one inode, you must call truncate_inode_pages.
322 *
323 * invalidate_mapping_pages() will not block on IO activity. It will not
324 * invalidate pages which are dirty, locked, under writeback or mapped into
325 * pagetables.
326 */
327unsigned long invalidate_mapping_pages(struct address_space *mapping,
328 pgoff_t start, pgoff_t end)
329{
330 return __invalidate_mapping_pages(mapping, start, end, false);
331}
Anton Altaparmakov54bc4852007-02-10 01:45:38 -0800332EXPORT_SYMBOL(invalidate_mapping_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700334/*
335 * This is like invalidate_complete_page(), except it ignores the page's
336 * refcount. We do this because invalidate_inode_pages2() needs stronger
337 * invalidation guarantees, and cannot afford to leave pages behind because
Anderson Briglia2706a1b2007-07-15 23:38:09 -0700338 * shrink_page_list() has a temp ref on them, or because they're transiently
339 * sitting in the lru_cache_add() pagevecs.
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700340 */
341static int
342invalidate_complete_page2(struct address_space *mapping, struct page *page)
343{
344 if (page->mapping != mapping)
345 return 0;
346
Trond Myklebust887ed2f2006-10-11 01:21:58 -0700347 if (PagePrivate(page) && !try_to_release_page(page, GFP_KERNEL))
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700348 return 0;
349
350 write_lock_irq(&mapping->tree_lock);
351 if (PageDirty(page))
352 goto failed;
353
354 BUG_ON(PagePrivate(page));
355 __remove_from_page_cache(page);
356 write_unlock_irq(&mapping->tree_lock);
357 ClearPageUptodate(page);
358 page_cache_release(page); /* pagecache ref */
359 return 1;
360failed:
361 write_unlock_irq(&mapping->tree_lock);
362 return 0;
363}
364
Trond Myklebuste3db7692007-01-10 23:15:39 -0800365static int do_launder_page(struct address_space *mapping, struct page *page)
366{
367 if (!PageDirty(page))
368 return 0;
369 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
370 return 0;
371 return mapping->a_ops->launder_page(page);
372}
373
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374/**
375 * invalidate_inode_pages2_range - remove range of pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700376 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 * @start: the page offset 'from' which to invalidate
378 * @end: the page offset 'to' which to invalidate (inclusive)
379 *
380 * Any pages which are found to be mapped into pagetables are unmapped prior to
381 * invalidation.
382 *
383 * Returns -EIO if any pages could not be invalidated.
384 */
385int invalidate_inode_pages2_range(struct address_space *mapping,
386 pgoff_t start, pgoff_t end)
387{
388 struct pagevec pvec;
389 pgoff_t next;
390 int i;
391 int ret = 0;
392 int did_range_unmap = 0;
393 int wrapped = 0;
394
395 pagevec_init(&pvec, 0);
396 next = start;
Trond Myklebust7b965e02007-02-28 20:13:55 -0800397 while (next <= end && !wrapped &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 pagevec_lookup(&pvec, mapping, next,
399 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
Trond Myklebust7b965e02007-02-28 20:13:55 -0800400 for (i = 0; i < pagevec_count(&pvec); i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 struct page *page = pvec.pages[i];
402 pgoff_t page_index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403
404 lock_page(page);
405 if (page->mapping != mapping) {
406 unlock_page(page);
407 continue;
408 }
409 page_index = page->index;
410 next = page_index + 1;
411 if (next == 0)
412 wrapped = 1;
413 if (page_index > end) {
414 unlock_page(page);
415 break;
416 }
417 wait_on_page_writeback(page);
Nick Piggind00806b2007-07-19 01:46:57 -0700418 if (page_mapped(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 if (!did_range_unmap) {
420 /*
421 * Zap the rest of the file in one hit.
422 */
423 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800424 (loff_t)page_index<<PAGE_CACHE_SHIFT,
425 (loff_t)(end - page_index + 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 << PAGE_CACHE_SHIFT,
427 0);
428 did_range_unmap = 1;
429 } else {
430 /*
431 * Just zap this page
432 */
433 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800434 (loff_t)page_index<<PAGE_CACHE_SHIFT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 PAGE_CACHE_SIZE, 0);
436 }
437 }
Nick Piggind00806b2007-07-19 01:46:57 -0700438 BUG_ON(page_mapped(page));
Trond Myklebuste3db7692007-01-10 23:15:39 -0800439 ret = do_launder_page(mapping, page);
440 if (ret == 0 && !invalidate_complete_page2(mapping, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 ret = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 unlock_page(page);
443 }
444 pagevec_release(&pvec);
445 cond_resched();
446 }
447 return ret;
448}
449EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
450
451/**
452 * invalidate_inode_pages2 - remove all pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700453 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 *
455 * Any pages which are found to be mapped into pagetables are unmapped prior to
456 * invalidation.
457 *
458 * Returns -EIO if any pages could not be invalidated.
459 */
460int invalidate_inode_pages2(struct address_space *mapping)
461{
462 return invalidate_inode_pages2_range(mapping, 0, -1);
463}
464EXPORT_SYMBOL_GPL(invalidate_inode_pages2);