blob: 4fbe1a2da5fb973acc352b1b17d87ca390aab376 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 akpm@zip.com.au
7 * Initial version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/mm.h>
Nick Piggin0fd0e6b2006-09-27 01:50:02 -070012#include <linux/swap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/module.h>
14#include <linux/pagemap.h>
Nate Diller01f27052007-05-09 02:35:07 -070015#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/pagevec.h>
Andrew Mortone08748c2006-12-10 02:19:31 -080017#include <linux/task_io_accounting_ops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/buffer_head.h> /* grr. try_to_release_page,
Jan Karaaaa40592005-10-30 15:00:16 -080019 do_invalidatepage */
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21
David Howellscf9a2ae2006-08-29 19:05:54 +010022/**
23 * do_invalidatepage - invalidate part of all of a page
24 * @page: the page which is affected
25 * @offset: the index of the truncation point
26 *
27 * do_invalidatepage() is called when all or part of the page has become
28 * invalidated by a truncate operation.
29 *
30 * do_invalidatepage() does not have to release all buffers, but it must
31 * ensure that no dirty buffer is left outside @offset and that no I/O
32 * is underway against any of the blocks which are outside the truncation
33 * point. Because the caller is about to free (and possibly reuse) those
34 * blocks on-disk.
35 */
36void do_invalidatepage(struct page *page, unsigned long offset)
37{
38 void (*invalidatepage)(struct page *, unsigned long);
39 invalidatepage = page->mapping->a_ops->invalidatepage;
David Howells93614012006-09-30 20:45:40 +020040#ifdef CONFIG_BLOCK
David Howellscf9a2ae2006-08-29 19:05:54 +010041 if (!invalidatepage)
42 invalidatepage = block_invalidatepage;
David Howells93614012006-09-30 20:45:40 +020043#endif
David Howellscf9a2ae2006-08-29 19:05:54 +010044 if (invalidatepage)
45 (*invalidatepage)(page, offset);
46}
47
Linus Torvalds1da177e2005-04-16 15:20:36 -070048static inline void truncate_partial_page(struct page *page, unsigned partial)
49{
Nate Diller01f27052007-05-09 02:35:07 -070050 zero_user_page(page, partial, PAGE_CACHE_SIZE - partial, KM_USER0);
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 if (PagePrivate(page))
52 do_invalidatepage(page, partial);
53}
54
Linus Torvaldsecdfc972007-01-26 12:47:06 -080055/*
56 * This cancels just the dirty bit on the kernel page itself, it
57 * does NOT actually remove dirty bits on any mmap's that may be
58 * around. It also leaves the page tagged dirty, so any sync
59 * activity will still find it on the dirty lists, and in particular,
60 * clear_page_dirty_for_io() will still look at the dirty bits in
61 * the VM.
62 *
63 * Doing this should *normally* only ever be done when a page
64 * is truncated, and is not actually mapped anywhere at all. However,
65 * fs/buffer.c does this when it notices that somebody has cleaned
66 * out all the buffers on a page without actually doing it through
67 * the VM. Can you say "ext3 is horribly ugly"? Tought you could.
68 */
Linus Torvaldsfba25912006-12-20 13:46:42 -080069void cancel_dirty_page(struct page *page, unsigned int account_size)
70{
Linus Torvalds8368e322006-12-23 09:25:04 -080071 if (TestClearPageDirty(page)) {
72 struct address_space *mapping = page->mapping;
73 if (mapping && mapping_cap_account_dirty(mapping)) {
74 dec_zone_page_state(page, NR_FILE_DIRTY);
75 if (account_size)
76 task_io_account_cancelled_write(account_size);
77 }
Andrew Morton3e67c092006-12-21 11:00:33 -080078 }
Linus Torvaldsfba25912006-12-20 13:46:42 -080079}
Linus Torvalds8368e322006-12-23 09:25:04 -080080EXPORT_SYMBOL(cancel_dirty_page);
Linus Torvaldsfba25912006-12-20 13:46:42 -080081
Linus Torvalds1da177e2005-04-16 15:20:36 -070082/*
83 * If truncate cannot remove the fs-private metadata from the page, the page
84 * becomes anonymous. It will be left on the LRU and may even be mapped into
85 * user pagetables if we're racing with filemap_nopage().
86 *
87 * We need to bale out if page->mapping is no longer equal to the original
88 * mapping. This happens a) when the VM reclaimed the page while we waited on
Andrew Mortonfc0ecff2007-02-10 01:45:39 -080089 * its lock, b) when a concurrent invalidate_mapping_pages got there first and
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
91 */
92static void
93truncate_complete_page(struct address_space *mapping, struct page *page)
94{
95 if (page->mapping != mapping)
96 return;
97
Andrew Morton3e67c092006-12-21 11:00:33 -080098 cancel_dirty_page(page, PAGE_CACHE_SIZE);
99
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 if (PagePrivate(page))
101 do_invalidatepage(page, 0);
102
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 ClearPageUptodate(page);
104 ClearPageMappedToDisk(page);
105 remove_from_page_cache(page);
106 page_cache_release(page); /* pagecache ref */
107}
108
109/*
Andrew Mortonfc0ecff2007-02-10 01:45:39 -0800110 * This is for invalidate_mapping_pages(). That function can be called at
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 * any time, and is not supposed to throw away dirty pages. But pages can
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700112 * be marked dirty at any time too, so use remove_mapping which safely
113 * discards clean, unused pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 *
115 * Returns non-zero if the page was successfully invalidated.
116 */
117static int
118invalidate_complete_page(struct address_space *mapping, struct page *page)
119{
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700120 int ret;
121
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 if (page->mapping != mapping)
123 return 0;
124
125 if (PagePrivate(page) && !try_to_release_page(page, 0))
126 return 0;
127
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700128 ret = remove_mapping(mapping, page);
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700129
130 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131}
132
133/**
Hans Reiserd7339072006-01-06 00:10:36 -0800134 * truncate_inode_pages - truncate range of pages specified by start and
135 * end byte offsets
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 * @mapping: mapping to truncate
137 * @lstart: offset from which to truncate
Hans Reiserd7339072006-01-06 00:10:36 -0800138 * @lend: offset to which to truncate
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 *
Hans Reiserd7339072006-01-06 00:10:36 -0800140 * Truncate the page cache, removing the pages that are between
141 * specified offsets (and zeroing out partial page
142 * (if lstart is not page aligned)).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143 *
144 * Truncate takes two passes - the first pass is nonblocking. It will not
145 * block on page locks and it will not block on writeback. The second pass
146 * will wait. This is to prevent as much IO as possible in the affected region.
147 * The first pass will remove most pages, so the search cost of the second pass
148 * is low.
149 *
150 * When looking at page->index outside the page lock we need to be careful to
151 * copy it into a local to avoid races (it could change at any time).
152 *
153 * We pass down the cache-hot hint to the page freeing code. Even if the
154 * mapping is large, it is probably the case that the final pages are the most
155 * recently touched, and freeing happens in ascending file offset order.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 */
Hans Reiserd7339072006-01-06 00:10:36 -0800157void truncate_inode_pages_range(struct address_space *mapping,
158 loff_t lstart, loff_t lend)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159{
160 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
Hans Reiserd7339072006-01-06 00:10:36 -0800161 pgoff_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
163 struct pagevec pvec;
164 pgoff_t next;
165 int i;
166
167 if (mapping->nrpages == 0)
168 return;
169
Hans Reiserd7339072006-01-06 00:10:36 -0800170 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
171 end = (lend >> PAGE_CACHE_SHIFT);
172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 pagevec_init(&pvec, 0);
174 next = start;
Hans Reiserd7339072006-01-06 00:10:36 -0800175 while (next <= end &&
176 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 for (i = 0; i < pagevec_count(&pvec); i++) {
178 struct page *page = pvec.pages[i];
179 pgoff_t page_index = page->index;
180
Hans Reiserd7339072006-01-06 00:10:36 -0800181 if (page_index > end) {
182 next = page_index;
183 break;
184 }
185
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 if (page_index > next)
187 next = page_index;
188 next++;
189 if (TestSetPageLocked(page))
190 continue;
191 if (PageWriteback(page)) {
192 unlock_page(page);
193 continue;
194 }
195 truncate_complete_page(mapping, page);
196 unlock_page(page);
197 }
198 pagevec_release(&pvec);
199 cond_resched();
200 }
201
202 if (partial) {
203 struct page *page = find_lock_page(mapping, start - 1);
204 if (page) {
205 wait_on_page_writeback(page);
206 truncate_partial_page(page, partial);
207 unlock_page(page);
208 page_cache_release(page);
209 }
210 }
211
212 next = start;
213 for ( ; ; ) {
214 cond_resched();
215 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
216 if (next == start)
217 break;
218 next = start;
219 continue;
220 }
Hans Reiserd7339072006-01-06 00:10:36 -0800221 if (pvec.pages[0]->index > end) {
222 pagevec_release(&pvec);
223 break;
224 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 for (i = 0; i < pagevec_count(&pvec); i++) {
226 struct page *page = pvec.pages[i];
227
Hans Reiserd7339072006-01-06 00:10:36 -0800228 if (page->index > end)
229 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 lock_page(page);
231 wait_on_page_writeback(page);
232 if (page->index > next)
233 next = page->index;
234 next++;
235 truncate_complete_page(mapping, page);
236 unlock_page(page);
237 }
238 pagevec_release(&pvec);
239 }
240}
Hans Reiserd7339072006-01-06 00:10:36 -0800241EXPORT_SYMBOL(truncate_inode_pages_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
Hans Reiserd7339072006-01-06 00:10:36 -0800243/**
244 * truncate_inode_pages - truncate *all* the pages from an offset
245 * @mapping: mapping to truncate
246 * @lstart: offset from which to truncate
247 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800248 * Called under (and serialised by) inode->i_mutex.
Hans Reiserd7339072006-01-06 00:10:36 -0800249 */
250void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
251{
252 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
253}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254EXPORT_SYMBOL(truncate_inode_pages);
255
256/**
257 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
258 * @mapping: the address_space which holds the pages to invalidate
259 * @start: the offset 'from' which to invalidate
260 * @end: the offset 'to' which to invalidate (inclusive)
261 *
262 * This function only removes the unlocked pages, if you want to
263 * remove all the pages of one inode, you must call truncate_inode_pages.
264 *
265 * invalidate_mapping_pages() will not block on IO activity. It will not
266 * invalidate pages which are dirty, locked, under writeback or mapped into
267 * pagetables.
268 */
269unsigned long invalidate_mapping_pages(struct address_space *mapping,
270 pgoff_t start, pgoff_t end)
271{
272 struct pagevec pvec;
273 pgoff_t next = start;
274 unsigned long ret = 0;
275 int i;
276
277 pagevec_init(&pvec, 0);
278 while (next <= end &&
279 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
280 for (i = 0; i < pagevec_count(&pvec); i++) {
281 struct page *page = pvec.pages[i];
NeilBrowne0f23602006-06-23 02:05:48 -0700282 pgoff_t index;
283 int lock_failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
NeilBrowne0f23602006-06-23 02:05:48 -0700285 lock_failed = TestSetPageLocked(page);
286
287 /*
288 * We really shouldn't be looking at the ->index of an
289 * unlocked page. But we're not allowed to lock these
290 * pages. So we rely upon nobody altering the ->index
291 * of this (pinned-by-us) page.
292 */
293 index = page->index;
294 if (index > next)
295 next = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 next++;
NeilBrowne0f23602006-06-23 02:05:48 -0700297 if (lock_failed)
298 continue;
299
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 if (PageDirty(page) || PageWriteback(page))
301 goto unlock;
302 if (page_mapped(page))
303 goto unlock;
304 ret += invalidate_complete_page(mapping, page);
305unlock:
306 unlock_page(page);
307 if (next > end)
308 break;
309 }
310 pagevec_release(&pvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 }
312 return ret;
313}
Anton Altaparmakov54bc4852007-02-10 01:45:38 -0800314EXPORT_SYMBOL(invalidate_mapping_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700316/*
317 * This is like invalidate_complete_page(), except it ignores the page's
318 * refcount. We do this because invalidate_inode_pages2() needs stronger
319 * invalidation guarantees, and cannot afford to leave pages behind because
320 * shrink_list() has a temp ref on them, or because they're transiently sitting
321 * in the lru_cache_add() pagevecs.
322 */
323static int
324invalidate_complete_page2(struct address_space *mapping, struct page *page)
325{
326 if (page->mapping != mapping)
327 return 0;
328
Trond Myklebust887ed2f2006-10-11 01:21:58 -0700329 if (PagePrivate(page) && !try_to_release_page(page, GFP_KERNEL))
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700330 return 0;
331
332 write_lock_irq(&mapping->tree_lock);
333 if (PageDirty(page))
334 goto failed;
335
336 BUG_ON(PagePrivate(page));
337 __remove_from_page_cache(page);
338 write_unlock_irq(&mapping->tree_lock);
339 ClearPageUptodate(page);
340 page_cache_release(page); /* pagecache ref */
341 return 1;
342failed:
343 write_unlock_irq(&mapping->tree_lock);
344 return 0;
345}
346
Trond Myklebuste3db7692007-01-10 23:15:39 -0800347static int do_launder_page(struct address_space *mapping, struct page *page)
348{
349 if (!PageDirty(page))
350 return 0;
351 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
352 return 0;
353 return mapping->a_ops->launder_page(page);
354}
355
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356/**
357 * invalidate_inode_pages2_range - remove range of pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700358 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 * @start: the page offset 'from' which to invalidate
360 * @end: the page offset 'to' which to invalidate (inclusive)
361 *
362 * Any pages which are found to be mapped into pagetables are unmapped prior to
363 * invalidation.
364 *
365 * Returns -EIO if any pages could not be invalidated.
366 */
367int invalidate_inode_pages2_range(struct address_space *mapping,
368 pgoff_t start, pgoff_t end)
369{
370 struct pagevec pvec;
371 pgoff_t next;
372 int i;
373 int ret = 0;
374 int did_range_unmap = 0;
375 int wrapped = 0;
376
377 pagevec_init(&pvec, 0);
378 next = start;
Trond Myklebust7b965e02007-02-28 20:13:55 -0800379 while (next <= end && !wrapped &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 pagevec_lookup(&pvec, mapping, next,
381 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
Trond Myklebust7b965e02007-02-28 20:13:55 -0800382 for (i = 0; i < pagevec_count(&pvec); i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 struct page *page = pvec.pages[i];
384 pgoff_t page_index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
386 lock_page(page);
387 if (page->mapping != mapping) {
388 unlock_page(page);
389 continue;
390 }
391 page_index = page->index;
392 next = page_index + 1;
393 if (next == 0)
394 wrapped = 1;
395 if (page_index > end) {
396 unlock_page(page);
397 break;
398 }
399 wait_on_page_writeback(page);
400 while (page_mapped(page)) {
401 if (!did_range_unmap) {
402 /*
403 * Zap the rest of the file in one hit.
404 */
405 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800406 (loff_t)page_index<<PAGE_CACHE_SHIFT,
407 (loff_t)(end - page_index + 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 << PAGE_CACHE_SHIFT,
409 0);
410 did_range_unmap = 1;
411 } else {
412 /*
413 * Just zap this page
414 */
415 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800416 (loff_t)page_index<<PAGE_CACHE_SHIFT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 PAGE_CACHE_SIZE, 0);
418 }
419 }
Trond Myklebuste3db7692007-01-10 23:15:39 -0800420 ret = do_launder_page(mapping, page);
421 if (ret == 0 && !invalidate_complete_page2(mapping, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 ret = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 unlock_page(page);
424 }
425 pagevec_release(&pvec);
426 cond_resched();
427 }
428 return ret;
429}
430EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
431
432/**
433 * invalidate_inode_pages2 - remove all pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700434 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 *
436 * Any pages which are found to be mapped into pagetables are unmapped prior to
437 * invalidation.
438 *
439 * Returns -EIO if any pages could not be invalidated.
440 */
441int invalidate_inode_pages2(struct address_space *mapping)
442{
443 return invalidate_inode_pages2_range(mapping, 0, -1);
444}
445EXPORT_SYMBOL_GPL(invalidate_inode_pages2);