blob: ebf3fcb4115b8c664e54e4c47e02968ed93ab0be [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 akpm@zip.com.au
7 * Initial version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/mm.h>
Nick Piggin0fd0e6b2006-09-27 01:50:02 -070012#include <linux/swap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/module.h>
14#include <linux/pagemap.h>
15#include <linux/pagevec.h>
Andrew Mortone08748c2006-12-10 02:19:31 -080016#include <linux/task_io_accounting_ops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/buffer_head.h> /* grr. try_to_release_page,
Jan Karaaaa40592005-10-30 15:00:16 -080018 do_invalidatepage */
Linus Torvalds1da177e2005-04-16 15:20:36 -070019
20
David Howellscf9a2ae2006-08-29 19:05:54 +010021/**
22 * do_invalidatepage - invalidate part of all of a page
23 * @page: the page which is affected
24 * @offset: the index of the truncation point
25 *
26 * do_invalidatepage() is called when all or part of the page has become
27 * invalidated by a truncate operation.
28 *
29 * do_invalidatepage() does not have to release all buffers, but it must
30 * ensure that no dirty buffer is left outside @offset and that no I/O
31 * is underway against any of the blocks which are outside the truncation
32 * point. Because the caller is about to free (and possibly reuse) those
33 * blocks on-disk.
34 */
35void do_invalidatepage(struct page *page, unsigned long offset)
36{
37 void (*invalidatepage)(struct page *, unsigned long);
38 invalidatepage = page->mapping->a_ops->invalidatepage;
David Howells93614012006-09-30 20:45:40 +020039#ifdef CONFIG_BLOCK
David Howellscf9a2ae2006-08-29 19:05:54 +010040 if (!invalidatepage)
41 invalidatepage = block_invalidatepage;
David Howells93614012006-09-30 20:45:40 +020042#endif
David Howellscf9a2ae2006-08-29 19:05:54 +010043 if (invalidatepage)
44 (*invalidatepage)(page, offset);
45}
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047static inline void truncate_partial_page(struct page *page, unsigned partial)
48{
49 memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
50 if (PagePrivate(page))
51 do_invalidatepage(page, partial);
52}
53
Linus Torvaldsecdfc972007-01-26 12:47:06 -080054/*
55 * This cancels just the dirty bit on the kernel page itself, it
56 * does NOT actually remove dirty bits on any mmap's that may be
57 * around. It also leaves the page tagged dirty, so any sync
58 * activity will still find it on the dirty lists, and in particular,
59 * clear_page_dirty_for_io() will still look at the dirty bits in
60 * the VM.
61 *
62 * Doing this should *normally* only ever be done when a page
63 * is truncated, and is not actually mapped anywhere at all. However,
64 * fs/buffer.c does this when it notices that somebody has cleaned
65 * out all the buffers on a page without actually doing it through
66 * the VM. Can you say "ext3 is horribly ugly"? Tought you could.
67 */
Linus Torvaldsfba25912006-12-20 13:46:42 -080068void cancel_dirty_page(struct page *page, unsigned int account_size)
69{
Linus Torvalds8368e322006-12-23 09:25:04 -080070 if (TestClearPageDirty(page)) {
71 struct address_space *mapping = page->mapping;
72 if (mapping && mapping_cap_account_dirty(mapping)) {
73 dec_zone_page_state(page, NR_FILE_DIRTY);
74 if (account_size)
75 task_io_account_cancelled_write(account_size);
76 }
Andrew Morton3e67c092006-12-21 11:00:33 -080077 }
Linus Torvaldsfba25912006-12-20 13:46:42 -080078}
Linus Torvalds8368e322006-12-23 09:25:04 -080079EXPORT_SYMBOL(cancel_dirty_page);
Linus Torvaldsfba25912006-12-20 13:46:42 -080080
Linus Torvalds1da177e2005-04-16 15:20:36 -070081/*
82 * If truncate cannot remove the fs-private metadata from the page, the page
83 * becomes anonymous. It will be left on the LRU and may even be mapped into
84 * user pagetables if we're racing with filemap_nopage().
85 *
86 * We need to bale out if page->mapping is no longer equal to the original
87 * mapping. This happens a) when the VM reclaimed the page while we waited on
Andrew Mortonfc0ecff2007-02-10 01:45:39 -080088 * its lock, b) when a concurrent invalidate_mapping_pages got there first and
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
90 */
91static void
92truncate_complete_page(struct address_space *mapping, struct page *page)
93{
94 if (page->mapping != mapping)
95 return;
96
Andrew Morton3e67c092006-12-21 11:00:33 -080097 cancel_dirty_page(page, PAGE_CACHE_SIZE);
98
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 if (PagePrivate(page))
100 do_invalidatepage(page, 0);
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 ClearPageUptodate(page);
103 ClearPageMappedToDisk(page);
104 remove_from_page_cache(page);
105 page_cache_release(page); /* pagecache ref */
106}
107
108/*
Andrew Mortonfc0ecff2007-02-10 01:45:39 -0800109 * This is for invalidate_mapping_pages(). That function can be called at
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 * any time, and is not supposed to throw away dirty pages. But pages can
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700111 * be marked dirty at any time too, so use remove_mapping which safely
112 * discards clean, unused pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 *
114 * Returns non-zero if the page was successfully invalidated.
115 */
116static int
117invalidate_complete_page(struct address_space *mapping, struct page *page)
118{
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700119 int ret;
120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 if (page->mapping != mapping)
122 return 0;
123
124 if (PagePrivate(page) && !try_to_release_page(page, 0))
125 return 0;
126
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700127 ret = remove_mapping(mapping, page);
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700128
129 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130}
131
132/**
Hans Reiserd7339072006-01-06 00:10:36 -0800133 * truncate_inode_pages - truncate range of pages specified by start and
134 * end byte offsets
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 * @mapping: mapping to truncate
136 * @lstart: offset from which to truncate
Hans Reiserd7339072006-01-06 00:10:36 -0800137 * @lend: offset to which to truncate
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 *
Hans Reiserd7339072006-01-06 00:10:36 -0800139 * Truncate the page cache, removing the pages that are between
140 * specified offsets (and zeroing out partial page
141 * (if lstart is not page aligned)).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142 *
143 * Truncate takes two passes - the first pass is nonblocking. It will not
144 * block on page locks and it will not block on writeback. The second pass
145 * will wait. This is to prevent as much IO as possible in the affected region.
146 * The first pass will remove most pages, so the search cost of the second pass
147 * is low.
148 *
149 * When looking at page->index outside the page lock we need to be careful to
150 * copy it into a local to avoid races (it could change at any time).
151 *
152 * We pass down the cache-hot hint to the page freeing code. Even if the
153 * mapping is large, it is probably the case that the final pages are the most
154 * recently touched, and freeing happens in ascending file offset order.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 */
Hans Reiserd7339072006-01-06 00:10:36 -0800156void truncate_inode_pages_range(struct address_space *mapping,
157 loff_t lstart, loff_t lend)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158{
159 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
Hans Reiserd7339072006-01-06 00:10:36 -0800160 pgoff_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
162 struct pagevec pvec;
163 pgoff_t next;
164 int i;
165
166 if (mapping->nrpages == 0)
167 return;
168
Hans Reiserd7339072006-01-06 00:10:36 -0800169 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
170 end = (lend >> PAGE_CACHE_SHIFT);
171
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 pagevec_init(&pvec, 0);
173 next = start;
Hans Reiserd7339072006-01-06 00:10:36 -0800174 while (next <= end &&
175 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 for (i = 0; i < pagevec_count(&pvec); i++) {
177 struct page *page = pvec.pages[i];
178 pgoff_t page_index = page->index;
179
Hans Reiserd7339072006-01-06 00:10:36 -0800180 if (page_index > end) {
181 next = page_index;
182 break;
183 }
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 if (page_index > next)
186 next = page_index;
187 next++;
188 if (TestSetPageLocked(page))
189 continue;
190 if (PageWriteback(page)) {
191 unlock_page(page);
192 continue;
193 }
194 truncate_complete_page(mapping, page);
195 unlock_page(page);
196 }
197 pagevec_release(&pvec);
198 cond_resched();
199 }
200
201 if (partial) {
202 struct page *page = find_lock_page(mapping, start - 1);
203 if (page) {
204 wait_on_page_writeback(page);
205 truncate_partial_page(page, partial);
206 unlock_page(page);
207 page_cache_release(page);
208 }
209 }
210
211 next = start;
212 for ( ; ; ) {
213 cond_resched();
214 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
215 if (next == start)
216 break;
217 next = start;
218 continue;
219 }
Hans Reiserd7339072006-01-06 00:10:36 -0800220 if (pvec.pages[0]->index > end) {
221 pagevec_release(&pvec);
222 break;
223 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 for (i = 0; i < pagevec_count(&pvec); i++) {
225 struct page *page = pvec.pages[i];
226
Hans Reiserd7339072006-01-06 00:10:36 -0800227 if (page->index > end)
228 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 lock_page(page);
230 wait_on_page_writeback(page);
231 if (page->index > next)
232 next = page->index;
233 next++;
234 truncate_complete_page(mapping, page);
235 unlock_page(page);
236 }
237 pagevec_release(&pvec);
238 }
239}
Hans Reiserd7339072006-01-06 00:10:36 -0800240EXPORT_SYMBOL(truncate_inode_pages_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241
Hans Reiserd7339072006-01-06 00:10:36 -0800242/**
243 * truncate_inode_pages - truncate *all* the pages from an offset
244 * @mapping: mapping to truncate
245 * @lstart: offset from which to truncate
246 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800247 * Called under (and serialised by) inode->i_mutex.
Hans Reiserd7339072006-01-06 00:10:36 -0800248 */
249void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
250{
251 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
252}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253EXPORT_SYMBOL(truncate_inode_pages);
254
255/**
256 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
257 * @mapping: the address_space which holds the pages to invalidate
258 * @start: the offset 'from' which to invalidate
259 * @end: the offset 'to' which to invalidate (inclusive)
260 *
261 * This function only removes the unlocked pages, if you want to
262 * remove all the pages of one inode, you must call truncate_inode_pages.
263 *
264 * invalidate_mapping_pages() will not block on IO activity. It will not
265 * invalidate pages which are dirty, locked, under writeback or mapped into
266 * pagetables.
267 */
268unsigned long invalidate_mapping_pages(struct address_space *mapping,
269 pgoff_t start, pgoff_t end)
270{
271 struct pagevec pvec;
272 pgoff_t next = start;
273 unsigned long ret = 0;
274 int i;
275
276 pagevec_init(&pvec, 0);
277 while (next <= end &&
278 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
279 for (i = 0; i < pagevec_count(&pvec); i++) {
280 struct page *page = pvec.pages[i];
NeilBrowne0f23602006-06-23 02:05:48 -0700281 pgoff_t index;
282 int lock_failed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283
NeilBrowne0f23602006-06-23 02:05:48 -0700284 lock_failed = TestSetPageLocked(page);
285
286 /*
287 * We really shouldn't be looking at the ->index of an
288 * unlocked page. But we're not allowed to lock these
289 * pages. So we rely upon nobody altering the ->index
290 * of this (pinned-by-us) page.
291 */
292 index = page->index;
293 if (index > next)
294 next = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 next++;
NeilBrowne0f23602006-06-23 02:05:48 -0700296 if (lock_failed)
297 continue;
298
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 if (PageDirty(page) || PageWriteback(page))
300 goto unlock;
301 if (page_mapped(page))
302 goto unlock;
303 ret += invalidate_complete_page(mapping, page);
304unlock:
305 unlock_page(page);
306 if (next > end)
307 break;
308 }
309 pagevec_release(&pvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 }
311 return ret;
312}
Anton Altaparmakov54bc4852007-02-10 01:45:38 -0800313EXPORT_SYMBOL(invalidate_mapping_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700315/*
316 * This is like invalidate_complete_page(), except it ignores the page's
317 * refcount. We do this because invalidate_inode_pages2() needs stronger
318 * invalidation guarantees, and cannot afford to leave pages behind because
319 * shrink_list() has a temp ref on them, or because they're transiently sitting
320 * in the lru_cache_add() pagevecs.
321 */
322static int
323invalidate_complete_page2(struct address_space *mapping, struct page *page)
324{
325 if (page->mapping != mapping)
326 return 0;
327
Trond Myklebust887ed2f2006-10-11 01:21:58 -0700328 if (PagePrivate(page) && !try_to_release_page(page, GFP_KERNEL))
Andrew Mortonbd4c8ce2006-09-30 23:29:29 -0700329 return 0;
330
331 write_lock_irq(&mapping->tree_lock);
332 if (PageDirty(page))
333 goto failed;
334
335 BUG_ON(PagePrivate(page));
336 __remove_from_page_cache(page);
337 write_unlock_irq(&mapping->tree_lock);
338 ClearPageUptodate(page);
339 page_cache_release(page); /* pagecache ref */
340 return 1;
341failed:
342 write_unlock_irq(&mapping->tree_lock);
343 return 0;
344}
345
Trond Myklebuste3db7692007-01-10 23:15:39 -0800346static int do_launder_page(struct address_space *mapping, struct page *page)
347{
348 if (!PageDirty(page))
349 return 0;
350 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
351 return 0;
352 return mapping->a_ops->launder_page(page);
353}
354
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355/**
356 * invalidate_inode_pages2_range - remove range of pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700357 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 * @start: the page offset 'from' which to invalidate
359 * @end: the page offset 'to' which to invalidate (inclusive)
360 *
361 * Any pages which are found to be mapped into pagetables are unmapped prior to
362 * invalidation.
363 *
364 * Returns -EIO if any pages could not be invalidated.
365 */
366int invalidate_inode_pages2_range(struct address_space *mapping,
367 pgoff_t start, pgoff_t end)
368{
369 struct pagevec pvec;
370 pgoff_t next;
371 int i;
372 int ret = 0;
373 int did_range_unmap = 0;
374 int wrapped = 0;
375
376 pagevec_init(&pvec, 0);
377 next = start;
378 while (next <= end && !ret && !wrapped &&
379 pagevec_lookup(&pvec, mapping, next,
380 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
381 for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
382 struct page *page = pvec.pages[i];
383 pgoff_t page_index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
385 lock_page(page);
386 if (page->mapping != mapping) {
387 unlock_page(page);
388 continue;
389 }
390 page_index = page->index;
391 next = page_index + 1;
392 if (next == 0)
393 wrapped = 1;
394 if (page_index > end) {
395 unlock_page(page);
396 break;
397 }
398 wait_on_page_writeback(page);
399 while (page_mapped(page)) {
400 if (!did_range_unmap) {
401 /*
402 * Zap the rest of the file in one hit.
403 */
404 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800405 (loff_t)page_index<<PAGE_CACHE_SHIFT,
406 (loff_t)(end - page_index + 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 << PAGE_CACHE_SHIFT,
408 0);
409 did_range_unmap = 1;
410 } else {
411 /*
412 * Just zap this page
413 */
414 unmap_mapping_range(mapping,
Oleg Drokin479ef592005-11-23 13:37:47 -0800415 (loff_t)page_index<<PAGE_CACHE_SHIFT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 PAGE_CACHE_SIZE, 0);
417 }
418 }
Trond Myklebuste3db7692007-01-10 23:15:39 -0800419 ret = do_launder_page(mapping, page);
420 if (ret == 0 && !invalidate_complete_page2(mapping, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 ret = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 unlock_page(page);
423 }
424 pagevec_release(&pvec);
425 cond_resched();
426 }
427 return ret;
428}
429EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
430
431/**
432 * invalidate_inode_pages2 - remove all pages from an address_space
Martin Waitz67be2dd2005-05-01 08:59:26 -0700433 * @mapping: the address_space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 *
435 * Any pages which are found to be mapped into pagetables are unmapped prior to
436 * invalidation.
437 *
438 * Returns -EIO if any pages could not be invalidated.
439 */
440int invalidate_inode_pages2(struct address_space *mapping)
441{
442 return invalidate_inode_pages2_range(mapping, 0, -1);
443}
444EXPORT_SYMBOL_GPL(invalidate_inode_pages2);