blob: 029dfad5a235753fab2b7f81dda1950e770e9f2d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Uwe Zeisbergerf30c2262006-10-03 23:01:26 +02002 * mm/page-writeback.c
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * Contains functions related to writing back dirty pages at the
7 * address_space level.
8 *
9 * 10Apr2002 akpm@zip.com.au
10 * Initial version
11 */
12
13#include <linux/kernel.h>
14#include <linux/module.h>
15#include <linux/spinlock.h>
16#include <linux/fs.h>
17#include <linux/mm.h>
18#include <linux/swap.h>
19#include <linux/slab.h>
20#include <linux/pagemap.h>
21#include <linux/writeback.h>
22#include <linux/init.h>
23#include <linux/backing-dev.h>
Andrew Morton55e829a2006-12-10 02:19:27 -080024#include <linux/task_io_accounting_ops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/blkdev.h>
26#include <linux/mpage.h>
Peter Zijlstrad08b3852006-09-25 23:30:57 -070027#include <linux/rmap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/percpu.h>
29#include <linux/notifier.h>
30#include <linux/smp.h>
31#include <linux/sysctl.h>
32#include <linux/cpu.h>
33#include <linux/syscalls.h>
David Howellscf9a2ae2006-08-29 19:05:54 +010034#include <linux/buffer_head.h>
David Howells811d7362006-08-29 19:06:09 +010035#include <linux/pagevec.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
37/*
38 * The maximum number of pages to writeout in a single bdflush/kupdate
39 * operation. We do this so we don't hold I_LOCK against an inode for
40 * enormous amounts of time, which would block a userspace task which has
41 * been forced to throttle against that inode. Also, the code reevaluates
42 * the dirty each time it has written this many pages.
43 */
44#define MAX_WRITEBACK_PAGES 1024
45
46/*
47 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
48 * will look to see if it needs to force writeback or throttling.
49 */
50static long ratelimit_pages = 32;
51
Andrew Mortone236a162006-01-18 17:42:26 -080052static int dirty_exceeded __cacheline_aligned_in_smp; /* Dirty mem may be over limit */
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54/*
55 * When balance_dirty_pages decides that the caller needs to perform some
56 * non-background writeback, this is how many pages it will attempt to write.
57 * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably
58 * large amounts of I/O are submitted.
59 */
60static inline long sync_writeback_pages(void)
61{
62 return ratelimit_pages + ratelimit_pages / 2;
63}
64
65/* The following parameters are exported via /proc/sys/vm */
66
67/*
68 * Start background writeback (via pdflush) at this percentage
69 */
Linus Torvalds07db59b2007-04-27 09:10:47 -070070int dirty_background_ratio = 5;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72/*
73 * The generator of dirty data starts writeback at this percentage
74 */
Linus Torvalds07db59b2007-04-27 09:10:47 -070075int vm_dirty_ratio = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
77/*
Coywolf Qi Huntfd5403c2006-04-10 22:54:35 -070078 * The interval between `kupdate'-style writebacks, in jiffies
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 */
Bart Samwelf6ef9432006-03-24 03:15:48 -080080int dirty_writeback_interval = 5 * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
82/*
Coywolf Qi Huntfd5403c2006-04-10 22:54:35 -070083 * The longest number of jiffies for which data is allowed to remain dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 */
Bart Samwelf6ef9432006-03-24 03:15:48 -080085int dirty_expire_interval = 30 * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
87/*
88 * Flag that makes the machine dump writes/reads and block dirtyings.
89 */
90int block_dump;
91
92/*
Bart Samweled5b43f2006-03-24 03:15:49 -080093 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
94 * a full sync is triggered after this time elapses without any disk activity.
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 */
96int laptop_mode;
97
98EXPORT_SYMBOL(laptop_mode);
99
100/* End of sysctl-exported parameters */
101
102
103static void background_writeout(unsigned long _min_pages);
104
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105/*
106 * Work out the current dirty-memory clamping and background writeout
107 * thresholds.
108 *
109 * The main aim here is to lower them aggressively if there is a lot of mapped
110 * memory around. To avoid stressing page reclaim with lots of unreclaimable
111 * pages. It is better to clamp down on writers than to start swapping, and
112 * performing lots of scanning.
113 *
114 * We only allow 1/2 of the currently-unmapped memory to be dirtied.
115 *
116 * We don't permit the clamping level to fall below 5% - that is getting rather
117 * excessive.
118 *
119 * We make sure that the background writeout level is below the adjusted
120 * clamping level.
121 */
Christoph Lameter1b424462007-05-06 14:48:59 -0700122
123static unsigned long highmem_dirtyable_memory(unsigned long total)
124{
125#ifdef CONFIG_HIGHMEM
126 int node;
127 unsigned long x = 0;
128
129 for_each_online_node(node) {
130 struct zone *z =
131 &NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
132
133 x += zone_page_state(z, NR_FREE_PAGES)
134 + zone_page_state(z, NR_INACTIVE)
135 + zone_page_state(z, NR_ACTIVE);
136 }
137 /*
138 * Make sure that the number of highmem pages is never larger
139 * than the number of the total dirtyable memory. This can only
140 * occur in very strange VM situations but we want to make sure
141 * that this does not occur.
142 */
143 return min(x, total);
144#else
145 return 0;
146#endif
147}
148
149static unsigned long determine_dirtyable_memory(void)
150{
151 unsigned long x;
152
153 x = global_page_state(NR_FREE_PAGES)
154 + global_page_state(NR_INACTIVE)
155 + global_page_state(NR_ACTIVE);
156 x -= highmem_dirtyable_memory(x);
157 return x + 1; /* Ensure that we never return 0 */
158}
159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160static void
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700161get_dirty_limits(long *pbackground, long *pdirty,
162 struct address_space *mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
164 int background_ratio; /* Percentages */
165 int dirty_ratio;
166 int unmapped_ratio;
167 long background;
168 long dirty;
Christoph Lameter1b424462007-05-06 14:48:59 -0700169 unsigned long available_memory = determine_dirtyable_memory();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 struct task_struct *tsk;
171
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700172 unmapped_ratio = 100 - ((global_page_state(NR_FILE_MAPPED) +
173 global_page_state(NR_ANON_PAGES)) * 100) /
Christoph Lameter1b424462007-05-06 14:48:59 -0700174 available_memory;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175
176 dirty_ratio = vm_dirty_ratio;
177 if (dirty_ratio > unmapped_ratio / 2)
178 dirty_ratio = unmapped_ratio / 2;
179
180 if (dirty_ratio < 5)
181 dirty_ratio = 5;
182
183 background_ratio = dirty_background_ratio;
184 if (background_ratio >= dirty_ratio)
185 background_ratio = dirty_ratio / 2;
186
187 background = (background_ratio * available_memory) / 100;
188 dirty = (dirty_ratio * available_memory) / 100;
189 tsk = current;
190 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
191 background += background / 4;
192 dirty += dirty / 4;
193 }
194 *pbackground = background;
195 *pdirty = dirty;
196}
197
198/*
199 * balance_dirty_pages() must be called by processes which are generating dirty
200 * data. It looks at the number of dirty pages in the machine and will force
201 * the caller to perform writeback if the system is over `vm_dirty_ratio'.
202 * If we're over `background_thresh' then pdflush is woken to perform some
203 * writeout.
204 */
205static void balance_dirty_pages(struct address_space *mapping)
206{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 long nr_reclaimable;
208 long background_thresh;
209 long dirty_thresh;
210 unsigned long pages_written = 0;
211 unsigned long write_chunk = sync_writeback_pages();
212
213 struct backing_dev_info *bdi = mapping->backing_dev_info;
214
215 for (;;) {
216 struct writeback_control wbc = {
217 .bdi = bdi,
218 .sync_mode = WB_SYNC_NONE,
219 .older_than_this = NULL,
220 .nr_to_write = write_chunk,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700221 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 };
223
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700224 get_dirty_limits(&background_thresh, &dirty_thresh, mapping);
225 nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
226 global_page_state(NR_UNSTABLE_NFS);
227 if (nr_reclaimable + global_page_state(NR_WRITEBACK) <=
228 dirty_thresh)
229 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Andrew Mortone236a162006-01-18 17:42:26 -0800231 if (!dirty_exceeded)
232 dirty_exceeded = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
235 * Unstable writes are a feature of certain networked
236 * filesystems (i.e. NFS) in which data may have been
237 * written to the server's write cache, but has not yet
238 * been flushed to permanent storage.
239 */
240 if (nr_reclaimable) {
241 writeback_inodes(&wbc);
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700242 get_dirty_limits(&background_thresh,
243 &dirty_thresh, mapping);
244 nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
245 global_page_state(NR_UNSTABLE_NFS);
246 if (nr_reclaimable +
247 global_page_state(NR_WRITEBACK)
248 <= dirty_thresh)
249 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 pages_written += write_chunk - wbc.nr_to_write;
251 if (pages_written >= write_chunk)
252 break; /* We've done our duty */
253 }
Andrew Morton3fcfab12006-10-19 23:28:16 -0700254 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 }
256
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700257 if (nr_reclaimable + global_page_state(NR_WRITEBACK)
258 <= dirty_thresh && dirty_exceeded)
259 dirty_exceeded = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
261 if (writeback_in_progress(bdi))
262 return; /* pdflush is already working this queue */
263
264 /*
265 * In laptop mode, we wait until hitting the higher threshold before
266 * starting background writeout, and then write out all the way down
267 * to the lower threshold. So slow writers cause minimal disk activity.
268 *
269 * In normal mode, we start background writeout at the lower
270 * background_thresh, to keep the amount of dirty memory low.
271 */
272 if ((laptop_mode && pages_written) ||
273 (!laptop_mode && (nr_reclaimable > background_thresh)))
274 pdflush_operation(background_writeout, 0);
275}
276
Peter Zijlstraedc79b22006-09-25 23:30:58 -0700277void set_page_dirty_balance(struct page *page)
278{
279 if (set_page_dirty(page)) {
280 struct address_space *mapping = page_mapping(page);
281
282 if (mapping)
283 balance_dirty_pages_ratelimited(mapping);
284 }
285}
286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287/**
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800288 * balance_dirty_pages_ratelimited_nr - balance dirty memory state
Martin Waitz67be2dd2005-05-01 08:59:26 -0700289 * @mapping: address_space which was dirtied
Martin Waitza5802902006-04-02 13:59:55 +0200290 * @nr_pages_dirtied: number of pages which the caller has just dirtied
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 *
292 * Processes which are dirtying memory should call in here once for each page
293 * which was newly dirtied. The function will periodically check the system's
294 * dirty state and will initiate writeback if needed.
295 *
296 * On really big machines, get_writeback_state is expensive, so try to avoid
297 * calling it too often (ratelimiting). But once we're over the dirty memory
298 * limit we decrease the ratelimiting by a lot, to prevent individual processes
299 * from overshooting the limit by (ratelimit_pages) each.
300 */
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800301void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
302 unsigned long nr_pages_dirtied)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303{
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800304 static DEFINE_PER_CPU(unsigned long, ratelimits) = 0;
305 unsigned long ratelimit;
306 unsigned long *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
308 ratelimit = ratelimit_pages;
309 if (dirty_exceeded)
310 ratelimit = 8;
311
312 /*
313 * Check the rate limiting. Also, we do not want to throttle real-time
314 * tasks in balance_dirty_pages(). Period.
315 */
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800316 preempt_disable();
317 p = &__get_cpu_var(ratelimits);
318 *p += nr_pages_dirtied;
319 if (unlikely(*p >= ratelimit)) {
320 *p = 0;
321 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 balance_dirty_pages(mapping);
323 return;
324 }
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800325 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326}
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800327EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
Andrew Morton232ea4d2007-02-28 20:13:21 -0800329void throttle_vm_writeout(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 long background_thresh;
332 long dirty_thresh;
333
Andrew Morton232ea4d2007-02-28 20:13:21 -0800334 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) {
335 /*
336 * The caller might hold locks which can prevent IO completion
337 * or progress in the filesystem. So we cannot just sit here
338 * waiting for IO to complete.
339 */
340 congestion_wait(WRITE, HZ/10);
341 return;
342 }
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 for ( ; ; ) {
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700345 get_dirty_limits(&background_thresh, &dirty_thresh, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347 /*
348 * Boost the allowable dirty threshold a bit for page
349 * allocators so they don't get DoS'ed by heavy writers
350 */
351 dirty_thresh += dirty_thresh / 10; /* wheeee... */
352
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700353 if (global_page_state(NR_UNSTABLE_NFS) +
354 global_page_state(NR_WRITEBACK) <= dirty_thresh)
355 break;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700356 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 }
358}
359
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360/*
361 * writeback at least _min_pages, and keep writing until the amount of dirty
362 * memory is less than the background threshold, or until we're all clean.
363 */
364static void background_writeout(unsigned long _min_pages)
365{
366 long min_pages = _min_pages;
367 struct writeback_control wbc = {
368 .bdi = NULL,
369 .sync_mode = WB_SYNC_NONE,
370 .older_than_this = NULL,
371 .nr_to_write = 0,
372 .nonblocking = 1,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700373 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 };
375
376 for ( ; ; ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 long background_thresh;
378 long dirty_thresh;
379
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700380 get_dirty_limits(&background_thresh, &dirty_thresh, NULL);
381 if (global_page_state(NR_FILE_DIRTY) +
382 global_page_state(NR_UNSTABLE_NFS) < background_thresh
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 && min_pages <= 0)
384 break;
385 wbc.encountered_congestion = 0;
386 wbc.nr_to_write = MAX_WRITEBACK_PAGES;
387 wbc.pages_skipped = 0;
388 writeback_inodes(&wbc);
389 min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
390 if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
391 /* Wrote less than expected */
Andrew Morton3fcfab12006-10-19 23:28:16 -0700392 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 if (!wbc.encountered_congestion)
394 break;
395 }
396 }
397}
398
399/*
400 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
401 * the whole world. Returns 0 if a pdflush thread was dispatched. Returns
402 * -1 if all pdflush threads were busy.
403 */
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700404int wakeup_pdflush(long nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405{
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700406 if (nr_pages == 0)
407 nr_pages = global_page_state(NR_FILE_DIRTY) +
408 global_page_state(NR_UNSTABLE_NFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 return pdflush_operation(background_writeout, nr_pages);
410}
411
412static void wb_timer_fn(unsigned long unused);
413static void laptop_timer_fn(unsigned long unused);
414
Ingo Molnar8d06afa2005-09-09 13:10:40 -0700415static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0);
416static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
418/*
419 * Periodic writeback of "old" data.
420 *
421 * Define "old": the first time one of an inode's pages is dirtied, we mark the
422 * dirtying-time in the inode's address_space. So this periodic writeback code
423 * just walks the superblock inode list, writing back any inodes which are
424 * older than a specific point in time.
425 *
Bart Samwelf6ef9432006-03-24 03:15:48 -0800426 * Try to run once per dirty_writeback_interval. But if a writeback event
427 * takes longer than a dirty_writeback_interval interval, then leave a
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 * one-second gap.
429 *
430 * older_than_this takes precedence over nr_to_write. So we'll only write back
431 * all dirty pages if they are all attached to "old" mappings.
432 */
433static void wb_kupdate(unsigned long arg)
434{
435 unsigned long oldest_jif;
436 unsigned long start_jif;
437 unsigned long next_jif;
438 long nr_to_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 struct writeback_control wbc = {
440 .bdi = NULL,
441 .sync_mode = WB_SYNC_NONE,
442 .older_than_this = &oldest_jif,
443 .nr_to_write = 0,
444 .nonblocking = 1,
445 .for_kupdate = 1,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700446 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 };
448
449 sync_supers();
450
Bart Samwelf6ef9432006-03-24 03:15:48 -0800451 oldest_jif = jiffies - dirty_expire_interval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 start_jif = jiffies;
Bart Samwelf6ef9432006-03-24 03:15:48 -0800453 next_jif = start_jif + dirty_writeback_interval;
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700454 nr_to_write = global_page_state(NR_FILE_DIRTY) +
455 global_page_state(NR_UNSTABLE_NFS) +
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 (inodes_stat.nr_inodes - inodes_stat.nr_unused);
457 while (nr_to_write > 0) {
458 wbc.encountered_congestion = 0;
459 wbc.nr_to_write = MAX_WRITEBACK_PAGES;
460 writeback_inodes(&wbc);
461 if (wbc.nr_to_write > 0) {
462 if (wbc.encountered_congestion)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700463 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 else
465 break; /* All the old data is written */
466 }
467 nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
468 }
469 if (time_before(next_jif, jiffies + HZ))
470 next_jif = jiffies + HZ;
Bart Samwelf6ef9432006-03-24 03:15:48 -0800471 if (dirty_writeback_interval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 mod_timer(&wb_timer, next_jif);
473}
474
475/*
476 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
477 */
478int dirty_writeback_centisecs_handler(ctl_table *table, int write,
479 struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
480{
Bart Samwelf6ef9432006-03-24 03:15:48 -0800481 proc_dointvec_userhz_jiffies(table, write, file, buffer, length, ppos);
482 if (dirty_writeback_interval) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 mod_timer(&wb_timer,
Bart Samwelf6ef9432006-03-24 03:15:48 -0800484 jiffies + dirty_writeback_interval);
485 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 del_timer(&wb_timer);
487 }
488 return 0;
489}
490
491static void wb_timer_fn(unsigned long unused)
492{
493 if (pdflush_operation(wb_kupdate, 0) < 0)
494 mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */
495}
496
497static void laptop_flush(unsigned long unused)
498{
499 sys_sync();
500}
501
502static void laptop_timer_fn(unsigned long unused)
503{
504 pdflush_operation(laptop_flush, 0);
505}
506
507/*
508 * We've spun up the disk and we're in laptop mode: schedule writeback
509 * of all dirty data a few seconds from now. If the flush is already scheduled
510 * then push it back - the user is still using the disk.
511 */
512void laptop_io_completion(void)
513{
Bart Samweled5b43f2006-03-24 03:15:49 -0800514 mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515}
516
517/*
518 * We're in laptop mode and we've just synced. The sync's writes will have
519 * caused another writeback to be scheduled by laptop_io_completion.
520 * Nothing needs to be written back anymore, so we unschedule the writeback.
521 */
522void laptop_sync_completion(void)
523{
524 del_timer(&laptop_mode_wb_timer);
525}
526
527/*
528 * If ratelimit_pages is too high then we can get into dirty-data overload
529 * if a large number of processes all perform writes at the same time.
530 * If it is too low then SMP machines will call the (expensive)
531 * get_writeback_state too often.
532 *
533 * Here we set ratelimit_pages to a level which ensures that when all CPUs are
534 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
535 * thresholds before writeback cuts in.
536 *
537 * But the limit should not be set too high. Because it also controls the
538 * amount of memory which the balance_dirty_pages() caller has to write back.
539 * If this is too large then the caller will block on the IO queue all the
540 * time. So limit it to four megabytes - the balance_dirty_pages() caller
541 * will write six megabyte chunks, max.
542 */
543
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700544void writeback_set_ratelimit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545{
Chandra Seetharaman40c99aa2006-09-29 02:01:24 -0700546 ratelimit_pages = vm_total_pages / (num_online_cpus() * 32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 if (ratelimit_pages < 16)
548 ratelimit_pages = 16;
549 if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024)
550 ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE;
551}
552
Chandra Seetharaman26c21432006-06-27 02:54:10 -0700553static int __cpuinit
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554ratelimit_handler(struct notifier_block *self, unsigned long u, void *v)
555{
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700556 writeback_set_ratelimit();
Paul E. McKenneyaa0f0302007-02-10 01:46:37 -0800557 return NOTIFY_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558}
559
Chandra Seetharaman74b85f32006-06-27 02:54:09 -0700560static struct notifier_block __cpuinitdata ratelimit_nb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 .notifier_call = ratelimit_handler,
562 .next = NULL,
563};
564
565/*
Linus Torvaldsdc6e29d2007-01-29 16:37:38 -0800566 * Called early on to tune the page writeback dirty limits.
567 *
568 * We used to scale dirty pages according to how total memory
569 * related to pages that could be allocated for buffers (by
570 * comparing nr_free_buffer_pages() to vm_total_pages.
571 *
572 * However, that was when we used "dirty_ratio" to scale with
573 * all memory, and we don't do that any more. "dirty_ratio"
574 * is now applied to total non-HIGHPAGE memory (by subtracting
575 * totalhigh_pages from vm_total_pages), and as such we can't
576 * get into the old insane situation any more where we had
577 * large amounts of dirty pages compared to a small amount of
578 * non-HIGHMEM memory.
579 *
580 * But we might still want to scale the dirty_ratio by how
581 * much memory the box has..
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 */
583void __init page_writeback_init(void)
584{
Bart Samwelf6ef9432006-03-24 03:15:48 -0800585 mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700586 writeback_set_ratelimit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 register_cpu_notifier(&ratelimit_nb);
588}
589
David Howells811d7362006-08-29 19:06:09 +0100590/**
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800591 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them.
David Howells811d7362006-08-29 19:06:09 +0100592 * @mapping: address space structure to write
593 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
594 *
595 * This is a library function, which implements the writepages()
596 * address_space_operation.
597 *
598 * If a page is already under I/O, generic_writepages() skips it, even
599 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
600 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
601 * and msync() need to guarantee that all the data which was dirty at the time
602 * the call was made get new I/O started against them. If wbc->sync_mode is
603 * WB_SYNC_ALL then we were called for data integrity and we must wait for
604 * existing IO to complete.
605 *
606 * Derived from mpage_writepages() - if you fix this you should check that
607 * also!
608 */
609int generic_writepages(struct address_space *mapping,
610 struct writeback_control *wbc)
611{
612 struct backing_dev_info *bdi = mapping->backing_dev_info;
613 int ret = 0;
614 int done = 0;
615 int (*writepage)(struct page *page, struct writeback_control *wbc);
616 struct pagevec pvec;
617 int nr_pages;
618 pgoff_t index;
619 pgoff_t end; /* Inclusive */
620 int scanned = 0;
621 int range_whole = 0;
622
623 if (wbc->nonblocking && bdi_write_congested(bdi)) {
624 wbc->encountered_congestion = 1;
625 return 0;
626 }
627
628 writepage = mapping->a_ops->writepage;
629
630 /* deal with chardevs and other special file */
631 if (!writepage)
632 return 0;
633
634 pagevec_init(&pvec, 0);
635 if (wbc->range_cyclic) {
636 index = mapping->writeback_index; /* Start from prev offset */
637 end = -1;
638 } else {
639 index = wbc->range_start >> PAGE_CACHE_SHIFT;
640 end = wbc->range_end >> PAGE_CACHE_SHIFT;
641 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
642 range_whole = 1;
643 scanned = 1;
644 }
645retry:
646 while (!done && (index <= end) &&
647 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
648 PAGECACHE_TAG_DIRTY,
649 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
650 unsigned i;
651
652 scanned = 1;
653 for (i = 0; i < nr_pages; i++) {
654 struct page *page = pvec.pages[i];
655
656 /*
657 * At this point we hold neither mapping->tree_lock nor
658 * lock on the page itself: the page may be truncated or
659 * invalidated (changing page->mapping to NULL), or even
660 * swizzled back from swapper_space to tmpfs file
661 * mapping
662 */
663 lock_page(page);
664
665 if (unlikely(page->mapping != mapping)) {
666 unlock_page(page);
667 continue;
668 }
669
670 if (!wbc->range_cyclic && page->index > end) {
671 done = 1;
672 unlock_page(page);
673 continue;
674 }
675
676 if (wbc->sync_mode != WB_SYNC_NONE)
677 wait_on_page_writeback(page);
678
679 if (PageWriteback(page) ||
680 !clear_page_dirty_for_io(page)) {
681 unlock_page(page);
682 continue;
683 }
684
685 ret = (*writepage)(page, wbc);
686 if (ret) {
687 if (ret == -ENOSPC)
688 set_bit(AS_ENOSPC, &mapping->flags);
689 else
690 set_bit(AS_EIO, &mapping->flags);
691 }
692
693 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE))
694 unlock_page(page);
695 if (ret || (--(wbc->nr_to_write) <= 0))
696 done = 1;
697 if (wbc->nonblocking && bdi_write_congested(bdi)) {
698 wbc->encountered_congestion = 1;
699 done = 1;
700 }
701 }
702 pagevec_release(&pvec);
703 cond_resched();
704 }
705 if (!scanned && !done) {
706 /*
707 * We hit the last page and there is more work to be done: wrap
708 * back to the start of the file
709 */
710 scanned = 1;
711 index = 0;
712 goto retry;
713 }
714 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
715 mapping->writeback_index = index;
716 return ret;
717}
718
719EXPORT_SYMBOL(generic_writepages);
720
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
722{
Andrew Morton22905f72005-11-16 15:07:01 -0800723 int ret;
724
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 if (wbc->nr_to_write <= 0)
726 return 0;
Andrew Morton22905f72005-11-16 15:07:01 -0800727 wbc->for_writepages = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 if (mapping->a_ops->writepages)
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700729 ret = mapping->a_ops->writepages(mapping, wbc);
Andrew Morton22905f72005-11-16 15:07:01 -0800730 else
731 ret = generic_writepages(mapping, wbc);
732 wbc->for_writepages = 0;
733 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734}
735
736/**
737 * write_one_page - write out a single page and optionally wait on I/O
Martin Waitz67be2dd2005-05-01 08:59:26 -0700738 * @page: the page to write
739 * @wait: if true, wait on writeout
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 *
741 * The page must be locked by the caller and will be unlocked upon return.
742 *
743 * write_one_page() returns a negative error code if I/O failed.
744 */
745int write_one_page(struct page *page, int wait)
746{
747 struct address_space *mapping = page->mapping;
748 int ret = 0;
749 struct writeback_control wbc = {
750 .sync_mode = WB_SYNC_ALL,
751 .nr_to_write = 1,
752 };
753
754 BUG_ON(!PageLocked(page));
755
756 if (wait)
757 wait_on_page_writeback(page);
758
759 if (clear_page_dirty_for_io(page)) {
760 page_cache_get(page);
761 ret = mapping->a_ops->writepage(page, &wbc);
762 if (ret == 0 && wait) {
763 wait_on_page_writeback(page);
764 if (PageError(page))
765 ret = -EIO;
766 }
767 page_cache_release(page);
768 } else {
769 unlock_page(page);
770 }
771 return ret;
772}
773EXPORT_SYMBOL(write_one_page);
774
775/*
Ken Chen76719322007-02-10 01:43:15 -0800776 * For address_spaces which do not use buffers nor write back.
777 */
778int __set_page_dirty_no_writeback(struct page *page)
779{
780 if (!PageDirty(page))
781 SetPageDirty(page);
782 return 0;
783}
784
785/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 * For address_spaces which do not use buffers. Just tag the page as dirty in
787 * its radix tree.
788 *
789 * This is also used when a single buffer is being dirtied: we want to set the
790 * page dirty in that case, but not all the buffers. This is a "bottom-up"
791 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
792 *
793 * Most callers have locked the page, which pins the address_space in memory.
794 * But zap_pte_range() does not lock the page, however in that case the
795 * mapping is pinned by the vma's ->vm_file reference.
796 *
797 * We take care to handle the case where the page was truncated from the
798 * mapping by re-checking page_mapping() insode tree_lock.
799 */
800int __set_page_dirty_nobuffers(struct page *page)
801{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 if (!TestSetPageDirty(page)) {
803 struct address_space *mapping = page_mapping(page);
804 struct address_space *mapping2;
805
Andrew Morton8c085402006-12-10 02:19:24 -0800806 if (!mapping)
807 return 1;
808
809 write_lock_irq(&mapping->tree_lock);
810 mapping2 = page_mapping(page);
811 if (mapping2) { /* Race with truncate? */
812 BUG_ON(mapping2 != mapping);
Andrew Morton55e829a2006-12-10 02:19:27 -0800813 if (mapping_cap_account_dirty(mapping)) {
Andrew Morton8c085402006-12-10 02:19:24 -0800814 __inc_zone_page_state(page, NR_FILE_DIRTY);
Andrew Morton55e829a2006-12-10 02:19:27 -0800815 task_io_account_write(PAGE_CACHE_SIZE);
816 }
Andrew Morton8c085402006-12-10 02:19:24 -0800817 radix_tree_tag_set(&mapping->page_tree,
818 page_index(page), PAGECACHE_TAG_DIRTY);
819 }
820 write_unlock_irq(&mapping->tree_lock);
821 if (mapping->host) {
822 /* !PageAnon && !swapper_space */
823 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800825 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800827 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828}
829EXPORT_SYMBOL(__set_page_dirty_nobuffers);
830
831/*
832 * When a writepage implementation decides that it doesn't want to write this
833 * page for some reason, it should redirty the locked page via
834 * redirty_page_for_writepage() and it should then unlock the page and return 0
835 */
836int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
837{
838 wbc->pages_skipped++;
839 return __set_page_dirty_nobuffers(page);
840}
841EXPORT_SYMBOL(redirty_page_for_writepage);
842
843/*
844 * If the mapping doesn't provide a set_page_dirty a_op, then
845 * just fall through and assume that it wants buffer_heads.
846 */
847int fastcall set_page_dirty(struct page *page)
848{
849 struct address_space *mapping = page_mapping(page);
850
851 if (likely(mapping)) {
852 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
David Howells93614012006-09-30 20:45:40 +0200853#ifdef CONFIG_BLOCK
854 if (!spd)
855 spd = __set_page_dirty_buffers;
856#endif
857 return (*spd)(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800859 if (!PageDirty(page)) {
860 if (!TestSetPageDirty(page))
861 return 1;
862 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 return 0;
864}
865EXPORT_SYMBOL(set_page_dirty);
866
867/*
868 * set_page_dirty() is racy if the caller has no reference against
869 * page->mapping->host, and if the page is unlocked. This is because another
870 * CPU could truncate the page off the mapping and then free the mapping.
871 *
872 * Usually, the page _is_ locked, or the caller is a user-space process which
873 * holds a reference on the inode by having an open file.
874 *
875 * In other cases, the page should be locked before running set_page_dirty().
876 */
877int set_page_dirty_lock(struct page *page)
878{
879 int ret;
880
Nick Piggindb376482006-09-25 23:31:24 -0700881 lock_page_nosync(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 ret = set_page_dirty(page);
883 unlock_page(page);
884 return ret;
885}
886EXPORT_SYMBOL(set_page_dirty_lock);
887
888/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 * Clear a page's dirty flag, while caring for dirty memory accounting.
890 * Returns true if the page was previously dirty.
891 *
892 * This is for preparing to put the page under writeout. We leave the page
893 * tagged as dirty in the radix tree so that a concurrent write-for-sync
894 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage
895 * implementation will run either set_page_writeback() or set_page_dirty(),
896 * at which stage we bring the page's dirty flag and radix-tree dirty tag
897 * back into sync.
898 *
899 * This incoherency between the page's dirty flag and radix-tree tag is
900 * unfortunate, but it only exists while the page is locked.
901 */
902int clear_page_dirty_for_io(struct page *page)
903{
904 struct address_space *mapping = page_mapping(page);
905
Linus Torvalds7658cc22006-12-29 10:00:58 -0800906 if (mapping && mapping_cap_account_dirty(mapping)) {
907 /*
908 * Yes, Virginia, this is indeed insane.
909 *
910 * We use this sequence to make sure that
911 * (a) we account for dirty stats properly
912 * (b) we tell the low-level filesystem to
913 * mark the whole page dirty if it was
914 * dirty in a pagetable. Only to then
915 * (c) clean the page again and return 1 to
916 * cause the writeback.
917 *
918 * This way we avoid all nasty races with the
919 * dirty bit in multiple places and clearing
920 * them concurrently from different threads.
921 *
922 * Note! Normally the "set_page_dirty(page)"
923 * has no effect on the actual dirty bit - since
924 * that will already usually be set. But we
925 * need the side effects, and it can help us
926 * avoid races.
927 *
928 * We basically use the page "master dirty bit"
929 * as a serialization point for all the different
930 * threads doing their things.
931 *
932 * FIXME! We still have a race here: if somebody
933 * adds the page back to the page tables in
934 * between the "page_mkclean()" and the "TestClearPageDirty()",
935 * we might have it mapped without the dirty bit set.
936 */
937 if (page_mkclean(page))
938 set_page_dirty(page);
939 if (TestClearPageDirty(page)) {
Andrew Morton8c085402006-12-10 02:19:24 -0800940 dec_zone_page_state(page, NR_FILE_DIRTY);
Linus Torvalds7658cc22006-12-29 10:00:58 -0800941 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 }
Linus Torvalds7658cc22006-12-29 10:00:58 -0800943 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 }
Linus Torvalds7658cc22006-12-29 10:00:58 -0800945 return TestClearPageDirty(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946}
Hans Reiser58bb01a2005-11-18 01:10:53 -0800947EXPORT_SYMBOL(clear_page_dirty_for_io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948
949int test_clear_page_writeback(struct page *page)
950{
951 struct address_space *mapping = page_mapping(page);
952 int ret;
953
954 if (mapping) {
955 unsigned long flags;
956
957 write_lock_irqsave(&mapping->tree_lock, flags);
958 ret = TestClearPageWriteback(page);
959 if (ret)
960 radix_tree_tag_clear(&mapping->page_tree,
961 page_index(page),
962 PAGECACHE_TAG_WRITEBACK);
963 write_unlock_irqrestore(&mapping->tree_lock, flags);
964 } else {
965 ret = TestClearPageWriteback(page);
966 }
967 return ret;
968}
969
970int test_set_page_writeback(struct page *page)
971{
972 struct address_space *mapping = page_mapping(page);
973 int ret;
974
975 if (mapping) {
976 unsigned long flags;
977
978 write_lock_irqsave(&mapping->tree_lock, flags);
979 ret = TestSetPageWriteback(page);
980 if (!ret)
981 radix_tree_tag_set(&mapping->page_tree,
982 page_index(page),
983 PAGECACHE_TAG_WRITEBACK);
984 if (!PageDirty(page))
985 radix_tree_tag_clear(&mapping->page_tree,
986 page_index(page),
987 PAGECACHE_TAG_DIRTY);
988 write_unlock_irqrestore(&mapping->tree_lock, flags);
989 } else {
990 ret = TestSetPageWriteback(page);
991 }
992 return ret;
993
994}
995EXPORT_SYMBOL(test_set_page_writeback);
996
997/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 * Return true if any of the pages in the mapping are marged with the
999 * passed tag.
1000 */
1001int mapping_tagged(struct address_space *mapping, int tag)
1002{
1003 unsigned long flags;
1004 int ret;
1005
1006 read_lock_irqsave(&mapping->tree_lock, flags);
1007 ret = radix_tree_tagged(&mapping->page_tree, tag);
1008 read_unlock_irqrestore(&mapping->tree_lock, flags);
1009 return ret;
1010}
1011EXPORT_SYMBOL(mapping_tagged);