blob: 63512a9ed57e7df17bec12f4dff1685f8944042a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Uwe Zeisbergerf30c2262006-10-03 23:01:26 +02002 * mm/page-writeback.c
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * Contains functions related to writing back dirty pages at the
7 * address_space level.
8 *
9 * 10Apr2002 akpm@zip.com.au
10 * Initial version
11 */
12
13#include <linux/kernel.h>
14#include <linux/module.h>
15#include <linux/spinlock.h>
16#include <linux/fs.h>
17#include <linux/mm.h>
18#include <linux/swap.h>
19#include <linux/slab.h>
20#include <linux/pagemap.h>
21#include <linux/writeback.h>
22#include <linux/init.h>
23#include <linux/backing-dev.h>
Andrew Morton55e829a2006-12-10 02:19:27 -080024#include <linux/task_io_accounting_ops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/blkdev.h>
26#include <linux/mpage.h>
Peter Zijlstrad08b3852006-09-25 23:30:57 -070027#include <linux/rmap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/percpu.h>
29#include <linux/notifier.h>
30#include <linux/smp.h>
31#include <linux/sysctl.h>
32#include <linux/cpu.h>
33#include <linux/syscalls.h>
David Howellscf9a2ae2006-08-29 19:05:54 +010034#include <linux/buffer_head.h>
David Howells811d7362006-08-29 19:06:09 +010035#include <linux/pagevec.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
37/*
38 * The maximum number of pages to writeout in a single bdflush/kupdate
39 * operation. We do this so we don't hold I_LOCK against an inode for
40 * enormous amounts of time, which would block a userspace task which has
41 * been forced to throttle against that inode. Also, the code reevaluates
42 * the dirty each time it has written this many pages.
43 */
44#define MAX_WRITEBACK_PAGES 1024
45
46/*
47 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
48 * will look to see if it needs to force writeback or throttling.
49 */
50static long ratelimit_pages = 32;
51
Andrew Mortone236a162006-01-18 17:42:26 -080052static int dirty_exceeded __cacheline_aligned_in_smp; /* Dirty mem may be over limit */
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54/*
55 * When balance_dirty_pages decides that the caller needs to perform some
56 * non-background writeback, this is how many pages it will attempt to write.
57 * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably
58 * large amounts of I/O are submitted.
59 */
60static inline long sync_writeback_pages(void)
61{
62 return ratelimit_pages + ratelimit_pages / 2;
63}
64
65/* The following parameters are exported via /proc/sys/vm */
66
67/*
68 * Start background writeback (via pdflush) at this percentage
69 */
Linus Torvalds07db59b2007-04-27 09:10:47 -070070int dirty_background_ratio = 5;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72/*
73 * The generator of dirty data starts writeback at this percentage
74 */
Linus Torvalds07db59b2007-04-27 09:10:47 -070075int vm_dirty_ratio = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
77/*
Coywolf Qi Huntfd5403c2006-04-10 22:54:35 -070078 * The interval between `kupdate'-style writebacks, in jiffies
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 */
Bart Samwelf6ef9432006-03-24 03:15:48 -080080int dirty_writeback_interval = 5 * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
82/*
Coywolf Qi Huntfd5403c2006-04-10 22:54:35 -070083 * The longest number of jiffies for which data is allowed to remain dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 */
Bart Samwelf6ef9432006-03-24 03:15:48 -080085int dirty_expire_interval = 30 * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
87/*
88 * Flag that makes the machine dump writes/reads and block dirtyings.
89 */
90int block_dump;
91
92/*
Bart Samweled5b43f2006-03-24 03:15:49 -080093 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
94 * a full sync is triggered after this time elapses without any disk activity.
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 */
96int laptop_mode;
97
98EXPORT_SYMBOL(laptop_mode);
99
100/* End of sysctl-exported parameters */
101
102
103static void background_writeout(unsigned long _min_pages);
104
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105/*
106 * Work out the current dirty-memory clamping and background writeout
107 * thresholds.
108 *
109 * The main aim here is to lower them aggressively if there is a lot of mapped
110 * memory around. To avoid stressing page reclaim with lots of unreclaimable
111 * pages. It is better to clamp down on writers than to start swapping, and
112 * performing lots of scanning.
113 *
114 * We only allow 1/2 of the currently-unmapped memory to be dirtied.
115 *
116 * We don't permit the clamping level to fall below 5% - that is getting rather
117 * excessive.
118 *
119 * We make sure that the background writeout level is below the adjusted
120 * clamping level.
121 */
Christoph Lameter1b424462007-05-06 14:48:59 -0700122
123static unsigned long highmem_dirtyable_memory(unsigned long total)
124{
125#ifdef CONFIG_HIGHMEM
126 int node;
127 unsigned long x = 0;
128
129 for_each_online_node(node) {
130 struct zone *z =
131 &NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
132
133 x += zone_page_state(z, NR_FREE_PAGES)
134 + zone_page_state(z, NR_INACTIVE)
135 + zone_page_state(z, NR_ACTIVE);
136 }
137 /*
138 * Make sure that the number of highmem pages is never larger
139 * than the number of the total dirtyable memory. This can only
140 * occur in very strange VM situations but we want to make sure
141 * that this does not occur.
142 */
143 return min(x, total);
144#else
145 return 0;
146#endif
147}
148
149static unsigned long determine_dirtyable_memory(void)
150{
151 unsigned long x;
152
153 x = global_page_state(NR_FREE_PAGES)
154 + global_page_state(NR_INACTIVE)
155 + global_page_state(NR_ACTIVE);
156 x -= highmem_dirtyable_memory(x);
157 return x + 1; /* Ensure that we never return 0 */
158}
159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160static void
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700161get_dirty_limits(long *pbackground, long *pdirty,
162 struct address_space *mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
164 int background_ratio; /* Percentages */
165 int dirty_ratio;
166 int unmapped_ratio;
167 long background;
168 long dirty;
Christoph Lameter1b424462007-05-06 14:48:59 -0700169 unsigned long available_memory = determine_dirtyable_memory();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 struct task_struct *tsk;
171
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700172 unmapped_ratio = 100 - ((global_page_state(NR_FILE_MAPPED) +
173 global_page_state(NR_ANON_PAGES)) * 100) /
Christoph Lameter1b424462007-05-06 14:48:59 -0700174 available_memory;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175
176 dirty_ratio = vm_dirty_ratio;
177 if (dirty_ratio > unmapped_ratio / 2)
178 dirty_ratio = unmapped_ratio / 2;
179
180 if (dirty_ratio < 5)
181 dirty_ratio = 5;
182
183 background_ratio = dirty_background_ratio;
184 if (background_ratio >= dirty_ratio)
185 background_ratio = dirty_ratio / 2;
186
187 background = (background_ratio * available_memory) / 100;
188 dirty = (dirty_ratio * available_memory) / 100;
189 tsk = current;
190 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
191 background += background / 4;
192 dirty += dirty / 4;
193 }
194 *pbackground = background;
195 *pdirty = dirty;
196}
197
198/*
199 * balance_dirty_pages() must be called by processes which are generating dirty
200 * data. It looks at the number of dirty pages in the machine and will force
201 * the caller to perform writeback if the system is over `vm_dirty_ratio'.
202 * If we're over `background_thresh' then pdflush is woken to perform some
203 * writeout.
204 */
205static void balance_dirty_pages(struct address_space *mapping)
206{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 long nr_reclaimable;
208 long background_thresh;
209 long dirty_thresh;
210 unsigned long pages_written = 0;
211 unsigned long write_chunk = sync_writeback_pages();
212
213 struct backing_dev_info *bdi = mapping->backing_dev_info;
214
215 for (;;) {
216 struct writeback_control wbc = {
217 .bdi = bdi,
218 .sync_mode = WB_SYNC_NONE,
219 .older_than_this = NULL,
220 .nr_to_write = write_chunk,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700221 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 };
223
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700224 get_dirty_limits(&background_thresh, &dirty_thresh, mapping);
225 nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
226 global_page_state(NR_UNSTABLE_NFS);
227 if (nr_reclaimable + global_page_state(NR_WRITEBACK) <=
228 dirty_thresh)
229 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Andrew Mortone236a162006-01-18 17:42:26 -0800231 if (!dirty_exceeded)
232 dirty_exceeded = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
235 * Unstable writes are a feature of certain networked
236 * filesystems (i.e. NFS) in which data may have been
237 * written to the server's write cache, but has not yet
238 * been flushed to permanent storage.
239 */
240 if (nr_reclaimable) {
241 writeback_inodes(&wbc);
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700242 get_dirty_limits(&background_thresh,
243 &dirty_thresh, mapping);
244 nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
245 global_page_state(NR_UNSTABLE_NFS);
246 if (nr_reclaimable +
247 global_page_state(NR_WRITEBACK)
248 <= dirty_thresh)
249 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 pages_written += write_chunk - wbc.nr_to_write;
251 if (pages_written >= write_chunk)
252 break; /* We've done our duty */
253 }
Andrew Morton3fcfab12006-10-19 23:28:16 -0700254 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 }
256
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700257 if (nr_reclaimable + global_page_state(NR_WRITEBACK)
258 <= dirty_thresh && dirty_exceeded)
259 dirty_exceeded = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
261 if (writeback_in_progress(bdi))
262 return; /* pdflush is already working this queue */
263
264 /*
265 * In laptop mode, we wait until hitting the higher threshold before
266 * starting background writeout, and then write out all the way down
267 * to the lower threshold. So slow writers cause minimal disk activity.
268 *
269 * In normal mode, we start background writeout at the lower
270 * background_thresh, to keep the amount of dirty memory low.
271 */
272 if ((laptop_mode && pages_written) ||
273 (!laptop_mode && (nr_reclaimable > background_thresh)))
274 pdflush_operation(background_writeout, 0);
275}
276
Peter Zijlstraedc79b22006-09-25 23:30:58 -0700277void set_page_dirty_balance(struct page *page)
278{
279 if (set_page_dirty(page)) {
280 struct address_space *mapping = page_mapping(page);
281
282 if (mapping)
283 balance_dirty_pages_ratelimited(mapping);
284 }
285}
286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287/**
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800288 * balance_dirty_pages_ratelimited_nr - balance dirty memory state
Martin Waitz67be2dd2005-05-01 08:59:26 -0700289 * @mapping: address_space which was dirtied
Martin Waitza5802902006-04-02 13:59:55 +0200290 * @nr_pages_dirtied: number of pages which the caller has just dirtied
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 *
292 * Processes which are dirtying memory should call in here once for each page
293 * which was newly dirtied. The function will periodically check the system's
294 * dirty state and will initiate writeback if needed.
295 *
296 * On really big machines, get_writeback_state is expensive, so try to avoid
297 * calling it too often (ratelimiting). But once we're over the dirty memory
298 * limit we decrease the ratelimiting by a lot, to prevent individual processes
299 * from overshooting the limit by (ratelimit_pages) each.
300 */
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800301void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
302 unsigned long nr_pages_dirtied)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303{
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800304 static DEFINE_PER_CPU(unsigned long, ratelimits) = 0;
305 unsigned long ratelimit;
306 unsigned long *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
308 ratelimit = ratelimit_pages;
309 if (dirty_exceeded)
310 ratelimit = 8;
311
312 /*
313 * Check the rate limiting. Also, we do not want to throttle real-time
314 * tasks in balance_dirty_pages(). Period.
315 */
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800316 preempt_disable();
317 p = &__get_cpu_var(ratelimits);
318 *p += nr_pages_dirtied;
319 if (unlikely(*p >= ratelimit)) {
320 *p = 0;
321 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 balance_dirty_pages(mapping);
323 return;
324 }
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800325 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326}
Andrew Mortonfa5a7342006-03-24 03:18:10 -0800327EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
Andrew Morton232ea4d2007-02-28 20:13:21 -0800329void throttle_vm_writeout(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 long background_thresh;
332 long dirty_thresh;
333
Andrew Morton232ea4d2007-02-28 20:13:21 -0800334 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) {
335 /*
336 * The caller might hold locks which can prevent IO completion
337 * or progress in the filesystem. So we cannot just sit here
338 * waiting for IO to complete.
339 */
340 congestion_wait(WRITE, HZ/10);
341 return;
342 }
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 for ( ; ; ) {
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700345 get_dirty_limits(&background_thresh, &dirty_thresh, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347 /*
348 * Boost the allowable dirty threshold a bit for page
349 * allocators so they don't get DoS'ed by heavy writers
350 */
351 dirty_thresh += dirty_thresh / 10; /* wheeee... */
352
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700353 if (global_page_state(NR_UNSTABLE_NFS) +
354 global_page_state(NR_WRITEBACK) <= dirty_thresh)
355 break;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700356 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 }
358}
359
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360/*
361 * writeback at least _min_pages, and keep writing until the amount of dirty
362 * memory is less than the background threshold, or until we're all clean.
363 */
364static void background_writeout(unsigned long _min_pages)
365{
366 long min_pages = _min_pages;
367 struct writeback_control wbc = {
368 .bdi = NULL,
369 .sync_mode = WB_SYNC_NONE,
370 .older_than_this = NULL,
371 .nr_to_write = 0,
372 .nonblocking = 1,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700373 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 };
375
376 for ( ; ; ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 long background_thresh;
378 long dirty_thresh;
379
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700380 get_dirty_limits(&background_thresh, &dirty_thresh, NULL);
381 if (global_page_state(NR_FILE_DIRTY) +
382 global_page_state(NR_UNSTABLE_NFS) < background_thresh
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 && min_pages <= 0)
384 break;
385 wbc.encountered_congestion = 0;
386 wbc.nr_to_write = MAX_WRITEBACK_PAGES;
387 wbc.pages_skipped = 0;
388 writeback_inodes(&wbc);
389 min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
390 if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
391 /* Wrote less than expected */
Andrew Morton3fcfab12006-10-19 23:28:16 -0700392 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 if (!wbc.encountered_congestion)
394 break;
395 }
396 }
397}
398
399/*
400 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
401 * the whole world. Returns 0 if a pdflush thread was dispatched. Returns
402 * -1 if all pdflush threads were busy.
403 */
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700404int wakeup_pdflush(long nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405{
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700406 if (nr_pages == 0)
407 nr_pages = global_page_state(NR_FILE_DIRTY) +
408 global_page_state(NR_UNSTABLE_NFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 return pdflush_operation(background_writeout, nr_pages);
410}
411
412static void wb_timer_fn(unsigned long unused);
413static void laptop_timer_fn(unsigned long unused);
414
Ingo Molnar8d06afa2005-09-09 13:10:40 -0700415static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0);
416static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
418/*
419 * Periodic writeback of "old" data.
420 *
421 * Define "old": the first time one of an inode's pages is dirtied, we mark the
422 * dirtying-time in the inode's address_space. So this periodic writeback code
423 * just walks the superblock inode list, writing back any inodes which are
424 * older than a specific point in time.
425 *
Bart Samwelf6ef9432006-03-24 03:15:48 -0800426 * Try to run once per dirty_writeback_interval. But if a writeback event
427 * takes longer than a dirty_writeback_interval interval, then leave a
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 * one-second gap.
429 *
430 * older_than_this takes precedence over nr_to_write. So we'll only write back
431 * all dirty pages if they are all attached to "old" mappings.
432 */
433static void wb_kupdate(unsigned long arg)
434{
435 unsigned long oldest_jif;
436 unsigned long start_jif;
437 unsigned long next_jif;
438 long nr_to_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 struct writeback_control wbc = {
440 .bdi = NULL,
441 .sync_mode = WB_SYNC_NONE,
442 .older_than_this = &oldest_jif,
443 .nr_to_write = 0,
444 .nonblocking = 1,
445 .for_kupdate = 1,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700446 .range_cyclic = 1,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 };
448
449 sync_supers();
450
Bart Samwelf6ef9432006-03-24 03:15:48 -0800451 oldest_jif = jiffies - dirty_expire_interval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 start_jif = jiffies;
Bart Samwelf6ef9432006-03-24 03:15:48 -0800453 next_jif = start_jif + dirty_writeback_interval;
Christoph Lameterc24f21b2006-06-30 01:55:42 -0700454 nr_to_write = global_page_state(NR_FILE_DIRTY) +
455 global_page_state(NR_UNSTABLE_NFS) +
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 (inodes_stat.nr_inodes - inodes_stat.nr_unused);
457 while (nr_to_write > 0) {
458 wbc.encountered_congestion = 0;
459 wbc.nr_to_write = MAX_WRITEBACK_PAGES;
460 writeback_inodes(&wbc);
461 if (wbc.nr_to_write > 0) {
462 if (wbc.encountered_congestion)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700463 congestion_wait(WRITE, HZ/10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 else
465 break; /* All the old data is written */
466 }
467 nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
468 }
469 if (time_before(next_jif, jiffies + HZ))
470 next_jif = jiffies + HZ;
Bart Samwelf6ef9432006-03-24 03:15:48 -0800471 if (dirty_writeback_interval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 mod_timer(&wb_timer, next_jif);
473}
474
475/*
476 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
477 */
478int dirty_writeback_centisecs_handler(ctl_table *table, int write,
Andrew Morton3e733f02007-07-15 23:41:05 -0700479 struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480{
Bart Samwelf6ef9432006-03-24 03:15:48 -0800481 proc_dointvec_userhz_jiffies(table, write, file, buffer, length, ppos);
Andrew Morton3e733f02007-07-15 23:41:05 -0700482 if (dirty_writeback_interval)
483 mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
484 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 del_timer(&wb_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 return 0;
487}
488
489static void wb_timer_fn(unsigned long unused)
490{
491 if (pdflush_operation(wb_kupdate, 0) < 0)
492 mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */
493}
494
495static void laptop_flush(unsigned long unused)
496{
497 sys_sync();
498}
499
500static void laptop_timer_fn(unsigned long unused)
501{
502 pdflush_operation(laptop_flush, 0);
503}
504
505/*
506 * We've spun up the disk and we're in laptop mode: schedule writeback
507 * of all dirty data a few seconds from now. If the flush is already scheduled
508 * then push it back - the user is still using the disk.
509 */
510void laptop_io_completion(void)
511{
Bart Samweled5b43f2006-03-24 03:15:49 -0800512 mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513}
514
515/*
516 * We're in laptop mode and we've just synced. The sync's writes will have
517 * caused another writeback to be scheduled by laptop_io_completion.
518 * Nothing needs to be written back anymore, so we unschedule the writeback.
519 */
520void laptop_sync_completion(void)
521{
522 del_timer(&laptop_mode_wb_timer);
523}
524
525/*
526 * If ratelimit_pages is too high then we can get into dirty-data overload
527 * if a large number of processes all perform writes at the same time.
528 * If it is too low then SMP machines will call the (expensive)
529 * get_writeback_state too often.
530 *
531 * Here we set ratelimit_pages to a level which ensures that when all CPUs are
532 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
533 * thresholds before writeback cuts in.
534 *
535 * But the limit should not be set too high. Because it also controls the
536 * amount of memory which the balance_dirty_pages() caller has to write back.
537 * If this is too large then the caller will block on the IO queue all the
538 * time. So limit it to four megabytes - the balance_dirty_pages() caller
539 * will write six megabyte chunks, max.
540 */
541
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700542void writeback_set_ratelimit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543{
Chandra Seetharaman40c99aa2006-09-29 02:01:24 -0700544 ratelimit_pages = vm_total_pages / (num_online_cpus() * 32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 if (ratelimit_pages < 16)
546 ratelimit_pages = 16;
547 if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024)
548 ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE;
549}
550
Chandra Seetharaman26c21432006-06-27 02:54:10 -0700551static int __cpuinit
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552ratelimit_handler(struct notifier_block *self, unsigned long u, void *v)
553{
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700554 writeback_set_ratelimit();
Paul E. McKenneyaa0f0302007-02-10 01:46:37 -0800555 return NOTIFY_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556}
557
Chandra Seetharaman74b85f32006-06-27 02:54:09 -0700558static struct notifier_block __cpuinitdata ratelimit_nb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 .notifier_call = ratelimit_handler,
560 .next = NULL,
561};
562
563/*
Linus Torvaldsdc6e29d2007-01-29 16:37:38 -0800564 * Called early on to tune the page writeback dirty limits.
565 *
566 * We used to scale dirty pages according to how total memory
567 * related to pages that could be allocated for buffers (by
568 * comparing nr_free_buffer_pages() to vm_total_pages.
569 *
570 * However, that was when we used "dirty_ratio" to scale with
571 * all memory, and we don't do that any more. "dirty_ratio"
572 * is now applied to total non-HIGHPAGE memory (by subtracting
573 * totalhigh_pages from vm_total_pages), and as such we can't
574 * get into the old insane situation any more where we had
575 * large amounts of dirty pages compared to a small amount of
576 * non-HIGHMEM memory.
577 *
578 * But we might still want to scale the dirty_ratio by how
579 * much memory the box has..
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 */
581void __init page_writeback_init(void)
582{
Bart Samwelf6ef9432006-03-24 03:15:48 -0800583 mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
Chandra Seetharaman2d1d43f2006-09-29 02:01:25 -0700584 writeback_set_ratelimit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 register_cpu_notifier(&ratelimit_nb);
586}
587
David Howells811d7362006-08-29 19:06:09 +0100588/**
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700589 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
David Howells811d7362006-08-29 19:06:09 +0100590 * @mapping: address space structure to write
591 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700592 * @writepage: function called for each page
593 * @data: data passed to writepage function
David Howells811d7362006-08-29 19:06:09 +0100594 *
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700595 * If a page is already under I/O, write_cache_pages() skips it, even
David Howells811d7362006-08-29 19:06:09 +0100596 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
597 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
598 * and msync() need to guarantee that all the data which was dirty at the time
599 * the call was made get new I/O started against them. If wbc->sync_mode is
600 * WB_SYNC_ALL then we were called for data integrity and we must wait for
601 * existing IO to complete.
David Howells811d7362006-08-29 19:06:09 +0100602 */
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700603int write_cache_pages(struct address_space *mapping,
604 struct writeback_control *wbc, writepage_t writepage,
605 void *data)
David Howells811d7362006-08-29 19:06:09 +0100606{
607 struct backing_dev_info *bdi = mapping->backing_dev_info;
608 int ret = 0;
609 int done = 0;
David Howells811d7362006-08-29 19:06:09 +0100610 struct pagevec pvec;
611 int nr_pages;
612 pgoff_t index;
613 pgoff_t end; /* Inclusive */
614 int scanned = 0;
615 int range_whole = 0;
616
617 if (wbc->nonblocking && bdi_write_congested(bdi)) {
618 wbc->encountered_congestion = 1;
619 return 0;
620 }
621
David Howells811d7362006-08-29 19:06:09 +0100622 pagevec_init(&pvec, 0);
623 if (wbc->range_cyclic) {
624 index = mapping->writeback_index; /* Start from prev offset */
625 end = -1;
626 } else {
627 index = wbc->range_start >> PAGE_CACHE_SHIFT;
628 end = wbc->range_end >> PAGE_CACHE_SHIFT;
629 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
630 range_whole = 1;
631 scanned = 1;
632 }
633retry:
634 while (!done && (index <= end) &&
635 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
636 PAGECACHE_TAG_DIRTY,
637 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
638 unsigned i;
639
640 scanned = 1;
641 for (i = 0; i < nr_pages; i++) {
642 struct page *page = pvec.pages[i];
643
644 /*
645 * At this point we hold neither mapping->tree_lock nor
646 * lock on the page itself: the page may be truncated or
647 * invalidated (changing page->mapping to NULL), or even
648 * swizzled back from swapper_space to tmpfs file
649 * mapping
650 */
651 lock_page(page);
652
653 if (unlikely(page->mapping != mapping)) {
654 unlock_page(page);
655 continue;
656 }
657
658 if (!wbc->range_cyclic && page->index > end) {
659 done = 1;
660 unlock_page(page);
661 continue;
662 }
663
664 if (wbc->sync_mode != WB_SYNC_NONE)
665 wait_on_page_writeback(page);
666
667 if (PageWriteback(page) ||
668 !clear_page_dirty_for_io(page)) {
669 unlock_page(page);
670 continue;
671 }
672
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700673 ret = (*writepage)(page, wbc, data);
David Howells811d7362006-08-29 19:06:09 +0100674
675 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE))
676 unlock_page(page);
677 if (ret || (--(wbc->nr_to_write) <= 0))
678 done = 1;
679 if (wbc->nonblocking && bdi_write_congested(bdi)) {
680 wbc->encountered_congestion = 1;
681 done = 1;
682 }
683 }
684 pagevec_release(&pvec);
685 cond_resched();
686 }
687 if (!scanned && !done) {
688 /*
689 * We hit the last page and there is more work to be done: wrap
690 * back to the start of the file
691 */
692 scanned = 1;
693 index = 0;
694 goto retry;
695 }
696 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
697 mapping->writeback_index = index;
698 return ret;
699}
Miklos Szeredi0ea97182007-05-10 22:22:51 -0700700EXPORT_SYMBOL(write_cache_pages);
701
702/*
703 * Function used by generic_writepages to call the real writepage
704 * function and set the mapping flags on error
705 */
706static int __writepage(struct page *page, struct writeback_control *wbc,
707 void *data)
708{
709 struct address_space *mapping = data;
710 int ret = mapping->a_ops->writepage(page, wbc);
711 mapping_set_error(mapping, ret);
712 return ret;
713}
714
715/**
716 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them.
717 * @mapping: address space structure to write
718 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
719 *
720 * This is a library function, which implements the writepages()
721 * address_space_operation.
722 */
723int generic_writepages(struct address_space *mapping,
724 struct writeback_control *wbc)
725{
726 /* deal with chardevs and other special file */
727 if (!mapping->a_ops->writepage)
728 return 0;
729
730 return write_cache_pages(mapping, wbc, __writepage, mapping);
731}
David Howells811d7362006-08-29 19:06:09 +0100732
733EXPORT_SYMBOL(generic_writepages);
734
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
736{
Andrew Morton22905f72005-11-16 15:07:01 -0800737 int ret;
738
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 if (wbc->nr_to_write <= 0)
740 return 0;
Andrew Morton22905f72005-11-16 15:07:01 -0800741 wbc->for_writepages = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 if (mapping->a_ops->writepages)
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700743 ret = mapping->a_ops->writepages(mapping, wbc);
Andrew Morton22905f72005-11-16 15:07:01 -0800744 else
745 ret = generic_writepages(mapping, wbc);
746 wbc->for_writepages = 0;
747 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748}
749
750/**
751 * write_one_page - write out a single page and optionally wait on I/O
Martin Waitz67be2dd2005-05-01 08:59:26 -0700752 * @page: the page to write
753 * @wait: if true, wait on writeout
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 *
755 * The page must be locked by the caller and will be unlocked upon return.
756 *
757 * write_one_page() returns a negative error code if I/O failed.
758 */
759int write_one_page(struct page *page, int wait)
760{
761 struct address_space *mapping = page->mapping;
762 int ret = 0;
763 struct writeback_control wbc = {
764 .sync_mode = WB_SYNC_ALL,
765 .nr_to_write = 1,
766 };
767
768 BUG_ON(!PageLocked(page));
769
770 if (wait)
771 wait_on_page_writeback(page);
772
773 if (clear_page_dirty_for_io(page)) {
774 page_cache_get(page);
775 ret = mapping->a_ops->writepage(page, &wbc);
776 if (ret == 0 && wait) {
777 wait_on_page_writeback(page);
778 if (PageError(page))
779 ret = -EIO;
780 }
781 page_cache_release(page);
782 } else {
783 unlock_page(page);
784 }
785 return ret;
786}
787EXPORT_SYMBOL(write_one_page);
788
789/*
Ken Chen76719322007-02-10 01:43:15 -0800790 * For address_spaces which do not use buffers nor write back.
791 */
792int __set_page_dirty_no_writeback(struct page *page)
793{
794 if (!PageDirty(page))
795 SetPageDirty(page);
796 return 0;
797}
798
799/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 * For address_spaces which do not use buffers. Just tag the page as dirty in
801 * its radix tree.
802 *
803 * This is also used when a single buffer is being dirtied: we want to set the
804 * page dirty in that case, but not all the buffers. This is a "bottom-up"
805 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
806 *
807 * Most callers have locked the page, which pins the address_space in memory.
808 * But zap_pte_range() does not lock the page, however in that case the
809 * mapping is pinned by the vma's ->vm_file reference.
810 *
811 * We take care to handle the case where the page was truncated from the
812 * mapping by re-checking page_mapping() insode tree_lock.
813 */
814int __set_page_dirty_nobuffers(struct page *page)
815{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 if (!TestSetPageDirty(page)) {
817 struct address_space *mapping = page_mapping(page);
818 struct address_space *mapping2;
819
Andrew Morton8c085402006-12-10 02:19:24 -0800820 if (!mapping)
821 return 1;
822
823 write_lock_irq(&mapping->tree_lock);
824 mapping2 = page_mapping(page);
825 if (mapping2) { /* Race with truncate? */
826 BUG_ON(mapping2 != mapping);
Nick Piggin787d2212007-07-17 04:03:34 -0700827 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
Andrew Morton55e829a2006-12-10 02:19:27 -0800828 if (mapping_cap_account_dirty(mapping)) {
Andrew Morton8c085402006-12-10 02:19:24 -0800829 __inc_zone_page_state(page, NR_FILE_DIRTY);
Andrew Morton55e829a2006-12-10 02:19:27 -0800830 task_io_account_write(PAGE_CACHE_SIZE);
831 }
Andrew Morton8c085402006-12-10 02:19:24 -0800832 radix_tree_tag_set(&mapping->page_tree,
833 page_index(page), PAGECACHE_TAG_DIRTY);
834 }
835 write_unlock_irq(&mapping->tree_lock);
836 if (mapping->host) {
837 /* !PageAnon && !swapper_space */
838 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800840 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800842 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843}
844EXPORT_SYMBOL(__set_page_dirty_nobuffers);
845
846/*
847 * When a writepage implementation decides that it doesn't want to write this
848 * page for some reason, it should redirty the locked page via
849 * redirty_page_for_writepage() and it should then unlock the page and return 0
850 */
851int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
852{
853 wbc->pages_skipped++;
854 return __set_page_dirty_nobuffers(page);
855}
856EXPORT_SYMBOL(redirty_page_for_writepage);
857
858/*
859 * If the mapping doesn't provide a set_page_dirty a_op, then
860 * just fall through and assume that it wants buffer_heads.
861 */
862int fastcall set_page_dirty(struct page *page)
863{
864 struct address_space *mapping = page_mapping(page);
865
866 if (likely(mapping)) {
867 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
David Howells93614012006-09-30 20:45:40 +0200868#ifdef CONFIG_BLOCK
869 if (!spd)
870 spd = __set_page_dirty_buffers;
871#endif
872 return (*spd)(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 }
Andrew Morton4741c9f2006-03-24 03:18:11 -0800874 if (!PageDirty(page)) {
875 if (!TestSetPageDirty(page))
876 return 1;
877 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 return 0;
879}
880EXPORT_SYMBOL(set_page_dirty);
881
882/*
883 * set_page_dirty() is racy if the caller has no reference against
884 * page->mapping->host, and if the page is unlocked. This is because another
885 * CPU could truncate the page off the mapping and then free the mapping.
886 *
887 * Usually, the page _is_ locked, or the caller is a user-space process which
888 * holds a reference on the inode by having an open file.
889 *
890 * In other cases, the page should be locked before running set_page_dirty().
891 */
892int set_page_dirty_lock(struct page *page)
893{
894 int ret;
895
Nick Piggindb376482006-09-25 23:31:24 -0700896 lock_page_nosync(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 ret = set_page_dirty(page);
898 unlock_page(page);
899 return ret;
900}
901EXPORT_SYMBOL(set_page_dirty_lock);
902
903/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 * Clear a page's dirty flag, while caring for dirty memory accounting.
905 * Returns true if the page was previously dirty.
906 *
907 * This is for preparing to put the page under writeout. We leave the page
908 * tagged as dirty in the radix tree so that a concurrent write-for-sync
909 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage
910 * implementation will run either set_page_writeback() or set_page_dirty(),
911 * at which stage we bring the page's dirty flag and radix-tree dirty tag
912 * back into sync.
913 *
914 * This incoherency between the page's dirty flag and radix-tree tag is
915 * unfortunate, but it only exists while the page is locked.
916 */
917int clear_page_dirty_for_io(struct page *page)
918{
919 struct address_space *mapping = page_mapping(page);
920
Nick Piggin79352892007-07-19 01:47:22 -0700921 BUG_ON(!PageLocked(page));
922
Fengguang Wufe3cba12007-07-19 01:48:07 -0700923 ClearPageReclaim(page);
Linus Torvalds7658cc22006-12-29 10:00:58 -0800924 if (mapping && mapping_cap_account_dirty(mapping)) {
925 /*
926 * Yes, Virginia, this is indeed insane.
927 *
928 * We use this sequence to make sure that
929 * (a) we account for dirty stats properly
930 * (b) we tell the low-level filesystem to
931 * mark the whole page dirty if it was
932 * dirty in a pagetable. Only to then
933 * (c) clean the page again and return 1 to
934 * cause the writeback.
935 *
936 * This way we avoid all nasty races with the
937 * dirty bit in multiple places and clearing
938 * them concurrently from different threads.
939 *
940 * Note! Normally the "set_page_dirty(page)"
941 * has no effect on the actual dirty bit - since
942 * that will already usually be set. But we
943 * need the side effects, and it can help us
944 * avoid races.
945 *
946 * We basically use the page "master dirty bit"
947 * as a serialization point for all the different
948 * threads doing their things.
Linus Torvalds7658cc22006-12-29 10:00:58 -0800949 */
950 if (page_mkclean(page))
951 set_page_dirty(page);
Nick Piggin79352892007-07-19 01:47:22 -0700952 /*
953 * We carefully synchronise fault handlers against
954 * installing a dirty pte and marking the page dirty
955 * at this point. We do this by having them hold the
956 * page lock at some point after installing their
957 * pte, but before marking the page dirty.
958 * Pages are always locked coming in here, so we get
959 * the desired exclusion. See mm/memory.c:do_wp_page()
960 * for more comments.
961 */
Linus Torvalds7658cc22006-12-29 10:00:58 -0800962 if (TestClearPageDirty(page)) {
Andrew Morton8c085402006-12-10 02:19:24 -0800963 dec_zone_page_state(page, NR_FILE_DIRTY);
Linus Torvalds7658cc22006-12-29 10:00:58 -0800964 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 }
Linus Torvalds7658cc22006-12-29 10:00:58 -0800966 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 }
Linus Torvalds7658cc22006-12-29 10:00:58 -0800968 return TestClearPageDirty(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969}
Hans Reiser58bb01a2005-11-18 01:10:53 -0800970EXPORT_SYMBOL(clear_page_dirty_for_io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971
972int test_clear_page_writeback(struct page *page)
973{
974 struct address_space *mapping = page_mapping(page);
975 int ret;
976
977 if (mapping) {
978 unsigned long flags;
979
980 write_lock_irqsave(&mapping->tree_lock, flags);
981 ret = TestClearPageWriteback(page);
982 if (ret)
983 radix_tree_tag_clear(&mapping->page_tree,
984 page_index(page),
985 PAGECACHE_TAG_WRITEBACK);
986 write_unlock_irqrestore(&mapping->tree_lock, flags);
987 } else {
988 ret = TestClearPageWriteback(page);
989 }
Andrew Mortond688abf2007-07-19 01:49:17 -0700990 if (ret)
991 dec_zone_page_state(page, NR_WRITEBACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 return ret;
993}
994
995int test_set_page_writeback(struct page *page)
996{
997 struct address_space *mapping = page_mapping(page);
998 int ret;
999
1000 if (mapping) {
1001 unsigned long flags;
1002
1003 write_lock_irqsave(&mapping->tree_lock, flags);
1004 ret = TestSetPageWriteback(page);
1005 if (!ret)
1006 radix_tree_tag_set(&mapping->page_tree,
1007 page_index(page),
1008 PAGECACHE_TAG_WRITEBACK);
1009 if (!PageDirty(page))
1010 radix_tree_tag_clear(&mapping->page_tree,
1011 page_index(page),
1012 PAGECACHE_TAG_DIRTY);
1013 write_unlock_irqrestore(&mapping->tree_lock, flags);
1014 } else {
1015 ret = TestSetPageWriteback(page);
1016 }
Andrew Mortond688abf2007-07-19 01:49:17 -07001017 if (!ret)
1018 inc_zone_page_state(page, NR_WRITEBACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 return ret;
1020
1021}
1022EXPORT_SYMBOL(test_set_page_writeback);
1023
1024/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 * Return true if any of the pages in the mapping are marged with the
1026 * passed tag.
1027 */
1028int mapping_tagged(struct address_space *mapping, int tag)
1029{
1030 unsigned long flags;
1031 int ret;
1032
1033 read_lock_irqsave(&mapping->tree_lock, flags);
1034 ret = radix_tree_tagged(&mapping->page_tree, tag);
1035 read_unlock_irqrestore(&mapping->tree_lock, flags);
1036 return ret;
1037}
1038EXPORT_SYMBOL(mapping_tagged);