blob: ea8592b906968913499b9847f35389a31e5dc1a3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * fs/fs-writeback.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * Contains all the functions related to writing back and waiting
7 * upon dirty inodes against superblocks, and writing back dirty
8 * pages against inodes. ie: data writeback. Writeout of the
9 * inode itself is not handled here.
10 *
Francois Camie1f8e872008-10-15 22:01:59 -070011 * 10Apr2002 Andrew Morton
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Split out of fs/inode.c
13 * Additions for address_space-based writeback
14 */
15
16#include <linux/kernel.h>
Jens Axboef5ff8422007-09-21 09:19:54 +020017#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/spinlock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/sched.h>
21#include <linux/fs.h>
22#include <linux/mm.h>
Jens Axboe03ba3782009-09-09 09:08:54 +020023#include <linux/kthread.h>
24#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/writeback.h>
26#include <linux/blkdev.h>
27#include <linux/backing-dev.h>
28#include <linux/buffer_head.h>
David Howells07f3f052006-09-30 20:52:18 +020029#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Jens Axboe66f3b8e2009-09-02 09:19:46 +020031#define inode_to_bdi(inode) ((inode)->i_mapping->backing_dev_info)
Adrian Bunkf11b00f2008-04-29 00:58:56 -070032
Jens Axboe03ba3782009-09-09 09:08:54 +020033/*
Jens Axboed0bceac2009-05-18 08:20:32 +020034 * We don't actually have pdflush, but this one is exported though /proc...
35 */
36int nr_pdflush_threads;
37
38/*
Jens Axboec4a77a62009-09-16 15:18:25 +020039 * Passed into wb_writeback(), essentially a subset of writeback_control
40 */
41struct wb_writeback_args {
42 long nr_pages;
43 struct super_block *sb;
44 enum writeback_sync_modes sync_mode;
H Hartley Sweeten52957fe2010-04-01 20:36:30 -050045 unsigned int for_kupdate:1;
46 unsigned int range_cyclic:1;
47 unsigned int for_background:1;
Linus Torvaldse8bebe22010-05-21 19:37:45 -070048 unsigned int sb_pinned:1;
Jens Axboec4a77a62009-09-16 15:18:25 +020049};
50
51/*
Jens Axboe03ba3782009-09-09 09:08:54 +020052 * Work items for the bdi_writeback threads
Adrian Bunkf11b00f2008-04-29 00:58:56 -070053 */
Jens Axboe03ba3782009-09-09 09:08:54 +020054struct bdi_work {
Jens Axboe8010c3b2009-09-15 20:04:57 +020055 struct list_head list; /* pending work list */
56 struct rcu_head rcu_head; /* for RCU free/clear of work */
Jens Axboe03ba3782009-09-09 09:08:54 +020057
Jens Axboe8010c3b2009-09-15 20:04:57 +020058 unsigned long seen; /* threads that have seen this work */
59 atomic_t pending; /* number of threads still to do work */
Jens Axboe03ba3782009-09-09 09:08:54 +020060
Jens Axboe8010c3b2009-09-15 20:04:57 +020061 struct wb_writeback_args args; /* writeback arguments */
Jens Axboe03ba3782009-09-09 09:08:54 +020062
Jens Axboe8010c3b2009-09-15 20:04:57 +020063 unsigned long state; /* flag bits, see WS_* */
Jens Axboe03ba3782009-09-09 09:08:54 +020064};
65
66enum {
67 WS_USED_B = 0,
68 WS_ONSTACK_B,
69};
70
71#define WS_USED (1 << WS_USED_B)
72#define WS_ONSTACK (1 << WS_ONSTACK_B)
73
74static inline bool bdi_work_on_stack(struct bdi_work *work)
Adrian Bunkf11b00f2008-04-29 00:58:56 -070075{
Jens Axboe03ba3782009-09-09 09:08:54 +020076 return test_bit(WS_ONSTACK_B, &work->state);
77}
78
79static inline void bdi_work_init(struct bdi_work *work,
Jens Axboeb6e51312009-09-16 15:13:54 +020080 struct wb_writeback_args *args)
Jens Axboe03ba3782009-09-09 09:08:54 +020081{
82 INIT_RCU_HEAD(&work->rcu_head);
Jens Axboeb6e51312009-09-16 15:13:54 +020083 work->args = *args;
Jens Axboe03ba3782009-09-09 09:08:54 +020084 work->state = WS_USED;
85}
86
Adrian Bunkf11b00f2008-04-29 00:58:56 -070087/**
88 * writeback_in_progress - determine whether there is writeback in progress
89 * @bdi: the device's backing_dev_info structure.
90 *
Jens Axboe03ba3782009-09-09 09:08:54 +020091 * Determine whether there is writeback waiting to be handled against a
92 * backing device.
Adrian Bunkf11b00f2008-04-29 00:58:56 -070093 */
94int writeback_in_progress(struct backing_dev_info *bdi)
95{
Jens Axboe03ba3782009-09-09 09:08:54 +020096 return !list_empty(&bdi->work_list);
Adrian Bunkf11b00f2008-04-29 00:58:56 -070097}
98
Jens Axboe03ba3782009-09-09 09:08:54 +020099static void bdi_work_clear(struct bdi_work *work)
Adrian Bunkf11b00f2008-04-29 00:58:56 -0700100{
Jens Axboe03ba3782009-09-09 09:08:54 +0200101 clear_bit(WS_USED_B, &work->state);
102 smp_mb__after_clear_bit();
Nick Piggin1ef7d9a2009-09-15 21:37:55 +0200103 /*
104 * work can have disappeared at this point. bit waitq functions
105 * should be able to tolerate this, provided bdi_sched_wait does
106 * not dereference it's pointer argument.
107 */
Jens Axboe03ba3782009-09-09 09:08:54 +0200108 wake_up_bit(&work->state, WS_USED_B);
Adrian Bunkf11b00f2008-04-29 00:58:56 -0700109}
110
Jens Axboe03ba3782009-09-09 09:08:54 +0200111static void bdi_work_free(struct rcu_head *head)
Nick Piggin4195f732009-05-28 09:01:15 +0200112{
Jens Axboe03ba3782009-09-09 09:08:54 +0200113 struct bdi_work *work = container_of(head, struct bdi_work, rcu_head);
Nick Piggin4195f732009-05-28 09:01:15 +0200114
Jens Axboe03ba3782009-09-09 09:08:54 +0200115 if (!bdi_work_on_stack(work))
116 kfree(work);
117 else
118 bdi_work_clear(work);
119}
120
121static void wb_work_complete(struct bdi_work *work)
122{
Jens Axboec4a77a62009-09-16 15:18:25 +0200123 const enum writeback_sync_modes sync_mode = work->args.sync_mode;
Nick Piggin77b9d052009-09-15 21:34:51 +0200124 int onstack = bdi_work_on_stack(work);
Jens Axboe03ba3782009-09-09 09:08:54 +0200125
126 /*
127 * For allocated work, we can clear the done/seen bit right here.
128 * For on-stack work, we need to postpone both the clear and free
129 * to after the RCU grace period, since the stack could be invalidated
130 * as soon as bdi_work_clear() has done the wakeup.
131 */
Nick Piggin77b9d052009-09-15 21:34:51 +0200132 if (!onstack)
Jens Axboe03ba3782009-09-09 09:08:54 +0200133 bdi_work_clear(work);
Nick Piggin77b9d052009-09-15 21:34:51 +0200134 if (sync_mode == WB_SYNC_NONE || onstack)
Jens Axboe03ba3782009-09-09 09:08:54 +0200135 call_rcu(&work->rcu_head, bdi_work_free);
136}
137
138static void wb_clear_pending(struct bdi_writeback *wb, struct bdi_work *work)
139{
140 /*
141 * The caller has retrieved the work arguments from this work,
142 * drop our reference. If this is the last ref, delete and free it
143 */
144 if (atomic_dec_and_test(&work->pending)) {
145 struct backing_dev_info *bdi = wb->bdi;
146
147 spin_lock(&bdi->wb_lock);
148 list_del_rcu(&work->list);
149 spin_unlock(&bdi->wb_lock);
150
151 wb_work_complete(work);
Nick Piggin4195f732009-05-28 09:01:15 +0200152 }
153}
154
Jens Axboe03ba3782009-09-09 09:08:54 +0200155static void bdi_queue_work(struct backing_dev_info *bdi, struct bdi_work *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156{
Jens Axboebcddc3f2009-09-13 20:07:36 +0200157 work->seen = bdi->wb_mask;
158 BUG_ON(!work->seen);
159 atomic_set(&work->pending, bdi->wb_cnt);
160 BUG_ON(!bdi->wb_cnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
Jens Axboebcddc3f2009-09-13 20:07:36 +0200162 /*
Nick Piggindeed62e2009-09-15 21:32:58 +0200163 * list_add_tail_rcu() contains the necessary barriers to
164 * make sure the above stores are seen before the item is
165 * noticed on the list
Jens Axboebcddc3f2009-09-13 20:07:36 +0200166 */
Jens Axboebcddc3f2009-09-13 20:07:36 +0200167 spin_lock(&bdi->wb_lock);
168 list_add_tail_rcu(&work->list, &bdi->work_list);
169 spin_unlock(&bdi->wb_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
171 /*
Jens Axboe03ba3782009-09-09 09:08:54 +0200172 * If the default thread isn't there, make sure we add it. When
173 * it gets created and wakes up, we'll run this work.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 */
Jens Axboe03ba3782009-09-09 09:08:54 +0200175 if (unlikely(list_empty_careful(&bdi->wb_list)))
176 wake_up_process(default_backing_dev_info.wb.task);
177 else {
178 struct bdi_writeback *wb = &bdi->wb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179
Nick Piggin1ef7d9a2009-09-15 21:37:55 +0200180 if (wb->task)
Jens Axboe03ba3782009-09-09 09:08:54 +0200181 wake_up_process(wb->task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183}
184
Jens Axboe03ba3782009-09-09 09:08:54 +0200185/*
186 * Used for on-stack allocated work items. The caller needs to wait until
187 * the wb threads have acked the work before it's safe to continue.
188 */
189static void bdi_wait_on_work_clear(struct bdi_work *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190{
Jens Axboe03ba3782009-09-09 09:08:54 +0200191 wait_on_bit(&work->state, WS_USED_B, bdi_sched_wait,
192 TASK_UNINTERRUPTIBLE);
193}
194
Jens Axboef11fcae2009-09-15 09:53:35 +0200195static void bdi_alloc_queue_work(struct backing_dev_info *bdi,
Jens Axboe7c8a3552010-05-18 14:29:29 +0200196 struct wb_writeback_args *args,
197 int wait)
Jens Axboe03ba3782009-09-09 09:08:54 +0200198{
199 struct bdi_work *work;
200
Jens Axboebcddc3f2009-09-13 20:07:36 +0200201 /*
202 * This is WB_SYNC_NONE writeback, so if allocation fails just
203 * wakeup the thread for old dirty data writeback
204 */
Jens Axboe03ba3782009-09-09 09:08:54 +0200205 work = kmalloc(sizeof(*work), GFP_ATOMIC);
Jens Axboebcddc3f2009-09-13 20:07:36 +0200206 if (work) {
Jens Axboeb6e51312009-09-16 15:13:54 +0200207 bdi_work_init(work, args);
Jens Axboebcddc3f2009-09-13 20:07:36 +0200208 bdi_queue_work(bdi, work);
Jens Axboe7c8a3552010-05-18 14:29:29 +0200209 if (wait)
210 bdi_wait_on_work_clear(work);
Jens Axboebcddc3f2009-09-13 20:07:36 +0200211 } else {
212 struct bdi_writeback *wb = &bdi->wb;
Jens Axboe03ba3782009-09-09 09:08:54 +0200213
Jens Axboebcddc3f2009-09-13 20:07:36 +0200214 if (wb->task)
215 wake_up_process(wb->task);
216 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200217}
218
Jens Axboeb6e51312009-09-16 15:13:54 +0200219/**
220 * bdi_sync_writeback - start and wait for writeback
221 * @bdi: the backing device to write from
222 * @sb: write inodes from this super_block
223 *
224 * Description:
225 * This does WB_SYNC_ALL data integrity writeback and waits for the
226 * IO to complete. Callers must hold the sb s_umount semaphore for
227 * reading, to avoid having the super disappear before we are done.
228 */
229static void bdi_sync_writeback(struct backing_dev_info *bdi,
230 struct super_block *sb)
Jens Axboe03ba3782009-09-09 09:08:54 +0200231{
Jens Axboeb6e51312009-09-16 15:13:54 +0200232 struct wb_writeback_args args = {
233 .sb = sb,
234 .sync_mode = WB_SYNC_ALL,
235 .nr_pages = LONG_MAX,
236 .range_cyclic = 0,
Jens Axboee913fc82010-05-17 12:55:07 +0200237 /*
238 * Setting sb_pinned is not necessary for WB_SYNC_ALL, but
239 * lets make it explicitly clear.
240 */
241 .sb_pinned = 1,
Jens Axboeb6e51312009-09-16 15:13:54 +0200242 };
243 struct bdi_work work;
Christoph Hellwigf0fad8a2009-09-11 09:47:56 +0200244
Jens Axboeb6e51312009-09-16 15:13:54 +0200245 bdi_work_init(&work, &args);
246 work.state |= WS_ONSTACK;
Christoph Hellwigf0fad8a2009-09-11 09:47:56 +0200247
Jens Axboeb6e51312009-09-16 15:13:54 +0200248 bdi_queue_work(bdi, &work);
249 bdi_wait_on_work_clear(&work);
250}
251
252/**
253 * bdi_start_writeback - start writeback
254 * @bdi: the backing device to write from
Jaswinder Singh Rajput4b6764f2010-01-01 20:35:23 -0800255 * @sb: write inodes from this super_block
Jens Axboeb6e51312009-09-16 15:13:54 +0200256 * @nr_pages: the number of pages to write
Jens Axboee913fc82010-05-17 12:55:07 +0200257 * @sb_locked: caller already holds sb umount sem.
Jens Axboeb6e51312009-09-16 15:13:54 +0200258 *
259 * Description:
260 * This does WB_SYNC_NONE opportunistic writeback. The IO is only
261 * started when this function returns, we make no guarentees on
Jens Axboee913fc82010-05-17 12:55:07 +0200262 * completion. Caller specifies whether sb umount sem is held already or not.
Jens Axboeb6e51312009-09-16 15:13:54 +0200263 *
264 */
Jens Axboea72bfd42009-09-26 00:07:46 +0200265void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
Jens Axboee913fc82010-05-17 12:55:07 +0200266 long nr_pages, int sb_locked)
Jens Axboeb6e51312009-09-16 15:13:54 +0200267{
268 struct wb_writeback_args args = {
Jens Axboea72bfd42009-09-26 00:07:46 +0200269 .sb = sb,
Jens Axboeb6e51312009-09-16 15:13:54 +0200270 .sync_mode = WB_SYNC_NONE,
271 .nr_pages = nr_pages,
272 .range_cyclic = 1,
Jens Axboee913fc82010-05-17 12:55:07 +0200273 .sb_pinned = sb_locked,
Jens Axboeb6e51312009-09-16 15:13:54 +0200274 };
275
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800276 /*
277 * We treat @nr_pages=0 as the special case to do background writeback,
278 * ie. to sync pages until the background dirty threshold is reached.
279 */
280 if (!nr_pages) {
281 args.nr_pages = LONG_MAX;
282 args.for_background = 1;
283 }
284
Jens Axboe7c8a3552010-05-18 14:29:29 +0200285 bdi_alloc_queue_work(bdi, &args, sb_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286}
287
288/*
Andrew Morton6610a0b2007-10-16 23:30:32 -0700289 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
290 * furthest end of its superblock's dirty-inode list.
291 *
292 * Before stamping the inode's ->dirtied_when, we check to see whether it is
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200293 * already the most-recently-dirtied inode on the b_dirty list. If that is
Andrew Morton6610a0b2007-10-16 23:30:32 -0700294 * the case then the inode must have been redirtied while it was being written
295 * out and we don't reset its dirtied_when.
296 */
297static void redirty_tail(struct inode *inode)
298{
Jens Axboe03ba3782009-09-09 09:08:54 +0200299 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
Andrew Morton6610a0b2007-10-16 23:30:32 -0700300
Jens Axboe03ba3782009-09-09 09:08:54 +0200301 if (!list_empty(&wb->b_dirty)) {
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200302 struct inode *tail;
Andrew Morton6610a0b2007-10-16 23:30:32 -0700303
Jens Axboe03ba3782009-09-09 09:08:54 +0200304 tail = list_entry(wb->b_dirty.next, struct inode, i_list);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200305 if (time_before(inode->dirtied_when, tail->dirtied_when))
Andrew Morton6610a0b2007-10-16 23:30:32 -0700306 inode->dirtied_when = jiffies;
307 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200308 list_move(&inode->i_list, &wb->b_dirty);
Andrew Morton6610a0b2007-10-16 23:30:32 -0700309}
310
311/*
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200312 * requeue inode for re-scanning after bdi->b_io list is exhausted.
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700313 */
Ken Chen0e0f4fc2007-10-16 23:30:38 -0700314static void requeue_io(struct inode *inode)
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700315{
Jens Axboe03ba3782009-09-09 09:08:54 +0200316 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
317
318 list_move(&inode->i_list, &wb->b_more_io);
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700319}
320
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700321static void inode_sync_complete(struct inode *inode)
322{
323 /*
324 * Prevent speculative execution through spin_unlock(&inode_lock);
325 */
326 smp_mb();
327 wake_up_bit(&inode->i_state, __I_SYNC);
328}
329
Jeff Laytond2caa3c2009-04-02 16:56:37 -0700330static bool inode_dirtied_after(struct inode *inode, unsigned long t)
331{
332 bool ret = time_after(inode->dirtied_when, t);
333#ifndef CONFIG_64BIT
334 /*
335 * For inodes being constantly redirtied, dirtied_when can get stuck.
336 * It _appears_ to be in the future, but is actually in distant past.
337 * This test is necessary to prevent such wrapped-around relative times
Jens Axboe5b0830c2009-09-23 19:37:09 +0200338 * from permanently stopping the whole bdi writeback.
Jeff Laytond2caa3c2009-04-02 16:56:37 -0700339 */
340 ret = ret && time_before_eq(inode->dirtied_when, jiffies);
341#endif
342 return ret;
343}
344
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700345/*
Fengguang Wu2c136572007-10-16 23:30:39 -0700346 * Move expired dirty inodes from @delaying_queue to @dispatch_queue.
347 */
348static void move_expired_inodes(struct list_head *delaying_queue,
349 struct list_head *dispatch_queue,
350 unsigned long *older_than_this)
351{
Shaohua Li5c034492009-09-24 14:42:33 +0200352 LIST_HEAD(tmp);
353 struct list_head *pos, *node;
Jens Axboecf137302009-09-24 15:12:57 +0200354 struct super_block *sb = NULL;
Shaohua Li5c034492009-09-24 14:42:33 +0200355 struct inode *inode;
Jens Axboecf137302009-09-24 15:12:57 +0200356 int do_sb_sort = 0;
Shaohua Li5c034492009-09-24 14:42:33 +0200357
Fengguang Wu2c136572007-10-16 23:30:39 -0700358 while (!list_empty(delaying_queue)) {
Shaohua Li5c034492009-09-24 14:42:33 +0200359 inode = list_entry(delaying_queue->prev, struct inode, i_list);
Fengguang Wu2c136572007-10-16 23:30:39 -0700360 if (older_than_this &&
Jeff Laytond2caa3c2009-04-02 16:56:37 -0700361 inode_dirtied_after(inode, *older_than_this))
Fengguang Wu2c136572007-10-16 23:30:39 -0700362 break;
Jens Axboecf137302009-09-24 15:12:57 +0200363 if (sb && sb != inode->i_sb)
364 do_sb_sort = 1;
365 sb = inode->i_sb;
Shaohua Li5c034492009-09-24 14:42:33 +0200366 list_move(&inode->i_list, &tmp);
367 }
368
Jens Axboecf137302009-09-24 15:12:57 +0200369 /* just one sb in list, splice to dispatch_queue and we're done */
370 if (!do_sb_sort) {
371 list_splice(&tmp, dispatch_queue);
372 return;
373 }
374
Shaohua Li5c034492009-09-24 14:42:33 +0200375 /* Move inodes from one superblock together */
376 while (!list_empty(&tmp)) {
377 inode = list_entry(tmp.prev, struct inode, i_list);
378 sb = inode->i_sb;
379 list_for_each_prev_safe(pos, node, &tmp) {
380 inode = list_entry(pos, struct inode, i_list);
381 if (inode->i_sb == sb)
382 list_move(&inode->i_list, dispatch_queue);
383 }
Fengguang Wu2c136572007-10-16 23:30:39 -0700384 }
385}
386
387/*
388 * Queue all expired dirty inodes for io, eldest first.
389 */
Jens Axboe03ba3782009-09-09 09:08:54 +0200390static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this)
Fengguang Wu2c136572007-10-16 23:30:39 -0700391{
Jens Axboe03ba3782009-09-09 09:08:54 +0200392 list_splice_init(&wb->b_more_io, wb->b_io.prev);
393 move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200394}
395
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100396static int write_inode(struct inode *inode, struct writeback_control *wbc)
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200397{
Jens Axboe03ba3782009-09-09 09:08:54 +0200398 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100399 return inode->i_sb->s_op->write_inode(inode, wbc);
Jens Axboe03ba3782009-09-09 09:08:54 +0200400 return 0;
Fengguang Wu2c136572007-10-16 23:30:39 -0700401}
402
403/*
Christoph Hellwig01c03192009-06-08 13:35:40 +0200404 * Wait for writeback on an inode to complete.
405 */
406static void inode_wait_for_writeback(struct inode *inode)
407{
408 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
409 wait_queue_head_t *wqh;
410
411 wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
Richard Kennedy58a9d3d2010-05-24 14:32:38 -0700412 while (inode->i_state & I_SYNC) {
Christoph Hellwig01c03192009-06-08 13:35:40 +0200413 spin_unlock(&inode_lock);
414 __wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
415 spin_lock(&inode_lock);
Richard Kennedy58a9d3d2010-05-24 14:32:38 -0700416 }
Christoph Hellwig01c03192009-06-08 13:35:40 +0200417}
418
419/*
420 * Write out an inode's dirty pages. Called under inode_lock. Either the
421 * caller has ref on the inode (either via __iget or via syscall against an fd)
422 * or the inode has I_WILL_FREE set (via generic_forget_inode)
423 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 * If `wait' is set, wait on the writeout.
425 *
426 * The whole writeout design is quite complex and fragile. We want to avoid
427 * starvation of particular inodes when others are being redirtied, prevent
428 * livelocks, etc.
429 *
430 * Called under inode_lock.
431 */
432static int
Christoph Hellwig01c03192009-06-08 13:35:40 +0200433writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 struct address_space *mapping = inode->i_mapping;
Christoph Hellwig01c03192009-06-08 13:35:40 +0200436 unsigned dirty;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 int ret;
438
Christoph Hellwig01c03192009-06-08 13:35:40 +0200439 if (!atomic_read(&inode->i_count))
440 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
441 else
442 WARN_ON(inode->i_state & I_WILL_FREE);
443
444 if (inode->i_state & I_SYNC) {
445 /*
446 * If this inode is locked for writeback and we are not doing
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200447 * writeback-for-data-integrity, move it to b_more_io so that
Christoph Hellwig01c03192009-06-08 13:35:40 +0200448 * writeback can proceed with the other inodes on s_io.
449 *
450 * We'll have another go at writing back this inode when we
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200451 * completed a full scan of b_io.
Christoph Hellwig01c03192009-06-08 13:35:40 +0200452 */
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100453 if (wbc->sync_mode != WB_SYNC_ALL) {
Christoph Hellwig01c03192009-06-08 13:35:40 +0200454 requeue_io(inode);
455 return 0;
456 }
457
458 /*
459 * It's a data-integrity sync. We must wait.
460 */
461 inode_wait_for_writeback(inode);
462 }
463
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700464 BUG_ON(inode->i_state & I_SYNC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
Dmitry Monakhov5547e8a2010-05-07 13:35:44 +0400466 /* Set I_SYNC, reset I_DIRTY_PAGES */
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700467 inode->i_state |= I_SYNC;
Dmitry Monakhov5547e8a2010-05-07 13:35:44 +0400468 inode->i_state &= ~I_DIRTY_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 spin_unlock(&inode_lock);
470
471 ret = do_writepages(mapping, wbc);
472
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100473 /*
474 * Make sure to wait on the data before writing out the metadata.
475 * This is important for filesystems that modify metadata on data
476 * I/O completion.
477 */
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100478 if (wbc->sync_mode == WB_SYNC_ALL) {
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100479 int err = filemap_fdatawait(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 if (ret == 0)
481 ret = err;
482 }
483
Dmitry Monakhov5547e8a2010-05-07 13:35:44 +0400484 /*
485 * Some filesystems may redirty the inode during the writeback
486 * due to delalloc, clear dirty metadata flags right before
487 * write_inode()
488 */
489 spin_lock(&inode_lock);
490 dirty = inode->i_state & I_DIRTY;
491 inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC);
492 spin_unlock(&inode_lock);
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100493 /* Don't write the inode if only I_DIRTY_PAGES was set */
494 if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100495 int err = write_inode(inode, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 if (ret == 0)
497 ret = err;
498 }
499
500 spin_lock(&inode_lock);
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700501 inode->i_state &= ~I_SYNC;
Wu Fengguang84a89242009-06-16 15:33:17 -0700502 if (!(inode->i_state & (I_FREEING | I_CLEAR))) {
Wu Fengguangb3af9462009-09-25 06:04:10 +0200503 if ((inode->i_state & I_DIRTY_PAGES) && wbc->for_kupdate) {
Wu Fengguangae1b7f72009-09-23 20:33:42 +0800504 /*
Wu Fengguangb3af9462009-09-25 06:04:10 +0200505 * More pages get dirtied by a fast dirtier.
506 */
507 goto select_queue;
508 } else if (inode->i_state & I_DIRTY) {
509 /*
510 * At least XFS will redirty the inode during the
511 * writeback (delalloc) and on io completion (isize).
Wu Fengguangae1b7f72009-09-23 20:33:42 +0800512 */
513 redirty_tail(inode);
514 } else if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 /*
516 * We didn't write back all the pages. nfs_writepages()
517 * sometimes bales out without doing anything. Redirty
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200518 * the inode; Move it from b_io onto b_more_io/b_dirty.
Andrew Morton1b43ef92007-10-16 23:30:35 -0700519 */
520 /*
521 * akpm: if the caller was the kupdate function we put
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200522 * this inode at the head of b_dirty so it gets first
Andrew Morton1b43ef92007-10-16 23:30:35 -0700523 * consideration. Otherwise, move it to the tail, for
524 * the reasons described there. I'm not really sure
525 * how much sense this makes. Presumably I had a good
526 * reasons for doing it this way, and I'd rather not
527 * muck with it at present.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 */
529 if (wbc->for_kupdate) {
530 /*
Fengguang Wu2c136572007-10-16 23:30:39 -0700531 * For the kupdate function we move the inode
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200532 * to b_more_io so it will get more writeout as
Fengguang Wu2c136572007-10-16 23:30:39 -0700533 * soon as the queue becomes uncongested.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 */
535 inode->i_state |= I_DIRTY_PAGES;
Wu Fengguangb3af9462009-09-25 06:04:10 +0200536select_queue:
Fengguang Wu8bc3be22008-02-04 22:29:36 -0800537 if (wbc->nr_to_write <= 0) {
538 /*
539 * slice used up: queue for next turn
540 */
541 requeue_io(inode);
542 } else {
543 /*
544 * somehow blocked: retry later
545 */
546 redirty_tail(inode);
547 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 } else {
549 /*
550 * Otherwise fully redirty the inode so that
551 * other inodes on this superblock will get some
552 * writeout. Otherwise heavy writing to one
553 * file would indefinitely suspend writeout of
554 * all the other files.
555 */
556 inode->i_state |= I_DIRTY_PAGES;
Andrew Morton1b43ef92007-10-16 23:30:35 -0700557 redirty_tail(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 } else if (atomic_read(&inode->i_count)) {
560 /*
561 * The inode is clean, inuse
562 */
563 list_move(&inode->i_list, &inode_in_use);
564 } else {
565 /*
566 * The inode is clean, unused
567 */
568 list_move(&inode->i_list, &inode_unused);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 }
570 }
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700571 inode_sync_complete(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 return ret;
573}
574
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800575static void unpin_sb_for_writeback(struct super_block *sb)
Jens Axboe9ecc2732009-09-24 15:25:11 +0200576{
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800577 up_read(&sb->s_umount);
578 put_super(sb);
Jens Axboe9ecc2732009-09-24 15:25:11 +0200579}
580
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800581enum sb_pin_state {
582 SB_PINNED,
583 SB_NOT_PINNED,
584 SB_PIN_FAILED
585};
586
Jens Axboe03ba3782009-09-09 09:08:54 +0200587/*
588 * For WB_SYNC_NONE writeback, the caller does not have the sb pinned
589 * before calling writeback. So make sure that we do pin it, so it doesn't
590 * go away while we are writing inodes from it.
Jens Axboe03ba3782009-09-09 09:08:54 +0200591 */
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800592static enum sb_pin_state pin_sb_for_writeback(struct writeback_control *wbc,
593 struct super_block *sb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594{
Jens Axboe9ecc2732009-09-24 15:25:11 +0200595 /*
Jens Axboe03ba3782009-09-09 09:08:54 +0200596 * Caller must already hold the ref for this
597 */
Jens Axboee913fc82010-05-17 12:55:07 +0200598 if (wbc->sync_mode == WB_SYNC_ALL || wbc->sb_pinned) {
Jens Axboe03ba3782009-09-09 09:08:54 +0200599 WARN_ON(!rwsem_is_locked(&sb->s_umount));
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800600 return SB_NOT_PINNED;
Jens Axboe03ba3782009-09-09 09:08:54 +0200601 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200602 spin_lock(&sb_lock);
603 sb->s_count++;
604 if (down_read_trylock(&sb->s_umount)) {
605 if (sb->s_root) {
606 spin_unlock(&sb_lock);
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800607 return SB_PINNED;
Jens Axboe03ba3782009-09-09 09:08:54 +0200608 }
609 /*
610 * umounted, drop rwsem again and fall through to failure
611 */
612 up_read(&sb->s_umount);
613 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200614 sb->s_count--;
615 spin_unlock(&sb_lock);
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800616 return SB_PIN_FAILED;
Jens Axboe03ba3782009-09-09 09:08:54 +0200617}
618
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800619/*
620 * Write a portion of b_io inodes which belong to @sb.
621 * If @wbc->sb != NULL, then find and write all such
622 * inodes. Otherwise write only ones which go sequentially
623 * in reverse order.
624 * Return 1, if the caller writeback routine should be
625 * interrupted. Otherwise return 0.
626 */
627static int writeback_sb_inodes(struct super_block *sb,
628 struct bdi_writeback *wb,
629 struct writeback_control *wbc)
Jens Axboe03ba3782009-09-09 09:08:54 +0200630{
Jens Axboe03ba3782009-09-09 09:08:54 +0200631 while (!list_empty(&wb->b_io)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 long pages_skipped;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800633 struct inode *inode = list_entry(wb->b_io.prev,
634 struct inode, i_list);
635 if (wbc->sb && sb != inode->i_sb) {
636 /* super block given and doesn't
637 match, skip this inode */
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200638 redirty_tail(inode);
639 continue;
640 }
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800641 if (sb != inode->i_sb)
642 /* finish with this superblock */
643 return 0;
Wu Fengguang84a89242009-06-16 15:33:17 -0700644 if (inode->i_state & (I_NEW | I_WILL_FREE)) {
Nick Piggin7ef0d732009-03-12 14:31:38 -0700645 requeue_io(inode);
646 continue;
647 }
Jeff Laytond2caa3c2009-04-02 16:56:37 -0700648 /*
649 * Was this inode dirtied after sync_sb_inodes was called?
650 * This keeps sync from extra jobs and livelock.
651 */
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800652 if (inode_dirtied_after(inode, wbc->wb_start))
653 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
Wu Fengguang84a89242009-06-16 15:33:17 -0700655 BUG_ON(inode->i_state & (I_FREEING | I_CLEAR));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 __iget(inode);
657 pages_skipped = wbc->pages_skipped;
Christoph Hellwig01c03192009-06-08 13:35:40 +0200658 writeback_single_inode(inode, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 if (wbc->pages_skipped != pages_skipped) {
660 /*
661 * writeback is not making progress due to locked
662 * buffers. Skip this inode for now.
663 */
Andrew Mortonf57b9b72007-10-16 23:30:34 -0700664 redirty_tail(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 }
666 spin_unlock(&inode_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667 iput(inode);
OGAWA Hirofumi4ffc8442006-03-25 03:07:44 -0800668 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 spin_lock(&inode_lock);
Fengguang Wu8bc3be22008-02-04 22:29:36 -0800670 if (wbc->nr_to_write <= 0) {
671 wbc->more_io = 1;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800672 return 1;
Fengguang Wu8bc3be22008-02-04 22:29:36 -0800673 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200674 if (!list_empty(&wb->b_more_io))
Fengguang Wu8bc3be22008-02-04 22:29:36 -0800675 wbc->more_io = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 }
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800677 /* b_io is empty */
678 return 1;
679}
Nick Piggin38f21972009-01-06 14:40:25 -0800680
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800681static void writeback_inodes_wb(struct bdi_writeback *wb,
682 struct writeback_control *wbc)
683{
684 int ret = 0;
Jens Axboe9ecc2732009-09-24 15:25:11 +0200685
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800686 wbc->wb_start = jiffies; /* livelock avoidance */
687 spin_lock(&inode_lock);
688 if (!wbc->for_kupdate || list_empty(&wb->b_io))
689 queue_io(wb, wbc->older_than_this);
690
691 while (!list_empty(&wb->b_io)) {
692 struct inode *inode = list_entry(wb->b_io.prev,
693 struct inode, i_list);
694 struct super_block *sb = inode->i_sb;
695 enum sb_pin_state state;
696
697 if (wbc->sb && sb != wbc->sb) {
698 /* super block given and doesn't
699 match, skip this inode */
700 redirty_tail(inode);
701 continue;
702 }
703 state = pin_sb_for_writeback(wbc, sb);
704
705 if (state == SB_PIN_FAILED) {
706 requeue_io(inode);
707 continue;
708 }
709 ret = writeback_sb_inodes(sb, wb, wbc);
710
711 if (state == SB_PINNED)
712 unpin_sb_for_writeback(sb);
713 if (ret)
714 break;
715 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200716 spin_unlock(&inode_lock);
717 /* Leave any unwritten inodes on b_io */
718}
719
Jens Axboe03ba3782009-09-09 09:08:54 +0200720void writeback_inodes_wbc(struct writeback_control *wbc)
721{
722 struct backing_dev_info *bdi = wbc->bdi;
723
724 writeback_inodes_wb(&bdi->wb, wbc);
725}
726
727/*
728 * The maximum number of pages to writeout in a single bdi flush/kupdate
729 * operation. We do this so we don't hold I_SYNC against an inode for
730 * enormous amounts of time, which would block a userspace task which has
731 * been forced to throttle against that inode. Also, the code reevaluates
732 * the dirty each time it has written this many pages.
733 */
734#define MAX_WRITEBACK_PAGES 1024
735
736static inline bool over_bground_thresh(void)
737{
738 unsigned long background_thresh, dirty_thresh;
739
740 get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
741
742 return (global_page_state(NR_FILE_DIRTY) +
743 global_page_state(NR_UNSTABLE_NFS) >= background_thresh);
744}
745
746/*
747 * Explicit flushing or periodic writeback of "old" data.
748 *
749 * Define "old": the first time one of an inode's pages is dirtied, we mark the
750 * dirtying-time in the inode's address_space. So this periodic writeback code
751 * just walks the superblock inode list, writing back any inodes which are
752 * older than a specific point in time.
753 *
754 * Try to run once per dirty_writeback_interval. But if a writeback event
755 * takes longer than a dirty_writeback_interval interval, then leave a
756 * one-second gap.
757 *
758 * older_than_this takes precedence over nr_to_write. So we'll only write back
759 * all dirty pages if they are all attached to "old" mappings.
760 */
Jens Axboec4a77a62009-09-16 15:18:25 +0200761static long wb_writeback(struct bdi_writeback *wb,
762 struct wb_writeback_args *args)
Jens Axboe03ba3782009-09-09 09:08:54 +0200763{
764 struct writeback_control wbc = {
765 .bdi = wb->bdi,
Jens Axboec4a77a62009-09-16 15:18:25 +0200766 .sb = args->sb,
767 .sync_mode = args->sync_mode,
Jens Axboe03ba3782009-09-09 09:08:54 +0200768 .older_than_this = NULL,
Jens Axboec4a77a62009-09-16 15:18:25 +0200769 .for_kupdate = args->for_kupdate,
Wu Fengguangb17621f2009-12-03 13:54:25 +0100770 .for_background = args->for_background,
Jens Axboec4a77a62009-09-16 15:18:25 +0200771 .range_cyclic = args->range_cyclic,
Jens Axboee913fc82010-05-17 12:55:07 +0200772 .sb_pinned = args->sb_pinned,
Jens Axboe03ba3782009-09-09 09:08:54 +0200773 };
774 unsigned long oldest_jif;
775 long wrote = 0;
Jan Karaa5989bd2009-09-16 19:22:48 +0200776 struct inode *inode;
Jens Axboe03ba3782009-09-09 09:08:54 +0200777
778 if (wbc.for_kupdate) {
779 wbc.older_than_this = &oldest_jif;
780 oldest_jif = jiffies -
781 msecs_to_jiffies(dirty_expire_interval * 10);
782 }
Jens Axboec4a77a62009-09-16 15:18:25 +0200783 if (!wbc.range_cyclic) {
784 wbc.range_start = 0;
785 wbc.range_end = LLONG_MAX;
786 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200787
788 for (;;) {
789 /*
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800790 * Stop writeback when nr_pages has been consumed
Jens Axboe03ba3782009-09-09 09:08:54 +0200791 */
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800792 if (args->nr_pages <= 0)
Jens Axboe03ba3782009-09-09 09:08:54 +0200793 break;
794
795 /*
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800796 * For background writeout, stop when we are below the
797 * background dirty threshold
Jens Axboe03ba3782009-09-09 09:08:54 +0200798 */
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800799 if (args->for_background && !over_bground_thresh())
Jens Axboe03ba3782009-09-09 09:08:54 +0200800 break;
801
802 wbc.more_io = 0;
Jens Axboe03ba3782009-09-09 09:08:54 +0200803 wbc.nr_to_write = MAX_WRITEBACK_PAGES;
804 wbc.pages_skipped = 0;
805 writeback_inodes_wb(wb, &wbc);
Jens Axboec4a77a62009-09-16 15:18:25 +0200806 args->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
Jens Axboe03ba3782009-09-09 09:08:54 +0200807 wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write;
808
809 /*
Jens Axboe71fd05a2009-09-23 19:32:26 +0200810 * If we consumed everything, see if we have more
Jens Axboe03ba3782009-09-09 09:08:54 +0200811 */
Jens Axboe71fd05a2009-09-23 19:32:26 +0200812 if (wbc.nr_to_write <= 0)
813 continue;
814 /*
815 * Didn't write everything and we don't have more IO, bail
816 */
817 if (!wbc.more_io)
Jens Axboe03ba3782009-09-09 09:08:54 +0200818 break;
Jens Axboe71fd05a2009-09-23 19:32:26 +0200819 /*
820 * Did we write something? Try for more
821 */
822 if (wbc.nr_to_write < MAX_WRITEBACK_PAGES)
823 continue;
824 /*
825 * Nothing written. Wait for some inode to
826 * become available for writeback. Otherwise
827 * we'll just busyloop.
828 */
829 spin_lock(&inode_lock);
830 if (!list_empty(&wb->b_more_io)) {
831 inode = list_entry(wb->b_more_io.prev,
832 struct inode, i_list);
833 inode_wait_for_writeback(inode);
Jens Axboe03ba3782009-09-09 09:08:54 +0200834 }
Jens Axboe71fd05a2009-09-23 19:32:26 +0200835 spin_unlock(&inode_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200836 }
837
838 return wrote;
839}
840
841/*
842 * Return the next bdi_work struct that hasn't been processed by this
Jens Axboe8010c3b2009-09-15 20:04:57 +0200843 * wb thread yet. ->seen is initially set for each thread that exists
844 * for this device, when a thread first notices a piece of work it
845 * clears its bit. Depending on writeback type, the thread will notify
846 * completion on either receiving the work (WB_SYNC_NONE) or after
847 * it is done (WB_SYNC_ALL).
Jens Axboe03ba3782009-09-09 09:08:54 +0200848 */
849static struct bdi_work *get_next_work_item(struct backing_dev_info *bdi,
850 struct bdi_writeback *wb)
851{
852 struct bdi_work *work, *ret = NULL;
853
854 rcu_read_lock();
855
856 list_for_each_entry_rcu(work, &bdi->work_list, list) {
Nick Piggin77fad5e2009-09-15 21:34:12 +0200857 if (!test_bit(wb->nr, &work->seen))
Jens Axboe03ba3782009-09-09 09:08:54 +0200858 continue;
Nick Piggin77fad5e2009-09-15 21:34:12 +0200859 clear_bit(wb->nr, &work->seen);
Jens Axboe03ba3782009-09-09 09:08:54 +0200860
861 ret = work;
862 break;
863 }
864
865 rcu_read_unlock();
866 return ret;
867}
868
869static long wb_check_old_data_flush(struct bdi_writeback *wb)
870{
871 unsigned long expired;
872 long nr_pages;
873
Jens Axboe69b62d02010-05-17 12:51:03 +0200874 /*
875 * When set to zero, disable periodic writeback
876 */
877 if (!dirty_writeback_interval)
878 return 0;
879
Jens Axboe03ba3782009-09-09 09:08:54 +0200880 expired = wb->last_old_flush +
881 msecs_to_jiffies(dirty_writeback_interval * 10);
882 if (time_before(jiffies, expired))
883 return 0;
884
885 wb->last_old_flush = jiffies;
886 nr_pages = global_page_state(NR_FILE_DIRTY) +
887 global_page_state(NR_UNSTABLE_NFS) +
888 (inodes_stat.nr_inodes - inodes_stat.nr_unused);
889
Jens Axboec4a77a62009-09-16 15:18:25 +0200890 if (nr_pages) {
891 struct wb_writeback_args args = {
892 .nr_pages = nr_pages,
893 .sync_mode = WB_SYNC_NONE,
894 .for_kupdate = 1,
895 .range_cyclic = 1,
896 };
897
898 return wb_writeback(wb, &args);
899 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200900
901 return 0;
902}
903
904/*
905 * Retrieve work items and do the writeback they describe
906 */
907long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
908{
909 struct backing_dev_info *bdi = wb->bdi;
910 struct bdi_work *work;
Jens Axboec4a77a62009-09-16 15:18:25 +0200911 long wrote = 0;
Jens Axboe03ba3782009-09-09 09:08:54 +0200912
913 while ((work = get_next_work_item(bdi, wb)) != NULL) {
Jens Axboec4a77a62009-09-16 15:18:25 +0200914 struct wb_writeback_args args = work->args;
Jens Axboe7c8a3552010-05-18 14:29:29 +0200915 int post_clear;
Jens Axboe03ba3782009-09-09 09:08:54 +0200916
917 /*
918 * Override sync mode, in case we must wait for completion
919 */
920 if (force_wait)
Jens Axboec4a77a62009-09-16 15:18:25 +0200921 work->args.sync_mode = args.sync_mode = WB_SYNC_ALL;
Jens Axboe03ba3782009-09-09 09:08:54 +0200922
Jens Axboe7c8a3552010-05-18 14:29:29 +0200923 post_clear = WB_SYNC_ALL || args.sb_pinned;
924
Jens Axboe03ba3782009-09-09 09:08:54 +0200925 /*
926 * If this isn't a data integrity operation, just notify
927 * that we have seen this work and we are now starting it.
928 */
Jens Axboe7c8a3552010-05-18 14:29:29 +0200929 if (!post_clear)
Jens Axboe03ba3782009-09-09 09:08:54 +0200930 wb_clear_pending(wb, work);
931
Jens Axboec4a77a62009-09-16 15:18:25 +0200932 wrote += wb_writeback(wb, &args);
Jens Axboe03ba3782009-09-09 09:08:54 +0200933
934 /*
935 * This is a data integrity writeback, so only do the
936 * notification when we have completed the work.
937 */
Jens Axboe7c8a3552010-05-18 14:29:29 +0200938 if (post_clear)
Jens Axboe03ba3782009-09-09 09:08:54 +0200939 wb_clear_pending(wb, work);
940 }
941
942 /*
943 * Check for periodic writeback, kupdated() style
944 */
945 wrote += wb_check_old_data_flush(wb);
946
947 return wrote;
948}
949
950/*
951 * Handle writeback of dirty data for the device backed by this bdi. Also
952 * wakes up periodically and does kupdated style flushing.
953 */
954int bdi_writeback_task(struct bdi_writeback *wb)
955{
956 unsigned long last_active = jiffies;
957 unsigned long wait_jiffies = -1UL;
958 long pages_written;
959
960 while (!kthread_should_stop()) {
961 pages_written = wb_do_writeback(wb, 0);
962
963 if (pages_written)
964 last_active = jiffies;
965 else if (wait_jiffies != -1UL) {
966 unsigned long max_idle;
967
968 /*
969 * Longest period of inactivity that we tolerate. If we
970 * see dirty data again later, the task will get
971 * recreated automatically.
972 */
973 max_idle = max(5UL * 60 * HZ, wait_jiffies);
974 if (time_after(jiffies, max_idle + last_active))
975 break;
976 }
977
Jens Axboe69b62d02010-05-17 12:51:03 +0200978 if (dirty_writeback_interval) {
979 wait_jiffies = msecs_to_jiffies(dirty_writeback_interval * 10);
980 schedule_timeout_interruptible(wait_jiffies);
Jens Axboef9eadbb2010-05-18 14:31:45 +0200981 } else {
982 set_current_state(TASK_INTERRUPTIBLE);
983 if (list_empty_careful(&wb->bdi->work_list) &&
984 !kthread_should_stop())
985 schedule();
986 __set_current_state(TASK_RUNNING);
987 }
Jens Axboe69b62d02010-05-17 12:51:03 +0200988
Jens Axboe03ba3782009-09-09 09:08:54 +0200989 try_to_freeze();
990 }
991
992 return 0;
993}
994
995/*
Jens Axboeb6e51312009-09-16 15:13:54 +0200996 * Schedule writeback for all backing devices. This does WB_SYNC_NONE
997 * writeback, for integrity writeback see bdi_sync_writeback().
Jens Axboe03ba3782009-09-09 09:08:54 +0200998 */
Jens Axboeb6e51312009-09-16 15:13:54 +0200999static void bdi_writeback_all(struct super_block *sb, long nr_pages)
Jens Axboe03ba3782009-09-09 09:08:54 +02001000{
Jens Axboeb6e51312009-09-16 15:13:54 +02001001 struct wb_writeback_args args = {
1002 .sb = sb,
1003 .nr_pages = nr_pages,
1004 .sync_mode = WB_SYNC_NONE,
1005 };
Jens Axboe03ba3782009-09-09 09:08:54 +02001006 struct backing_dev_info *bdi;
Jens Axboe03ba3782009-09-09 09:08:54 +02001007
Jens Axboecfc4ba52009-09-14 13:12:40 +02001008 rcu_read_lock();
Jens Axboe03ba3782009-09-09 09:08:54 +02001009
Jens Axboecfc4ba52009-09-14 13:12:40 +02001010 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
Jens Axboe03ba3782009-09-09 09:08:54 +02001011 if (!bdi_has_dirty_io(bdi))
1012 continue;
1013
Jens Axboe7c8a3552010-05-18 14:29:29 +02001014 bdi_alloc_queue_work(bdi, &args, 0);
Jens Axboe03ba3782009-09-09 09:08:54 +02001015 }
1016
Jens Axboecfc4ba52009-09-14 13:12:40 +02001017 rcu_read_unlock();
Jens Axboe03ba3782009-09-09 09:08:54 +02001018}
1019
1020/*
1021 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
1022 * the whole world.
1023 */
1024void wakeup_flusher_threads(long nr_pages)
1025{
Jens Axboe03ba3782009-09-09 09:08:54 +02001026 if (nr_pages == 0)
1027 nr_pages = global_page_state(NR_FILE_DIRTY) +
1028 global_page_state(NR_UNSTABLE_NFS);
Jens Axboeb6e51312009-09-16 15:13:54 +02001029 bdi_writeback_all(NULL, nr_pages);
Jens Axboe03ba3782009-09-09 09:08:54 +02001030}
1031
1032static noinline void block_dump___mark_inode_dirty(struct inode *inode)
1033{
1034 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
1035 struct dentry *dentry;
1036 const char *name = "?";
1037
1038 dentry = d_find_alias(inode);
1039 if (dentry) {
1040 spin_lock(&dentry->d_lock);
1041 name = (const char *) dentry->d_name.name;
1042 }
1043 printk(KERN_DEBUG
1044 "%s(%d): dirtied inode %lu (%s) on %s\n",
1045 current->comm, task_pid_nr(current), inode->i_ino,
1046 name, inode->i_sb->s_id);
1047 if (dentry) {
1048 spin_unlock(&dentry->d_lock);
1049 dput(dentry);
1050 }
1051 }
1052}
1053
1054/**
1055 * __mark_inode_dirty - internal function
1056 * @inode: inode to mark
1057 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC)
1058 * Mark an inode as dirty. Callers should use mark_inode_dirty or
1059 * mark_inode_dirty_sync.
1060 *
1061 * Put the inode on the super block's dirty list.
1062 *
1063 * CAREFUL! We mark it dirty unconditionally, but move it onto the
1064 * dirty list only if it is hashed or if it refers to a blockdev.
1065 * If it was not hashed, it will never be added to the dirty list
1066 * even if it is later hashed, as it will have been marked dirty already.
1067 *
1068 * In short, make sure you hash any inodes _before_ you start marking
1069 * them dirty.
1070 *
1071 * This function *must* be atomic for the I_DIRTY_PAGES case -
1072 * set_page_dirty() is called under spinlock in several places.
1073 *
1074 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
1075 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
1076 * the kernel-internal blockdev inode represents the dirtying time of the
1077 * blockdev's pages. This is why for I_DIRTY_PAGES we always use
1078 * page->mapping->host, so the page-dirtying time is recorded in the internal
1079 * blockdev inode.
1080 */
1081void __mark_inode_dirty(struct inode *inode, int flags)
1082{
1083 struct super_block *sb = inode->i_sb;
1084
1085 /*
1086 * Don't do this for I_DIRTY_PAGES - that doesn't actually
1087 * dirty the inode itself
1088 */
1089 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
1090 if (sb->s_op->dirty_inode)
1091 sb->s_op->dirty_inode(inode);
1092 }
1093
1094 /*
1095 * make sure that changes are seen by all cpus before we test i_state
1096 * -- mikulas
1097 */
1098 smp_mb();
1099
1100 /* avoid the locking if we can */
1101 if ((inode->i_state & flags) == flags)
1102 return;
1103
1104 if (unlikely(block_dump))
1105 block_dump___mark_inode_dirty(inode);
1106
1107 spin_lock(&inode_lock);
1108 if ((inode->i_state & flags) != flags) {
1109 const int was_dirty = inode->i_state & I_DIRTY;
1110
1111 inode->i_state |= flags;
1112
1113 /*
1114 * If the inode is being synced, just update its dirty state.
1115 * The unlocker will place the inode on the appropriate
1116 * superblock list, based upon its state.
1117 */
1118 if (inode->i_state & I_SYNC)
1119 goto out;
1120
1121 /*
1122 * Only add valid (hashed) inodes to the superblock's
1123 * dirty list. Add blockdev inodes as well.
1124 */
1125 if (!S_ISBLK(inode->i_mode)) {
1126 if (hlist_unhashed(&inode->i_hash))
1127 goto out;
1128 }
1129 if (inode->i_state & (I_FREEING|I_CLEAR))
1130 goto out;
1131
1132 /*
1133 * If the inode was already on b_dirty/b_io/b_more_io, don't
1134 * reposition it (that would break b_dirty time-ordering).
1135 */
1136 if (!was_dirty) {
1137 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
Jens Axboe500b0672009-09-09 09:10:25 +02001138 struct backing_dev_info *bdi = wb->bdi;
1139
1140 if (bdi_cap_writeback_dirty(bdi) &&
1141 !test_bit(BDI_registered, &bdi->state)) {
1142 WARN_ON(1);
1143 printk(KERN_ERR "bdi-%s not registered\n",
1144 bdi->name);
1145 }
Jens Axboe03ba3782009-09-09 09:08:54 +02001146
1147 inode->dirtied_when = jiffies;
1148 list_move(&inode->i_list, &wb->b_dirty);
1149 }
1150 }
1151out:
1152 spin_unlock(&inode_lock);
1153}
1154EXPORT_SYMBOL(__mark_inode_dirty);
1155
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001156/*
1157 * Write out a superblock's list of dirty inodes. A wait will be performed
1158 * upon no inodes, all inodes or the final one, depending upon sync_mode.
1159 *
1160 * If older_than_this is non-NULL, then only write out inodes which
1161 * had their first dirtying at a time earlier than *older_than_this.
1162 *
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001163 * If `bdi' is non-zero then we're being asked to writeback a specific queue.
1164 * This function assumes that the blockdev superblock's inodes are backed by
1165 * a variety of queues, so all inodes are searched. For other superblocks,
1166 * assume that all inodes are backed by the same queue.
1167 *
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001168 * The inodes to be written are parked on bdi->b_io. They are moved back onto
1169 * bdi->b_dirty as they are selected for writing. This way, none can be missed
1170 * on the writer throttling path, and we get decent balancing between many
1171 * throttled threads: we don't want them all piling up on inode_sync_wait.
1172 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001173static void wait_sb_inodes(struct super_block *sb)
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001174{
Jens Axboe03ba3782009-09-09 09:08:54 +02001175 struct inode *inode, *old_inode = NULL;
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001176
Jens Axboe03ba3782009-09-09 09:08:54 +02001177 /*
1178 * We need to be protected against the filesystem going from
1179 * r/o to r/w or vice versa.
1180 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001181 WARN_ON(!rwsem_is_locked(&sb->s_umount));
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001182
Jens Axboe03ba3782009-09-09 09:08:54 +02001183 spin_lock(&inode_lock);
1184
1185 /*
1186 * Data integrity sync. Must wait for all pages under writeback,
1187 * because there may have been pages dirtied before our sync
1188 * call, but which had writeout started before we write it out.
1189 * In which case, the inode may not be on the dirty list, but
1190 * we still have to wait for that writeout.
1191 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001192 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
Jens Axboe03ba3782009-09-09 09:08:54 +02001193 struct address_space *mapping;
1194
1195 if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE|I_NEW))
1196 continue;
1197 mapping = inode->i_mapping;
1198 if (mapping->nrpages == 0)
1199 continue;
1200 __iget(inode);
1201 spin_unlock(&inode_lock);
1202 /*
1203 * We hold a reference to 'inode' so it couldn't have
1204 * been removed from s_inodes list while we dropped the
1205 * inode_lock. We cannot iput the inode now as we can
1206 * be holding the last reference and we cannot iput it
1207 * under inode_lock. So we keep the reference and iput
1208 * it later.
1209 */
1210 iput(old_inode);
1211 old_inode = inode;
1212
1213 filemap_fdatawait(mapping);
1214
1215 cond_resched();
Nick Piggin38f21972009-01-06 14:40:25 -08001216
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001217 spin_lock(&inode_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001218 }
Jens Axboe03ba3782009-09-09 09:08:54 +02001219 spin_unlock(&inode_lock);
1220 iput(old_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221}
1222
Jens Axboee913fc82010-05-17 12:55:07 +02001223static void __writeback_inodes_sb(struct super_block *sb, int sb_locked)
1224{
1225 unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY);
1226 unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS);
1227 long nr_to_write;
1228
1229 nr_to_write = nr_dirty + nr_unstable +
1230 (inodes_stat.nr_inodes - inodes_stat.nr_unused);
1231
1232 bdi_start_writeback(sb->s_bdi, sb, nr_to_write, sb_locked);
1233}
1234
Jens Axboed8a85592009-09-02 12:34:32 +02001235/**
1236 * writeback_inodes_sb - writeback dirty inodes from given super_block
1237 * @sb: the superblock
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 *
Jens Axboed8a85592009-09-02 12:34:32 +02001239 * Start writeback on some inodes on this super_block. No guarantees are made
1240 * on how many (if any) will be written, and this function does not wait
1241 * for IO completion of submitted IO. The number of pages submitted is
1242 * returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001244void writeback_inodes_sb(struct super_block *sb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245{
Jens Axboee913fc82010-05-17 12:55:07 +02001246 __writeback_inodes_sb(sb, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247}
Jens Axboed8a85592009-09-02 12:34:32 +02001248EXPORT_SYMBOL(writeback_inodes_sb);
1249
1250/**
Jens Axboee913fc82010-05-17 12:55:07 +02001251 * writeback_inodes_sb_locked - writeback dirty inodes from given super_block
1252 * @sb: the superblock
1253 *
1254 * Like writeback_inodes_sb(), except the caller already holds the
1255 * sb umount sem.
1256 */
1257void writeback_inodes_sb_locked(struct super_block *sb)
1258{
1259 __writeback_inodes_sb(sb, 1);
1260}
1261
1262/**
Eric Sandeen17bd55d2009-12-23 07:57:07 -05001263 * writeback_inodes_sb_if_idle - start writeback if none underway
1264 * @sb: the superblock
1265 *
1266 * Invoke writeback_inodes_sb if no writeback is currently underway.
1267 * Returns 1 if writeback was started, 0 if not.
1268 */
1269int writeback_inodes_sb_if_idle(struct super_block *sb)
1270{
1271 if (!writeback_in_progress(sb->s_bdi)) {
1272 writeback_inodes_sb(sb);
1273 return 1;
1274 } else
1275 return 0;
1276}
1277EXPORT_SYMBOL(writeback_inodes_sb_if_idle);
1278
1279/**
Jens Axboed8a85592009-09-02 12:34:32 +02001280 * sync_inodes_sb - sync sb inode pages
1281 * @sb: the superblock
1282 *
1283 * This function writes and waits on any dirty inode belonging to this
1284 * super_block. The number of pages synced is returned.
1285 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001286void sync_inodes_sb(struct super_block *sb)
Jens Axboed8a85592009-09-02 12:34:32 +02001287{
Jens Axboeb6e51312009-09-16 15:13:54 +02001288 bdi_sync_writeback(sb->s_bdi, sb);
1289 wait_sb_inodes(sb);
Jens Axboed8a85592009-09-02 12:34:32 +02001290}
1291EXPORT_SYMBOL(sync_inodes_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293/**
Andrea Arcangeli7f04c262005-10-30 15:03:05 -08001294 * write_inode_now - write an inode to disk
1295 * @inode: inode to write to disk
1296 * @sync: whether the write should be synchronous or not
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297 *
Andrea Arcangeli7f04c262005-10-30 15:03:05 -08001298 * This function commits an inode to disk immediately if it is dirty. This is
1299 * primarily needed by knfsd.
1300 *
1301 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303int write_inode_now(struct inode *inode, int sync)
1304{
1305 int ret;
1306 struct writeback_control wbc = {
1307 .nr_to_write = LONG_MAX,
Mike Galbraith18914b12008-02-08 04:20:23 -08001308 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -07001309 .range_start = 0,
1310 .range_end = LLONG_MAX,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 };
1312
1313 if (!mapping_cap_writeback_dirty(inode->i_mapping))
Andrew Morton49364ce2005-11-07 00:59:15 -08001314 wbc.nr_to_write = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315
1316 might_sleep();
1317 spin_lock(&inode_lock);
Christoph Hellwig01c03192009-06-08 13:35:40 +02001318 ret = writeback_single_inode(inode, &wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 spin_unlock(&inode_lock);
1320 if (sync)
Joern Engel1c0eeaf2007-10-16 23:30:44 -07001321 inode_sync_wait(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 return ret;
1323}
1324EXPORT_SYMBOL(write_inode_now);
1325
1326/**
1327 * sync_inode - write an inode and its pages to disk.
1328 * @inode: the inode to sync
1329 * @wbc: controls the writeback mode
1330 *
1331 * sync_inode() will write an inode and its pages to disk. It will also
1332 * correctly update the inode on its superblock's dirty inode lists and will
1333 * update inode->i_state.
1334 *
1335 * The caller must have a ref on the inode.
1336 */
1337int sync_inode(struct inode *inode, struct writeback_control *wbc)
1338{
1339 int ret;
1340
1341 spin_lock(&inode_lock);
Christoph Hellwig01c03192009-06-08 13:35:40 +02001342 ret = writeback_single_inode(inode, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 spin_unlock(&inode_lock);
1344 return ret;
1345}
1346EXPORT_SYMBOL(sync_inode);