blob: cabdece4cb391d200e140c907b658da98d1793cb [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * fs/fs-writeback.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * Contains all the functions related to writing back and waiting
7 * upon dirty inodes against superblocks, and writing back dirty
8 * pages against inodes. ie: data writeback. Writeout of the
9 * inode itself is not handled here.
10 *
Francois Camie1f8e872008-10-15 22:01:59 -070011 * 10Apr2002 Andrew Morton
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Split out of fs/inode.c
13 * Additions for address_space-based writeback
14 */
15
16#include <linux/kernel.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050017#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/spinlock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/sched.h>
21#include <linux/fs.h>
22#include <linux/mm.h>
Wu Fengguangbc31b862012-01-07 20:41:55 -060023#include <linux/pagemap.h>
Jens Axboe03ba3782009-09-09 09:08:54 +020024#include <linux/kthread.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/writeback.h>
26#include <linux/blkdev.h>
27#include <linux/backing-dev.h>
Dave Chinner455b2862010-07-07 13:24:06 +100028#include <linux/tracepoint.h>
David Howells07f3f052006-09-30 20:52:18 +020029#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Jens Axboed0bceac2009-05-18 08:20:32 +020031/*
Wu Fengguangbc31b862012-01-07 20:41:55 -060032 * 4MB minimal write chunk size
33 */
34#define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_CACHE_SHIFT - 10))
35
36/*
Jens Axboec4a77a62009-09-16 15:18:25 +020037 * Passed into wb_writeback(), essentially a subset of writeback_control
38 */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +020039struct wb_writeback_work {
Jens Axboec4a77a62009-09-16 15:18:25 +020040 long nr_pages;
41 struct super_block *sb;
Wu Fengguangd46db3d2011-05-04 19:54:37 -060042 unsigned long *older_than_this;
Jens Axboec4a77a62009-09-16 15:18:25 +020043 enum writeback_sync_modes sync_mode;
Wu Fengguang6e6938b2010-06-06 10:38:15 -060044 unsigned int tagged_writepages:1;
H Hartley Sweeten52957fe2010-04-01 20:36:30 -050045 unsigned int for_kupdate:1;
46 unsigned int range_cyclic:1;
47 unsigned int for_background:1;
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -060048 enum wb_reason reason; /* why was writeback initiated? */
Jens Axboec4a77a62009-09-16 15:18:25 +020049
Jens Axboe8010c3b2009-09-15 20:04:57 +020050 struct list_head list; /* pending work list */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +020051 struct completion *done; /* set if the caller waits */
Jens Axboe03ba3782009-09-09 09:08:54 +020052};
53
Adrian Bunkf11b00f2008-04-29 00:58:56 -070054/**
55 * writeback_in_progress - determine whether there is writeback in progress
56 * @bdi: the device's backing_dev_info structure.
57 *
Jens Axboe03ba3782009-09-09 09:08:54 +020058 * Determine whether there is writeback waiting to be handled against a
59 * backing device.
Adrian Bunkf11b00f2008-04-29 00:58:56 -070060 */
61int writeback_in_progress(struct backing_dev_info *bdi)
62{
Jan Kara81d73a32010-08-11 14:17:44 -070063 return test_bit(BDI_writeback_running, &bdi->state);
Adrian Bunkf11b00f2008-04-29 00:58:56 -070064}
Theodore Ts'o00d4e732012-09-19 22:42:36 -040065EXPORT_SYMBOL(writeback_in_progress);
Adrian Bunkf11b00f2008-04-29 00:58:56 -070066
Jan Kara692ebd12010-09-21 11:51:01 +020067static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
68{
69 struct super_block *sb = inode->i_sb;
Jan Kara692ebd12010-09-21 11:51:01 +020070
Jan Karaa8855992013-07-09 22:36:45 +080071 if (sb_is_blkdev_sb(sb))
Christoph Hellwigaaead252010-10-04 14:25:33 +020072 return inode->i_mapping->backing_dev_info;
73
74 return sb->s_bdi;
Jan Kara692ebd12010-09-21 11:51:01 +020075}
76
Nick Piggin7ccf19a2010-10-21 11:49:30 +110077static inline struct inode *wb_inode(struct list_head *head)
78{
79 return list_entry(head, struct inode, i_wb_list);
80}
81
Wu Fengguang15eb77a2012-01-17 11:18:56 -060082/*
83 * Include the creation of the trace points after defining the
84 * wb_writeback_work structure and inline functions so that the definition
85 * remains local to this file.
86 */
87#define CREATE_TRACE_POINTS
88#include <trace/events/writeback.h>
89
Jan Kara65850272011-01-13 15:45:44 -080090static void bdi_queue_work(struct backing_dev_info *bdi,
91 struct wb_writeback_work *work)
92{
93 trace_writeback_queue(bdi, work);
94
95 spin_lock_bh(&bdi->wb_lock);
96 list_add_tail(&work->list, &bdi->work_list);
Artem Bityutskiy64677162010-07-25 14:29:22 +030097 spin_unlock_bh(&bdi->wb_lock);
Tejun Heo839a8e82013-04-01 19:08:06 -070098
99 mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100}
101
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200102static void
103__bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600104 bool range_cyclic, enum wb_reason reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105{
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200106 struct wb_writeback_work *work;
Jens Axboe03ba3782009-09-09 09:08:54 +0200107
Jens Axboebcddc3f2009-09-13 20:07:36 +0200108 /*
109 * This is WB_SYNC_NONE writeback, so if allocation fails just
110 * wakeup the thread for old dirty data writeback
111 */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200112 work = kzalloc(sizeof(*work), GFP_ATOMIC);
113 if (!work) {
Tejun Heo839a8e82013-04-01 19:08:06 -0700114 trace_writeback_nowork(bdi);
115 mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200116 return;
Jens Axboebcddc3f2009-09-13 20:07:36 +0200117 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200118
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200119 work->sync_mode = WB_SYNC_NONE;
120 work->nr_pages = nr_pages;
121 work->range_cyclic = range_cyclic;
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600122 work->reason = reason;
Christoph Hellwigf0fad8a2009-09-11 09:47:56 +0200123
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200124 bdi_queue_work(bdi, work);
Jens Axboeb6e51312009-09-16 15:13:54 +0200125}
126
127/**
128 * bdi_start_writeback - start writeback
129 * @bdi: the backing device to write from
130 * @nr_pages: the number of pages to write
Marcos Paulo de Souza786228a2011-11-23 20:56:45 +0800131 * @reason: reason why some writeback work was initiated
Jens Axboeb6e51312009-09-16 15:13:54 +0200132 *
133 * Description:
134 * This does WB_SYNC_NONE opportunistic writeback. The IO is only
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300135 * started when this function returns, we make no guarantees on
Jens Axboe0e3c9a22010-06-01 11:08:43 +0200136 * completion. Caller need not hold sb s_umount semaphore.
Jens Axboeb6e51312009-09-16 15:13:54 +0200137 *
138 */
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600139void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
140 enum wb_reason reason)
Jens Axboeb6e51312009-09-16 15:13:54 +0200141{
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600142 __bdi_start_writeback(bdi, nr_pages, true, reason);
Christoph Hellwigc5444192010-06-08 18:15:15 +0200143}
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800144
Christoph Hellwigc5444192010-06-08 18:15:15 +0200145/**
146 * bdi_start_background_writeback - start background writeback
147 * @bdi: the backing device to write from
148 *
149 * Description:
Jan Kara65850272011-01-13 15:45:44 -0800150 * This makes sure WB_SYNC_NONE background writeback happens. When
151 * this function returns, it is only guaranteed that for given BDI
152 * some IO is happening if we are over background dirty threshold.
153 * Caller need not hold sb s_umount semaphore.
Christoph Hellwigc5444192010-06-08 18:15:15 +0200154 */
155void bdi_start_background_writeback(struct backing_dev_info *bdi)
156{
Jan Kara65850272011-01-13 15:45:44 -0800157 /*
158 * We just wake up the flusher thread. It will perform background
159 * writeback as soon as there is no other work to do.
160 */
Wu Fengguang71927e82011-01-13 15:45:46 -0800161 trace_writeback_wake_background(bdi);
Tejun Heo839a8e82013-04-01 19:08:06 -0700162 mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163}
164
165/*
Dave Chinnera66979a2011-03-22 22:23:41 +1100166 * Remove the inode from the writeback list it is on.
167 */
168void inode_wb_list_del(struct inode *inode)
169{
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600170 struct backing_dev_info *bdi = inode_to_bdi(inode);
Dave Chinnera66979a2011-03-22 22:23:41 +1100171
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600172 spin_lock(&bdi->wb.list_lock);
173 list_del_init(&inode->i_wb_list);
174 spin_unlock(&bdi->wb.list_lock);
175}
Dave Chinnera66979a2011-03-22 22:23:41 +1100176
177/*
Andrew Morton6610a0b2007-10-16 23:30:32 -0700178 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
179 * furthest end of its superblock's dirty-inode list.
180 *
181 * Before stamping the inode's ->dirtied_when, we check to see whether it is
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200182 * already the most-recently-dirtied inode on the b_dirty list. If that is
Andrew Morton6610a0b2007-10-16 23:30:32 -0700183 * the case then the inode must have been redirtied while it was being written
184 * out and we don't reset its dirtied_when.
185 */
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600186static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
Andrew Morton6610a0b2007-10-16 23:30:32 -0700187{
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600188 assert_spin_locked(&wb->list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200189 if (!list_empty(&wb->b_dirty)) {
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200190 struct inode *tail;
Andrew Morton6610a0b2007-10-16 23:30:32 -0700191
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100192 tail = wb_inode(wb->b_dirty.next);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200193 if (time_before(inode->dirtied_when, tail->dirtied_when))
Andrew Morton6610a0b2007-10-16 23:30:32 -0700194 inode->dirtied_when = jiffies;
195 }
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100196 list_move(&inode->i_wb_list, &wb->b_dirty);
Andrew Morton6610a0b2007-10-16 23:30:32 -0700197}
198
199/*
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200200 * requeue inode for re-scanning after bdi->b_io list is exhausted.
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700201 */
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600202static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700203{
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600204 assert_spin_locked(&wb->list_lock);
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100205 list_move(&inode->i_wb_list, &wb->b_more_io);
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700206}
207
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700208static void inode_sync_complete(struct inode *inode)
209{
Jan Kara365b94ae2012-05-03 14:47:55 +0200210 inode->i_state &= ~I_SYNC;
Jan Kara4eff96d2012-11-26 16:29:51 -0800211 /* If inode is clean an unused, put it into LRU now... */
212 inode_add_lru(inode);
Jan Kara365b94ae2012-05-03 14:47:55 +0200213 /* Waiters must see I_SYNC cleared before being woken up */
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700214 smp_mb();
215 wake_up_bit(&inode->i_state, __I_SYNC);
216}
217
Jeff Laytond2caa3c52009-04-02 16:56:37 -0700218static bool inode_dirtied_after(struct inode *inode, unsigned long t)
219{
220 bool ret = time_after(inode->dirtied_when, t);
221#ifndef CONFIG_64BIT
222 /*
223 * For inodes being constantly redirtied, dirtied_when can get stuck.
224 * It _appears_ to be in the future, but is actually in distant past.
225 * This test is necessary to prevent such wrapped-around relative times
Jens Axboe5b0830c2009-09-23 19:37:09 +0200226 * from permanently stopping the whole bdi writeback.
Jeff Laytond2caa3c52009-04-02 16:56:37 -0700227 */
228 ret = ret && time_before_eq(inode->dirtied_when, jiffies);
229#endif
230 return ret;
231}
232
Andrew Mortonc986d1e2007-10-16 23:30:34 -0700233/*
Wang Sheng-Hui0e2f2b22012-09-11 08:28:18 +0800234 * Move expired (dirtied before work->older_than_this) dirty inodes from
Jan Kara697e6fe2012-03-09 07:26:22 -0800235 * @delaying_queue to @dispatch_queue.
Fengguang Wu2c136572007-10-16 23:30:39 -0700236 */
Wu Fengguange84d0a42011-04-23 12:27:27 -0600237static int move_expired_inodes(struct list_head *delaying_queue,
Fengguang Wu2c136572007-10-16 23:30:39 -0700238 struct list_head *dispatch_queue,
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600239 struct wb_writeback_work *work)
Fengguang Wu2c136572007-10-16 23:30:39 -0700240{
Shaohua Li5c034492009-09-24 14:42:33 +0200241 LIST_HEAD(tmp);
242 struct list_head *pos, *node;
Jens Axboecf137302009-09-24 15:12:57 +0200243 struct super_block *sb = NULL;
Shaohua Li5c034492009-09-24 14:42:33 +0200244 struct inode *inode;
Jens Axboecf137302009-09-24 15:12:57 +0200245 int do_sb_sort = 0;
Wu Fengguange84d0a42011-04-23 12:27:27 -0600246 int moved = 0;
Shaohua Li5c034492009-09-24 14:42:33 +0200247
Fengguang Wu2c136572007-10-16 23:30:39 -0700248 while (!list_empty(delaying_queue)) {
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100249 inode = wb_inode(delaying_queue->prev);
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600250 if (work->older_than_this &&
251 inode_dirtied_after(inode, *work->older_than_this))
Fengguang Wu2c136572007-10-16 23:30:39 -0700252 break;
Jan Karaa8855992013-07-09 22:36:45 +0800253 list_move(&inode->i_wb_list, &tmp);
254 moved++;
255 if (sb_is_blkdev_sb(inode->i_sb))
256 continue;
Jens Axboecf137302009-09-24 15:12:57 +0200257 if (sb && sb != inode->i_sb)
258 do_sb_sort = 1;
259 sb = inode->i_sb;
Shaohua Li5c034492009-09-24 14:42:33 +0200260 }
261
Jens Axboecf137302009-09-24 15:12:57 +0200262 /* just one sb in list, splice to dispatch_queue and we're done */
263 if (!do_sb_sort) {
264 list_splice(&tmp, dispatch_queue);
Wu Fengguange84d0a42011-04-23 12:27:27 -0600265 goto out;
Jens Axboecf137302009-09-24 15:12:57 +0200266 }
267
Shaohua Li5c034492009-09-24 14:42:33 +0200268 /* Move inodes from one superblock together */
269 while (!list_empty(&tmp)) {
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100270 sb = wb_inode(tmp.prev)->i_sb;
Shaohua Li5c034492009-09-24 14:42:33 +0200271 list_for_each_prev_safe(pos, node, &tmp) {
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100272 inode = wb_inode(pos);
Shaohua Li5c034492009-09-24 14:42:33 +0200273 if (inode->i_sb == sb)
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100274 list_move(&inode->i_wb_list, dispatch_queue);
Shaohua Li5c034492009-09-24 14:42:33 +0200275 }
Fengguang Wu2c136572007-10-16 23:30:39 -0700276 }
Wu Fengguange84d0a42011-04-23 12:27:27 -0600277out:
278 return moved;
Fengguang Wu2c136572007-10-16 23:30:39 -0700279}
280
281/*
282 * Queue all expired dirty inodes for io, eldest first.
Wu Fengguang4ea879b2010-08-11 14:17:42 -0700283 * Before
284 * newly dirtied b_dirty b_io b_more_io
285 * =============> gf edc BA
286 * After
287 * newly dirtied b_dirty b_io b_more_io
288 * =============> g fBAedc
289 * |
290 * +--> dequeue for IO
Fengguang Wu2c136572007-10-16 23:30:39 -0700291 */
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600292static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work)
Fengguang Wu2c136572007-10-16 23:30:39 -0700293{
Wu Fengguange84d0a42011-04-23 12:27:27 -0600294 int moved;
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600295 assert_spin_locked(&wb->list_lock);
Wu Fengguang4ea879b2010-08-11 14:17:42 -0700296 list_splice_init(&wb->b_more_io, &wb->b_io);
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600297 moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, work);
298 trace_writeback_queue_io(wb, work, moved);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200299}
300
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100301static int write_inode(struct inode *inode, struct writeback_control *wbc)
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200302{
Tejun Heo9fb0a7d2013-01-11 13:06:37 -0800303 int ret;
304
305 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
306 trace_writeback_write_inode_start(inode, wbc);
307 ret = inode->i_sb->s_op->write_inode(inode, wbc);
308 trace_writeback_write_inode(inode, wbc);
309 return ret;
310 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200311 return 0;
Fengguang Wu2c136572007-10-16 23:30:39 -0700312}
313
314/*
Jan Kara169ebd92012-05-03 14:48:03 +0200315 * Wait for writeback on an inode to complete. Called with i_lock held.
316 * Caller must make sure inode cannot go away when we drop i_lock.
Christoph Hellwig01c03192009-06-08 13:35:40 +0200317 */
Jan Kara169ebd92012-05-03 14:48:03 +0200318static void __inode_wait_for_writeback(struct inode *inode)
319 __releases(inode->i_lock)
320 __acquires(inode->i_lock)
Christoph Hellwig01c03192009-06-08 13:35:40 +0200321{
322 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
323 wait_queue_head_t *wqh;
324
325 wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
Dave Chinner250df6e2011-03-22 22:23:36 +1100326 while (inode->i_state & I_SYNC) {
327 spin_unlock(&inode->i_lock);
Christoph Hellwig01c03192009-06-08 13:35:40 +0200328 __wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
Dave Chinner250df6e2011-03-22 22:23:36 +1100329 spin_lock(&inode->i_lock);
Richard Kennedy58a9d3d82010-05-24 14:32:38 -0700330 }
Christoph Hellwig01c03192009-06-08 13:35:40 +0200331}
332
333/*
Jan Kara169ebd92012-05-03 14:48:03 +0200334 * Wait for writeback on an inode to complete. Caller must have inode pinned.
335 */
336void inode_wait_for_writeback(struct inode *inode)
337{
338 spin_lock(&inode->i_lock);
339 __inode_wait_for_writeback(inode);
340 spin_unlock(&inode->i_lock);
341}
342
343/*
344 * Sleep until I_SYNC is cleared. This function must be called with i_lock
345 * held and drops it. It is aimed for callers not holding any inode reference
346 * so once i_lock is dropped, inode can go away.
347 */
348static void inode_sleep_on_writeback(struct inode *inode)
349 __releases(inode->i_lock)
350{
351 DEFINE_WAIT(wait);
352 wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
353 int sleep;
354
355 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
356 sleep = inode->i_state & I_SYNC;
357 spin_unlock(&inode->i_lock);
358 if (sleep)
359 schedule();
360 finish_wait(wqh, &wait);
361}
362
363/*
Jan Karaccb26b52012-05-03 14:47:58 +0200364 * Find proper writeback list for the inode depending on its current state and
365 * possibly also change of its state while we were doing writeback. Here we
366 * handle things such as livelock prevention or fairness of writeback among
367 * inodes. This function can be called only by flusher thread - noone else
368 * processes all inodes in writeback lists and requeueing inodes behind flusher
369 * thread's back can have unexpected consequences.
370 */
371static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
372 struct writeback_control *wbc)
373{
374 if (inode->i_state & I_FREEING)
375 return;
376
377 /*
378 * Sync livelock prevention. Each inode is tagged and synced in one
379 * shot. If still dirty, it will be redirty_tail()'ed below. Update
380 * the dirty time to prevent enqueue and sync it again.
381 */
382 if ((inode->i_state & I_DIRTY) &&
383 (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
384 inode->dirtied_when = jiffies;
385
Jan Kara4f8ad652012-05-03 14:48:00 +0200386 if (wbc->pages_skipped) {
387 /*
388 * writeback is not making progress due to locked
389 * buffers. Skip this inode for now.
390 */
391 redirty_tail(inode, wb);
392 return;
393 }
394
Jan Karaccb26b52012-05-03 14:47:58 +0200395 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
396 /*
397 * We didn't write back all the pages. nfs_writepages()
398 * sometimes bales out without doing anything.
399 */
400 if (wbc->nr_to_write <= 0) {
401 /* Slice used up. Queue for next turn. */
402 requeue_io(inode, wb);
403 } else {
404 /*
405 * Writeback blocked by something other than
406 * congestion. Delay the inode for some time to
407 * avoid spinning on the CPU (100% iowait)
408 * retrying writeback of the dirty page/inode
409 * that cannot be performed immediately.
410 */
411 redirty_tail(inode, wb);
412 }
413 } else if (inode->i_state & I_DIRTY) {
414 /*
415 * Filesystems can dirty the inode during writeback operations,
416 * such as delayed allocation during submission or metadata
417 * updates after data IO completion.
418 */
419 redirty_tail(inode, wb);
420 } else {
421 /* The inode is clean. Remove from writeback lists. */
422 list_del_init(&inode->i_wb_list);
423 }
424}
425
426/*
Jan Kara4f8ad652012-05-03 14:48:00 +0200427 * Write out an inode and its dirty pages. Do not update the writeback list
428 * linkage. That is left to the caller. The caller is also responsible for
429 * setting I_SYNC flag and calling inode_sync_complete() to clear it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 */
431static int
Yan Hongcd8ed2a2012-10-08 16:33:45 -0700432__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 struct address_space *mapping = inode->i_mapping;
Wu Fengguang251d6a42010-12-01 17:33:37 -0600435 long nr_to_write = wbc->nr_to_write;
Christoph Hellwig01c03192009-06-08 13:35:40 +0200436 unsigned dirty;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 int ret;
438
Jan Kara4f8ad652012-05-03 14:48:00 +0200439 WARN_ON(!(inode->i_state & I_SYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440
Tejun Heo9fb0a7d2013-01-11 13:06:37 -0800441 trace_writeback_single_inode_start(inode, wbc, nr_to_write);
442
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 ret = do_writepages(mapping, wbc);
444
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100445 /*
446 * Make sure to wait on the data before writing out the metadata.
447 * This is important for filesystems that modify metadata on data
448 * I/O completion.
449 */
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100450 if (wbc->sync_mode == WB_SYNC_ALL) {
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100451 int err = filemap_fdatawait(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 if (ret == 0)
453 ret = err;
454 }
455
Dmitry Monakhov5547e8a2010-05-07 13:35:44 +0400456 /*
457 * Some filesystems may redirty the inode during the writeback
458 * due to delalloc, clear dirty metadata flags right before
459 * write_inode()
460 */
Dave Chinner250df6e2011-03-22 22:23:36 +1100461 spin_lock(&inode->i_lock);
Jan Kara6290be12012-05-03 14:47:57 +0200462 /* Clear I_DIRTY_PAGES if we've written out all dirty pages */
463 if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
464 inode->i_state &= ~I_DIRTY_PAGES;
Dmitry Monakhov5547e8a2010-05-07 13:35:44 +0400465 dirty = inode->i_state & I_DIRTY;
466 inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC);
Dave Chinner250df6e2011-03-22 22:23:36 +1100467 spin_unlock(&inode->i_lock);
Christoph Hellwig26821ed2010-03-05 09:21:21 +0100468 /* Don't write the inode if only I_DIRTY_PAGES was set */
469 if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
Christoph Hellwiga9185b42010-03-05 09:21:37 +0100470 int err = write_inode(inode, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 if (ret == 0)
472 ret = err;
473 }
Jan Kara4f8ad652012-05-03 14:48:00 +0200474 trace_writeback_single_inode(inode, wbc, nr_to_write);
475 return ret;
476}
477
478/*
479 * Write out an inode's dirty pages. Either the caller has an active reference
480 * on the inode or the inode has I_WILL_FREE set.
481 *
482 * This function is designed to be called for writing back one inode which
483 * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
484 * and does more profound writeback list handling in writeback_sb_inodes().
485 */
486static int
487writeback_single_inode(struct inode *inode, struct bdi_writeback *wb,
488 struct writeback_control *wbc)
489{
490 int ret = 0;
491
492 spin_lock(&inode->i_lock);
493 if (!atomic_read(&inode->i_count))
494 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
495 else
496 WARN_ON(inode->i_state & I_WILL_FREE);
497
498 if (inode->i_state & I_SYNC) {
499 if (wbc->sync_mode != WB_SYNC_ALL)
500 goto out;
501 /*
Jan Kara169ebd92012-05-03 14:48:03 +0200502 * It's a data-integrity sync. We must wait. Since callers hold
503 * inode reference or inode has I_WILL_FREE set, it cannot go
504 * away under us.
Jan Kara4f8ad652012-05-03 14:48:00 +0200505 */
Jan Kara169ebd92012-05-03 14:48:03 +0200506 __inode_wait_for_writeback(inode);
Jan Kara4f8ad652012-05-03 14:48:00 +0200507 }
508 WARN_ON(inode->i_state & I_SYNC);
509 /*
510 * Skip inode if it is clean. We don't want to mess with writeback
511 * lists in this function since flusher thread may be doing for example
512 * sync in parallel and if we move the inode, it could get skipped. So
513 * here we make sure inode is on some writeback list and leave it there
514 * unless we have completely cleaned the inode.
515 */
516 if (!(inode->i_state & I_DIRTY))
517 goto out;
518 inode->i_state |= I_SYNC;
519 spin_unlock(&inode->i_lock);
520
Yan Hongcd8ed2a2012-10-08 16:33:45 -0700521 ret = __writeback_single_inode(inode, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600523 spin_lock(&wb->list_lock);
Dave Chinner250df6e2011-03-22 22:23:36 +1100524 spin_lock(&inode->i_lock);
Jan Kara4f8ad652012-05-03 14:48:00 +0200525 /*
526 * If inode is clean, remove it from writeback lists. Otherwise don't
527 * touch it. See comment above for explanation.
528 */
529 if (!(inode->i_state & I_DIRTY))
530 list_del_init(&inode->i_wb_list);
531 spin_unlock(&wb->list_lock);
Joern Engel1c0eeaf2007-10-16 23:30:44 -0700532 inode_sync_complete(inode);
Jan Kara4f8ad652012-05-03 14:48:00 +0200533out:
534 spin_unlock(&inode->i_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 return ret;
536}
537
Wu Fengguang1a12d8b2010-08-29 13:28:09 -0600538static long writeback_chunk_size(struct backing_dev_info *bdi,
539 struct wb_writeback_work *work)
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600540{
541 long pages;
542
543 /*
544 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
545 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
546 * here avoids calling into writeback_inodes_wb() more than once.
547 *
548 * The intended call sequence for WB_SYNC_ALL writeback is:
549 *
550 * wb_writeback()
551 * writeback_sb_inodes() <== called only once
552 * write_cache_pages() <== called once for each inode
553 * (quickly) tag currently dirty pages
554 * (maybe slowly) sync all tagged pages
555 */
556 if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
557 pages = LONG_MAX;
Wu Fengguang1a12d8b2010-08-29 13:28:09 -0600558 else {
559 pages = min(bdi->avg_write_bandwidth / 2,
560 global_dirty_limit / DIRTY_SCOPE);
561 pages = min(pages, work->nr_pages);
562 pages = round_down(pages + MIN_WRITEBACK_PAGES,
563 MIN_WRITEBACK_PAGES);
564 }
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600565
566 return pages;
567}
568
Jens Axboe03ba3782009-09-09 09:08:54 +0200569/*
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800570 * Write a portion of b_io inodes which belong to @sb.
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200571 *
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600572 * Return the number of pages and/or inodes written.
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800573 */
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600574static long writeback_sb_inodes(struct super_block *sb,
575 struct bdi_writeback *wb,
576 struct wb_writeback_work *work)
Jens Axboe03ba3782009-09-09 09:08:54 +0200577{
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600578 struct writeback_control wbc = {
579 .sync_mode = work->sync_mode,
580 .tagged_writepages = work->tagged_writepages,
581 .for_kupdate = work->for_kupdate,
582 .for_background = work->for_background,
583 .range_cyclic = work->range_cyclic,
584 .range_start = 0,
585 .range_end = LLONG_MAX,
586 };
587 unsigned long start_time = jiffies;
588 long write_chunk;
589 long wrote = 0; /* count both pages and inodes */
590
Jens Axboe03ba3782009-09-09 09:08:54 +0200591 while (!list_empty(&wb->b_io)) {
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100592 struct inode *inode = wb_inode(wb->b_io.prev);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200593
594 if (inode->i_sb != sb) {
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600595 if (work->sb) {
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200596 /*
597 * We only want to write back data for this
598 * superblock, move all inodes not belonging
599 * to it back onto the dirty list.
600 */
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600601 redirty_tail(inode, wb);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200602 continue;
603 }
604
605 /*
606 * The inode belongs to a different superblock.
607 * Bounce back to the caller to unpin this and
608 * pin the next superblock.
609 */
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600610 break;
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200611 }
612
Christoph Hellwig9843b762010-10-24 19:40:46 +0200613 /*
Wanpeng Li331cbde2012-06-09 11:10:55 +0800614 * Don't bother with new inodes or inodes being freed, first
615 * kind does not need periodic writeout yet, and for the latter
Christoph Hellwig9843b762010-10-24 19:40:46 +0200616 * kind writeout is handled by the freer.
617 */
Dave Chinner250df6e2011-03-22 22:23:36 +1100618 spin_lock(&inode->i_lock);
Christoph Hellwig9843b762010-10-24 19:40:46 +0200619 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
Dave Chinner250df6e2011-03-22 22:23:36 +1100620 spin_unlock(&inode->i_lock);
Wu Fengguangfcc5c222011-07-11 23:08:50 -0700621 redirty_tail(inode, wb);
Nick Piggin7ef0d732009-03-12 14:31:38 -0700622 continue;
623 }
Jan Karacc1676d2012-05-03 14:47:56 +0200624 if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
625 /*
626 * If this inode is locked for writeback and we are not
627 * doing writeback-for-data-integrity, move it to
628 * b_more_io so that writeback can proceed with the
629 * other inodes on s_io.
630 *
631 * We'll have another go at writing back this inode
632 * when we completed a full scan of b_io.
633 */
634 spin_unlock(&inode->i_lock);
635 requeue_io(inode, wb);
636 trace_writeback_sb_inodes_requeue(inode);
637 continue;
638 }
Jan Karaf0d07b72012-05-03 14:47:59 +0200639 spin_unlock(&wb->list_lock);
640
Jan Kara4f8ad652012-05-03 14:48:00 +0200641 /*
642 * We already requeued the inode if it had I_SYNC set and we
643 * are doing WB_SYNC_NONE writeback. So this catches only the
644 * WB_SYNC_ALL case.
645 */
Jan Kara169ebd92012-05-03 14:48:03 +0200646 if (inode->i_state & I_SYNC) {
647 /* Wait for I_SYNC. This function drops i_lock... */
648 inode_sleep_on_writeback(inode);
649 /* Inode may be gone, start again */
Jan Karaead188f2012-06-08 17:07:36 +0200650 spin_lock(&wb->list_lock);
Jan Kara169ebd92012-05-03 14:48:03 +0200651 continue;
652 }
Jan Kara4f8ad652012-05-03 14:48:00 +0200653 inode->i_state |= I_SYNC;
654 spin_unlock(&inode->i_lock);
Jan Kara169ebd92012-05-03 14:48:03 +0200655
Wu Fengguang1a12d8b2010-08-29 13:28:09 -0600656 write_chunk = writeback_chunk_size(wb->bdi, work);
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600657 wbc.nr_to_write = write_chunk;
658 wbc.pages_skipped = 0;
Dave Chinner250df6e2011-03-22 22:23:36 +1100659
Jan Kara169ebd92012-05-03 14:48:03 +0200660 /*
661 * We use I_SYNC to pin the inode in memory. While it is set
662 * evict_inode() will wait so the inode cannot be freed.
663 */
Yan Hongcd8ed2a2012-10-08 16:33:45 -0700664 __writeback_single_inode(inode, &wbc);
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600665
666 work->nr_pages -= write_chunk - wbc.nr_to_write;
667 wrote += write_chunk - wbc.nr_to_write;
Jan Kara4f8ad652012-05-03 14:48:00 +0200668 spin_lock(&wb->list_lock);
669 spin_lock(&inode->i_lock);
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600670 if (!(inode->i_state & I_DIRTY))
671 wrote++;
Jan Kara4f8ad652012-05-03 14:48:00 +0200672 requeue_inode(inode, wb, &wbc);
673 inode_sync_complete(inode);
Dave Chinner0f1b1fd2011-03-22 22:23:43 +1100674 spin_unlock(&inode->i_lock);
Jan Kara169ebd92012-05-03 14:48:03 +0200675 cond_resched_lock(&wb->list_lock);
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600676 /*
677 * bail out to wb_writeback() often enough to check
678 * background threshold and other termination conditions.
679 */
680 if (wrote) {
681 if (time_is_before_jiffies(start_time + HZ / 10UL))
682 break;
683 if (work->nr_pages <= 0)
684 break;
Fengguang Wu8bc3be22008-02-04 22:29:36 -0800685 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 }
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600687 return wrote;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800688}
Nick Piggin38f21972009-01-06 14:40:25 -0800689
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600690static long __writeback_inodes_wb(struct bdi_writeback *wb,
691 struct wb_writeback_work *work)
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800692{
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600693 unsigned long start_time = jiffies;
694 long wrote = 0;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800695
696 while (!list_empty(&wb->b_io)) {
Nick Piggin7ccf19a2010-10-21 11:49:30 +1100697 struct inode *inode = wb_inode(wb->b_io.prev);
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800698 struct super_block *sb = inode->i_sb;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800699
Dave Chinner12ad3ab2011-07-08 14:14:41 +1000700 if (!grab_super_passive(sb)) {
Wu Fengguang0e9958162011-07-29 22:14:35 -0600701 /*
702 * grab_super_passive() may fail consistently due to
703 * s_umount being grabbed by someone else. Don't use
704 * requeue_io() to avoid busy retrying the inode/sb.
705 */
706 redirty_tail(inode, wb);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200707 continue;
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800708 }
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600709 wrote += writeback_sb_inodes(sb, wb, work);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200710 drop_super(sb);
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800711
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600712 /* refer to the same tests at the end of writeback_sb_inodes */
713 if (wrote) {
714 if (time_is_before_jiffies(start_time + HZ / 10UL))
715 break;
716 if (work->nr_pages <= 0)
717 break;
718 }
Edward Shishkinf11c9c52010-03-11 14:09:47 -0800719 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200720 /* Leave any unwritten inodes on b_io */
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600721 return wrote;
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200722}
723
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600724long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
725 enum wb_reason reason)
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200726{
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600727 struct wb_writeback_work work = {
728 .nr_pages = nr_pages,
729 .sync_mode = WB_SYNC_NONE,
730 .range_cyclic = 1,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600731 .reason = reason,
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600732 };
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200733
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600734 spin_lock(&wb->list_lock);
Wu Fengguang424b3512010-07-21 20:11:53 -0600735 if (list_empty(&wb->b_io))
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600736 queue_io(wb, &work);
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600737 __writeback_inodes_wb(wb, &work);
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600738 spin_unlock(&wb->list_lock);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200739
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600740 return nr_pages - work.nr_pages;
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200741}
742
Wu Fengguangb00949a2010-11-18 14:38:33 -0600743static bool over_bground_thresh(struct backing_dev_info *bdi)
Jens Axboe03ba3782009-09-09 09:08:54 +0200744{
745 unsigned long background_thresh, dirty_thresh;
746
Wu Fengguang16c40422010-08-11 14:17:39 -0700747 global_dirty_limits(&background_thresh, &dirty_thresh);
Jens Axboe03ba3782009-09-09 09:08:54 +0200748
Wu Fengguangb00949a2010-11-18 14:38:33 -0600749 if (global_page_state(NR_FILE_DIRTY) +
750 global_page_state(NR_UNSTABLE_NFS) > background_thresh)
751 return true;
752
753 if (bdi_stat(bdi, BDI_RECLAIMABLE) >
754 bdi_dirty_limit(bdi, background_thresh))
755 return true;
756
757 return false;
Jens Axboe03ba3782009-09-09 09:08:54 +0200758}
759
760/*
Wu Fengguange98be2d2010-08-29 11:22:30 -0600761 * Called under wb->list_lock. If there are multiple wb per bdi,
762 * only the flusher working on the first wb should do it.
763 */
764static void wb_update_bandwidth(struct bdi_writeback *wb,
765 unsigned long start_time)
766{
Wu Fengguangaf6a3112011-10-03 20:46:17 -0600767 __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
Wu Fengguange98be2d2010-08-29 11:22:30 -0600768}
769
770/*
Jens Axboe03ba3782009-09-09 09:08:54 +0200771 * Explicit flushing or periodic writeback of "old" data.
772 *
773 * Define "old": the first time one of an inode's pages is dirtied, we mark the
774 * dirtying-time in the inode's address_space. So this periodic writeback code
775 * just walks the superblock inode list, writing back any inodes which are
776 * older than a specific point in time.
777 *
778 * Try to run once per dirty_writeback_interval. But if a writeback event
779 * takes longer than a dirty_writeback_interval interval, then leave a
780 * one-second gap.
781 *
782 * older_than_this takes precedence over nr_to_write. So we'll only write back
783 * all dirty pages if they are all attached to "old" mappings.
784 */
Jens Axboec4a77a62009-09-16 15:18:25 +0200785static long wb_writeback(struct bdi_writeback *wb,
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200786 struct wb_writeback_work *work)
Jens Axboe03ba3782009-09-09 09:08:54 +0200787{
Wu Fengguange98be2d2010-08-29 11:22:30 -0600788 unsigned long wb_start = jiffies;
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600789 long nr_pages = work->nr_pages;
Jens Axboe03ba3782009-09-09 09:08:54 +0200790 unsigned long oldest_jif;
Jan Karaa5989bd2009-09-16 19:22:48 +0200791 struct inode *inode;
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600792 long progress;
Jens Axboe03ba3782009-09-09 09:08:54 +0200793
Wu Fengguange185dda2011-04-23 11:26:07 -0600794 oldest_jif = jiffies;
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600795 work->older_than_this = &oldest_jif;
Jens Axboe03ba3782009-09-09 09:08:54 +0200796
Wu Fengguange8dfc302011-04-21 12:06:32 -0600797 spin_lock(&wb->list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200798 for (;;) {
799 /*
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800800 * Stop writeback when nr_pages has been consumed
Jens Axboe03ba3782009-09-09 09:08:54 +0200801 */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200802 if (work->nr_pages <= 0)
Jens Axboe03ba3782009-09-09 09:08:54 +0200803 break;
804
805 /*
Jan Karaaa373cf2011-01-13 15:45:47 -0800806 * Background writeout and kupdate-style writeback may
807 * run forever. Stop them if there is other work to do
808 * so that e.g. sync can proceed. They'll be restarted
809 * after the other works are all done.
810 */
811 if ((work->for_background || work->for_kupdate) &&
812 !list_empty(&wb->bdi->work_list))
813 break;
814
815 /*
Wu Fengguangd3ddec72009-09-23 20:33:40 +0800816 * For background writeout, stop when we are below the
817 * background dirty threshold
Jens Axboe03ba3782009-09-09 09:08:54 +0200818 */
Wu Fengguangb00949a2010-11-18 14:38:33 -0600819 if (work->for_background && !over_bground_thresh(wb->bdi))
Jens Axboe03ba3782009-09-09 09:08:54 +0200820 break;
821
Jan Kara1bc36b62011-10-19 11:44:41 +0200822 /*
823 * Kupdate and background works are special and we want to
824 * include all inodes that need writing. Livelock avoidance is
825 * handled by these works yielding to any other work so we are
826 * safe.
827 */
Wu Fengguangba9aa832010-07-21 20:32:30 -0600828 if (work->for_kupdate) {
829 oldest_jif = jiffies -
830 msecs_to_jiffies(dirty_expire_interval * 10);
Jan Kara1bc36b62011-10-19 11:44:41 +0200831 } else if (work->for_background)
832 oldest_jif = jiffies;
Dave Chinner028c2dd2010-07-07 13:24:07 +1000833
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600834 trace_writeback_start(wb->bdi, work);
Wu Fengguange8dfc302011-04-21 12:06:32 -0600835 if (list_empty(&wb->b_io))
Curt Wohlgemuthad4e38d2011-10-07 21:51:56 -0600836 queue_io(wb, work);
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200837 if (work->sb)
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600838 progress = writeback_sb_inodes(work->sb, wb, work);
Christoph Hellwigedadfb12010-06-10 12:07:54 +0200839 else
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600840 progress = __writeback_inodes_wb(wb, work);
841 trace_writeback_written(wb->bdi, work);
Dave Chinner028c2dd2010-07-07 13:24:07 +1000842
Wu Fengguange98be2d2010-08-29 11:22:30 -0600843 wb_update_bandwidth(wb, wb_start);
Jens Axboe03ba3782009-09-09 09:08:54 +0200844
845 /*
Jens Axboe71fd05a2009-09-23 19:32:26 +0200846 * Did we write something? Try for more
Wu Fengguange6fb6da2010-07-22 10:23:44 -0600847 *
848 * Dirty inodes are moved to b_io for writeback in batches.
849 * The completion of the current batch does not necessarily
850 * mean the overall work is done. So we keep looping as long
851 * as made some progress on cleaning pages or inodes.
Jens Axboe71fd05a2009-09-23 19:32:26 +0200852 */
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600853 if (progress)
Jens Axboe71fd05a2009-09-23 19:32:26 +0200854 continue;
855 /*
Wu Fengguange6fb6da2010-07-22 10:23:44 -0600856 * No more inodes for IO, bail
Jens Axboe71fd05a2009-09-23 19:32:26 +0200857 */
Wu Fengguangb7a24412010-07-21 22:19:51 -0600858 if (list_empty(&wb->b_more_io))
Jens Axboe03ba3782009-09-09 09:08:54 +0200859 break;
860 /*
Jens Axboe8010c3b2009-09-15 20:04:57 +0200861 * Nothing written. Wait for some inode to
862 * become available for writeback. Otherwise
863 * we'll just busyloop.
Jens Axboe03ba3782009-09-09 09:08:54 +0200864 */
Jens Axboe03ba3782009-09-09 09:08:54 +0200865 if (!list_empty(&wb->b_more_io)) {
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600866 trace_writeback_wait(wb->bdi, work);
Jens Axboe03ba3782009-09-09 09:08:54 +0200867 inode = wb_inode(wb->b_more_io.prev);
Dave Chinner250df6e2011-03-22 22:23:36 +1100868 spin_lock(&inode->i_lock);
Jan Karaf0d07b72012-05-03 14:47:59 +0200869 spin_unlock(&wb->list_lock);
Jan Kara169ebd92012-05-03 14:48:03 +0200870 /* This function drops i_lock... */
871 inode_sleep_on_writeback(inode);
Jan Karaf0d07b72012-05-03 14:47:59 +0200872 spin_lock(&wb->list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200873 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200874 }
Wu Fengguange8dfc302011-04-21 12:06:32 -0600875 spin_unlock(&wb->list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200876
Wu Fengguangd46db3d2011-05-04 19:54:37 -0600877 return nr_pages - work->nr_pages;
Jens Axboe03ba3782009-09-09 09:08:54 +0200878}
879
880/*
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200881 * Return the next wb_writeback_work struct that hasn't been processed yet.
Jens Axboe03ba3782009-09-09 09:08:54 +0200882 */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200883static struct wb_writeback_work *
Minchan Kim08852b62010-08-03 12:51:16 +0200884get_next_work_item(struct backing_dev_info *bdi)
Jens Axboe03ba3782009-09-09 09:08:54 +0200885{
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200886 struct wb_writeback_work *work = NULL;
Jens Axboe03ba3782009-09-09 09:08:54 +0200887
Artem Bityutskiy64677162010-07-25 14:29:22 +0300888 spin_lock_bh(&bdi->wb_lock);
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200889 if (!list_empty(&bdi->work_list)) {
890 work = list_entry(bdi->work_list.next,
891 struct wb_writeback_work, list);
892 list_del_init(&work->list);
Jens Axboe03ba3782009-09-09 09:08:54 +0200893 }
Artem Bityutskiy64677162010-07-25 14:29:22 +0300894 spin_unlock_bh(&bdi->wb_lock);
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200895 return work;
Jens Axboe03ba3782009-09-09 09:08:54 +0200896}
897
Linus Torvaldscdf01dd2010-10-30 08:55:52 -0700898/*
899 * Add in the number of potentially dirty inodes, because each inode
900 * write can dirty pagecache in the underlying blockdev.
901 */
902static unsigned long get_nr_dirty_pages(void)
903{
904 return global_page_state(NR_FILE_DIRTY) +
905 global_page_state(NR_UNSTABLE_NFS) +
906 get_nr_dirty_inodes();
907}
908
Jan Kara65850272011-01-13 15:45:44 -0800909static long wb_check_background_flush(struct bdi_writeback *wb)
910{
Wu Fengguangb00949a2010-11-18 14:38:33 -0600911 if (over_bground_thresh(wb->bdi)) {
Jan Kara65850272011-01-13 15:45:44 -0800912
913 struct wb_writeback_work work = {
914 .nr_pages = LONG_MAX,
915 .sync_mode = WB_SYNC_NONE,
916 .for_background = 1,
917 .range_cyclic = 1,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600918 .reason = WB_REASON_BACKGROUND,
Jan Kara65850272011-01-13 15:45:44 -0800919 };
920
921 return wb_writeback(wb, &work);
922 }
923
924 return 0;
925}
926
Jens Axboe03ba3782009-09-09 09:08:54 +0200927static long wb_check_old_data_flush(struct bdi_writeback *wb)
928{
929 unsigned long expired;
930 long nr_pages;
931
Jens Axboe69b62d02010-05-17 12:51:03 +0200932 /*
933 * When set to zero, disable periodic writeback
934 */
935 if (!dirty_writeback_interval)
936 return 0;
937
Jens Axboe03ba3782009-09-09 09:08:54 +0200938 expired = wb->last_old_flush +
939 msecs_to_jiffies(dirty_writeback_interval * 10);
940 if (time_before(jiffies, expired))
941 return 0;
942
943 wb->last_old_flush = jiffies;
Linus Torvaldscdf01dd2010-10-30 08:55:52 -0700944 nr_pages = get_nr_dirty_pages();
Jens Axboe03ba3782009-09-09 09:08:54 +0200945
Jens Axboec4a77a62009-09-16 15:18:25 +0200946 if (nr_pages) {
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200947 struct wb_writeback_work work = {
Jens Axboec4a77a62009-09-16 15:18:25 +0200948 .nr_pages = nr_pages,
949 .sync_mode = WB_SYNC_NONE,
950 .for_kupdate = 1,
951 .range_cyclic = 1,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -0600952 .reason = WB_REASON_PERIODIC,
Jens Axboec4a77a62009-09-16 15:18:25 +0200953 };
954
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200955 return wb_writeback(wb, &work);
Jens Axboec4a77a62009-09-16 15:18:25 +0200956 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200957
958 return 0;
959}
960
961/*
962 * Retrieve work items and do the writeback they describe
963 */
964long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
965{
966 struct backing_dev_info *bdi = wb->bdi;
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200967 struct wb_writeback_work *work;
Jens Axboec4a77a62009-09-16 15:18:25 +0200968 long wrote = 0;
Jens Axboe03ba3782009-09-09 09:08:54 +0200969
Jan Kara81d73a32010-08-11 14:17:44 -0700970 set_bit(BDI_writeback_running, &wb->bdi->state);
Minchan Kim08852b62010-08-03 12:51:16 +0200971 while ((work = get_next_work_item(bdi)) != NULL) {
Jens Axboe03ba3782009-09-09 09:08:54 +0200972 /*
973 * Override sync mode, in case we must wait for completion
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200974 * because this thread is exiting now.
Jens Axboe03ba3782009-09-09 09:08:54 +0200975 */
976 if (force_wait)
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200977 work->sync_mode = WB_SYNC_ALL;
978
Dave Chinner455b2862010-07-07 13:24:06 +1000979 trace_writeback_exec(bdi, work);
980
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200981 wrote += wb_writeback(wb, work);
Jens Axboe03ba3782009-09-09 09:08:54 +0200982
983 /*
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200984 * Notify the caller of completion if this is a synchronous
985 * work item, otherwise just free it.
Jens Axboe03ba3782009-09-09 09:08:54 +0200986 */
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200987 if (work->done)
988 complete(work->done);
989 else
990 kfree(work);
Jens Axboe03ba3782009-09-09 09:08:54 +0200991 }
992
993 /*
994 * Check for periodic writeback, kupdated() style
995 */
996 wrote += wb_check_old_data_flush(wb);
Jan Kara65850272011-01-13 15:45:44 -0800997 wrote += wb_check_background_flush(wb);
Jan Kara81d73a32010-08-11 14:17:44 -0700998 clear_bit(BDI_writeback_running, &wb->bdi->state);
Jens Axboe03ba3782009-09-09 09:08:54 +0200999
1000 return wrote;
1001}
1002
1003/*
1004 * Handle writeback of dirty data for the device backed by this bdi. Also
Tejun Heo839a8e82013-04-01 19:08:06 -07001005 * reschedules periodically and does kupdated style flushing.
Jens Axboe03ba3782009-09-09 09:08:54 +02001006 */
Tejun Heo839a8e82013-04-01 19:08:06 -07001007void bdi_writeback_workfn(struct work_struct *work)
Jens Axboe03ba3782009-09-09 09:08:54 +02001008{
Tejun Heo839a8e82013-04-01 19:08:06 -07001009 struct bdi_writeback *wb = container_of(to_delayed_work(work),
1010 struct bdi_writeback, dwork);
Christoph Hellwig08243902010-06-19 23:08:22 +02001011 struct backing_dev_info *bdi = wb->bdi;
Jens Axboe03ba3782009-09-09 09:08:54 +02001012 long pages_written;
1013
Tejun Heoef3b1012013-04-30 15:27:24 -07001014 set_worker_desc("flush-%s", dev_name(bdi->dev));
Peter Zijlstra766f9162010-10-26 14:22:45 -07001015 current->flags |= PF_SWAPWRITE;
Christoph Hellwig08243902010-06-19 23:08:22 +02001016
Tejun Heo839a8e82013-04-01 19:08:06 -07001017 if (likely(!current_is_workqueue_rescuer() ||
1018 list_empty(&bdi->bdi_list))) {
Artem Bityutskiy64677162010-07-25 14:29:22 +03001019 /*
Tejun Heo839a8e82013-04-01 19:08:06 -07001020 * The normal path. Keep writing back @bdi until its
1021 * work_list is empty. Note that this path is also taken
1022 * if @bdi is shutting down even when we're running off the
1023 * rescuer as work_list needs to be drained.
Artem Bityutskiy64677162010-07-25 14:29:22 +03001024 */
Tejun Heo839a8e82013-04-01 19:08:06 -07001025 do {
1026 pages_written = wb_do_writeback(wb, 0);
1027 trace_writeback_pages_written(pages_written);
1028 } while (!list_empty(&bdi->work_list));
1029 } else {
1030 /*
1031 * bdi_wq can't get enough workers and we're running off
1032 * the emergency worker. Don't hog it. Hopefully, 1024 is
1033 * enough for efficient IO.
1034 */
1035 pages_written = writeback_inodes_wb(&bdi->wb, 1024,
1036 WB_REASON_FORKER_THREAD);
Dave Chinner455b2862010-07-07 13:24:06 +10001037 trace_writeback_pages_written(pages_written);
Jens Axboe03ba3782009-09-09 09:08:54 +02001038 }
1039
Tejun Heo839a8e82013-04-01 19:08:06 -07001040 if (!list_empty(&bdi->work_list) ||
1041 (wb_has_dirty_io(wb) && dirty_writeback_interval))
1042 queue_delayed_work(bdi_wq, &wb->dwork,
1043 msecs_to_jiffies(dirty_writeback_interval * 10));
Dave Chinner455b2862010-07-07 13:24:06 +10001044
Tejun Heo839a8e82013-04-01 19:08:06 -07001045 current->flags &= ~PF_SWAPWRITE;
Jens Axboe03ba3782009-09-09 09:08:54 +02001046}
1047
1048/*
Jens Axboe03ba3782009-09-09 09:08:54 +02001049 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
1050 * the whole world.
1051 */
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001052void wakeup_flusher_threads(long nr_pages, enum wb_reason reason)
Jens Axboe03ba3782009-09-09 09:08:54 +02001053{
Christoph Hellwigb8c2f342010-06-08 18:15:07 +02001054 struct backing_dev_info *bdi;
Christoph Hellwigb8c2f342010-06-08 18:15:07 +02001055
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001056 if (!nr_pages) {
1057 nr_pages = global_page_state(NR_FILE_DIRTY) +
Jens Axboe03ba3782009-09-09 09:08:54 +02001058 global_page_state(NR_UNSTABLE_NFS);
Christoph Hellwigb8c2f342010-06-08 18:15:07 +02001059 }
1060
1061 rcu_read_lock();
1062 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
1063 if (!bdi_has_dirty_io(bdi))
1064 continue;
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001065 __bdi_start_writeback(bdi, nr_pages, false, reason);
Christoph Hellwigb8c2f342010-06-08 18:15:07 +02001066 }
1067 rcu_read_unlock();
Jens Axboe03ba3782009-09-09 09:08:54 +02001068}
1069
1070static noinline void block_dump___mark_inode_dirty(struct inode *inode)
1071{
1072 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
1073 struct dentry *dentry;
1074 const char *name = "?";
1075
1076 dentry = d_find_alias(inode);
1077 if (dentry) {
1078 spin_lock(&dentry->d_lock);
1079 name = (const char *) dentry->d_name.name;
1080 }
1081 printk(KERN_DEBUG
1082 "%s(%d): dirtied inode %lu (%s) on %s\n",
1083 current->comm, task_pid_nr(current), inode->i_ino,
1084 name, inode->i_sb->s_id);
1085 if (dentry) {
1086 spin_unlock(&dentry->d_lock);
1087 dput(dentry);
1088 }
1089 }
1090}
1091
1092/**
1093 * __mark_inode_dirty - internal function
1094 * @inode: inode to mark
1095 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC)
1096 * Mark an inode as dirty. Callers should use mark_inode_dirty or
1097 * mark_inode_dirty_sync.
1098 *
1099 * Put the inode on the super block's dirty list.
1100 *
1101 * CAREFUL! We mark it dirty unconditionally, but move it onto the
1102 * dirty list only if it is hashed or if it refers to a blockdev.
1103 * If it was not hashed, it will never be added to the dirty list
1104 * even if it is later hashed, as it will have been marked dirty already.
1105 *
1106 * In short, make sure you hash any inodes _before_ you start marking
1107 * them dirty.
1108 *
Jens Axboe03ba3782009-09-09 09:08:54 +02001109 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
1110 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
1111 * the kernel-internal blockdev inode represents the dirtying time of the
1112 * blockdev's pages. This is why for I_DIRTY_PAGES we always use
1113 * page->mapping->host, so the page-dirtying time is recorded in the internal
1114 * blockdev inode.
1115 */
1116void __mark_inode_dirty(struct inode *inode, int flags)
1117{
1118 struct super_block *sb = inode->i_sb;
Artem Bityutskiy253c34e2010-07-25 14:29:21 +03001119 struct backing_dev_info *bdi = NULL;
Jens Axboe03ba3782009-09-09 09:08:54 +02001120
1121 /*
1122 * Don't do this for I_DIRTY_PAGES - that doesn't actually
1123 * dirty the inode itself
1124 */
1125 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
Tejun Heo9fb0a7d2013-01-11 13:06:37 -08001126 trace_writeback_dirty_inode_start(inode, flags);
1127
Jens Axboe03ba3782009-09-09 09:08:54 +02001128 if (sb->s_op->dirty_inode)
Christoph Hellwigaa385722011-05-27 06:53:02 -04001129 sb->s_op->dirty_inode(inode, flags);
Tejun Heo9fb0a7d2013-01-11 13:06:37 -08001130
1131 trace_writeback_dirty_inode(inode, flags);
Jens Axboe03ba3782009-09-09 09:08:54 +02001132 }
1133
1134 /*
1135 * make sure that changes are seen by all cpus before we test i_state
1136 * -- mikulas
1137 */
1138 smp_mb();
1139
1140 /* avoid the locking if we can */
1141 if ((inode->i_state & flags) == flags)
1142 return;
1143
1144 if (unlikely(block_dump))
1145 block_dump___mark_inode_dirty(inode);
1146
Dave Chinner250df6e2011-03-22 22:23:36 +11001147 spin_lock(&inode->i_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +02001148 if ((inode->i_state & flags) != flags) {
1149 const int was_dirty = inode->i_state & I_DIRTY;
1150
1151 inode->i_state |= flags;
1152
1153 /*
1154 * If the inode is being synced, just update its dirty state.
1155 * The unlocker will place the inode on the appropriate
1156 * superblock list, based upon its state.
1157 */
1158 if (inode->i_state & I_SYNC)
Dave Chinner250df6e2011-03-22 22:23:36 +11001159 goto out_unlock_inode;
Jens Axboe03ba3782009-09-09 09:08:54 +02001160
1161 /*
1162 * Only add valid (hashed) inodes to the superblock's
1163 * dirty list. Add blockdev inodes as well.
1164 */
1165 if (!S_ISBLK(inode->i_mode)) {
Al Viro1d3382cb2010-10-23 15:19:20 -04001166 if (inode_unhashed(inode))
Dave Chinner250df6e2011-03-22 22:23:36 +11001167 goto out_unlock_inode;
Jens Axboe03ba3782009-09-09 09:08:54 +02001168 }
Al Viroa4ffdde2010-06-02 17:38:30 -04001169 if (inode->i_state & I_FREEING)
Dave Chinner250df6e2011-03-22 22:23:36 +11001170 goto out_unlock_inode;
Jens Axboe03ba3782009-09-09 09:08:54 +02001171
1172 /*
1173 * If the inode was already on b_dirty/b_io/b_more_io, don't
1174 * reposition it (that would break b_dirty time-ordering).
1175 */
1176 if (!was_dirty) {
Dave Chinnera66979a2011-03-22 22:23:41 +11001177 bool wakeup_bdi = false;
Artem Bityutskiy253c34e2010-07-25 14:29:21 +03001178 bdi = inode_to_bdi(inode);
Jens Axboe500b0672009-09-09 09:10:25 +02001179
Artem Bityutskiy253c34e2010-07-25 14:29:21 +03001180 if (bdi_cap_writeback_dirty(bdi)) {
1181 WARN(!test_bit(BDI_registered, &bdi->state),
1182 "bdi-%s not registered\n", bdi->name);
1183
1184 /*
1185 * If this is the first dirty inode for this
1186 * bdi, we have to wake-up the corresponding
1187 * bdi thread to make sure background
1188 * write-back happens later.
1189 */
1190 if (!wb_has_dirty_io(&bdi->wb))
1191 wakeup_bdi = true;
Jens Axboe500b0672009-09-09 09:10:25 +02001192 }
Jens Axboe03ba3782009-09-09 09:08:54 +02001193
Dave Chinnera66979a2011-03-22 22:23:41 +11001194 spin_unlock(&inode->i_lock);
Christoph Hellwigf758eea2011-04-21 18:19:44 -06001195 spin_lock(&bdi->wb.list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +02001196 inode->dirtied_when = jiffies;
Nick Piggin7ccf19a2010-10-21 11:49:30 +11001197 list_move(&inode->i_wb_list, &bdi->wb.b_dirty);
Christoph Hellwigf758eea2011-04-21 18:19:44 -06001198 spin_unlock(&bdi->wb.list_lock);
Dave Chinnera66979a2011-03-22 22:23:41 +11001199
1200 if (wakeup_bdi)
1201 bdi_wakeup_thread_delayed(bdi);
1202 return;
Jens Axboe03ba3782009-09-09 09:08:54 +02001203 }
1204 }
Dave Chinner250df6e2011-03-22 22:23:36 +11001205out_unlock_inode:
1206 spin_unlock(&inode->i_lock);
Artem Bityutskiy253c34e2010-07-25 14:29:21 +03001207
Jens Axboe03ba3782009-09-09 09:08:54 +02001208}
1209EXPORT_SYMBOL(__mark_inode_dirty);
1210
Jens Axboeb6e51312009-09-16 15:13:54 +02001211static void wait_sb_inodes(struct super_block *sb)
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001212{
Jens Axboe03ba3782009-09-09 09:08:54 +02001213 struct inode *inode, *old_inode = NULL;
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001214
Jens Axboe03ba3782009-09-09 09:08:54 +02001215 /*
1216 * We need to be protected against the filesystem going from
1217 * r/o to r/w or vice versa.
1218 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001219 WARN_ON(!rwsem_is_locked(&sb->s_umount));
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001220
Dave Chinner55fa6092011-03-22 22:23:40 +11001221 spin_lock(&inode_sb_list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +02001222
1223 /*
1224 * Data integrity sync. Must wait for all pages under writeback,
1225 * because there may have been pages dirtied before our sync
1226 * call, but which had writeout started before we write it out.
1227 * In which case, the inode may not be on the dirty list, but
1228 * we still have to wait for that writeout.
1229 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001230 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
Dave Chinner250df6e2011-03-22 22:23:36 +11001231 struct address_space *mapping = inode->i_mapping;
Jens Axboe03ba3782009-09-09 09:08:54 +02001232
Dave Chinner250df6e2011-03-22 22:23:36 +11001233 spin_lock(&inode->i_lock);
1234 if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
1235 (mapping->nrpages == 0)) {
1236 spin_unlock(&inode->i_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +02001237 continue;
Dave Chinner250df6e2011-03-22 22:23:36 +11001238 }
Jens Axboe03ba3782009-09-09 09:08:54 +02001239 __iget(inode);
Dave Chinner250df6e2011-03-22 22:23:36 +11001240 spin_unlock(&inode->i_lock);
Dave Chinner55fa6092011-03-22 22:23:40 +11001241 spin_unlock(&inode_sb_list_lock);
1242
Jens Axboe03ba3782009-09-09 09:08:54 +02001243 /*
Dave Chinner55fa6092011-03-22 22:23:40 +11001244 * We hold a reference to 'inode' so it couldn't have been
1245 * removed from s_inodes list while we dropped the
1246 * inode_sb_list_lock. We cannot iput the inode now as we can
1247 * be holding the last reference and we cannot iput it under
1248 * inode_sb_list_lock. So we keep the reference and iput it
1249 * later.
Jens Axboe03ba3782009-09-09 09:08:54 +02001250 */
1251 iput(old_inode);
1252 old_inode = inode;
1253
1254 filemap_fdatawait(mapping);
1255
1256 cond_resched();
Nick Piggin38f21972009-01-06 14:40:25 -08001257
Dave Chinner55fa6092011-03-22 22:23:40 +11001258 spin_lock(&inode_sb_list_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +02001259 }
Dave Chinner55fa6092011-03-22 22:23:40 +11001260 spin_unlock(&inode_sb_list_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +02001261 iput(old_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262}
1263
Jens Axboed8a85592009-09-02 12:34:32 +02001264/**
Chris Mason3259f8b2010-10-29 11:16:17 -04001265 * writeback_inodes_sb_nr - writeback dirty inodes from given super_block
Jens Axboed8a85592009-09-02 12:34:32 +02001266 * @sb: the superblock
Chris Mason3259f8b2010-10-29 11:16:17 -04001267 * @nr: the number of pages to write
Marcos Paulo de Souza786228a2011-11-23 20:56:45 +08001268 * @reason: reason why some writeback work initiated
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 *
Jens Axboed8a85592009-09-02 12:34:32 +02001270 * Start writeback on some inodes on this super_block. No guarantees are made
1271 * on how many (if any) will be written, and this function does not wait
Chris Mason3259f8b2010-10-29 11:16:17 -04001272 * for IO completion of submitted IO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273 */
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001274void writeback_inodes_sb_nr(struct super_block *sb,
1275 unsigned long nr,
1276 enum wb_reason reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277{
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001278 DECLARE_COMPLETION_ONSTACK(done);
1279 struct wb_writeback_work work = {
Wu Fengguang6e6938b2010-06-06 10:38:15 -06001280 .sb = sb,
1281 .sync_mode = WB_SYNC_NONE,
1282 .tagged_writepages = 1,
1283 .done = &done,
1284 .nr_pages = nr,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001285 .reason = reason,
Christoph Hellwig3c4d7162010-06-08 18:14:43 +02001286 };
Jens Axboe0e3c9a22010-06-01 11:08:43 +02001287
Jan Kara6eedc702012-07-03 16:45:27 +02001288 if (sb->s_bdi == &noop_backing_dev_info)
1289 return;
Christoph Hellwigcf37e972010-06-08 18:14:51 +02001290 WARN_ON(!rwsem_is_locked(&sb->s_umount));
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001291 bdi_queue_work(sb->s_bdi, &work);
1292 wait_for_completion(&done);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293}
Chris Mason3259f8b2010-10-29 11:16:17 -04001294EXPORT_SYMBOL(writeback_inodes_sb_nr);
1295
1296/**
1297 * writeback_inodes_sb - writeback dirty inodes from given super_block
1298 * @sb: the superblock
Marcos Paulo de Souza786228a2011-11-23 20:56:45 +08001299 * @reason: reason why some writeback work was initiated
Chris Mason3259f8b2010-10-29 11:16:17 -04001300 *
1301 * Start writeback on some inodes on this super_block. No guarantees are made
1302 * on how many (if any) will be written, and this function does not wait
1303 * for IO completion of submitted IO.
1304 */
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001305void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
Chris Mason3259f8b2010-10-29 11:16:17 -04001306{
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001307 return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
Chris Mason3259f8b2010-10-29 11:16:17 -04001308}
Jens Axboed8a85592009-09-02 12:34:32 +02001309EXPORT_SYMBOL(writeback_inodes_sb);
1310
1311/**
Miao Xie10ee27a2013-01-10 13:47:57 +08001312 * try_to_writeback_inodes_sb_nr - try to start writeback if none underway
Chris Mason3259f8b2010-10-29 11:16:17 -04001313 * @sb: the superblock
1314 * @nr: the number of pages to write
Miao Xie10ee27a2013-01-10 13:47:57 +08001315 * @reason: the reason of writeback
Chris Mason3259f8b2010-10-29 11:16:17 -04001316 *
Miao Xie10ee27a2013-01-10 13:47:57 +08001317 * Invoke writeback_inodes_sb_nr if no writeback is currently underway.
Chris Mason3259f8b2010-10-29 11:16:17 -04001318 * Returns 1 if writeback was started, 0 if not.
1319 */
Miao Xie10ee27a2013-01-10 13:47:57 +08001320int try_to_writeback_inodes_sb_nr(struct super_block *sb,
1321 unsigned long nr,
1322 enum wb_reason reason)
Chris Mason3259f8b2010-10-29 11:16:17 -04001323{
Miao Xie10ee27a2013-01-10 13:47:57 +08001324 if (writeback_in_progress(sb->s_bdi))
Chris Mason3259f8b2010-10-29 11:16:17 -04001325 return 1;
Miao Xie10ee27a2013-01-10 13:47:57 +08001326
1327 if (!down_read_trylock(&sb->s_umount))
Chris Mason3259f8b2010-10-29 11:16:17 -04001328 return 0;
Miao Xie10ee27a2013-01-10 13:47:57 +08001329
1330 writeback_inodes_sb_nr(sb, nr, reason);
1331 up_read(&sb->s_umount);
1332 return 1;
Chris Mason3259f8b2010-10-29 11:16:17 -04001333}
Miao Xie10ee27a2013-01-10 13:47:57 +08001334EXPORT_SYMBOL(try_to_writeback_inodes_sb_nr);
1335
1336/**
1337 * try_to_writeback_inodes_sb - try to start writeback if none underway
1338 * @sb: the superblock
1339 * @reason: reason why some writeback work was initiated
1340 *
1341 * Implement by try_to_writeback_inodes_sb_nr()
1342 * Returns 1 if writeback was started, 0 if not.
1343 */
1344int try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
1345{
1346 return try_to_writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
1347}
1348EXPORT_SYMBOL(try_to_writeback_inodes_sb);
Chris Mason3259f8b2010-10-29 11:16:17 -04001349
1350/**
Jens Axboed8a85592009-09-02 12:34:32 +02001351 * sync_inodes_sb - sync sb inode pages
1352 * @sb: the superblock
1353 *
1354 * This function writes and waits on any dirty inode belonging to this
Stefan Hajnoczicb9ef8d2011-01-13 15:47:26 -08001355 * super_block.
Jens Axboed8a85592009-09-02 12:34:32 +02001356 */
Jens Axboeb6e51312009-09-16 15:13:54 +02001357void sync_inodes_sb(struct super_block *sb)
Jens Axboed8a85592009-09-02 12:34:32 +02001358{
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001359 DECLARE_COMPLETION_ONSTACK(done);
1360 struct wb_writeback_work work = {
Christoph Hellwig3c4d7162010-06-08 18:14:43 +02001361 .sb = sb,
1362 .sync_mode = WB_SYNC_ALL,
1363 .nr_pages = LONG_MAX,
1364 .range_cyclic = 0,
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001365 .done = &done,
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06001366 .reason = WB_REASON_SYNC,
Christoph Hellwig3c4d7162010-06-08 18:14:43 +02001367 };
1368
Jan Kara6eedc702012-07-03 16:45:27 +02001369 /* Nothing to do? */
1370 if (sb->s_bdi == &noop_backing_dev_info)
1371 return;
Christoph Hellwigcf37e972010-06-08 18:14:51 +02001372 WARN_ON(!rwsem_is_locked(&sb->s_umount));
1373
Christoph Hellwig83ba7b02010-07-06 08:59:53 +02001374 bdi_queue_work(sb->s_bdi, &work);
1375 wait_for_completion(&done);
1376
Jens Axboeb6e51312009-09-16 15:13:54 +02001377 wait_sb_inodes(sb);
Jens Axboed8a85592009-09-02 12:34:32 +02001378}
1379EXPORT_SYMBOL(sync_inodes_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381/**
Andrea Arcangeli7f04c262005-10-30 15:03:05 -08001382 * write_inode_now - write an inode to disk
1383 * @inode: inode to write to disk
1384 * @sync: whether the write should be synchronous or not
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385 *
Andrea Arcangeli7f04c262005-10-30 15:03:05 -08001386 * This function commits an inode to disk immediately if it is dirty. This is
1387 * primarily needed by knfsd.
1388 *
1389 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391int write_inode_now(struct inode *inode, int sync)
1392{
Christoph Hellwigf758eea2011-04-21 18:19:44 -06001393 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394 struct writeback_control wbc = {
1395 .nr_to_write = LONG_MAX,
Mike Galbraith18914b12008-02-08 04:20:23 -08001396 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -07001397 .range_start = 0,
1398 .range_end = LLONG_MAX,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 };
1400
1401 if (!mapping_cap_writeback_dirty(inode->i_mapping))
Andrew Morton49364ce2005-11-07 00:59:15 -08001402 wbc.nr_to_write = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403
1404 might_sleep();
Jan Kara4f8ad652012-05-03 14:48:00 +02001405 return writeback_single_inode(inode, wb, &wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406}
1407EXPORT_SYMBOL(write_inode_now);
1408
1409/**
1410 * sync_inode - write an inode and its pages to disk.
1411 * @inode: the inode to sync
1412 * @wbc: controls the writeback mode
1413 *
1414 * sync_inode() will write an inode and its pages to disk. It will also
1415 * correctly update the inode on its superblock's dirty inode lists and will
1416 * update inode->i_state.
1417 *
1418 * The caller must have a ref on the inode.
1419 */
1420int sync_inode(struct inode *inode, struct writeback_control *wbc)
1421{
Jan Kara4f8ad652012-05-03 14:48:00 +02001422 return writeback_single_inode(inode, &inode_to_bdi(inode)->wb, wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423}
1424EXPORT_SYMBOL(sync_inode);
Christoph Hellwigc37650162010-10-06 10:48:20 +02001425
1426/**
Andrew Mortonc691b9d2011-01-13 15:45:48 -08001427 * sync_inode_metadata - write an inode to disk
Christoph Hellwigc37650162010-10-06 10:48:20 +02001428 * @inode: the inode to sync
1429 * @wait: wait for I/O to complete.
1430 *
Andrew Mortonc691b9d2011-01-13 15:45:48 -08001431 * Write an inode to disk and adjust its dirty state after completion.
Christoph Hellwigc37650162010-10-06 10:48:20 +02001432 *
1433 * Note: only writes the actual inode, no associated data or other metadata.
1434 */
1435int sync_inode_metadata(struct inode *inode, int wait)
1436{
1437 struct writeback_control wbc = {
1438 .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
1439 .nr_to_write = 0, /* metadata-only */
1440 };
1441
1442 return sync_inode(inode, &wbc);
1443}
1444EXPORT_SYMBOL(sync_inode_metadata);