blob: 123bcef13e51806ce3d57232d4ad665a60d7d172 [file] [log] [blame]
Andrew Morton3fcfab12006-10-19 23:28:16 -07001
2#include <linux/wait.h>
3#include <linux/backing-dev.h>
Jens Axboe03ba3782009-09-09 09:08:54 +02004#include <linux/kthread.h>
5#include <linux/freezer.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -07006#include <linux/fs.h>
Jens Axboe26160152009-03-17 09:35:06 +01007#include <linux/pagemap.h>
Jens Axboe03ba3782009-09-09 09:08:54 +02008#include <linux/mm.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -07009#include <linux/sched.h>
10#include <linux/module.h>
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070011#include <linux/writeback.h>
12#include <linux/device.h>
13
Jens Axboec3c53202010-04-22 11:37:01 +020014static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
15
Jens Axboe26160152009-03-17 09:35:06 +010016void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
17{
18}
19EXPORT_SYMBOL(default_unplug_io_fn);
20
21struct backing_dev_info default_backing_dev_info = {
Jens Axboed9938312009-06-12 14:45:52 +020022 .name = "default",
Jens Axboe26160152009-03-17 09:35:06 +010023 .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
24 .state = 0,
25 .capabilities = BDI_CAP_MAP_COPY,
26 .unplug_io_fn = default_unplug_io_fn,
27};
28EXPORT_SYMBOL_GPL(default_backing_dev_info);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070029
Jörn Engel5129a462010-04-25 08:54:42 +020030struct backing_dev_info noop_backing_dev_info = {
31 .name = "noop",
32};
33EXPORT_SYMBOL_GPL(noop_backing_dev_info);
34
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070035static struct class *bdi_class;
Jens Axboecfc4ba52009-09-14 13:12:40 +020036
37/*
38 * bdi_lock protects updates to bdi_list and bdi_pending_list, as well as
39 * reader side protection for bdi_pending_list. bdi_list has RCU reader side
40 * locking.
41 */
Jens Axboe03ba3782009-09-09 09:08:54 +020042DEFINE_SPINLOCK(bdi_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +020043LIST_HEAD(bdi_list);
Jens Axboe03ba3782009-09-09 09:08:54 +020044LIST_HEAD(bdi_pending_list);
45
46static struct task_struct *sync_supers_tsk;
47static struct timer_list sync_supers_timer;
48
49static int bdi_sync_supers(void *);
50static void sync_supers_timer_fn(unsigned long);
Jens Axboe03ba3782009-09-09 09:08:54 +020051
52static void bdi_add_default_flusher_task(struct backing_dev_info *bdi);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070053
Miklos Szeredi76f14182008-04-30 00:54:36 -070054#ifdef CONFIG_DEBUG_FS
55#include <linux/debugfs.h>
56#include <linux/seq_file.h>
57
58static struct dentry *bdi_debug_root;
59
60static void bdi_debug_init(void)
61{
62 bdi_debug_root = debugfs_create_dir("bdi", NULL);
63}
64
65static int bdi_debug_stats_show(struct seq_file *m, void *v)
66{
67 struct backing_dev_info *bdi = m->private;
Jens Axboef09b00d2009-05-25 09:08:21 +020068 struct bdi_writeback *wb;
David Rientjes364aeb22009-01-06 14:39:29 -080069 unsigned long background_thresh;
70 unsigned long dirty_thresh;
71 unsigned long bdi_thresh;
Jens Axboef09b00d2009-05-25 09:08:21 +020072 unsigned long nr_dirty, nr_io, nr_more_io, nr_wb;
73 struct inode *inode;
74
75 /*
76 * inode lock is enough here, the bdi->wb_list is protected by
77 * RCU on the reader side
78 */
79 nr_wb = nr_dirty = nr_io = nr_more_io = 0;
80 spin_lock(&inode_lock);
81 list_for_each_entry(wb, &bdi->wb_list, list) {
82 nr_wb++;
83 list_for_each_entry(inode, &wb->b_dirty, i_list)
84 nr_dirty++;
85 list_for_each_entry(inode, &wb->b_io, i_list)
86 nr_io++;
87 list_for_each_entry(inode, &wb->b_more_io, i_list)
88 nr_more_io++;
89 }
90 spin_unlock(&inode_lock);
Miklos Szeredi76f14182008-04-30 00:54:36 -070091
92 get_dirty_limits(&background_thresh, &dirty_thresh, &bdi_thresh, bdi);
93
94#define K(x) ((x) << (PAGE_SHIFT - 10))
95 seq_printf(m,
96 "BdiWriteback: %8lu kB\n"
97 "BdiReclaimable: %8lu kB\n"
98 "BdiDirtyThresh: %8lu kB\n"
99 "DirtyThresh: %8lu kB\n"
Jens Axboef09b00d2009-05-25 09:08:21 +0200100 "BackgroundThresh: %8lu kB\n"
Wu Fengguang961515f2009-10-09 13:01:27 +0200101 "WritebackThreads: %8lu\n"
Jens Axboef09b00d2009-05-25 09:08:21 +0200102 "b_dirty: %8lu\n"
103 "b_io: %8lu\n"
104 "b_more_io: %8lu\n"
105 "bdi_list: %8u\n"
106 "state: %8lx\n"
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200107 "wb_list: %8u\n",
Miklos Szeredi76f14182008-04-30 00:54:36 -0700108 (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)),
109 (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)),
Jens Axboef09b00d2009-05-25 09:08:21 +0200110 K(bdi_thresh), K(dirty_thresh),
111 K(background_thresh), nr_wb, nr_dirty, nr_io, nr_more_io,
Christoph Hellwig83ba7b02010-07-06 08:59:53 +0200112 !list_empty(&bdi->bdi_list), bdi->state,
113 !list_empty(&bdi->wb_list));
Miklos Szeredi76f14182008-04-30 00:54:36 -0700114#undef K
115
116 return 0;
117}
118
119static int bdi_debug_stats_open(struct inode *inode, struct file *file)
120{
121 return single_open(file, bdi_debug_stats_show, inode->i_private);
122}
123
124static const struct file_operations bdi_debug_stats_fops = {
125 .open = bdi_debug_stats_open,
126 .read = seq_read,
127 .llseek = seq_lseek,
128 .release = single_release,
129};
130
131static void bdi_debug_register(struct backing_dev_info *bdi, const char *name)
132{
133 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
134 bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
135 bdi, &bdi_debug_stats_fops);
136}
137
138static void bdi_debug_unregister(struct backing_dev_info *bdi)
139{
140 debugfs_remove(bdi->debug_stats);
141 debugfs_remove(bdi->debug_dir);
142}
143#else
144static inline void bdi_debug_init(void)
145{
146}
147static inline void bdi_debug_register(struct backing_dev_info *bdi,
148 const char *name)
149{
150}
151static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
152{
153}
154#endif
155
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700156static ssize_t read_ahead_kb_store(struct device *dev,
157 struct device_attribute *attr,
158 const char *buf, size_t count)
159{
160 struct backing_dev_info *bdi = dev_get_drvdata(dev);
161 char *end;
162 unsigned long read_ahead_kb;
163 ssize_t ret = -EINVAL;
164
165 read_ahead_kb = simple_strtoul(buf, &end, 10);
166 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
167 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);
168 ret = count;
169 }
170 return ret;
171}
172
173#define K(pages) ((pages) << (PAGE_SHIFT - 10))
174
175#define BDI_SHOW(name, expr) \
176static ssize_t name##_show(struct device *dev, \
177 struct device_attribute *attr, char *page) \
178{ \
179 struct backing_dev_info *bdi = dev_get_drvdata(dev); \
180 \
181 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \
182}
183
184BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))
185
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700186static ssize_t min_ratio_store(struct device *dev,
187 struct device_attribute *attr, const char *buf, size_t count)
188{
189 struct backing_dev_info *bdi = dev_get_drvdata(dev);
190 char *end;
191 unsigned int ratio;
192 ssize_t ret = -EINVAL;
193
194 ratio = simple_strtoul(buf, &end, 10);
195 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
196 ret = bdi_set_min_ratio(bdi, ratio);
197 if (!ret)
198 ret = count;
199 }
200 return ret;
201}
202BDI_SHOW(min_ratio, bdi->min_ratio)
203
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700204static ssize_t max_ratio_store(struct device *dev,
205 struct device_attribute *attr, const char *buf, size_t count)
206{
207 struct backing_dev_info *bdi = dev_get_drvdata(dev);
208 char *end;
209 unsigned int ratio;
210 ssize_t ret = -EINVAL;
211
212 ratio = simple_strtoul(buf, &end, 10);
213 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
214 ret = bdi_set_max_ratio(bdi, ratio);
215 if (!ret)
216 ret = count;
217 }
218 return ret;
219}
220BDI_SHOW(max_ratio, bdi->max_ratio)
221
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700222#define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store)
223
224static struct device_attribute bdi_dev_attrs[] = {
225 __ATTR_RW(read_ahead_kb),
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700226 __ATTR_RW(min_ratio),
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700227 __ATTR_RW(max_ratio),
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700228 __ATTR_NULL,
229};
230
231static __init int bdi_class_init(void)
232{
233 bdi_class = class_create(THIS_MODULE, "bdi");
Anton Blanchard14421452010-04-02 09:46:55 +0200234 if (IS_ERR(bdi_class))
235 return PTR_ERR(bdi_class);
236
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700237 bdi_class->dev_attrs = bdi_dev_attrs;
Miklos Szeredi76f14182008-04-30 00:54:36 -0700238 bdi_debug_init();
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700239 return 0;
240}
Miklos Szeredi76f14182008-04-30 00:54:36 -0700241postcore_initcall(bdi_class_init);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700242
Jens Axboe26160152009-03-17 09:35:06 +0100243static int __init default_bdi_init(void)
244{
245 int err;
246
Jens Axboe03ba3782009-09-09 09:08:54 +0200247 sync_supers_tsk = kthread_run(bdi_sync_supers, NULL, "sync_supers");
248 BUG_ON(IS_ERR(sync_supers_tsk));
249
250 init_timer(&sync_supers_timer);
251 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0);
Jens Axboe64231042010-05-21 20:00:35 +0200252 bdi_arm_supers_timer();
Jens Axboe03ba3782009-09-09 09:08:54 +0200253
Jens Axboe26160152009-03-17 09:35:06 +0100254 err = bdi_init(&default_backing_dev_info);
255 if (!err)
256 bdi_register(&default_backing_dev_info, NULL, "default");
257
258 return err;
259}
260subsys_initcall(default_bdi_init);
261
Jens Axboe03ba3782009-09-09 09:08:54 +0200262static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi)
263{
264 memset(wb, 0, sizeof(*wb));
265
266 wb->bdi = bdi;
267 wb->last_old_flush = jiffies;
268 INIT_LIST_HEAD(&wb->b_dirty);
269 INIT_LIST_HEAD(&wb->b_io);
270 INIT_LIST_HEAD(&wb->b_more_io);
271}
272
273static void bdi_task_init(struct backing_dev_info *bdi,
274 struct bdi_writeback *wb)
275{
276 struct task_struct *tsk = current;
277
278 spin_lock(&bdi->wb_lock);
279 list_add_tail_rcu(&wb->list, &bdi->wb_list);
280 spin_unlock(&bdi->wb_lock);
281
282 tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
283 set_freezable();
284
285 /*
286 * Our parent may run at a different priority, just set us to normal
287 */
288 set_user_nice(tsk, 0);
289}
290
291static int bdi_start_fn(void *ptr)
292{
293 struct bdi_writeback *wb = ptr;
294 struct backing_dev_info *bdi = wb->bdi;
295 int ret;
296
297 /*
298 * Add us to the active bdi_list
299 */
Jens Axboecfc4ba52009-09-14 13:12:40 +0200300 spin_lock_bh(&bdi_lock);
301 list_add_rcu(&bdi->bdi_list, &bdi_list);
302 spin_unlock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200303
304 bdi_task_init(bdi, wb);
305
306 /*
307 * Clear pending bit and wakeup anybody waiting to tear us down
308 */
309 clear_bit(BDI_pending, &bdi->state);
310 smp_mb__after_clear_bit();
311 wake_up_bit(&bdi->state, BDI_pending);
312
313 ret = bdi_writeback_task(wb);
314
315 /*
316 * Remove us from the list
317 */
318 spin_lock(&bdi->wb_lock);
319 list_del_rcu(&wb->list);
320 spin_unlock(&bdi->wb_lock);
321
322 /*
323 * Flush any work that raced with us exiting. No new work
324 * will be added, since this bdi isn't discoverable anymore.
325 */
326 if (!list_empty(&bdi->work_list))
327 wb_do_writeback(wb, 1);
328
329 wb->task = NULL;
330 return ret;
331}
332
333int bdi_has_dirty_io(struct backing_dev_info *bdi)
334{
335 return wb_has_dirty_io(&bdi->wb);
336}
337
338static void bdi_flush_io(struct backing_dev_info *bdi)
339{
340 struct writeback_control wbc = {
Jens Axboe03ba3782009-09-09 09:08:54 +0200341 .sync_mode = WB_SYNC_NONE,
342 .older_than_this = NULL,
343 .range_cyclic = 1,
344 .nr_to_write = 1024,
345 };
346
Christoph Hellwig9c3a8ee2010-06-10 12:07:27 +0200347 writeback_inodes_wb(&bdi->wb, &wbc);
Jens Axboe03ba3782009-09-09 09:08:54 +0200348}
349
350/*
351 * kupdated() used to do this. We cannot do it from the bdi_forker_task()
352 * or we risk deadlocking on ->s_umount. The longer term solution would be
353 * to implement sync_supers_bdi() or similar and simply do it from the
354 * bdi writeback tasks individually.
355 */
356static int bdi_sync_supers(void *unused)
357{
358 set_user_nice(current, 0);
359
360 while (!kthread_should_stop()) {
361 set_current_state(TASK_INTERRUPTIBLE);
362 schedule();
363
364 /*
365 * Do this periodically, like kupdated() did before.
366 */
367 sync_supers();
368 }
369
370 return 0;
371}
372
Jens Axboe64231042010-05-21 20:00:35 +0200373void bdi_arm_supers_timer(void)
Jens Axboe03ba3782009-09-09 09:08:54 +0200374{
375 unsigned long next;
376
Jens Axboe64231042010-05-21 20:00:35 +0200377 if (!dirty_writeback_interval)
378 return;
379
Jens Axboe03ba3782009-09-09 09:08:54 +0200380 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies;
381 mod_timer(&sync_supers_timer, round_jiffies_up(next));
382}
383
384static void sync_supers_timer_fn(unsigned long unused)
385{
386 wake_up_process(sync_supers_tsk);
Jens Axboe64231042010-05-21 20:00:35 +0200387 bdi_arm_supers_timer();
Jens Axboe03ba3782009-09-09 09:08:54 +0200388}
389
390static int bdi_forker_task(void *ptr)
391{
392 struct bdi_writeback *me = ptr;
393
394 bdi_task_init(me->bdi, me);
395
396 for (;;) {
397 struct backing_dev_info *bdi, *tmp;
398 struct bdi_writeback *wb;
399
400 /*
401 * Temporary measure, we want to make sure we don't see
402 * dirty data on the default backing_dev_info
403 */
404 if (wb_has_dirty_io(me) || !list_empty(&me->bdi->work_list))
405 wb_do_writeback(me, 0);
406
Jens Axboecfc4ba52009-09-14 13:12:40 +0200407 spin_lock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200408
409 /*
410 * Check if any existing bdi's have dirty data without
411 * a thread registered. If so, set that up.
412 */
413 list_for_each_entry_safe(bdi, tmp, &bdi_list, bdi_list) {
414 if (bdi->wb.task)
415 continue;
416 if (list_empty(&bdi->work_list) &&
417 !bdi_has_dirty_io(bdi))
418 continue;
419
420 bdi_add_default_flusher_task(bdi);
421 }
422
423 set_current_state(TASK_INTERRUPTIBLE);
424
425 if (list_empty(&bdi_pending_list)) {
426 unsigned long wait;
427
Jens Axboecfc4ba52009-09-14 13:12:40 +0200428 spin_unlock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200429 wait = msecs_to_jiffies(dirty_writeback_interval * 10);
Jens Axboe64231042010-05-21 20:00:35 +0200430 if (wait)
431 schedule_timeout(wait);
432 else
433 schedule();
Jens Axboe03ba3782009-09-09 09:08:54 +0200434 try_to_freeze();
435 continue;
436 }
437
438 __set_current_state(TASK_RUNNING);
439
440 /*
441 * This is our real job - check for pending entries in
442 * bdi_pending_list, and create the tasks that got added
443 */
444 bdi = list_entry(bdi_pending_list.next, struct backing_dev_info,
445 bdi_list);
446 list_del_init(&bdi->bdi_list);
Jens Axboecfc4ba52009-09-14 13:12:40 +0200447 spin_unlock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200448
449 wb = &bdi->wb;
450 wb->task = kthread_run(bdi_start_fn, wb, "flush-%s",
451 dev_name(bdi->dev));
452 /*
453 * If task creation fails, then readd the bdi to
454 * the pending list and force writeout of the bdi
455 * from this forker thread. That will free some memory
456 * and we can try again.
457 */
458 if (IS_ERR(wb->task)) {
459 wb->task = NULL;
460
461 /*
462 * Add this 'bdi' to the back, so we get
463 * a chance to flush other bdi's to free
464 * memory.
465 */
Jens Axboecfc4ba52009-09-14 13:12:40 +0200466 spin_lock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200467 list_add_tail(&bdi->bdi_list, &bdi_pending_list);
Jens Axboecfc4ba52009-09-14 13:12:40 +0200468 spin_unlock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200469
470 bdi_flush_io(bdi);
471 }
472 }
473
474 return 0;
475}
476
Jens Axboecfc4ba52009-09-14 13:12:40 +0200477static void bdi_add_to_pending(struct rcu_head *head)
478{
479 struct backing_dev_info *bdi;
480
481 bdi = container_of(head, struct backing_dev_info, rcu_head);
482 INIT_LIST_HEAD(&bdi->bdi_list);
483
484 spin_lock(&bdi_lock);
485 list_add_tail(&bdi->bdi_list, &bdi_pending_list);
486 spin_unlock(&bdi_lock);
487
488 /*
489 * We are now on the pending list, wake up bdi_forker_task()
490 * to finish the job and add us back to the active bdi_list
491 */
492 wake_up_process(default_backing_dev_info.wb.task);
493}
494
Jens Axboe03ba3782009-09-09 09:08:54 +0200495/*
496 * Add the default flusher task that gets created for any bdi
497 * that has dirty data pending writeout
498 */
499void static bdi_add_default_flusher_task(struct backing_dev_info *bdi)
500{
501 if (!bdi_cap_writeback_dirty(bdi))
502 return;
503
Jens Axboe500b0672009-09-09 09:10:25 +0200504 if (WARN_ON(!test_bit(BDI_registered, &bdi->state))) {
505 printk(KERN_ERR "bdi %p/%s is not registered!\n",
506 bdi, bdi->name);
507 return;
508 }
509
Jens Axboe03ba3782009-09-09 09:08:54 +0200510 /*
511 * Check with the helper whether to proceed adding a task. Will only
512 * abort if we two or more simultanous calls to
513 * bdi_add_default_flusher_task() occured, further additions will block
514 * waiting for previous additions to finish.
515 */
516 if (!test_and_set_bit(BDI_pending, &bdi->state)) {
Jens Axboecfc4ba52009-09-14 13:12:40 +0200517 list_del_rcu(&bdi->bdi_list);
Jens Axboe03ba3782009-09-09 09:08:54 +0200518
519 /*
Jens Axboecfc4ba52009-09-14 13:12:40 +0200520 * We must wait for the current RCU period to end before
521 * moving to the pending list. So schedule that operation
522 * from an RCU callback.
Jens Axboe03ba3782009-09-09 09:08:54 +0200523 */
Jens Axboecfc4ba52009-09-14 13:12:40 +0200524 call_rcu(&bdi->rcu_head, bdi_add_to_pending);
Jens Axboe03ba3782009-09-09 09:08:54 +0200525 }
526}
527
Jens Axboecfc4ba52009-09-14 13:12:40 +0200528/*
529 * Remove bdi from bdi_list, and ensure that it is no longer visible
530 */
531static void bdi_remove_from_list(struct backing_dev_info *bdi)
532{
533 spin_lock_bh(&bdi_lock);
534 list_del_rcu(&bdi->bdi_list);
535 spin_unlock_bh(&bdi_lock);
536
537 synchronize_rcu();
538}
539
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700540int bdi_register(struct backing_dev_info *bdi, struct device *parent,
541 const char *fmt, ...)
542{
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700543 va_list args;
544 int ret = 0;
545 struct device *dev;
546
Andrew Morton69fc2082008-12-09 13:14:06 -0800547 if (bdi->dev) /* The driver needs to use separate queues per device */
Kay Sieversf1d0b062008-12-02 10:31:50 -0800548 goto exit;
549
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700550 va_start(args, fmt);
Greg Kroah-Hartman19051c52008-05-15 13:44:08 -0700551 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700552 va_end(args);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700553 if (IS_ERR(dev)) {
554 ret = PTR_ERR(dev);
555 goto exit;
556 }
557
Jens Axboecfc4ba52009-09-14 13:12:40 +0200558 spin_lock_bh(&bdi_lock);
559 list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
560 spin_unlock_bh(&bdi_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200561
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700562 bdi->dev = dev;
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700563
Jens Axboe03ba3782009-09-09 09:08:54 +0200564 /*
565 * Just start the forker thread for our default backing_dev_info,
566 * and add other bdi's to the list. They will get a thread created
567 * on-demand when they need it.
568 */
569 if (bdi_cap_flush_forker(bdi)) {
570 struct bdi_writeback *wb = &bdi->wb;
571
572 wb->task = kthread_run(bdi_forker_task, wb, "bdi-%s",
573 dev_name(dev));
574 if (IS_ERR(wb->task)) {
575 wb->task = NULL;
576 ret = -ENOMEM;
577
Jens Axboecfc4ba52009-09-14 13:12:40 +0200578 bdi_remove_from_list(bdi);
Jens Axboe03ba3782009-09-09 09:08:54 +0200579 goto exit;
580 }
581 }
582
583 bdi_debug_register(bdi, dev_name(dev));
Jens Axboe500b0672009-09-09 09:10:25 +0200584 set_bit(BDI_registered, &bdi->state);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700585exit:
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700586 return ret;
587}
588EXPORT_SYMBOL(bdi_register);
589
590int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
591{
592 return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
593}
594EXPORT_SYMBOL(bdi_register_dev);
595
Jens Axboe03ba3782009-09-09 09:08:54 +0200596/*
597 * Remove bdi from the global list and shutdown any threads we have running
598 */
599static void bdi_wb_shutdown(struct backing_dev_info *bdi)
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200600{
Jens Axboe03ba3782009-09-09 09:08:54 +0200601 struct bdi_writeback *wb;
602
603 if (!bdi_cap_writeback_dirty(bdi))
604 return;
605
606 /*
607 * If setup is pending, wait for that to complete first
608 */
609 wait_on_bit(&bdi->state, BDI_pending, bdi_sched_wait,
610 TASK_UNINTERRUPTIBLE);
611
612 /*
613 * Make sure nobody finds us on the bdi_list anymore
614 */
Jens Axboecfc4ba52009-09-14 13:12:40 +0200615 bdi_remove_from_list(bdi);
Jens Axboe03ba3782009-09-09 09:08:54 +0200616
617 /*
618 * Finally, kill the kernel threads. We don't need to be RCU
Romit Dasguptac62b17a2009-11-12 13:08:11 +0100619 * safe anymore, since the bdi is gone from visibility. Force
620 * unfreeze of the thread before calling kthread_stop(), otherwise
621 * it would never exet if it is currently stuck in the refrigerator.
Jens Axboe03ba3782009-09-09 09:08:54 +0200622 */
Romit Dasguptac62b17a2009-11-12 13:08:11 +0100623 list_for_each_entry(wb, &bdi->wb_list, list) {
OGAWA Hirofumibf7ec5b2009-12-03 13:49:43 +0100624 thaw_process(wb->task);
Jens Axboe03ba3782009-09-09 09:08:54 +0200625 kthread_stop(wb->task);
Romit Dasguptac62b17a2009-11-12 13:08:11 +0100626 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200627}
628
Jens Axboe592b09a2009-10-29 11:46:12 +0100629/*
630 * This bdi is going away now, make sure that no super_blocks point to it
631 */
632static void bdi_prune_sb(struct backing_dev_info *bdi)
633{
634 struct super_block *sb;
635
636 spin_lock(&sb_lock);
637 list_for_each_entry(sb, &super_blocks, s_list) {
638 if (sb->s_bdi == bdi)
639 sb->s_bdi = NULL;
640 }
641 spin_unlock(&sb_lock);
642}
643
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700644void bdi_unregister(struct backing_dev_info *bdi)
645{
646 if (bdi->dev) {
Jens Axboe8c4db332009-11-03 20:18:44 +0100647 bdi_prune_sb(bdi);
648
Jens Axboe03ba3782009-09-09 09:08:54 +0200649 if (!bdi_cap_flush_forker(bdi))
650 bdi_wb_shutdown(bdi);
Miklos Szeredi76f14182008-04-30 00:54:36 -0700651 bdi_debug_unregister(bdi);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700652 device_unregister(bdi->dev);
653 bdi->dev = NULL;
654 }
655}
656EXPORT_SYMBOL(bdi_unregister);
Andrew Morton3fcfab12006-10-19 23:28:16 -0700657
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700658int bdi_init(struct backing_dev_info *bdi)
659{
Jens Axboe03ba3782009-09-09 09:08:54 +0200660 int i, err;
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700661
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700662 bdi->dev = NULL;
663
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700664 bdi->min_ratio = 0;
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700665 bdi->max_ratio = 100;
666 bdi->max_prop_frac = PROP_FRAC_BASE;
Jens Axboe03ba3782009-09-09 09:08:54 +0200667 spin_lock_init(&bdi->wb_lock);
Jens Axboecfc4ba52009-09-14 13:12:40 +0200668 INIT_RCU_HEAD(&bdi->rcu_head);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200669 INIT_LIST_HEAD(&bdi->bdi_list);
Jens Axboe03ba3782009-09-09 09:08:54 +0200670 INIT_LIST_HEAD(&bdi->wb_list);
671 INIT_LIST_HEAD(&bdi->work_list);
672
673 bdi_wb_init(&bdi->wb, bdi);
674
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700675 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
Peter Zijlstraea319512008-12-26 15:08:55 +0100676 err = percpu_counter_init(&bdi->bdi_stat[i], 0);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700677 if (err)
678 goto err;
679 }
680
681 bdi->dirty_exceeded = 0;
682 err = prop_local_init_percpu(&bdi->completions);
683
684 if (err) {
685err:
Denis Cheng4b01a0b2007-12-04 23:45:07 -0800686 while (i--)
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700687 percpu_counter_destroy(&bdi->bdi_stat[i]);
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700688 }
689
690 return err;
691}
692EXPORT_SYMBOL(bdi_init);
693
694void bdi_destroy(struct backing_dev_info *bdi)
695{
696 int i;
697
Jens Axboece5f8e72009-09-14 12:57:56 +0200698 /*
699 * Splice our entries to the default_backing_dev_info, if this
700 * bdi disappears
701 */
702 if (bdi_has_dirty_io(bdi)) {
703 struct bdi_writeback *dst = &default_backing_dev_info.wb;
704
705 spin_lock(&inode_lock);
706 list_splice(&bdi->wb.b_dirty, &dst->b_dirty);
707 list_splice(&bdi->wb.b_io, &dst->b_io);
708 list_splice(&bdi->wb.b_more_io, &dst->b_more_io);
709 spin_unlock(&inode_lock);
710 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200711
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700712 bdi_unregister(bdi);
713
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700714 for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
715 percpu_counter_destroy(&bdi->bdi_stat[i]);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700716
717 prop_local_destroy_percpu(&bdi->completions);
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700718}
719EXPORT_SYMBOL(bdi_destroy);
720
Jens Axboec3c53202010-04-22 11:37:01 +0200721/*
722 * For use from filesystems to quickly init and register a bdi associated
723 * with dirty writeback
724 */
725int bdi_setup_and_register(struct backing_dev_info *bdi, char *name,
726 unsigned int cap)
727{
728 char tmp[32];
729 int err;
730
731 bdi->name = name;
732 bdi->capabilities = cap;
733 err = bdi_init(bdi);
734 if (err)
735 return err;
736
737 sprintf(tmp, "%.28s%s", name, "-%d");
738 err = bdi_register(bdi, NULL, tmp, atomic_long_inc_return(&bdi_seq));
739 if (err) {
740 bdi_destroy(bdi);
741 return err;
742 }
743
744 return 0;
745}
746EXPORT_SYMBOL(bdi_setup_and_register);
747
Andrew Morton3fcfab12006-10-19 23:28:16 -0700748static wait_queue_head_t congestion_wqh[2] = {
749 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
750 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
751 };
752
Jens Axboe1faa16d2009-04-06 14:48:01 +0200753void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700754{
755 enum bdi_state bit;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200756 wait_queue_head_t *wqh = &congestion_wqh[sync];
Andrew Morton3fcfab12006-10-19 23:28:16 -0700757
Jens Axboe1faa16d2009-04-06 14:48:01 +0200758 bit = sync ? BDI_sync_congested : BDI_async_congested;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700759 clear_bit(bit, &bdi->state);
760 smp_mb__after_clear_bit();
761 if (waitqueue_active(wqh))
762 wake_up(wqh);
763}
764EXPORT_SYMBOL(clear_bdi_congested);
765
Jens Axboe1faa16d2009-04-06 14:48:01 +0200766void set_bdi_congested(struct backing_dev_info *bdi, int sync)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700767{
768 enum bdi_state bit;
769
Jens Axboe1faa16d2009-04-06 14:48:01 +0200770 bit = sync ? BDI_sync_congested : BDI_async_congested;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700771 set_bit(bit, &bdi->state);
772}
773EXPORT_SYMBOL(set_bdi_congested);
774
775/**
776 * congestion_wait - wait for a backing_dev to become uncongested
Jens Axboe8aa7e842009-07-09 14:52:32 +0200777 * @sync: SYNC or ASYNC IO
Andrew Morton3fcfab12006-10-19 23:28:16 -0700778 * @timeout: timeout in jiffies
779 *
780 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
781 * write congestion. If no backing_devs are congested then just wait for the
782 * next write to be completed.
783 */
Jens Axboe8aa7e842009-07-09 14:52:32 +0200784long congestion_wait(int sync, long timeout)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700785{
786 long ret;
787 DEFINE_WAIT(wait);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200788 wait_queue_head_t *wqh = &congestion_wqh[sync];
Andrew Morton3fcfab12006-10-19 23:28:16 -0700789
790 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
791 ret = io_schedule_timeout(timeout);
792 finish_wait(wqh, &wait);
793 return ret;
794}
795EXPORT_SYMBOL(congestion_wait);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700796