blob: 1c62203a4906ec1c7ba2ca9ede00e2a2c498aeda [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
21#include <linux/config.h>
22#include <linux/kernel.h>
23#include <linux/syscalls.h>
24#include <linux/fs.h>
25#include <linux/mm.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/smp_lock.h>
29#include <linux/blkdev.h>
30#include <linux/file.h>
31#include <linux/quotaops.h>
32#include <linux/highmem.h>
33#include <linux/module.h>
34#include <linux/writeback.h>
35#include <linux/hash.h>
36#include <linux/suspend.h>
37#include <linux/buffer_head.h>
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
43
44static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
45static void invalidate_bh_lrus(void);
46
47#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48
49inline void
50init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51{
52 bh->b_end_io = handler;
53 bh->b_private = private;
54}
55
56static int sync_buffer(void *word)
57{
58 struct block_device *bd;
59 struct buffer_head *bh
60 = container_of(word, struct buffer_head, b_state);
61
62 smp_mb();
63 bd = bh->b_bdev;
64 if (bd)
65 blk_run_address_space(bd->bd_inode->i_mapping);
66 io_schedule();
67 return 0;
68}
69
70void fastcall __lock_buffer(struct buffer_head *bh)
71{
72 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
73 TASK_UNINTERRUPTIBLE);
74}
75EXPORT_SYMBOL(__lock_buffer);
76
77void fastcall unlock_buffer(struct buffer_head *bh)
78{
79 clear_buffer_locked(bh);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh->b_state, BH_Lock);
82}
83
84/*
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
88 */
89void __wait_on_buffer(struct buffer_head * bh)
90{
91 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
92}
93
94static void
95__clear_page_buffers(struct page *page)
96{
97 ClearPagePrivate(page);
98 page->private = 0;
99 page_cache_release(page);
100}
101
102static void buffer_io_error(struct buffer_head *bh)
103{
104 char b[BDEVNAME_SIZE];
105
106 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
107 bdevname(bh->b_bdev, b),
108 (unsigned long long)bh->b_blocknr);
109}
110
111/*
112 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
113 * unlock the buffer. This is what ll_rw_block uses too.
114 */
115void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
116{
117 if (uptodate) {
118 set_buffer_uptodate(bh);
119 } else {
120 /* This happens, due to failed READA attempts. */
121 clear_buffer_uptodate(bh);
122 }
123 unlock_buffer(bh);
124 put_bh(bh);
125}
126
127void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
128{
129 char b[BDEVNAME_SIZE];
130
131 if (uptodate) {
132 set_buffer_uptodate(bh);
133 } else {
134 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
135 buffer_io_error(bh);
136 printk(KERN_WARNING "lost page write due to "
137 "I/O error on %s\n",
138 bdevname(bh->b_bdev, b));
139 }
140 set_buffer_write_io_error(bh);
141 clear_buffer_uptodate(bh);
142 }
143 unlock_buffer(bh);
144 put_bh(bh);
145}
146
147/*
148 * Write out and wait upon all the dirty data associated with a block
149 * device via its mapping. Does not take the superblock lock.
150 */
151int sync_blockdev(struct block_device *bdev)
152{
153 int ret = 0;
154
155 if (bdev) {
156 int err;
157
158 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
159 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
160 if (!ret)
161 ret = err;
162 }
163 return ret;
164}
165EXPORT_SYMBOL(sync_blockdev);
166
167/*
168 * Write out and wait upon all dirty data associated with this
169 * superblock. Filesystem data as well as the underlying block
170 * device. Takes the superblock lock.
171 */
172int fsync_super(struct super_block *sb)
173{
174 sync_inodes_sb(sb, 0);
175 DQUOT_SYNC(sb);
176 lock_super(sb);
177 if (sb->s_dirt && sb->s_op->write_super)
178 sb->s_op->write_super(sb);
179 unlock_super(sb);
180 if (sb->s_op->sync_fs)
181 sb->s_op->sync_fs(sb, 1);
182 sync_blockdev(sb->s_bdev);
183 sync_inodes_sb(sb, 1);
184
185 return sync_blockdev(sb->s_bdev);
186}
187
188/*
189 * Write out and wait upon all dirty data associated with this
190 * device. Filesystem data as well as the underlying block
191 * device. Takes the superblock lock.
192 */
193int fsync_bdev(struct block_device *bdev)
194{
195 struct super_block *sb = get_super(bdev);
196 if (sb) {
197 int res = fsync_super(sb);
198 drop_super(sb);
199 return res;
200 }
201 return sync_blockdev(bdev);
202}
203
204/**
205 * freeze_bdev -- lock a filesystem and force it into a consistent state
206 * @bdev: blockdevice to lock
207 *
208 * This takes the block device bd_mount_sem to make sure no new mounts
209 * happen on bdev until thaw_bdev() is called.
210 * If a superblock is found on this device, we take the s_umount semaphore
211 * on it to make sure nobody unmounts until the snapshot creation is done.
212 */
213struct super_block *freeze_bdev(struct block_device *bdev)
214{
215 struct super_block *sb;
216
217 down(&bdev->bd_mount_sem);
218 sb = get_super(bdev);
219 if (sb && !(sb->s_flags & MS_RDONLY)) {
220 sb->s_frozen = SB_FREEZE_WRITE;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700221 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
223 sync_inodes_sb(sb, 0);
224 DQUOT_SYNC(sb);
225
226 lock_super(sb);
227 if (sb->s_dirt && sb->s_op->write_super)
228 sb->s_op->write_super(sb);
229 unlock_super(sb);
230
231 if (sb->s_op->sync_fs)
232 sb->s_op->sync_fs(sb, 1);
233
234 sync_blockdev(sb->s_bdev);
235 sync_inodes_sb(sb, 1);
236
237 sb->s_frozen = SB_FREEZE_TRANS;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700238 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
240 sync_blockdev(sb->s_bdev);
241
242 if (sb->s_op->write_super_lockfs)
243 sb->s_op->write_super_lockfs(sb);
244 }
245
246 sync_blockdev(bdev);
247 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
248}
249EXPORT_SYMBOL(freeze_bdev);
250
251/**
252 * thaw_bdev -- unlock filesystem
253 * @bdev: blockdevice to unlock
254 * @sb: associated superblock
255 *
256 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
257 */
258void thaw_bdev(struct block_device *bdev, struct super_block *sb)
259{
260 if (sb) {
261 BUG_ON(sb->s_bdev != bdev);
262
263 if (sb->s_op->unlockfs)
264 sb->s_op->unlockfs(sb);
265 sb->s_frozen = SB_UNFROZEN;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700266 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 wake_up(&sb->s_wait_unfrozen);
268 drop_super(sb);
269 }
270
271 up(&bdev->bd_mount_sem);
272}
273EXPORT_SYMBOL(thaw_bdev);
274
275/*
276 * sync everything. Start out by waking pdflush, because that writes back
277 * all queues in parallel.
278 */
279static void do_sync(unsigned long wait)
280{
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700281 wakeup_pdflush(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 sync_inodes(0); /* All mappings, inodes and their blockdevs */
283 DQUOT_SYNC(NULL);
284 sync_supers(); /* Write the superblocks */
285 sync_filesystems(0); /* Start syncing the filesystems */
286 sync_filesystems(wait); /* Waitingly sync the filesystems */
287 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
288 if (!wait)
289 printk("Emergency Sync complete\n");
290 if (unlikely(laptop_mode))
291 laptop_sync_completion();
292}
293
294asmlinkage long sys_sync(void)
295{
296 do_sync(1);
297 return 0;
298}
299
300void emergency_sync(void)
301{
302 pdflush_operation(do_sync, 0);
303}
304
305/*
306 * Generic function to fsync a file.
307 *
308 * filp may be NULL if called via the msync of a vma.
309 */
310
311int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
312{
313 struct inode * inode = dentry->d_inode;
314 struct super_block * sb;
315 int ret, err;
316
317 /* sync the inode to buffers */
318 ret = write_inode_now(inode, 0);
319
320 /* sync the superblock to buffers */
321 sb = inode->i_sb;
322 lock_super(sb);
323 if (sb->s_op->write_super)
324 sb->s_op->write_super(sb);
325 unlock_super(sb);
326
327 /* .. finally sync the buffers to disk */
328 err = sync_blockdev(sb->s_bdev);
329 if (!ret)
330 ret = err;
331 return ret;
332}
333
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700334static long do_fsync(unsigned int fd, int datasync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
336 struct file * file;
337 struct address_space *mapping;
338 int ret, err;
339
340 ret = -EBADF;
341 file = fget(fd);
342 if (!file)
343 goto out;
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 ret = -EINVAL;
346 if (!file->f_op || !file->f_op->fsync) {
347 /* Why? We can still call filemap_fdatawrite */
348 goto out_putf;
349 }
350
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700351 mapping = file->f_mapping;
352
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 current->flags |= PF_SYNCWRITE;
354 ret = filemap_fdatawrite(mapping);
355
356 /*
357 * We need to protect against concurrent writers,
358 * which could cause livelocks in fsync_buffers_list
359 */
360 down(&mapping->host->i_sem);
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700361 err = file->f_op->fsync(file, file->f_dentry, datasync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 if (!ret)
363 ret = err;
364 up(&mapping->host->i_sem);
365 err = filemap_fdatawait(mapping);
366 if (!ret)
367 ret = err;
368 current->flags &= ~PF_SYNCWRITE;
369
370out_putf:
371 fput(file);
372out:
373 return ret;
374}
375
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700376asmlinkage long sys_fsync(unsigned int fd)
377{
378 return do_fsync(fd, 0);
379}
380
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381asmlinkage long sys_fdatasync(unsigned int fd)
382{
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700383 return do_fsync(fd, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385
386/*
387 * Various filesystems appear to want __find_get_block to be non-blocking.
388 * But it's the page lock which protects the buffers. To get around this,
389 * we get exclusion from try_to_free_buffers with the blockdev mapping's
390 * private_lock.
391 *
392 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
393 * may be quite high. This code could TryLock the page, and if that
394 * succeeds, there is no need to take private_lock. (But if
395 * private_lock is contended then so is mapping->tree_lock).
396 */
397static struct buffer_head *
398__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
399{
400 struct inode *bd_inode = bdev->bd_inode;
401 struct address_space *bd_mapping = bd_inode->i_mapping;
402 struct buffer_head *ret = NULL;
403 pgoff_t index;
404 struct buffer_head *bh;
405 struct buffer_head *head;
406 struct page *page;
407 int all_mapped = 1;
408
409 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
410 page = find_get_page(bd_mapping, index);
411 if (!page)
412 goto out;
413
414 spin_lock(&bd_mapping->private_lock);
415 if (!page_has_buffers(page))
416 goto out_unlock;
417 head = page_buffers(page);
418 bh = head;
419 do {
420 if (bh->b_blocknr == block) {
421 ret = bh;
422 get_bh(bh);
423 goto out_unlock;
424 }
425 if (!buffer_mapped(bh))
426 all_mapped = 0;
427 bh = bh->b_this_page;
428 } while (bh != head);
429
430 /* we might be here because some of the buffers on this page are
431 * not mapped. This is due to various races between
432 * file io on the block device and getblk. It gets dealt with
433 * elsewhere, don't buffer_error if we had some unmapped buffers
434 */
435 if (all_mapped) {
436 printk("__find_get_block_slow() failed. "
437 "block=%llu, b_blocknr=%llu\n",
438 (unsigned long long)block, (unsigned long long)bh->b_blocknr);
439 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
440 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
441 }
442out_unlock:
443 spin_unlock(&bd_mapping->private_lock);
444 page_cache_release(page);
445out:
446 return ret;
447}
448
449/* If invalidate_buffers() will trash dirty buffers, it means some kind
450 of fs corruption is going on. Trashing dirty data always imply losing
451 information that was supposed to be just stored on the physical layer
452 by the user.
453
454 Thus invalidate_buffers in general usage is not allwowed to trash
455 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
456 be preserved. These buffers are simply skipped.
457
458 We also skip buffers which are still in use. For example this can
459 happen if a userspace program is reading the block device.
460
461 NOTE: In the case where the user removed a removable-media-disk even if
462 there's still dirty data not synced on disk (due a bug in the device driver
463 or due an error of the user), by not destroying the dirty buffers we could
464 generate corruption also on the next media inserted, thus a parameter is
465 necessary to handle this case in the most safe way possible (trying
466 to not corrupt also the new disk inserted with the data belonging to
467 the old now corrupted disk). Also for the ramdisk the natural thing
468 to do in order to release the ramdisk memory is to destroy dirty buffers.
469
470 These are two special cases. Normal usage imply the device driver
471 to issue a sync on the device (without waiting I/O completion) and
472 then an invalidate_buffers call that doesn't trash dirty buffers.
473
474 For handling cache coherency with the blkdev pagecache the 'update' case
475 is been introduced. It is needed to re-read from disk any pinned
476 buffer. NOTE: re-reading from disk is destructive so we can do it only
477 when we assume nobody is changing the buffercache under our I/O and when
478 we think the disk contains more recent information than the buffercache.
479 The update == 1 pass marks the buffers we need to update, the update == 2
480 pass does the actual I/O. */
481void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
482{
483 invalidate_bh_lrus();
484 /*
485 * FIXME: what about destroy_dirty_buffers?
486 * We really want to use invalidate_inode_pages2() for
487 * that, but not until that's cleaned up.
488 */
489 invalidate_inode_pages(bdev->bd_inode->i_mapping);
490}
491
492/*
493 * Kick pdflush then try to free up some ZONE_NORMAL memory.
494 */
495static void free_more_memory(void)
496{
497 struct zone **zones;
498 pg_data_t *pgdat;
499
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700500 wakeup_pdflush(1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 yield();
502
503 for_each_pgdat(pgdat) {
504 zones = pgdat->node_zonelists[GFP_NOFS&GFP_ZONEMASK].zones;
505 if (*zones)
Darren Hart1ad539b2005-06-21 17:14:53 -0700506 try_to_free_pages(zones, GFP_NOFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 }
508}
509
510/*
511 * I/O completion handler for block_read_full_page() - pages
512 * which come unlocked at the end of I/O.
513 */
514static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
515{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700517 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 struct buffer_head *tmp;
519 struct page *page;
520 int page_uptodate = 1;
521
522 BUG_ON(!buffer_async_read(bh));
523
524 page = bh->b_page;
525 if (uptodate) {
526 set_buffer_uptodate(bh);
527 } else {
528 clear_buffer_uptodate(bh);
529 if (printk_ratelimit())
530 buffer_io_error(bh);
531 SetPageError(page);
532 }
533
534 /*
535 * Be _very_ careful from here on. Bad things can happen if
536 * two buffer heads end IO at almost the same time and both
537 * decide that the page is now completely done.
538 */
Nick Piggina3972202005-07-07 17:56:56 -0700539 first = page_buffers(page);
540 local_irq_save(flags);
541 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 clear_buffer_async_read(bh);
543 unlock_buffer(bh);
544 tmp = bh;
545 do {
546 if (!buffer_uptodate(tmp))
547 page_uptodate = 0;
548 if (buffer_async_read(tmp)) {
549 BUG_ON(!buffer_locked(tmp));
550 goto still_busy;
551 }
552 tmp = tmp->b_this_page;
553 } while (tmp != bh);
Nick Piggina3972202005-07-07 17:56:56 -0700554 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
555 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
557 /*
558 * If none of the buffers had errors and they are all
559 * uptodate then we can set the page uptodate.
560 */
561 if (page_uptodate && !PageError(page))
562 SetPageUptodate(page);
563 unlock_page(page);
564 return;
565
566still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700567 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
568 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 return;
570}
571
572/*
573 * Completion handler for block_write_full_page() - pages which are unlocked
574 * during I/O, and which have PageWriteback cleared upon I/O completion.
575 */
576void end_buffer_async_write(struct buffer_head *bh, int uptodate)
577{
578 char b[BDEVNAME_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700580 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 struct buffer_head *tmp;
582 struct page *page;
583
584 BUG_ON(!buffer_async_write(bh));
585
586 page = bh->b_page;
587 if (uptodate) {
588 set_buffer_uptodate(bh);
589 } else {
590 if (printk_ratelimit()) {
591 buffer_io_error(bh);
592 printk(KERN_WARNING "lost page write due to "
593 "I/O error on %s\n",
594 bdevname(bh->b_bdev, b));
595 }
596 set_bit(AS_EIO, &page->mapping->flags);
597 clear_buffer_uptodate(bh);
598 SetPageError(page);
599 }
600
Nick Piggina3972202005-07-07 17:56:56 -0700601 first = page_buffers(page);
602 local_irq_save(flags);
603 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
604
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 clear_buffer_async_write(bh);
606 unlock_buffer(bh);
607 tmp = bh->b_this_page;
608 while (tmp != bh) {
609 if (buffer_async_write(tmp)) {
610 BUG_ON(!buffer_locked(tmp));
611 goto still_busy;
612 }
613 tmp = tmp->b_this_page;
614 }
Nick Piggina3972202005-07-07 17:56:56 -0700615 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
616 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 end_page_writeback(page);
618 return;
619
620still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700621 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
622 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 return;
624}
625
626/*
627 * If a page's buffers are under async readin (end_buffer_async_read
628 * completion) then there is a possibility that another thread of
629 * control could lock one of the buffers after it has completed
630 * but while some of the other buffers have not completed. This
631 * locked buffer would confuse end_buffer_async_read() into not unlocking
632 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
633 * that this buffer is not under async I/O.
634 *
635 * The page comes unlocked when it has no locked buffer_async buffers
636 * left.
637 *
638 * PageLocked prevents anyone starting new async I/O reads any of
639 * the buffers.
640 *
641 * PageWriteback is used to prevent simultaneous writeout of the same
642 * page.
643 *
644 * PageLocked prevents anyone from starting writeback of a page which is
645 * under read I/O (PageWriteback is only ever set against a locked page).
646 */
647static void mark_buffer_async_read(struct buffer_head *bh)
648{
649 bh->b_end_io = end_buffer_async_read;
650 set_buffer_async_read(bh);
651}
652
653void mark_buffer_async_write(struct buffer_head *bh)
654{
655 bh->b_end_io = end_buffer_async_write;
656 set_buffer_async_write(bh);
657}
658EXPORT_SYMBOL(mark_buffer_async_write);
659
660
661/*
662 * fs/buffer.c contains helper functions for buffer-backed address space's
663 * fsync functions. A common requirement for buffer-based filesystems is
664 * that certain data from the backing blockdev needs to be written out for
665 * a successful fsync(). For example, ext2 indirect blocks need to be
666 * written back and waited upon before fsync() returns.
667 *
668 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
669 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
670 * management of a list of dependent buffers at ->i_mapping->private_list.
671 *
672 * Locking is a little subtle: try_to_free_buffers() will remove buffers
673 * from their controlling inode's queue when they are being freed. But
674 * try_to_free_buffers() will be operating against the *blockdev* mapping
675 * at the time, not against the S_ISREG file which depends on those buffers.
676 * So the locking for private_list is via the private_lock in the address_space
677 * which backs the buffers. Which is different from the address_space
678 * against which the buffers are listed. So for a particular address_space,
679 * mapping->private_lock does *not* protect mapping->private_list! In fact,
680 * mapping->private_list will always be protected by the backing blockdev's
681 * ->private_lock.
682 *
683 * Which introduces a requirement: all buffers on an address_space's
684 * ->private_list must be from the same address_space: the blockdev's.
685 *
686 * address_spaces which do not place buffers at ->private_list via these
687 * utility functions are free to use private_lock and private_list for
688 * whatever they want. The only requirement is that list_empty(private_list)
689 * be true at clear_inode() time.
690 *
691 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
692 * filesystems should do that. invalidate_inode_buffers() should just go
693 * BUG_ON(!list_empty).
694 *
695 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
696 * take an address_space, not an inode. And it should be called
697 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
698 * queued up.
699 *
700 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
701 * list if it is already on a list. Because if the buffer is on a list,
702 * it *must* already be on the right one. If not, the filesystem is being
703 * silly. This will save a ton of locking. But first we have to ensure
704 * that buffers are taken *off* the old inode's list when they are freed
705 * (presumably in truncate). That requires careful auditing of all
706 * filesystems (do it inside bforget()). It could also be done by bringing
707 * b_inode back.
708 */
709
710/*
711 * The buffer's backing address_space's private_lock must be held
712 */
713static inline void __remove_assoc_queue(struct buffer_head *bh)
714{
715 list_del_init(&bh->b_assoc_buffers);
716}
717
718int inode_has_buffers(struct inode *inode)
719{
720 return !list_empty(&inode->i_data.private_list);
721}
722
723/*
724 * osync is designed to support O_SYNC io. It waits synchronously for
725 * all already-submitted IO to complete, but does not queue any new
726 * writes to the disk.
727 *
728 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
729 * you dirty the buffers, and then use osync_inode_buffers to wait for
730 * completion. Any other dirty buffers which are not yet queued for
731 * write will not be flushed to disk by the osync.
732 */
733static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
734{
735 struct buffer_head *bh;
736 struct list_head *p;
737 int err = 0;
738
739 spin_lock(lock);
740repeat:
741 list_for_each_prev(p, list) {
742 bh = BH_ENTRY(p);
743 if (buffer_locked(bh)) {
744 get_bh(bh);
745 spin_unlock(lock);
746 wait_on_buffer(bh);
747 if (!buffer_uptodate(bh))
748 err = -EIO;
749 brelse(bh);
750 spin_lock(lock);
751 goto repeat;
752 }
753 }
754 spin_unlock(lock);
755 return err;
756}
757
758/**
759 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
760 * buffers
Martin Waitz67be2dd2005-05-01 08:59:26 -0700761 * @mapping: the mapping which wants those buffers written
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 *
763 * Starts I/O against the buffers at mapping->private_list, and waits upon
764 * that I/O.
765 *
Martin Waitz67be2dd2005-05-01 08:59:26 -0700766 * Basically, this is a convenience function for fsync().
767 * @mapping is a file or directory which needs those buffers to be written for
768 * a successful fsync().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 */
770int sync_mapping_buffers(struct address_space *mapping)
771{
772 struct address_space *buffer_mapping = mapping->assoc_mapping;
773
774 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
775 return 0;
776
777 return fsync_buffers_list(&buffer_mapping->private_lock,
778 &mapping->private_list);
779}
780EXPORT_SYMBOL(sync_mapping_buffers);
781
782/*
783 * Called when we've recently written block `bblock', and it is known that
784 * `bblock' was for a buffer_boundary() buffer. This means that the block at
785 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
786 * dirty, schedule it for IO. So that indirects merge nicely with their data.
787 */
788void write_boundary_block(struct block_device *bdev,
789 sector_t bblock, unsigned blocksize)
790{
791 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
792 if (bh) {
793 if (buffer_dirty(bh))
794 ll_rw_block(WRITE, 1, &bh);
795 put_bh(bh);
796 }
797}
798
799void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
800{
801 struct address_space *mapping = inode->i_mapping;
802 struct address_space *buffer_mapping = bh->b_page->mapping;
803
804 mark_buffer_dirty(bh);
805 if (!mapping->assoc_mapping) {
806 mapping->assoc_mapping = buffer_mapping;
807 } else {
808 if (mapping->assoc_mapping != buffer_mapping)
809 BUG();
810 }
811 if (list_empty(&bh->b_assoc_buffers)) {
812 spin_lock(&buffer_mapping->private_lock);
813 list_move_tail(&bh->b_assoc_buffers,
814 &mapping->private_list);
815 spin_unlock(&buffer_mapping->private_lock);
816 }
817}
818EXPORT_SYMBOL(mark_buffer_dirty_inode);
819
820/*
821 * Add a page to the dirty page list.
822 *
823 * It is a sad fact of life that this function is called from several places
824 * deeply under spinlocking. It may not sleep.
825 *
826 * If the page has buffers, the uptodate buffers are set dirty, to preserve
827 * dirty-state coherency between the page and the buffers. It the page does
828 * not have buffers then when they are later attached they will all be set
829 * dirty.
830 *
831 * The buffers are dirtied before the page is dirtied. There's a small race
832 * window in which a writepage caller may see the page cleanness but not the
833 * buffer dirtiness. That's fine. If this code were to set the page dirty
834 * before the buffers, a concurrent writepage caller could clear the page dirty
835 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
836 * page on the dirty page list.
837 *
838 * We use private_lock to lock against try_to_free_buffers while using the
839 * page's buffer list. Also use this to protect against clean buffers being
840 * added to the page after it was set dirty.
841 *
842 * FIXME: may need to call ->reservepage here as well. That's rather up to the
843 * address_space though.
844 */
845int __set_page_dirty_buffers(struct page *page)
846{
847 struct address_space * const mapping = page->mapping;
848
849 spin_lock(&mapping->private_lock);
850 if (page_has_buffers(page)) {
851 struct buffer_head *head = page_buffers(page);
852 struct buffer_head *bh = head;
853
854 do {
855 set_buffer_dirty(bh);
856 bh = bh->b_this_page;
857 } while (bh != head);
858 }
859 spin_unlock(&mapping->private_lock);
860
861 if (!TestSetPageDirty(page)) {
862 write_lock_irq(&mapping->tree_lock);
863 if (page->mapping) { /* Race with truncate? */
864 if (mapping_cap_account_dirty(mapping))
865 inc_page_state(nr_dirty);
866 radix_tree_tag_set(&mapping->page_tree,
867 page_index(page),
868 PAGECACHE_TAG_DIRTY);
869 }
870 write_unlock_irq(&mapping->tree_lock);
871 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
872 }
873
874 return 0;
875}
876EXPORT_SYMBOL(__set_page_dirty_buffers);
877
878/*
879 * Write out and wait upon a list of buffers.
880 *
881 * We have conflicting pressures: we want to make sure that all
882 * initially dirty buffers get waited on, but that any subsequently
883 * dirtied buffers don't. After all, we don't want fsync to last
884 * forever if somebody is actively writing to the file.
885 *
886 * Do this in two main stages: first we copy dirty buffers to a
887 * temporary inode list, queueing the writes as we go. Then we clean
888 * up, waiting for those writes to complete.
889 *
890 * During this second stage, any subsequent updates to the file may end
891 * up refiling the buffer on the original inode's dirty list again, so
892 * there is a chance we will end up with a buffer queued for write but
893 * not yet completed on that list. So, as a final cleanup we go through
894 * the osync code to catch these locked, dirty buffers without requeuing
895 * any newly dirty buffers for write.
896 */
897static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
898{
899 struct buffer_head *bh;
900 struct list_head tmp;
901 int err = 0, err2;
902
903 INIT_LIST_HEAD(&tmp);
904
905 spin_lock(lock);
906 while (!list_empty(list)) {
907 bh = BH_ENTRY(list->next);
908 list_del_init(&bh->b_assoc_buffers);
909 if (buffer_dirty(bh) || buffer_locked(bh)) {
910 list_add(&bh->b_assoc_buffers, &tmp);
911 if (buffer_dirty(bh)) {
912 get_bh(bh);
913 spin_unlock(lock);
914 /*
915 * Ensure any pending I/O completes so that
916 * ll_rw_block() actually writes the current
917 * contents - it is a noop if I/O is still in
918 * flight on potentially older contents.
919 */
Jan Karaa7662232005-09-06 15:19:10 -0700920 ll_rw_block(SWRITE, 1, &bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921 brelse(bh);
922 spin_lock(lock);
923 }
924 }
925 }
926
927 while (!list_empty(&tmp)) {
928 bh = BH_ENTRY(tmp.prev);
929 __remove_assoc_queue(bh);
930 get_bh(bh);
931 spin_unlock(lock);
932 wait_on_buffer(bh);
933 if (!buffer_uptodate(bh))
934 err = -EIO;
935 brelse(bh);
936 spin_lock(lock);
937 }
938
939 spin_unlock(lock);
940 err2 = osync_buffers_list(lock, list);
941 if (err)
942 return err;
943 else
944 return err2;
945}
946
947/*
948 * Invalidate any and all dirty buffers on a given inode. We are
949 * probably unmounting the fs, but that doesn't mean we have already
950 * done a sync(). Just drop the buffers from the inode list.
951 *
952 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
953 * assumes that all the buffers are against the blockdev. Not true
954 * for reiserfs.
955 */
956void invalidate_inode_buffers(struct inode *inode)
957{
958 if (inode_has_buffers(inode)) {
959 struct address_space *mapping = &inode->i_data;
960 struct list_head *list = &mapping->private_list;
961 struct address_space *buffer_mapping = mapping->assoc_mapping;
962
963 spin_lock(&buffer_mapping->private_lock);
964 while (!list_empty(list))
965 __remove_assoc_queue(BH_ENTRY(list->next));
966 spin_unlock(&buffer_mapping->private_lock);
967 }
968}
969
970/*
971 * Remove any clean buffers from the inode's buffer list. This is called
972 * when we're trying to free the inode itself. Those buffers can pin it.
973 *
974 * Returns true if all buffers were removed.
975 */
976int remove_inode_buffers(struct inode *inode)
977{
978 int ret = 1;
979
980 if (inode_has_buffers(inode)) {
981 struct address_space *mapping = &inode->i_data;
982 struct list_head *list = &mapping->private_list;
983 struct address_space *buffer_mapping = mapping->assoc_mapping;
984
985 spin_lock(&buffer_mapping->private_lock);
986 while (!list_empty(list)) {
987 struct buffer_head *bh = BH_ENTRY(list->next);
988 if (buffer_dirty(bh)) {
989 ret = 0;
990 break;
991 }
992 __remove_assoc_queue(bh);
993 }
994 spin_unlock(&buffer_mapping->private_lock);
995 }
996 return ret;
997}
998
999/*
1000 * Create the appropriate buffers when given a page for data area and
1001 * the size of each buffer.. Use the bh->b_this_page linked list to
1002 * follow the buffers created. Return NULL if unable to create more
1003 * buffers.
1004 *
1005 * The retry flag is used to differentiate async IO (paging, swapping)
1006 * which may not fail from ordinary buffer allocations.
1007 */
1008struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1009 int retry)
1010{
1011 struct buffer_head *bh, *head;
1012 long offset;
1013
1014try_again:
1015 head = NULL;
1016 offset = PAGE_SIZE;
1017 while ((offset -= size) >= 0) {
1018 bh = alloc_buffer_head(GFP_NOFS);
1019 if (!bh)
1020 goto no_grow;
1021
1022 bh->b_bdev = NULL;
1023 bh->b_this_page = head;
1024 bh->b_blocknr = -1;
1025 head = bh;
1026
1027 bh->b_state = 0;
1028 atomic_set(&bh->b_count, 0);
1029 bh->b_size = size;
1030
1031 /* Link the buffer to its page */
1032 set_bh_page(bh, page, offset);
1033
1034 bh->b_end_io = NULL;
1035 }
1036 return head;
1037/*
1038 * In case anything failed, we just free everything we got.
1039 */
1040no_grow:
1041 if (head) {
1042 do {
1043 bh = head;
1044 head = head->b_this_page;
1045 free_buffer_head(bh);
1046 } while (head);
1047 }
1048
1049 /*
1050 * Return failure for non-async IO requests. Async IO requests
1051 * are not allowed to fail, so we have to wait until buffer heads
1052 * become available. But we don't want tasks sleeping with
1053 * partially complete buffers, so all were released above.
1054 */
1055 if (!retry)
1056 return NULL;
1057
1058 /* We're _really_ low on memory. Now we just
1059 * wait for old buffer heads to become free due to
1060 * finishing IO. Since this is an async request and
1061 * the reserve list is empty, we're sure there are
1062 * async buffer heads in use.
1063 */
1064 free_more_memory();
1065 goto try_again;
1066}
1067EXPORT_SYMBOL_GPL(alloc_page_buffers);
1068
1069static inline void
1070link_dev_buffers(struct page *page, struct buffer_head *head)
1071{
1072 struct buffer_head *bh, *tail;
1073
1074 bh = head;
1075 do {
1076 tail = bh;
1077 bh = bh->b_this_page;
1078 } while (bh);
1079 tail->b_this_page = head;
1080 attach_page_buffers(page, head);
1081}
1082
1083/*
1084 * Initialise the state of a blockdev page's buffers.
1085 */
1086static void
1087init_page_buffers(struct page *page, struct block_device *bdev,
1088 sector_t block, int size)
1089{
1090 struct buffer_head *head = page_buffers(page);
1091 struct buffer_head *bh = head;
1092 int uptodate = PageUptodate(page);
1093
1094 do {
1095 if (!buffer_mapped(bh)) {
1096 init_buffer(bh, NULL, NULL);
1097 bh->b_bdev = bdev;
1098 bh->b_blocknr = block;
1099 if (uptodate)
1100 set_buffer_uptodate(bh);
1101 set_buffer_mapped(bh);
1102 }
1103 block++;
1104 bh = bh->b_this_page;
1105 } while (bh != head);
1106}
1107
1108/*
1109 * Create the page-cache page that contains the requested block.
1110 *
1111 * This is user purely for blockdev mappings.
1112 */
1113static struct page *
1114grow_dev_page(struct block_device *bdev, sector_t block,
1115 pgoff_t index, int size)
1116{
1117 struct inode *inode = bdev->bd_inode;
1118 struct page *page;
1119 struct buffer_head *bh;
1120
1121 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1122 if (!page)
1123 return NULL;
1124
1125 if (!PageLocked(page))
1126 BUG();
1127
1128 if (page_has_buffers(page)) {
1129 bh = page_buffers(page);
1130 if (bh->b_size == size) {
1131 init_page_buffers(page, bdev, block, size);
1132 return page;
1133 }
1134 if (!try_to_free_buffers(page))
1135 goto failed;
1136 }
1137
1138 /*
1139 * Allocate some buffers for this page
1140 */
1141 bh = alloc_page_buffers(page, size, 0);
1142 if (!bh)
1143 goto failed;
1144
1145 /*
1146 * Link the page to the buffers and initialise them. Take the
1147 * lock to be atomic wrt __find_get_block(), which does not
1148 * run under the page lock.
1149 */
1150 spin_lock(&inode->i_mapping->private_lock);
1151 link_dev_buffers(page, bh);
1152 init_page_buffers(page, bdev, block, size);
1153 spin_unlock(&inode->i_mapping->private_lock);
1154 return page;
1155
1156failed:
1157 BUG();
1158 unlock_page(page);
1159 page_cache_release(page);
1160 return NULL;
1161}
1162
1163/*
1164 * Create buffers for the specified block device block's page. If
1165 * that page was dirty, the buffers are set dirty also.
1166 *
1167 * Except that's a bug. Attaching dirty buffers to a dirty
1168 * blockdev's page can result in filesystem corruption, because
1169 * some of those buffers may be aliases of filesystem data.
1170 * grow_dev_page() will go BUG() if this happens.
1171 */
1172static inline int
1173grow_buffers(struct block_device *bdev, sector_t block, int size)
1174{
1175 struct page *page;
1176 pgoff_t index;
1177 int sizebits;
1178
1179 sizebits = -1;
1180 do {
1181 sizebits++;
1182 } while ((size << sizebits) < PAGE_SIZE);
1183
1184 index = block >> sizebits;
1185 block = index << sizebits;
1186
1187 /* Create a page with the proper size buffers.. */
1188 page = grow_dev_page(bdev, block, index, size);
1189 if (!page)
1190 return 0;
1191 unlock_page(page);
1192 page_cache_release(page);
1193 return 1;
1194}
1195
Adrian Bunk75c96f82005-05-05 16:16:09 -07001196static struct buffer_head *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197__getblk_slow(struct block_device *bdev, sector_t block, int size)
1198{
1199 /* Size must be multiple of hard sectorsize */
1200 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1201 (size < 512 || size > PAGE_SIZE))) {
1202 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1203 size);
1204 printk(KERN_ERR "hardsect size: %d\n",
1205 bdev_hardsect_size(bdev));
1206
1207 dump_stack();
1208 return NULL;
1209 }
1210
1211 for (;;) {
1212 struct buffer_head * bh;
1213
1214 bh = __find_get_block(bdev, block, size);
1215 if (bh)
1216 return bh;
1217
1218 if (!grow_buffers(bdev, block, size))
1219 free_more_memory();
1220 }
1221}
1222
1223/*
1224 * The relationship between dirty buffers and dirty pages:
1225 *
1226 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1227 * the page is tagged dirty in its radix tree.
1228 *
1229 * At all times, the dirtiness of the buffers represents the dirtiness of
1230 * subsections of the page. If the page has buffers, the page dirty bit is
1231 * merely a hint about the true dirty state.
1232 *
1233 * When a page is set dirty in its entirety, all its buffers are marked dirty
1234 * (if the page has buffers).
1235 *
1236 * When a buffer is marked dirty, its page is dirtied, but the page's other
1237 * buffers are not.
1238 *
1239 * Also. When blockdev buffers are explicitly read with bread(), they
1240 * individually become uptodate. But their backing page remains not
1241 * uptodate - even if all of its buffers are uptodate. A subsequent
1242 * block_read_full_page() against that page will discover all the uptodate
1243 * buffers, will set the page uptodate and will perform no I/O.
1244 */
1245
1246/**
1247 * mark_buffer_dirty - mark a buffer_head as needing writeout
Martin Waitz67be2dd2005-05-01 08:59:26 -07001248 * @bh: the buffer_head to mark dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 *
1250 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1251 * backing page dirty, then tag the page as dirty in its address_space's radix
1252 * tree and then attach the address_space's inode to its superblock's dirty
1253 * inode list.
1254 *
1255 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1256 * mapping->tree_lock and the global inode_lock.
1257 */
1258void fastcall mark_buffer_dirty(struct buffer_head *bh)
1259{
1260 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1261 __set_page_dirty_nobuffers(bh->b_page);
1262}
1263
1264/*
1265 * Decrement a buffer_head's reference count. If all buffers against a page
1266 * have zero reference count, are clean and unlocked, and if the page is clean
1267 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1268 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1269 * a page but it ends up not being freed, and buffers may later be reattached).
1270 */
1271void __brelse(struct buffer_head * buf)
1272{
1273 if (atomic_read(&buf->b_count)) {
1274 put_bh(buf);
1275 return;
1276 }
1277 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1278 WARN_ON(1);
1279}
1280
1281/*
1282 * bforget() is like brelse(), except it discards any
1283 * potentially dirty data.
1284 */
1285void __bforget(struct buffer_head *bh)
1286{
1287 clear_buffer_dirty(bh);
1288 if (!list_empty(&bh->b_assoc_buffers)) {
1289 struct address_space *buffer_mapping = bh->b_page->mapping;
1290
1291 spin_lock(&buffer_mapping->private_lock);
1292 list_del_init(&bh->b_assoc_buffers);
1293 spin_unlock(&buffer_mapping->private_lock);
1294 }
1295 __brelse(bh);
1296}
1297
1298static struct buffer_head *__bread_slow(struct buffer_head *bh)
1299{
1300 lock_buffer(bh);
1301 if (buffer_uptodate(bh)) {
1302 unlock_buffer(bh);
1303 return bh;
1304 } else {
1305 get_bh(bh);
1306 bh->b_end_io = end_buffer_read_sync;
1307 submit_bh(READ, bh);
1308 wait_on_buffer(bh);
1309 if (buffer_uptodate(bh))
1310 return bh;
1311 }
1312 brelse(bh);
1313 return NULL;
1314}
1315
1316/*
1317 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1318 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1319 * refcount elevated by one when they're in an LRU. A buffer can only appear
1320 * once in a particular CPU's LRU. A single buffer can be present in multiple
1321 * CPU's LRUs at the same time.
1322 *
1323 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1324 * sb_find_get_block().
1325 *
1326 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1327 * a local interrupt disable for that.
1328 */
1329
1330#define BH_LRU_SIZE 8
1331
1332struct bh_lru {
1333 struct buffer_head *bhs[BH_LRU_SIZE];
1334};
1335
1336static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1337
1338#ifdef CONFIG_SMP
1339#define bh_lru_lock() local_irq_disable()
1340#define bh_lru_unlock() local_irq_enable()
1341#else
1342#define bh_lru_lock() preempt_disable()
1343#define bh_lru_unlock() preempt_enable()
1344#endif
1345
1346static inline void check_irqs_on(void)
1347{
1348#ifdef irqs_disabled
1349 BUG_ON(irqs_disabled());
1350#endif
1351}
1352
1353/*
1354 * The LRU management algorithm is dopey-but-simple. Sorry.
1355 */
1356static void bh_lru_install(struct buffer_head *bh)
1357{
1358 struct buffer_head *evictee = NULL;
1359 struct bh_lru *lru;
1360
1361 check_irqs_on();
1362 bh_lru_lock();
1363 lru = &__get_cpu_var(bh_lrus);
1364 if (lru->bhs[0] != bh) {
1365 struct buffer_head *bhs[BH_LRU_SIZE];
1366 int in;
1367 int out = 0;
1368
1369 get_bh(bh);
1370 bhs[out++] = bh;
1371 for (in = 0; in < BH_LRU_SIZE; in++) {
1372 struct buffer_head *bh2 = lru->bhs[in];
1373
1374 if (bh2 == bh) {
1375 __brelse(bh2);
1376 } else {
1377 if (out >= BH_LRU_SIZE) {
1378 BUG_ON(evictee != NULL);
1379 evictee = bh2;
1380 } else {
1381 bhs[out++] = bh2;
1382 }
1383 }
1384 }
1385 while (out < BH_LRU_SIZE)
1386 bhs[out++] = NULL;
1387 memcpy(lru->bhs, bhs, sizeof(bhs));
1388 }
1389 bh_lru_unlock();
1390
1391 if (evictee)
1392 __brelse(evictee);
1393}
1394
1395/*
1396 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1397 */
1398static inline struct buffer_head *
1399lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1400{
1401 struct buffer_head *ret = NULL;
1402 struct bh_lru *lru;
1403 int i;
1404
1405 check_irqs_on();
1406 bh_lru_lock();
1407 lru = &__get_cpu_var(bh_lrus);
1408 for (i = 0; i < BH_LRU_SIZE; i++) {
1409 struct buffer_head *bh = lru->bhs[i];
1410
1411 if (bh && bh->b_bdev == bdev &&
1412 bh->b_blocknr == block && bh->b_size == size) {
1413 if (i) {
1414 while (i) {
1415 lru->bhs[i] = lru->bhs[i - 1];
1416 i--;
1417 }
1418 lru->bhs[0] = bh;
1419 }
1420 get_bh(bh);
1421 ret = bh;
1422 break;
1423 }
1424 }
1425 bh_lru_unlock();
1426 return ret;
1427}
1428
1429/*
1430 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1431 * it in the LRU and mark it as accessed. If it is not present then return
1432 * NULL
1433 */
1434struct buffer_head *
1435__find_get_block(struct block_device *bdev, sector_t block, int size)
1436{
1437 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1438
1439 if (bh == NULL) {
1440 bh = __find_get_block_slow(bdev, block, size);
1441 if (bh)
1442 bh_lru_install(bh);
1443 }
1444 if (bh)
1445 touch_buffer(bh);
1446 return bh;
1447}
1448EXPORT_SYMBOL(__find_get_block);
1449
1450/*
1451 * __getblk will locate (and, if necessary, create) the buffer_head
1452 * which corresponds to the passed block_device, block and size. The
1453 * returned buffer has its reference count incremented.
1454 *
1455 * __getblk() cannot fail - it just keeps trying. If you pass it an
1456 * illegal block number, __getblk() will happily return a buffer_head
1457 * which represents the non-existent block. Very weird.
1458 *
1459 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1460 * attempt is failing. FIXME, perhaps?
1461 */
1462struct buffer_head *
1463__getblk(struct block_device *bdev, sector_t block, int size)
1464{
1465 struct buffer_head *bh = __find_get_block(bdev, block, size);
1466
1467 might_sleep();
1468 if (bh == NULL)
1469 bh = __getblk_slow(bdev, block, size);
1470 return bh;
1471}
1472EXPORT_SYMBOL(__getblk);
1473
1474/*
1475 * Do async read-ahead on a buffer..
1476 */
1477void __breadahead(struct block_device *bdev, sector_t block, int size)
1478{
1479 struct buffer_head *bh = __getblk(bdev, block, size);
1480 ll_rw_block(READA, 1, &bh);
1481 brelse(bh);
1482}
1483EXPORT_SYMBOL(__breadahead);
1484
1485/**
1486 * __bread() - reads a specified block and returns the bh
Martin Waitz67be2dd2005-05-01 08:59:26 -07001487 * @bdev: the block_device to read from
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 * @block: number of block
1489 * @size: size (in bytes) to read
1490 *
1491 * Reads a specified block, and returns buffer head that contains it.
1492 * It returns NULL if the block was unreadable.
1493 */
1494struct buffer_head *
1495__bread(struct block_device *bdev, sector_t block, int size)
1496{
1497 struct buffer_head *bh = __getblk(bdev, block, size);
1498
1499 if (!buffer_uptodate(bh))
1500 bh = __bread_slow(bh);
1501 return bh;
1502}
1503EXPORT_SYMBOL(__bread);
1504
1505/*
1506 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1507 * This doesn't race because it runs in each cpu either in irq
1508 * or with preempt disabled.
1509 */
1510static void invalidate_bh_lru(void *arg)
1511{
1512 struct bh_lru *b = &get_cpu_var(bh_lrus);
1513 int i;
1514
1515 for (i = 0; i < BH_LRU_SIZE; i++) {
1516 brelse(b->bhs[i]);
1517 b->bhs[i] = NULL;
1518 }
1519 put_cpu_var(bh_lrus);
1520}
1521
1522static void invalidate_bh_lrus(void)
1523{
1524 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1525}
1526
1527void set_bh_page(struct buffer_head *bh,
1528 struct page *page, unsigned long offset)
1529{
1530 bh->b_page = page;
1531 if (offset >= PAGE_SIZE)
1532 BUG();
1533 if (PageHighMem(page))
1534 /*
1535 * This catches illegal uses and preserves the offset:
1536 */
1537 bh->b_data = (char *)(0 + offset);
1538 else
1539 bh->b_data = page_address(page) + offset;
1540}
1541EXPORT_SYMBOL(set_bh_page);
1542
1543/*
1544 * Called when truncating a buffer on a page completely.
1545 */
1546static inline void discard_buffer(struct buffer_head * bh)
1547{
1548 lock_buffer(bh);
1549 clear_buffer_dirty(bh);
1550 bh->b_bdev = NULL;
1551 clear_buffer_mapped(bh);
1552 clear_buffer_req(bh);
1553 clear_buffer_new(bh);
1554 clear_buffer_delay(bh);
1555 unlock_buffer(bh);
1556}
1557
1558/**
1559 * try_to_release_page() - release old fs-specific metadata on a page
1560 *
1561 * @page: the page which the kernel is trying to free
1562 * @gfp_mask: memory allocation flags (and I/O mode)
1563 *
1564 * The address_space is to try to release any data against the page
1565 * (presumably at page->private). If the release was successful, return `1'.
1566 * Otherwise return zero.
1567 *
1568 * The @gfp_mask argument specifies whether I/O may be performed to release
1569 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1570 *
1571 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1572 */
1573int try_to_release_page(struct page *page, int gfp_mask)
1574{
1575 struct address_space * const mapping = page->mapping;
1576
1577 BUG_ON(!PageLocked(page));
1578 if (PageWriteback(page))
1579 return 0;
1580
1581 if (mapping && mapping->a_ops->releasepage)
1582 return mapping->a_ops->releasepage(page, gfp_mask);
1583 return try_to_free_buffers(page);
1584}
1585EXPORT_SYMBOL(try_to_release_page);
1586
1587/**
1588 * block_invalidatepage - invalidate part of all of a buffer-backed page
1589 *
1590 * @page: the page which is affected
1591 * @offset: the index of the truncation point
1592 *
1593 * block_invalidatepage() is called when all or part of the page has become
1594 * invalidatedby a truncate operation.
1595 *
1596 * block_invalidatepage() does not have to release all buffers, but it must
1597 * ensure that no dirty buffer is left outside @offset and that no I/O
1598 * is underway against any of the blocks which are outside the truncation
1599 * point. Because the caller is about to free (and possibly reuse) those
1600 * blocks on-disk.
1601 */
1602int block_invalidatepage(struct page *page, unsigned long offset)
1603{
1604 struct buffer_head *head, *bh, *next;
1605 unsigned int curr_off = 0;
1606 int ret = 1;
1607
1608 BUG_ON(!PageLocked(page));
1609 if (!page_has_buffers(page))
1610 goto out;
1611
1612 head = page_buffers(page);
1613 bh = head;
1614 do {
1615 unsigned int next_off = curr_off + bh->b_size;
1616 next = bh->b_this_page;
1617
1618 /*
1619 * is this block fully invalidated?
1620 */
1621 if (offset <= curr_off)
1622 discard_buffer(bh);
1623 curr_off = next_off;
1624 bh = next;
1625 } while (bh != head);
1626
1627 /*
1628 * We release buffers only if the entire page is being invalidated.
1629 * The get_block cached value has been unconditionally invalidated,
1630 * so real IO is not possible anymore.
1631 */
1632 if (offset == 0)
1633 ret = try_to_release_page(page, 0);
1634out:
1635 return ret;
1636}
1637EXPORT_SYMBOL(block_invalidatepage);
1638
1639/*
1640 * We attach and possibly dirty the buffers atomically wrt
1641 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1642 * is already excluded via the page lock.
1643 */
1644void create_empty_buffers(struct page *page,
1645 unsigned long blocksize, unsigned long b_state)
1646{
1647 struct buffer_head *bh, *head, *tail;
1648
1649 head = alloc_page_buffers(page, blocksize, 1);
1650 bh = head;
1651 do {
1652 bh->b_state |= b_state;
1653 tail = bh;
1654 bh = bh->b_this_page;
1655 } while (bh);
1656 tail->b_this_page = head;
1657
1658 spin_lock(&page->mapping->private_lock);
1659 if (PageUptodate(page) || PageDirty(page)) {
1660 bh = head;
1661 do {
1662 if (PageDirty(page))
1663 set_buffer_dirty(bh);
1664 if (PageUptodate(page))
1665 set_buffer_uptodate(bh);
1666 bh = bh->b_this_page;
1667 } while (bh != head);
1668 }
1669 attach_page_buffers(page, head);
1670 spin_unlock(&page->mapping->private_lock);
1671}
1672EXPORT_SYMBOL(create_empty_buffers);
1673
1674/*
1675 * We are taking a block for data and we don't want any output from any
1676 * buffer-cache aliases starting from return from that function and
1677 * until the moment when something will explicitly mark the buffer
1678 * dirty (hopefully that will not happen until we will free that block ;-)
1679 * We don't even need to mark it not-uptodate - nobody can expect
1680 * anything from a newly allocated buffer anyway. We used to used
1681 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1682 * don't want to mark the alias unmapped, for example - it would confuse
1683 * anyone who might pick it with bread() afterwards...
1684 *
1685 * Also.. Note that bforget() doesn't lock the buffer. So there can
1686 * be writeout I/O going on against recently-freed buffers. We don't
1687 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1688 * only if we really need to. That happens here.
1689 */
1690void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1691{
1692 struct buffer_head *old_bh;
1693
1694 might_sleep();
1695
1696 old_bh = __find_get_block_slow(bdev, block, 0);
1697 if (old_bh) {
1698 clear_buffer_dirty(old_bh);
1699 wait_on_buffer(old_bh);
1700 clear_buffer_req(old_bh);
1701 __brelse(old_bh);
1702 }
1703}
1704EXPORT_SYMBOL(unmap_underlying_metadata);
1705
1706/*
1707 * NOTE! All mapped/uptodate combinations are valid:
1708 *
1709 * Mapped Uptodate Meaning
1710 *
1711 * No No "unknown" - must do get_block()
1712 * No Yes "hole" - zero-filled
1713 * Yes No "allocated" - allocated on disk, not read in
1714 * Yes Yes "valid" - allocated and up-to-date in memory.
1715 *
1716 * "Dirty" is valid only with the last case (mapped+uptodate).
1717 */
1718
1719/*
1720 * While block_write_full_page is writing back the dirty buffers under
1721 * the page lock, whoever dirtied the buffers may decide to clean them
1722 * again at any time. We handle that by only looking at the buffer
1723 * state inside lock_buffer().
1724 *
1725 * If block_write_full_page() is called for regular writeback
1726 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1727 * locked buffer. This only can happen if someone has written the buffer
1728 * directly, with submit_bh(). At the address_space level PageWriteback
1729 * prevents this contention from occurring.
1730 */
1731static int __block_write_full_page(struct inode *inode, struct page *page,
1732 get_block_t *get_block, struct writeback_control *wbc)
1733{
1734 int err;
1735 sector_t block;
1736 sector_t last_block;
Andrew Mortonf0fbd5f2005-05-05 16:15:48 -07001737 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 int nr_underway = 0;
1739
1740 BUG_ON(!PageLocked(page));
1741
1742 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1743
1744 if (!page_has_buffers(page)) {
1745 create_empty_buffers(page, 1 << inode->i_blkbits,
1746 (1 << BH_Dirty)|(1 << BH_Uptodate));
1747 }
1748
1749 /*
1750 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1751 * here, and the (potentially unmapped) buffers may become dirty at
1752 * any time. If a buffer becomes dirty here after we've inspected it
1753 * then we just miss that fact, and the page stays dirty.
1754 *
1755 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1756 * handle that here by just cleaning them.
1757 */
1758
1759 block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1760 head = page_buffers(page);
1761 bh = head;
1762
1763 /*
1764 * Get all the dirty buffers mapped to disk addresses and
1765 * handle any aliases from the underlying blockdev's mapping.
1766 */
1767 do {
1768 if (block > last_block) {
1769 /*
1770 * mapped buffers outside i_size will occur, because
1771 * this page can be outside i_size when there is a
1772 * truncate in progress.
1773 */
1774 /*
1775 * The buffer was zeroed by block_write_full_page()
1776 */
1777 clear_buffer_dirty(bh);
1778 set_buffer_uptodate(bh);
1779 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1780 err = get_block(inode, block, bh, 1);
1781 if (err)
1782 goto recover;
1783 if (buffer_new(bh)) {
1784 /* blockdev mappings never come here */
1785 clear_buffer_new(bh);
1786 unmap_underlying_metadata(bh->b_bdev,
1787 bh->b_blocknr);
1788 }
1789 }
1790 bh = bh->b_this_page;
1791 block++;
1792 } while (bh != head);
1793
1794 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795 if (!buffer_mapped(bh))
1796 continue;
1797 /*
1798 * If it's a fully non-blocking write attempt and we cannot
1799 * lock the buffer then redirty the page. Note that this can
1800 * potentially cause a busy-wait loop from pdflush and kswapd
1801 * activity, but those code paths have their own higher-level
1802 * throttling.
1803 */
1804 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1805 lock_buffer(bh);
1806 } else if (test_set_buffer_locked(bh)) {
1807 redirty_page_for_writepage(wbc, page);
1808 continue;
1809 }
1810 if (test_clear_buffer_dirty(bh)) {
1811 mark_buffer_async_write(bh);
1812 } else {
1813 unlock_buffer(bh);
1814 }
1815 } while ((bh = bh->b_this_page) != head);
1816
1817 /*
1818 * The page and its buffers are protected by PageWriteback(), so we can
1819 * drop the bh refcounts early.
1820 */
1821 BUG_ON(PageWriteback(page));
1822 set_page_writeback(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823
1824 do {
1825 struct buffer_head *next = bh->b_this_page;
1826 if (buffer_async_write(bh)) {
1827 submit_bh(WRITE, bh);
1828 nr_underway++;
1829 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 bh = next;
1831 } while (bh != head);
Andrew Morton05937ba2005-05-05 16:15:47 -07001832 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833
1834 err = 0;
1835done:
1836 if (nr_underway == 0) {
1837 /*
1838 * The page was marked dirty, but the buffers were
1839 * clean. Someone wrote them back by hand with
1840 * ll_rw_block/submit_bh. A rare case.
1841 */
1842 int uptodate = 1;
1843 do {
1844 if (!buffer_uptodate(bh)) {
1845 uptodate = 0;
1846 break;
1847 }
1848 bh = bh->b_this_page;
1849 } while (bh != head);
1850 if (uptodate)
1851 SetPageUptodate(page);
1852 end_page_writeback(page);
1853 /*
1854 * The page and buffer_heads can be released at any time from
1855 * here on.
1856 */
1857 wbc->pages_skipped++; /* We didn't write this page */
1858 }
1859 return err;
1860
1861recover:
1862 /*
1863 * ENOSPC, or some other error. We may already have added some
1864 * blocks to the file, so we need to write these out to avoid
1865 * exposing stale data.
1866 * The page is currently locked and not marked for writeback
1867 */
1868 bh = head;
1869 /* Recovery: lock and submit the mapped buffers */
1870 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001871 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1872 lock_buffer(bh);
1873 mark_buffer_async_write(bh);
1874 } else {
1875 /*
1876 * The buffer may have been set dirty during
1877 * attachment to a dirty page.
1878 */
1879 clear_buffer_dirty(bh);
1880 }
1881 } while ((bh = bh->b_this_page) != head);
1882 SetPageError(page);
1883 BUG_ON(PageWriteback(page));
1884 set_page_writeback(page);
1885 unlock_page(page);
1886 do {
1887 struct buffer_head *next = bh->b_this_page;
1888 if (buffer_async_write(bh)) {
1889 clear_buffer_dirty(bh);
1890 submit_bh(WRITE, bh);
1891 nr_underway++;
1892 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893 bh = next;
1894 } while (bh != head);
1895 goto done;
1896}
1897
1898static int __block_prepare_write(struct inode *inode, struct page *page,
1899 unsigned from, unsigned to, get_block_t *get_block)
1900{
1901 unsigned block_start, block_end;
1902 sector_t block;
1903 int err = 0;
1904 unsigned blocksize, bbits;
1905 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1906
1907 BUG_ON(!PageLocked(page));
1908 BUG_ON(from > PAGE_CACHE_SIZE);
1909 BUG_ON(to > PAGE_CACHE_SIZE);
1910 BUG_ON(from > to);
1911
1912 blocksize = 1 << inode->i_blkbits;
1913 if (!page_has_buffers(page))
1914 create_empty_buffers(page, blocksize, 0);
1915 head = page_buffers(page);
1916
1917 bbits = inode->i_blkbits;
1918 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1919
1920 for(bh = head, block_start = 0; bh != head || !block_start;
1921 block++, block_start=block_end, bh = bh->b_this_page) {
1922 block_end = block_start + blocksize;
1923 if (block_end <= from || block_start >= to) {
1924 if (PageUptodate(page)) {
1925 if (!buffer_uptodate(bh))
1926 set_buffer_uptodate(bh);
1927 }
1928 continue;
1929 }
1930 if (buffer_new(bh))
1931 clear_buffer_new(bh);
1932 if (!buffer_mapped(bh)) {
1933 err = get_block(inode, block, bh, 1);
1934 if (err)
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001935 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001936 if (buffer_new(bh)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 unmap_underlying_metadata(bh->b_bdev,
1938 bh->b_blocknr);
1939 if (PageUptodate(page)) {
1940 set_buffer_uptodate(bh);
1941 continue;
1942 }
1943 if (block_end > to || block_start < from) {
1944 void *kaddr;
1945
1946 kaddr = kmap_atomic(page, KM_USER0);
1947 if (block_end > to)
1948 memset(kaddr+to, 0,
1949 block_end-to);
1950 if (block_start < from)
1951 memset(kaddr+block_start,
1952 0, from-block_start);
1953 flush_dcache_page(page);
1954 kunmap_atomic(kaddr, KM_USER0);
1955 }
1956 continue;
1957 }
1958 }
1959 if (PageUptodate(page)) {
1960 if (!buffer_uptodate(bh))
1961 set_buffer_uptodate(bh);
1962 continue;
1963 }
1964 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1965 (block_start < from || block_end > to)) {
1966 ll_rw_block(READ, 1, &bh);
1967 *wait_bh++=bh;
1968 }
1969 }
1970 /*
1971 * If we issued read requests - let them complete.
1972 */
1973 while(wait_bh > wait) {
1974 wait_on_buffer(*--wait_bh);
1975 if (!buffer_uptodate(*wait_bh))
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001976 err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 }
Anton Altaparmakov152becd2005-06-23 00:10:21 -07001978 if (!err) {
1979 bh = head;
1980 do {
1981 if (buffer_new(bh))
1982 clear_buffer_new(bh);
1983 } while ((bh = bh->b_this_page) != head);
1984 return 0;
1985 }
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001986 /* Error case: */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987 /*
1988 * Zero out any newly allocated blocks to avoid exposing stale
1989 * data. If BH_New is set, we know that the block was newly
1990 * allocated in the above loop.
1991 */
1992 bh = head;
1993 block_start = 0;
1994 do {
1995 block_end = block_start+blocksize;
1996 if (block_end <= from)
1997 goto next_bh;
1998 if (block_start >= to)
1999 break;
2000 if (buffer_new(bh)) {
2001 void *kaddr;
2002
2003 clear_buffer_new(bh);
2004 kaddr = kmap_atomic(page, KM_USER0);
2005 memset(kaddr+block_start, 0, bh->b_size);
2006 kunmap_atomic(kaddr, KM_USER0);
2007 set_buffer_uptodate(bh);
2008 mark_buffer_dirty(bh);
2009 }
2010next_bh:
2011 block_start = block_end;
2012 bh = bh->b_this_page;
2013 } while (bh != head);
2014 return err;
2015}
2016
2017static int __block_commit_write(struct inode *inode, struct page *page,
2018 unsigned from, unsigned to)
2019{
2020 unsigned block_start, block_end;
2021 int partial = 0;
2022 unsigned blocksize;
2023 struct buffer_head *bh, *head;
2024
2025 blocksize = 1 << inode->i_blkbits;
2026
2027 for(bh = head = page_buffers(page), block_start = 0;
2028 bh != head || !block_start;
2029 block_start=block_end, bh = bh->b_this_page) {
2030 block_end = block_start + blocksize;
2031 if (block_end <= from || block_start >= to) {
2032 if (!buffer_uptodate(bh))
2033 partial = 1;
2034 } else {
2035 set_buffer_uptodate(bh);
2036 mark_buffer_dirty(bh);
2037 }
2038 }
2039
2040 /*
2041 * If this is a partial write which happened to make all buffers
2042 * uptodate then we can optimize away a bogus readpage() for
2043 * the next read(). Here we 'discover' whether the page went
2044 * uptodate as a result of this (potentially partial) write.
2045 */
2046 if (!partial)
2047 SetPageUptodate(page);
2048 return 0;
2049}
2050
2051/*
2052 * Generic "read page" function for block devices that have the normal
2053 * get_block functionality. This is most of the block device filesystems.
2054 * Reads the page asynchronously --- the unlock_buffer() and
2055 * set/clear_buffer_uptodate() functions propagate buffer state into the
2056 * page struct once IO has completed.
2057 */
2058int block_read_full_page(struct page *page, get_block_t *get_block)
2059{
2060 struct inode *inode = page->mapping->host;
2061 sector_t iblock, lblock;
2062 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2063 unsigned int blocksize;
2064 int nr, i;
2065 int fully_mapped = 1;
2066
Matt Mackallcd7619d2005-05-01 08:59:01 -07002067 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068 blocksize = 1 << inode->i_blkbits;
2069 if (!page_has_buffers(page))
2070 create_empty_buffers(page, blocksize, 0);
2071 head = page_buffers(page);
2072
2073 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2074 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2075 bh = head;
2076 nr = 0;
2077 i = 0;
2078
2079 do {
2080 if (buffer_uptodate(bh))
2081 continue;
2082
2083 if (!buffer_mapped(bh)) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002084 int err = 0;
2085
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 fully_mapped = 0;
2087 if (iblock < lblock) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002088 err = get_block(inode, iblock, bh, 0);
2089 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002090 SetPageError(page);
2091 }
2092 if (!buffer_mapped(bh)) {
2093 void *kaddr = kmap_atomic(page, KM_USER0);
2094 memset(kaddr + i * blocksize, 0, blocksize);
2095 flush_dcache_page(page);
2096 kunmap_atomic(kaddr, KM_USER0);
Andrew Mortonc64610b2005-05-16 21:53:49 -07002097 if (!err)
2098 set_buffer_uptodate(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002099 continue;
2100 }
2101 /*
2102 * get_block() might have updated the buffer
2103 * synchronously
2104 */
2105 if (buffer_uptodate(bh))
2106 continue;
2107 }
2108 arr[nr++] = bh;
2109 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2110
2111 if (fully_mapped)
2112 SetPageMappedToDisk(page);
2113
2114 if (!nr) {
2115 /*
2116 * All buffers are uptodate - we can set the page uptodate
2117 * as well. But not if get_block() returned an error.
2118 */
2119 if (!PageError(page))
2120 SetPageUptodate(page);
2121 unlock_page(page);
2122 return 0;
2123 }
2124
2125 /* Stage two: lock the buffers */
2126 for (i = 0; i < nr; i++) {
2127 bh = arr[i];
2128 lock_buffer(bh);
2129 mark_buffer_async_read(bh);
2130 }
2131
2132 /*
2133 * Stage 3: start the IO. Check for uptodateness
2134 * inside the buffer lock in case another process reading
2135 * the underlying blockdev brought it uptodate (the sct fix).
2136 */
2137 for (i = 0; i < nr; i++) {
2138 bh = arr[i];
2139 if (buffer_uptodate(bh))
2140 end_buffer_async_read(bh, 1);
2141 else
2142 submit_bh(READ, bh);
2143 }
2144 return 0;
2145}
2146
2147/* utility function for filesystems that need to do work on expanding
2148 * truncates. Uses prepare/commit_write to allow the filesystem to
2149 * deal with the hole.
2150 */
2151int generic_cont_expand(struct inode *inode, loff_t size)
2152{
2153 struct address_space *mapping = inode->i_mapping;
2154 struct page *page;
2155 unsigned long index, offset, limit;
2156 int err;
2157
2158 err = -EFBIG;
2159 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2160 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2161 send_sig(SIGXFSZ, current, 0);
2162 goto out;
2163 }
2164 if (size > inode->i_sb->s_maxbytes)
2165 goto out;
2166
2167 offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2168
2169 /* ugh. in prepare/commit_write, if from==to==start of block, we
2170 ** skip the prepare. make sure we never send an offset for the start
2171 ** of a block
2172 */
2173 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2174 offset++;
2175 }
2176 index = size >> PAGE_CACHE_SHIFT;
2177 err = -ENOMEM;
2178 page = grab_cache_page(mapping, index);
2179 if (!page)
2180 goto out;
2181 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2182 if (!err) {
2183 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2184 }
2185 unlock_page(page);
2186 page_cache_release(page);
2187 if (err > 0)
2188 err = 0;
2189out:
2190 return err;
2191}
2192
2193/*
2194 * For moronic filesystems that do not allow holes in file.
2195 * We may have to extend the file.
2196 */
2197
2198int cont_prepare_write(struct page *page, unsigned offset,
2199 unsigned to, get_block_t *get_block, loff_t *bytes)
2200{
2201 struct address_space *mapping = page->mapping;
2202 struct inode *inode = mapping->host;
2203 struct page *new_page;
2204 pgoff_t pgpos;
2205 long status;
2206 unsigned zerofrom;
2207 unsigned blocksize = 1 << inode->i_blkbits;
2208 void *kaddr;
2209
2210 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2211 status = -ENOMEM;
2212 new_page = grab_cache_page(mapping, pgpos);
2213 if (!new_page)
2214 goto out;
2215 /* we might sleep */
2216 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2217 unlock_page(new_page);
2218 page_cache_release(new_page);
2219 continue;
2220 }
2221 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2222 if (zerofrom & (blocksize-1)) {
2223 *bytes |= (blocksize-1);
2224 (*bytes)++;
2225 }
2226 status = __block_prepare_write(inode, new_page, zerofrom,
2227 PAGE_CACHE_SIZE, get_block);
2228 if (status)
2229 goto out_unmap;
2230 kaddr = kmap_atomic(new_page, KM_USER0);
2231 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2232 flush_dcache_page(new_page);
2233 kunmap_atomic(kaddr, KM_USER0);
2234 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2235 unlock_page(new_page);
2236 page_cache_release(new_page);
2237 }
2238
2239 if (page->index < pgpos) {
2240 /* completely inside the area */
2241 zerofrom = offset;
2242 } else {
2243 /* page covers the boundary, find the boundary offset */
2244 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2245
2246 /* if we will expand the thing last block will be filled */
2247 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2248 *bytes |= (blocksize-1);
2249 (*bytes)++;
2250 }
2251
2252 /* starting below the boundary? Nothing to zero out */
2253 if (offset <= zerofrom)
2254 zerofrom = offset;
2255 }
2256 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2257 if (status)
2258 goto out1;
2259 if (zerofrom < offset) {
2260 kaddr = kmap_atomic(page, KM_USER0);
2261 memset(kaddr+zerofrom, 0, offset-zerofrom);
2262 flush_dcache_page(page);
2263 kunmap_atomic(kaddr, KM_USER0);
2264 __block_commit_write(inode, page, zerofrom, offset);
2265 }
2266 return 0;
2267out1:
2268 ClearPageUptodate(page);
2269 return status;
2270
2271out_unmap:
2272 ClearPageUptodate(new_page);
2273 unlock_page(new_page);
2274 page_cache_release(new_page);
2275out:
2276 return status;
2277}
2278
2279int block_prepare_write(struct page *page, unsigned from, unsigned to,
2280 get_block_t *get_block)
2281{
2282 struct inode *inode = page->mapping->host;
2283 int err = __block_prepare_write(inode, page, from, to, get_block);
2284 if (err)
2285 ClearPageUptodate(page);
2286 return err;
2287}
2288
2289int block_commit_write(struct page *page, unsigned from, unsigned to)
2290{
2291 struct inode *inode = page->mapping->host;
2292 __block_commit_write(inode,page,from,to);
2293 return 0;
2294}
2295
2296int generic_commit_write(struct file *file, struct page *page,
2297 unsigned from, unsigned to)
2298{
2299 struct inode *inode = page->mapping->host;
2300 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2301 __block_commit_write(inode,page,from,to);
2302 /*
2303 * No need to use i_size_read() here, the i_size
2304 * cannot change under us because we hold i_sem.
2305 */
2306 if (pos > inode->i_size) {
2307 i_size_write(inode, pos);
2308 mark_inode_dirty(inode);
2309 }
2310 return 0;
2311}
2312
2313
2314/*
2315 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2316 * immediately, while under the page lock. So it needs a special end_io
2317 * handler which does not touch the bh after unlocking it.
2318 *
2319 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2320 * a race there is benign: unlock_buffer() only use the bh's address for
2321 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2322 * itself.
2323 */
2324static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2325{
2326 if (uptodate) {
2327 set_buffer_uptodate(bh);
2328 } else {
2329 /* This happens, due to failed READA attempts. */
2330 clear_buffer_uptodate(bh);
2331 }
2332 unlock_buffer(bh);
2333}
2334
2335/*
2336 * On entry, the page is fully not uptodate.
2337 * On exit the page is fully uptodate in the areas outside (from,to)
2338 */
2339int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2340 get_block_t *get_block)
2341{
2342 struct inode *inode = page->mapping->host;
2343 const unsigned blkbits = inode->i_blkbits;
2344 const unsigned blocksize = 1 << blkbits;
2345 struct buffer_head map_bh;
2346 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2347 unsigned block_in_page;
2348 unsigned block_start;
2349 sector_t block_in_file;
2350 char *kaddr;
2351 int nr_reads = 0;
2352 int i;
2353 int ret = 0;
2354 int is_mapped_to_disk = 1;
2355 int dirtied_it = 0;
2356
2357 if (PageMappedToDisk(page))
2358 return 0;
2359
2360 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2361 map_bh.b_page = page;
2362
2363 /*
2364 * We loop across all blocks in the page, whether or not they are
2365 * part of the affected region. This is so we can discover if the
2366 * page is fully mapped-to-disk.
2367 */
2368 for (block_start = 0, block_in_page = 0;
2369 block_start < PAGE_CACHE_SIZE;
2370 block_in_page++, block_start += blocksize) {
2371 unsigned block_end = block_start + blocksize;
2372 int create;
2373
2374 map_bh.b_state = 0;
2375 create = 1;
2376 if (block_start >= to)
2377 create = 0;
2378 ret = get_block(inode, block_in_file + block_in_page,
2379 &map_bh, create);
2380 if (ret)
2381 goto failed;
2382 if (!buffer_mapped(&map_bh))
2383 is_mapped_to_disk = 0;
2384 if (buffer_new(&map_bh))
2385 unmap_underlying_metadata(map_bh.b_bdev,
2386 map_bh.b_blocknr);
2387 if (PageUptodate(page))
2388 continue;
2389 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2390 kaddr = kmap_atomic(page, KM_USER0);
2391 if (block_start < from) {
2392 memset(kaddr+block_start, 0, from-block_start);
2393 dirtied_it = 1;
2394 }
2395 if (block_end > to) {
2396 memset(kaddr + to, 0, block_end - to);
2397 dirtied_it = 1;
2398 }
2399 flush_dcache_page(page);
2400 kunmap_atomic(kaddr, KM_USER0);
2401 continue;
2402 }
2403 if (buffer_uptodate(&map_bh))
2404 continue; /* reiserfs does this */
2405 if (block_start < from || block_end > to) {
2406 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2407
2408 if (!bh) {
2409 ret = -ENOMEM;
2410 goto failed;
2411 }
2412 bh->b_state = map_bh.b_state;
2413 atomic_set(&bh->b_count, 0);
2414 bh->b_this_page = NULL;
2415 bh->b_page = page;
2416 bh->b_blocknr = map_bh.b_blocknr;
2417 bh->b_size = blocksize;
2418 bh->b_data = (char *)(long)block_start;
2419 bh->b_bdev = map_bh.b_bdev;
2420 bh->b_private = NULL;
2421 read_bh[nr_reads++] = bh;
2422 }
2423 }
2424
2425 if (nr_reads) {
2426 struct buffer_head *bh;
2427
2428 /*
2429 * The page is locked, so these buffers are protected from
2430 * any VM or truncate activity. Hence we don't need to care
2431 * for the buffer_head refcounts.
2432 */
2433 for (i = 0; i < nr_reads; i++) {
2434 bh = read_bh[i];
2435 lock_buffer(bh);
2436 bh->b_end_io = end_buffer_read_nobh;
2437 submit_bh(READ, bh);
2438 }
2439 for (i = 0; i < nr_reads; i++) {
2440 bh = read_bh[i];
2441 wait_on_buffer(bh);
2442 if (!buffer_uptodate(bh))
2443 ret = -EIO;
2444 free_buffer_head(bh);
2445 read_bh[i] = NULL;
2446 }
2447 if (ret)
2448 goto failed;
2449 }
2450
2451 if (is_mapped_to_disk)
2452 SetPageMappedToDisk(page);
2453 SetPageUptodate(page);
2454
2455 /*
2456 * Setting the page dirty here isn't necessary for the prepare_write
2457 * function - commit_write will do that. But if/when this function is
2458 * used within the pagefault handler to ensure that all mmapped pages
2459 * have backing space in the filesystem, we will need to dirty the page
2460 * if its contents were altered.
2461 */
2462 if (dirtied_it)
2463 set_page_dirty(page);
2464
2465 return 0;
2466
2467failed:
2468 for (i = 0; i < nr_reads; i++) {
2469 if (read_bh[i])
2470 free_buffer_head(read_bh[i]);
2471 }
2472
2473 /*
2474 * Error recovery is pretty slack. Clear the page and mark it dirty
2475 * so we'll later zero out any blocks which _were_ allocated.
2476 */
2477 kaddr = kmap_atomic(page, KM_USER0);
2478 memset(kaddr, 0, PAGE_CACHE_SIZE);
2479 kunmap_atomic(kaddr, KM_USER0);
2480 SetPageUptodate(page);
2481 set_page_dirty(page);
2482 return ret;
2483}
2484EXPORT_SYMBOL(nobh_prepare_write);
2485
2486int nobh_commit_write(struct file *file, struct page *page,
2487 unsigned from, unsigned to)
2488{
2489 struct inode *inode = page->mapping->host;
2490 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2491
2492 set_page_dirty(page);
2493 if (pos > inode->i_size) {
2494 i_size_write(inode, pos);
2495 mark_inode_dirty(inode);
2496 }
2497 return 0;
2498}
2499EXPORT_SYMBOL(nobh_commit_write);
2500
2501/*
2502 * nobh_writepage() - based on block_full_write_page() except
2503 * that it tries to operate without attaching bufferheads to
2504 * the page.
2505 */
2506int nobh_writepage(struct page *page, get_block_t *get_block,
2507 struct writeback_control *wbc)
2508{
2509 struct inode * const inode = page->mapping->host;
2510 loff_t i_size = i_size_read(inode);
2511 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2512 unsigned offset;
2513 void *kaddr;
2514 int ret;
2515
2516 /* Is the page fully inside i_size? */
2517 if (page->index < end_index)
2518 goto out;
2519
2520 /* Is the page fully outside i_size? (truncate in progress) */
2521 offset = i_size & (PAGE_CACHE_SIZE-1);
2522 if (page->index >= end_index+1 || !offset) {
2523 /*
2524 * The page may have dirty, unmapped buffers. For example,
2525 * they may have been added in ext3_writepage(). Make them
2526 * freeable here, so the page does not leak.
2527 */
2528#if 0
2529 /* Not really sure about this - do we need this ? */
2530 if (page->mapping->a_ops->invalidatepage)
2531 page->mapping->a_ops->invalidatepage(page, offset);
2532#endif
2533 unlock_page(page);
2534 return 0; /* don't care */
2535 }
2536
2537 /*
2538 * The page straddles i_size. It must be zeroed out on each and every
2539 * writepage invocation because it may be mmapped. "A file is mapped
2540 * in multiples of the page size. For a file that is not a multiple of
2541 * the page size, the remaining memory is zeroed when mapped, and
2542 * writes to that region are not written out to the file."
2543 */
2544 kaddr = kmap_atomic(page, KM_USER0);
2545 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2546 flush_dcache_page(page);
2547 kunmap_atomic(kaddr, KM_USER0);
2548out:
2549 ret = mpage_writepage(page, get_block, wbc);
2550 if (ret == -EAGAIN)
2551 ret = __block_write_full_page(inode, page, get_block, wbc);
2552 return ret;
2553}
2554EXPORT_SYMBOL(nobh_writepage);
2555
2556/*
2557 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2558 */
2559int nobh_truncate_page(struct address_space *mapping, loff_t from)
2560{
2561 struct inode *inode = mapping->host;
2562 unsigned blocksize = 1 << inode->i_blkbits;
2563 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2564 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2565 unsigned to;
2566 struct page *page;
2567 struct address_space_operations *a_ops = mapping->a_ops;
2568 char *kaddr;
2569 int ret = 0;
2570
2571 if ((offset & (blocksize - 1)) == 0)
2572 goto out;
2573
2574 ret = -ENOMEM;
2575 page = grab_cache_page(mapping, index);
2576 if (!page)
2577 goto out;
2578
2579 to = (offset + blocksize) & ~(blocksize - 1);
2580 ret = a_ops->prepare_write(NULL, page, offset, to);
2581 if (ret == 0) {
2582 kaddr = kmap_atomic(page, KM_USER0);
2583 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2584 flush_dcache_page(page);
2585 kunmap_atomic(kaddr, KM_USER0);
2586 set_page_dirty(page);
2587 }
2588 unlock_page(page);
2589 page_cache_release(page);
2590out:
2591 return ret;
2592}
2593EXPORT_SYMBOL(nobh_truncate_page);
2594
2595int block_truncate_page(struct address_space *mapping,
2596 loff_t from, get_block_t *get_block)
2597{
2598 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2599 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2600 unsigned blocksize;
2601 pgoff_t iblock;
2602 unsigned length, pos;
2603 struct inode *inode = mapping->host;
2604 struct page *page;
2605 struct buffer_head *bh;
2606 void *kaddr;
2607 int err;
2608
2609 blocksize = 1 << inode->i_blkbits;
2610 length = offset & (blocksize - 1);
2611
2612 /* Block boundary? Nothing to do */
2613 if (!length)
2614 return 0;
2615
2616 length = blocksize - length;
2617 iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2618
2619 page = grab_cache_page(mapping, index);
2620 err = -ENOMEM;
2621 if (!page)
2622 goto out;
2623
2624 if (!page_has_buffers(page))
2625 create_empty_buffers(page, blocksize, 0);
2626
2627 /* Find the buffer that contains "offset" */
2628 bh = page_buffers(page);
2629 pos = blocksize;
2630 while (offset >= pos) {
2631 bh = bh->b_this_page;
2632 iblock++;
2633 pos += blocksize;
2634 }
2635
2636 err = 0;
2637 if (!buffer_mapped(bh)) {
2638 err = get_block(inode, iblock, bh, 0);
2639 if (err)
2640 goto unlock;
2641 /* unmapped? It's a hole - nothing to do */
2642 if (!buffer_mapped(bh))
2643 goto unlock;
2644 }
2645
2646 /* Ok, it's mapped. Make sure it's up-to-date */
2647 if (PageUptodate(page))
2648 set_buffer_uptodate(bh);
2649
2650 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2651 err = -EIO;
2652 ll_rw_block(READ, 1, &bh);
2653 wait_on_buffer(bh);
2654 /* Uhhuh. Read error. Complain and punt. */
2655 if (!buffer_uptodate(bh))
2656 goto unlock;
2657 }
2658
2659 kaddr = kmap_atomic(page, KM_USER0);
2660 memset(kaddr + offset, 0, length);
2661 flush_dcache_page(page);
2662 kunmap_atomic(kaddr, KM_USER0);
2663
2664 mark_buffer_dirty(bh);
2665 err = 0;
2666
2667unlock:
2668 unlock_page(page);
2669 page_cache_release(page);
2670out:
2671 return err;
2672}
2673
2674/*
2675 * The generic ->writepage function for buffer-backed address_spaces
2676 */
2677int block_write_full_page(struct page *page, get_block_t *get_block,
2678 struct writeback_control *wbc)
2679{
2680 struct inode * const inode = page->mapping->host;
2681 loff_t i_size = i_size_read(inode);
2682 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2683 unsigned offset;
2684 void *kaddr;
2685
2686 /* Is the page fully inside i_size? */
2687 if (page->index < end_index)
2688 return __block_write_full_page(inode, page, get_block, wbc);
2689
2690 /* Is the page fully outside i_size? (truncate in progress) */
2691 offset = i_size & (PAGE_CACHE_SIZE-1);
2692 if (page->index >= end_index+1 || !offset) {
2693 /*
2694 * The page may have dirty, unmapped buffers. For example,
2695 * they may have been added in ext3_writepage(). Make them
2696 * freeable here, so the page does not leak.
2697 */
2698 block_invalidatepage(page, 0);
2699 unlock_page(page);
2700 return 0; /* don't care */
2701 }
2702
2703 /*
2704 * The page straddles i_size. It must be zeroed out on each and every
2705 * writepage invokation because it may be mmapped. "A file is mapped
2706 * in multiples of the page size. For a file that is not a multiple of
2707 * the page size, the remaining memory is zeroed when mapped, and
2708 * writes to that region are not written out to the file."
2709 */
2710 kaddr = kmap_atomic(page, KM_USER0);
2711 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2712 flush_dcache_page(page);
2713 kunmap_atomic(kaddr, KM_USER0);
2714 return __block_write_full_page(inode, page, get_block, wbc);
2715}
2716
2717sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2718 get_block_t *get_block)
2719{
2720 struct buffer_head tmp;
2721 struct inode *inode = mapping->host;
2722 tmp.b_state = 0;
2723 tmp.b_blocknr = 0;
2724 get_block(inode, block, &tmp, 0);
2725 return tmp.b_blocknr;
2726}
2727
2728static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2729{
2730 struct buffer_head *bh = bio->bi_private;
2731
2732 if (bio->bi_size)
2733 return 1;
2734
2735 if (err == -EOPNOTSUPP) {
2736 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2737 set_bit(BH_Eopnotsupp, &bh->b_state);
2738 }
2739
2740 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2741 bio_put(bio);
2742 return 0;
2743}
2744
2745int submit_bh(int rw, struct buffer_head * bh)
2746{
2747 struct bio *bio;
2748 int ret = 0;
2749
2750 BUG_ON(!buffer_locked(bh));
2751 BUG_ON(!buffer_mapped(bh));
2752 BUG_ON(!bh->b_end_io);
2753
2754 if (buffer_ordered(bh) && (rw == WRITE))
2755 rw = WRITE_BARRIER;
2756
2757 /*
2758 * Only clear out a write error when rewriting, should this
2759 * include WRITE_SYNC as well?
2760 */
2761 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2762 clear_buffer_write_io_error(bh);
2763
2764 /*
2765 * from here on down, it's all bio -- do the initial mapping,
2766 * submit_bio -> generic_make_request may further map this bio around
2767 */
2768 bio = bio_alloc(GFP_NOIO, 1);
2769
2770 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2771 bio->bi_bdev = bh->b_bdev;
2772 bio->bi_io_vec[0].bv_page = bh->b_page;
2773 bio->bi_io_vec[0].bv_len = bh->b_size;
2774 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2775
2776 bio->bi_vcnt = 1;
2777 bio->bi_idx = 0;
2778 bio->bi_size = bh->b_size;
2779
2780 bio->bi_end_io = end_bio_bh_io_sync;
2781 bio->bi_private = bh;
2782
2783 bio_get(bio);
2784 submit_bio(rw, bio);
2785
2786 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2787 ret = -EOPNOTSUPP;
2788
2789 bio_put(bio);
2790 return ret;
2791}
2792
2793/**
2794 * ll_rw_block: low-level access to block devices (DEPRECATED)
Jan Karaa7662232005-09-06 15:19:10 -07002795 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002796 * @nr: number of &struct buffer_heads in the array
2797 * @bhs: array of pointers to &struct buffer_head
2798 *
Jan Karaa7662232005-09-06 15:19:10 -07002799 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2800 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2801 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2802 * are sent to disk. The fourth %READA option is described in the documentation
2803 * for generic_make_request() which ll_rw_block() calls.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002804 *
2805 * This function drops any buffer that it cannot get a lock on (with the
Jan Karaa7662232005-09-06 15:19:10 -07002806 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2807 * clean when doing a write request, and any buffer that appears to be
2808 * up-to-date when doing read request. Further it marks as clean buffers that
2809 * are processed for writing (the buffer cache won't assume that they are
2810 * actually clean until the buffer gets unlocked).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002811 *
2812 * ll_rw_block sets b_end_io to simple completion handler that marks
2813 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2814 * any waiters.
2815 *
2816 * All of the buffers must be for the same device, and must also be a
2817 * multiple of the current approved size for the device.
2818 */
2819void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2820{
2821 int i;
2822
2823 for (i = 0; i < nr; i++) {
2824 struct buffer_head *bh = bhs[i];
2825
Jan Karaa7662232005-09-06 15:19:10 -07002826 if (rw == SWRITE)
2827 lock_buffer(bh);
2828 else if (test_set_buffer_locked(bh))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829 continue;
2830
2831 get_bh(bh);
Jan Karaa7662232005-09-06 15:19:10 -07002832 if (rw == WRITE || rw == SWRITE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833 if (test_clear_buffer_dirty(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002834 bh->b_end_io = end_buffer_write_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 submit_bh(WRITE, bh);
2836 continue;
2837 }
2838 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 if (!buffer_uptodate(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002840 bh->b_end_io = end_buffer_read_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841 submit_bh(rw, bh);
2842 continue;
2843 }
2844 }
2845 unlock_buffer(bh);
2846 put_bh(bh);
2847 }
2848}
2849
2850/*
2851 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2852 * and then start new I/O and then wait upon it. The caller must have a ref on
2853 * the buffer_head.
2854 */
2855int sync_dirty_buffer(struct buffer_head *bh)
2856{
2857 int ret = 0;
2858
2859 WARN_ON(atomic_read(&bh->b_count) < 1);
2860 lock_buffer(bh);
2861 if (test_clear_buffer_dirty(bh)) {
2862 get_bh(bh);
2863 bh->b_end_io = end_buffer_write_sync;
2864 ret = submit_bh(WRITE, bh);
2865 wait_on_buffer(bh);
2866 if (buffer_eopnotsupp(bh)) {
2867 clear_buffer_eopnotsupp(bh);
2868 ret = -EOPNOTSUPP;
2869 }
2870 if (!ret && !buffer_uptodate(bh))
2871 ret = -EIO;
2872 } else {
2873 unlock_buffer(bh);
2874 }
2875 return ret;
2876}
2877
2878/*
2879 * try_to_free_buffers() checks if all the buffers on this particular page
2880 * are unused, and releases them if so.
2881 *
2882 * Exclusion against try_to_free_buffers may be obtained by either
2883 * locking the page or by holding its mapping's private_lock.
2884 *
2885 * If the page is dirty but all the buffers are clean then we need to
2886 * be sure to mark the page clean as well. This is because the page
2887 * may be against a block device, and a later reattachment of buffers
2888 * to a dirty page will set *all* buffers dirty. Which would corrupt
2889 * filesystem data on the same device.
2890 *
2891 * The same applies to regular filesystem pages: if all the buffers are
2892 * clean then we set the page clean and proceed. To do that, we require
2893 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2894 * private_lock.
2895 *
2896 * try_to_free_buffers() is non-blocking.
2897 */
2898static inline int buffer_busy(struct buffer_head *bh)
2899{
2900 return atomic_read(&bh->b_count) |
2901 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2902}
2903
2904static int
2905drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2906{
2907 struct buffer_head *head = page_buffers(page);
2908 struct buffer_head *bh;
2909
2910 bh = head;
2911 do {
akpm@osdl.orgde7d5a32005-05-01 08:58:39 -07002912 if (buffer_write_io_error(bh) && page->mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002913 set_bit(AS_EIO, &page->mapping->flags);
2914 if (buffer_busy(bh))
2915 goto failed;
2916 bh = bh->b_this_page;
2917 } while (bh != head);
2918
2919 do {
2920 struct buffer_head *next = bh->b_this_page;
2921
2922 if (!list_empty(&bh->b_assoc_buffers))
2923 __remove_assoc_queue(bh);
2924 bh = next;
2925 } while (bh != head);
2926 *buffers_to_free = head;
2927 __clear_page_buffers(page);
2928 return 1;
2929failed:
2930 return 0;
2931}
2932
2933int try_to_free_buffers(struct page *page)
2934{
2935 struct address_space * const mapping = page->mapping;
2936 struct buffer_head *buffers_to_free = NULL;
2937 int ret = 0;
2938
2939 BUG_ON(!PageLocked(page));
2940 if (PageWriteback(page))
2941 return 0;
2942
2943 if (mapping == NULL) { /* can this still happen? */
2944 ret = drop_buffers(page, &buffers_to_free);
2945 goto out;
2946 }
2947
2948 spin_lock(&mapping->private_lock);
2949 ret = drop_buffers(page, &buffers_to_free);
2950 if (ret) {
2951 /*
2952 * If the filesystem writes its buffers by hand (eg ext3)
2953 * then we can have clean buffers against a dirty page. We
2954 * clean the page here; otherwise later reattachment of buffers
2955 * could encounter a non-uptodate page, which is unresolvable.
2956 * This only applies in the rare case where try_to_free_buffers
2957 * succeeds but the page is not freed.
2958 */
2959 clear_page_dirty(page);
2960 }
2961 spin_unlock(&mapping->private_lock);
2962out:
2963 if (buffers_to_free) {
2964 struct buffer_head *bh = buffers_to_free;
2965
2966 do {
2967 struct buffer_head *next = bh->b_this_page;
2968 free_buffer_head(bh);
2969 bh = next;
2970 } while (bh != buffers_to_free);
2971 }
2972 return ret;
2973}
2974EXPORT_SYMBOL(try_to_free_buffers);
2975
2976int block_sync_page(struct page *page)
2977{
2978 struct address_space *mapping;
2979
2980 smp_mb();
2981 mapping = page_mapping(page);
2982 if (mapping)
2983 blk_run_backing_dev(mapping->backing_dev_info, page);
2984 return 0;
2985}
2986
2987/*
2988 * There are no bdflush tunables left. But distributions are
2989 * still running obsolete flush daemons, so we terminate them here.
2990 *
2991 * Use of bdflush() is deprecated and will be removed in a future kernel.
2992 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2993 */
2994asmlinkage long sys_bdflush(int func, long data)
2995{
2996 static int msg_count;
2997
2998 if (!capable(CAP_SYS_ADMIN))
2999 return -EPERM;
3000
3001 if (msg_count < 5) {
3002 msg_count++;
3003 printk(KERN_INFO
3004 "warning: process `%s' used the obsolete bdflush"
3005 " system call\n", current->comm);
3006 printk(KERN_INFO "Fix your initscripts?\n");
3007 }
3008
3009 if (func == 1)
3010 do_exit(0);
3011 return 0;
3012}
3013
3014/*
3015 * Buffer-head allocation
3016 */
3017static kmem_cache_t *bh_cachep;
3018
3019/*
3020 * Once the number of bh's in the machine exceeds this level, we start
3021 * stripping them in writeback.
3022 */
3023static int max_buffer_heads;
3024
3025int buffer_heads_over_limit;
3026
3027struct bh_accounting {
3028 int nr; /* Number of live bh's */
3029 int ratelimit; /* Limit cacheline bouncing */
3030};
3031
3032static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3033
3034static void recalc_bh_state(void)
3035{
3036 int i;
3037 int tot = 0;
3038
3039 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3040 return;
3041 __get_cpu_var(bh_accounting).ratelimit = 0;
3042 for_each_cpu(i)
3043 tot += per_cpu(bh_accounting, i).nr;
3044 buffer_heads_over_limit = (tot > max_buffer_heads);
3045}
3046
3047struct buffer_head *alloc_buffer_head(unsigned int __nocast gfp_flags)
3048{
3049 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3050 if (ret) {
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003051 get_cpu_var(bh_accounting).nr++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003052 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003053 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003054 }
3055 return ret;
3056}
3057EXPORT_SYMBOL(alloc_buffer_head);
3058
3059void free_buffer_head(struct buffer_head *bh)
3060{
3061 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3062 kmem_cache_free(bh_cachep, bh);
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003063 get_cpu_var(bh_accounting).nr--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003065 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003066}
3067EXPORT_SYMBOL(free_buffer_head);
3068
3069static void
3070init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3071{
3072 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3073 SLAB_CTOR_CONSTRUCTOR) {
3074 struct buffer_head * bh = (struct buffer_head *)data;
3075
3076 memset(bh, 0, sizeof(*bh));
3077 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3078 }
3079}
3080
3081#ifdef CONFIG_HOTPLUG_CPU
3082static void buffer_exit_cpu(int cpu)
3083{
3084 int i;
3085 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3086
3087 for (i = 0; i < BH_LRU_SIZE; i++) {
3088 brelse(b->bhs[i]);
3089 b->bhs[i] = NULL;
3090 }
3091}
3092
3093static int buffer_cpu_notify(struct notifier_block *self,
3094 unsigned long action, void *hcpu)
3095{
3096 if (action == CPU_DEAD)
3097 buffer_exit_cpu((unsigned long)hcpu);
3098 return NOTIFY_OK;
3099}
3100#endif /* CONFIG_HOTPLUG_CPU */
3101
3102void __init buffer_init(void)
3103{
3104 int nrpages;
3105
3106 bh_cachep = kmem_cache_create("buffer_head",
3107 sizeof(struct buffer_head), 0,
Andrea Arcangelie422fd22005-05-05 16:15:04 -07003108 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109
3110 /*
3111 * Limit the bh occupancy to 10% of ZONE_NORMAL
3112 */
3113 nrpages = (nr_free_buffer_pages() * 10) / 100;
3114 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3115 hotcpu_notifier(buffer_cpu_notify, 0);
3116}
3117
3118EXPORT_SYMBOL(__bforget);
3119EXPORT_SYMBOL(__brelse);
3120EXPORT_SYMBOL(__wait_on_buffer);
3121EXPORT_SYMBOL(block_commit_write);
3122EXPORT_SYMBOL(block_prepare_write);
3123EXPORT_SYMBOL(block_read_full_page);
3124EXPORT_SYMBOL(block_sync_page);
3125EXPORT_SYMBOL(block_truncate_page);
3126EXPORT_SYMBOL(block_write_full_page);
3127EXPORT_SYMBOL(cont_prepare_write);
3128EXPORT_SYMBOL(end_buffer_async_write);
3129EXPORT_SYMBOL(end_buffer_read_sync);
3130EXPORT_SYMBOL(end_buffer_write_sync);
3131EXPORT_SYMBOL(file_fsync);
3132EXPORT_SYMBOL(fsync_bdev);
3133EXPORT_SYMBOL(generic_block_bmap);
3134EXPORT_SYMBOL(generic_commit_write);
3135EXPORT_SYMBOL(generic_cont_expand);
3136EXPORT_SYMBOL(init_buffer);
3137EXPORT_SYMBOL(invalidate_bdev);
3138EXPORT_SYMBOL(ll_rw_block);
3139EXPORT_SYMBOL(mark_buffer_dirty);
3140EXPORT_SYMBOL(submit_bh);
3141EXPORT_SYMBOL(sync_dirty_buffer);
3142EXPORT_SYMBOL(unlock_buffer);