blob: 561e63a149667abafef2c92aabe287e0eae2f27d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
21#include <linux/config.h>
22#include <linux/kernel.h>
23#include <linux/syscalls.h>
24#include <linux/fs.h>
25#include <linux/mm.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/smp_lock.h>
29#include <linux/blkdev.h>
30#include <linux/file.h>
31#include <linux/quotaops.h>
32#include <linux/highmem.h>
33#include <linux/module.h>
34#include <linux/writeback.h>
35#include <linux/hash.h>
36#include <linux/suspend.h>
37#include <linux/buffer_head.h>
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
43
44static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
45static void invalidate_bh_lrus(void);
46
47#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48
49inline void
50init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51{
52 bh->b_end_io = handler;
53 bh->b_private = private;
54}
55
56static int sync_buffer(void *word)
57{
58 struct block_device *bd;
59 struct buffer_head *bh
60 = container_of(word, struct buffer_head, b_state);
61
62 smp_mb();
63 bd = bh->b_bdev;
64 if (bd)
65 blk_run_address_space(bd->bd_inode->i_mapping);
66 io_schedule();
67 return 0;
68}
69
70void fastcall __lock_buffer(struct buffer_head *bh)
71{
72 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
73 TASK_UNINTERRUPTIBLE);
74}
75EXPORT_SYMBOL(__lock_buffer);
76
77void fastcall unlock_buffer(struct buffer_head *bh)
78{
79 clear_buffer_locked(bh);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh->b_state, BH_Lock);
82}
83
84/*
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
88 */
89void __wait_on_buffer(struct buffer_head * bh)
90{
91 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
92}
93
94static void
95__clear_page_buffers(struct page *page)
96{
97 ClearPagePrivate(page);
98 page->private = 0;
99 page_cache_release(page);
100}
101
102static void buffer_io_error(struct buffer_head *bh)
103{
104 char b[BDEVNAME_SIZE];
105
106 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
107 bdevname(bh->b_bdev, b),
108 (unsigned long long)bh->b_blocknr);
109}
110
111/*
112 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
113 * unlock the buffer. This is what ll_rw_block uses too.
114 */
115void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
116{
117 if (uptodate) {
118 set_buffer_uptodate(bh);
119 } else {
120 /* This happens, due to failed READA attempts. */
121 clear_buffer_uptodate(bh);
122 }
123 unlock_buffer(bh);
124 put_bh(bh);
125}
126
127void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
128{
129 char b[BDEVNAME_SIZE];
130
131 if (uptodate) {
132 set_buffer_uptodate(bh);
133 } else {
134 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
135 buffer_io_error(bh);
136 printk(KERN_WARNING "lost page write due to "
137 "I/O error on %s\n",
138 bdevname(bh->b_bdev, b));
139 }
140 set_buffer_write_io_error(bh);
141 clear_buffer_uptodate(bh);
142 }
143 unlock_buffer(bh);
144 put_bh(bh);
145}
146
147/*
148 * Write out and wait upon all the dirty data associated with a block
149 * device via its mapping. Does not take the superblock lock.
150 */
151int sync_blockdev(struct block_device *bdev)
152{
153 int ret = 0;
154
155 if (bdev) {
156 int err;
157
158 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
159 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
160 if (!ret)
161 ret = err;
162 }
163 return ret;
164}
165EXPORT_SYMBOL(sync_blockdev);
166
167/*
168 * Write out and wait upon all dirty data associated with this
169 * superblock. Filesystem data as well as the underlying block
170 * device. Takes the superblock lock.
171 */
172int fsync_super(struct super_block *sb)
173{
174 sync_inodes_sb(sb, 0);
175 DQUOT_SYNC(sb);
176 lock_super(sb);
177 if (sb->s_dirt && sb->s_op->write_super)
178 sb->s_op->write_super(sb);
179 unlock_super(sb);
180 if (sb->s_op->sync_fs)
181 sb->s_op->sync_fs(sb, 1);
182 sync_blockdev(sb->s_bdev);
183 sync_inodes_sb(sb, 1);
184
185 return sync_blockdev(sb->s_bdev);
186}
187
188/*
189 * Write out and wait upon all dirty data associated with this
190 * device. Filesystem data as well as the underlying block
191 * device. Takes the superblock lock.
192 */
193int fsync_bdev(struct block_device *bdev)
194{
195 struct super_block *sb = get_super(bdev);
196 if (sb) {
197 int res = fsync_super(sb);
198 drop_super(sb);
199 return res;
200 }
201 return sync_blockdev(bdev);
202}
203
204/**
205 * freeze_bdev -- lock a filesystem and force it into a consistent state
206 * @bdev: blockdevice to lock
207 *
208 * This takes the block device bd_mount_sem to make sure no new mounts
209 * happen on bdev until thaw_bdev() is called.
210 * If a superblock is found on this device, we take the s_umount semaphore
211 * on it to make sure nobody unmounts until the snapshot creation is done.
212 */
213struct super_block *freeze_bdev(struct block_device *bdev)
214{
215 struct super_block *sb;
216
217 down(&bdev->bd_mount_sem);
218 sb = get_super(bdev);
219 if (sb && !(sb->s_flags & MS_RDONLY)) {
220 sb->s_frozen = SB_FREEZE_WRITE;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700221 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
223 sync_inodes_sb(sb, 0);
224 DQUOT_SYNC(sb);
225
226 lock_super(sb);
227 if (sb->s_dirt && sb->s_op->write_super)
228 sb->s_op->write_super(sb);
229 unlock_super(sb);
230
231 if (sb->s_op->sync_fs)
232 sb->s_op->sync_fs(sb, 1);
233
234 sync_blockdev(sb->s_bdev);
235 sync_inodes_sb(sb, 1);
236
237 sb->s_frozen = SB_FREEZE_TRANS;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700238 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
240 sync_blockdev(sb->s_bdev);
241
242 if (sb->s_op->write_super_lockfs)
243 sb->s_op->write_super_lockfs(sb);
244 }
245
246 sync_blockdev(bdev);
247 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
248}
249EXPORT_SYMBOL(freeze_bdev);
250
251/**
252 * thaw_bdev -- unlock filesystem
253 * @bdev: blockdevice to unlock
254 * @sb: associated superblock
255 *
256 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
257 */
258void thaw_bdev(struct block_device *bdev, struct super_block *sb)
259{
260 if (sb) {
261 BUG_ON(sb->s_bdev != bdev);
262
263 if (sb->s_op->unlockfs)
264 sb->s_op->unlockfs(sb);
265 sb->s_frozen = SB_UNFROZEN;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700266 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 wake_up(&sb->s_wait_unfrozen);
268 drop_super(sb);
269 }
270
271 up(&bdev->bd_mount_sem);
272}
273EXPORT_SYMBOL(thaw_bdev);
274
275/*
276 * sync everything. Start out by waking pdflush, because that writes back
277 * all queues in parallel.
278 */
279static void do_sync(unsigned long wait)
280{
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700281 wakeup_pdflush(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 sync_inodes(0); /* All mappings, inodes and their blockdevs */
283 DQUOT_SYNC(NULL);
284 sync_supers(); /* Write the superblocks */
285 sync_filesystems(0); /* Start syncing the filesystems */
286 sync_filesystems(wait); /* Waitingly sync the filesystems */
287 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
288 if (!wait)
289 printk("Emergency Sync complete\n");
290 if (unlikely(laptop_mode))
291 laptop_sync_completion();
292}
293
294asmlinkage long sys_sync(void)
295{
296 do_sync(1);
297 return 0;
298}
299
300void emergency_sync(void)
301{
302 pdflush_operation(do_sync, 0);
303}
304
305/*
306 * Generic function to fsync a file.
307 *
308 * filp may be NULL if called via the msync of a vma.
309 */
310
311int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
312{
313 struct inode * inode = dentry->d_inode;
314 struct super_block * sb;
315 int ret, err;
316
317 /* sync the inode to buffers */
318 ret = write_inode_now(inode, 0);
319
320 /* sync the superblock to buffers */
321 sb = inode->i_sb;
322 lock_super(sb);
323 if (sb->s_op->write_super)
324 sb->s_op->write_super(sb);
325 unlock_super(sb);
326
327 /* .. finally sync the buffers to disk */
328 err = sync_blockdev(sb->s_bdev);
329 if (!ret)
330 ret = err;
331 return ret;
332}
333
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700334static long do_fsync(unsigned int fd, int datasync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
336 struct file * file;
337 struct address_space *mapping;
338 int ret, err;
339
340 ret = -EBADF;
341 file = fget(fd);
342 if (!file)
343 goto out;
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 ret = -EINVAL;
346 if (!file->f_op || !file->f_op->fsync) {
347 /* Why? We can still call filemap_fdatawrite */
348 goto out_putf;
349 }
350
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700351 mapping = file->f_mapping;
352
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 current->flags |= PF_SYNCWRITE;
354 ret = filemap_fdatawrite(mapping);
355
356 /*
357 * We need to protect against concurrent writers,
358 * which could cause livelocks in fsync_buffers_list
359 */
360 down(&mapping->host->i_sem);
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700361 err = file->f_op->fsync(file, file->f_dentry, datasync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 if (!ret)
363 ret = err;
364 up(&mapping->host->i_sem);
365 err = filemap_fdatawait(mapping);
366 if (!ret)
367 ret = err;
368 current->flags &= ~PF_SYNCWRITE;
369
370out_putf:
371 fput(file);
372out:
373 return ret;
374}
375
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700376asmlinkage long sys_fsync(unsigned int fd)
377{
378 return do_fsync(fd, 0);
379}
380
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381asmlinkage long sys_fdatasync(unsigned int fd)
382{
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700383 return do_fsync(fd, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385
386/*
387 * Various filesystems appear to want __find_get_block to be non-blocking.
388 * But it's the page lock which protects the buffers. To get around this,
389 * we get exclusion from try_to_free_buffers with the blockdev mapping's
390 * private_lock.
391 *
392 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
393 * may be quite high. This code could TryLock the page, and if that
394 * succeeds, there is no need to take private_lock. (But if
395 * private_lock is contended then so is mapping->tree_lock).
396 */
397static struct buffer_head *
398__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
399{
400 struct inode *bd_inode = bdev->bd_inode;
401 struct address_space *bd_mapping = bd_inode->i_mapping;
402 struct buffer_head *ret = NULL;
403 pgoff_t index;
404 struct buffer_head *bh;
405 struct buffer_head *head;
406 struct page *page;
407 int all_mapped = 1;
408
409 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
410 page = find_get_page(bd_mapping, index);
411 if (!page)
412 goto out;
413
414 spin_lock(&bd_mapping->private_lock);
415 if (!page_has_buffers(page))
416 goto out_unlock;
417 head = page_buffers(page);
418 bh = head;
419 do {
420 if (bh->b_blocknr == block) {
421 ret = bh;
422 get_bh(bh);
423 goto out_unlock;
424 }
425 if (!buffer_mapped(bh))
426 all_mapped = 0;
427 bh = bh->b_this_page;
428 } while (bh != head);
429
430 /* we might be here because some of the buffers on this page are
431 * not mapped. This is due to various races between
432 * file io on the block device and getblk. It gets dealt with
433 * elsewhere, don't buffer_error if we had some unmapped buffers
434 */
435 if (all_mapped) {
436 printk("__find_get_block_slow() failed. "
437 "block=%llu, b_blocknr=%llu\n",
438 (unsigned long long)block, (unsigned long long)bh->b_blocknr);
439 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
440 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
441 }
442out_unlock:
443 spin_unlock(&bd_mapping->private_lock);
444 page_cache_release(page);
445out:
446 return ret;
447}
448
449/* If invalidate_buffers() will trash dirty buffers, it means some kind
450 of fs corruption is going on. Trashing dirty data always imply losing
451 information that was supposed to be just stored on the physical layer
452 by the user.
453
454 Thus invalidate_buffers in general usage is not allwowed to trash
455 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
456 be preserved. These buffers are simply skipped.
457
458 We also skip buffers which are still in use. For example this can
459 happen if a userspace program is reading the block device.
460
461 NOTE: In the case where the user removed a removable-media-disk even if
462 there's still dirty data not synced on disk (due a bug in the device driver
463 or due an error of the user), by not destroying the dirty buffers we could
464 generate corruption also on the next media inserted, thus a parameter is
465 necessary to handle this case in the most safe way possible (trying
466 to not corrupt also the new disk inserted with the data belonging to
467 the old now corrupted disk). Also for the ramdisk the natural thing
468 to do in order to release the ramdisk memory is to destroy dirty buffers.
469
470 These are two special cases. Normal usage imply the device driver
471 to issue a sync on the device (without waiting I/O completion) and
472 then an invalidate_buffers call that doesn't trash dirty buffers.
473
474 For handling cache coherency with the blkdev pagecache the 'update' case
475 is been introduced. It is needed to re-read from disk any pinned
476 buffer. NOTE: re-reading from disk is destructive so we can do it only
477 when we assume nobody is changing the buffercache under our I/O and when
478 we think the disk contains more recent information than the buffercache.
479 The update == 1 pass marks the buffers we need to update, the update == 2
480 pass does the actual I/O. */
481void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
482{
483 invalidate_bh_lrus();
484 /*
485 * FIXME: what about destroy_dirty_buffers?
486 * We really want to use invalidate_inode_pages2() for
487 * that, but not until that's cleaned up.
488 */
489 invalidate_inode_pages(bdev->bd_inode->i_mapping);
490}
491
492/*
493 * Kick pdflush then try to free up some ZONE_NORMAL memory.
494 */
495static void free_more_memory(void)
496{
497 struct zone **zones;
498 pg_data_t *pgdat;
499
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700500 wakeup_pdflush(1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 yield();
502
503 for_each_pgdat(pgdat) {
504 zones = pgdat->node_zonelists[GFP_NOFS&GFP_ZONEMASK].zones;
505 if (*zones)
Darren Hart1ad539b2005-06-21 17:14:53 -0700506 try_to_free_pages(zones, GFP_NOFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 }
508}
509
510/*
511 * I/O completion handler for block_read_full_page() - pages
512 * which come unlocked at the end of I/O.
513 */
514static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
515{
516 static DEFINE_SPINLOCK(page_uptodate_lock);
517 unsigned long flags;
518 struct buffer_head *tmp;
519 struct page *page;
520 int page_uptodate = 1;
521
522 BUG_ON(!buffer_async_read(bh));
523
524 page = bh->b_page;
525 if (uptodate) {
526 set_buffer_uptodate(bh);
527 } else {
528 clear_buffer_uptodate(bh);
529 if (printk_ratelimit())
530 buffer_io_error(bh);
531 SetPageError(page);
532 }
533
534 /*
535 * Be _very_ careful from here on. Bad things can happen if
536 * two buffer heads end IO at almost the same time and both
537 * decide that the page is now completely done.
538 */
539 spin_lock_irqsave(&page_uptodate_lock, flags);
540 clear_buffer_async_read(bh);
541 unlock_buffer(bh);
542 tmp = bh;
543 do {
544 if (!buffer_uptodate(tmp))
545 page_uptodate = 0;
546 if (buffer_async_read(tmp)) {
547 BUG_ON(!buffer_locked(tmp));
548 goto still_busy;
549 }
550 tmp = tmp->b_this_page;
551 } while (tmp != bh);
552 spin_unlock_irqrestore(&page_uptodate_lock, flags);
553
554 /*
555 * If none of the buffers had errors and they are all
556 * uptodate then we can set the page uptodate.
557 */
558 if (page_uptodate && !PageError(page))
559 SetPageUptodate(page);
560 unlock_page(page);
561 return;
562
563still_busy:
564 spin_unlock_irqrestore(&page_uptodate_lock, flags);
565 return;
566}
567
568/*
569 * Completion handler for block_write_full_page() - pages which are unlocked
570 * during I/O, and which have PageWriteback cleared upon I/O completion.
571 */
572void end_buffer_async_write(struct buffer_head *bh, int uptodate)
573{
574 char b[BDEVNAME_SIZE];
575 static DEFINE_SPINLOCK(page_uptodate_lock);
576 unsigned long flags;
577 struct buffer_head *tmp;
578 struct page *page;
579
580 BUG_ON(!buffer_async_write(bh));
581
582 page = bh->b_page;
583 if (uptodate) {
584 set_buffer_uptodate(bh);
585 } else {
586 if (printk_ratelimit()) {
587 buffer_io_error(bh);
588 printk(KERN_WARNING "lost page write due to "
589 "I/O error on %s\n",
590 bdevname(bh->b_bdev, b));
591 }
592 set_bit(AS_EIO, &page->mapping->flags);
593 clear_buffer_uptodate(bh);
594 SetPageError(page);
595 }
596
597 spin_lock_irqsave(&page_uptodate_lock, flags);
598 clear_buffer_async_write(bh);
599 unlock_buffer(bh);
600 tmp = bh->b_this_page;
601 while (tmp != bh) {
602 if (buffer_async_write(tmp)) {
603 BUG_ON(!buffer_locked(tmp));
604 goto still_busy;
605 }
606 tmp = tmp->b_this_page;
607 }
608 spin_unlock_irqrestore(&page_uptodate_lock, flags);
609 end_page_writeback(page);
610 return;
611
612still_busy:
613 spin_unlock_irqrestore(&page_uptodate_lock, flags);
614 return;
615}
616
617/*
618 * If a page's buffers are under async readin (end_buffer_async_read
619 * completion) then there is a possibility that another thread of
620 * control could lock one of the buffers after it has completed
621 * but while some of the other buffers have not completed. This
622 * locked buffer would confuse end_buffer_async_read() into not unlocking
623 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
624 * that this buffer is not under async I/O.
625 *
626 * The page comes unlocked when it has no locked buffer_async buffers
627 * left.
628 *
629 * PageLocked prevents anyone starting new async I/O reads any of
630 * the buffers.
631 *
632 * PageWriteback is used to prevent simultaneous writeout of the same
633 * page.
634 *
635 * PageLocked prevents anyone from starting writeback of a page which is
636 * under read I/O (PageWriteback is only ever set against a locked page).
637 */
638static void mark_buffer_async_read(struct buffer_head *bh)
639{
640 bh->b_end_io = end_buffer_async_read;
641 set_buffer_async_read(bh);
642}
643
644void mark_buffer_async_write(struct buffer_head *bh)
645{
646 bh->b_end_io = end_buffer_async_write;
647 set_buffer_async_write(bh);
648}
649EXPORT_SYMBOL(mark_buffer_async_write);
650
651
652/*
653 * fs/buffer.c contains helper functions for buffer-backed address space's
654 * fsync functions. A common requirement for buffer-based filesystems is
655 * that certain data from the backing blockdev needs to be written out for
656 * a successful fsync(). For example, ext2 indirect blocks need to be
657 * written back and waited upon before fsync() returns.
658 *
659 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
660 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
661 * management of a list of dependent buffers at ->i_mapping->private_list.
662 *
663 * Locking is a little subtle: try_to_free_buffers() will remove buffers
664 * from their controlling inode's queue when they are being freed. But
665 * try_to_free_buffers() will be operating against the *blockdev* mapping
666 * at the time, not against the S_ISREG file which depends on those buffers.
667 * So the locking for private_list is via the private_lock in the address_space
668 * which backs the buffers. Which is different from the address_space
669 * against which the buffers are listed. So for a particular address_space,
670 * mapping->private_lock does *not* protect mapping->private_list! In fact,
671 * mapping->private_list will always be protected by the backing blockdev's
672 * ->private_lock.
673 *
674 * Which introduces a requirement: all buffers on an address_space's
675 * ->private_list must be from the same address_space: the blockdev's.
676 *
677 * address_spaces which do not place buffers at ->private_list via these
678 * utility functions are free to use private_lock and private_list for
679 * whatever they want. The only requirement is that list_empty(private_list)
680 * be true at clear_inode() time.
681 *
682 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
683 * filesystems should do that. invalidate_inode_buffers() should just go
684 * BUG_ON(!list_empty).
685 *
686 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
687 * take an address_space, not an inode. And it should be called
688 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
689 * queued up.
690 *
691 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
692 * list if it is already on a list. Because if the buffer is on a list,
693 * it *must* already be on the right one. If not, the filesystem is being
694 * silly. This will save a ton of locking. But first we have to ensure
695 * that buffers are taken *off* the old inode's list when they are freed
696 * (presumably in truncate). That requires careful auditing of all
697 * filesystems (do it inside bforget()). It could also be done by bringing
698 * b_inode back.
699 */
700
701/*
702 * The buffer's backing address_space's private_lock must be held
703 */
704static inline void __remove_assoc_queue(struct buffer_head *bh)
705{
706 list_del_init(&bh->b_assoc_buffers);
707}
708
709int inode_has_buffers(struct inode *inode)
710{
711 return !list_empty(&inode->i_data.private_list);
712}
713
714/*
715 * osync is designed to support O_SYNC io. It waits synchronously for
716 * all already-submitted IO to complete, but does not queue any new
717 * writes to the disk.
718 *
719 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
720 * you dirty the buffers, and then use osync_inode_buffers to wait for
721 * completion. Any other dirty buffers which are not yet queued for
722 * write will not be flushed to disk by the osync.
723 */
724static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
725{
726 struct buffer_head *bh;
727 struct list_head *p;
728 int err = 0;
729
730 spin_lock(lock);
731repeat:
732 list_for_each_prev(p, list) {
733 bh = BH_ENTRY(p);
734 if (buffer_locked(bh)) {
735 get_bh(bh);
736 spin_unlock(lock);
737 wait_on_buffer(bh);
738 if (!buffer_uptodate(bh))
739 err = -EIO;
740 brelse(bh);
741 spin_lock(lock);
742 goto repeat;
743 }
744 }
745 spin_unlock(lock);
746 return err;
747}
748
749/**
750 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
751 * buffers
Martin Waitz67be2dd2005-05-01 08:59:26 -0700752 * @mapping: the mapping which wants those buffers written
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 *
754 * Starts I/O against the buffers at mapping->private_list, and waits upon
755 * that I/O.
756 *
Martin Waitz67be2dd2005-05-01 08:59:26 -0700757 * Basically, this is a convenience function for fsync().
758 * @mapping is a file or directory which needs those buffers to be written for
759 * a successful fsync().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 */
761int sync_mapping_buffers(struct address_space *mapping)
762{
763 struct address_space *buffer_mapping = mapping->assoc_mapping;
764
765 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
766 return 0;
767
768 return fsync_buffers_list(&buffer_mapping->private_lock,
769 &mapping->private_list);
770}
771EXPORT_SYMBOL(sync_mapping_buffers);
772
773/*
774 * Called when we've recently written block `bblock', and it is known that
775 * `bblock' was for a buffer_boundary() buffer. This means that the block at
776 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
777 * dirty, schedule it for IO. So that indirects merge nicely with their data.
778 */
779void write_boundary_block(struct block_device *bdev,
780 sector_t bblock, unsigned blocksize)
781{
782 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
783 if (bh) {
784 if (buffer_dirty(bh))
785 ll_rw_block(WRITE, 1, &bh);
786 put_bh(bh);
787 }
788}
789
790void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
791{
792 struct address_space *mapping = inode->i_mapping;
793 struct address_space *buffer_mapping = bh->b_page->mapping;
794
795 mark_buffer_dirty(bh);
796 if (!mapping->assoc_mapping) {
797 mapping->assoc_mapping = buffer_mapping;
798 } else {
799 if (mapping->assoc_mapping != buffer_mapping)
800 BUG();
801 }
802 if (list_empty(&bh->b_assoc_buffers)) {
803 spin_lock(&buffer_mapping->private_lock);
804 list_move_tail(&bh->b_assoc_buffers,
805 &mapping->private_list);
806 spin_unlock(&buffer_mapping->private_lock);
807 }
808}
809EXPORT_SYMBOL(mark_buffer_dirty_inode);
810
811/*
812 * Add a page to the dirty page list.
813 *
814 * It is a sad fact of life that this function is called from several places
815 * deeply under spinlocking. It may not sleep.
816 *
817 * If the page has buffers, the uptodate buffers are set dirty, to preserve
818 * dirty-state coherency between the page and the buffers. It the page does
819 * not have buffers then when they are later attached they will all be set
820 * dirty.
821 *
822 * The buffers are dirtied before the page is dirtied. There's a small race
823 * window in which a writepage caller may see the page cleanness but not the
824 * buffer dirtiness. That's fine. If this code were to set the page dirty
825 * before the buffers, a concurrent writepage caller could clear the page dirty
826 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
827 * page on the dirty page list.
828 *
829 * We use private_lock to lock against try_to_free_buffers while using the
830 * page's buffer list. Also use this to protect against clean buffers being
831 * added to the page after it was set dirty.
832 *
833 * FIXME: may need to call ->reservepage here as well. That's rather up to the
834 * address_space though.
835 */
836int __set_page_dirty_buffers(struct page *page)
837{
838 struct address_space * const mapping = page->mapping;
839
840 spin_lock(&mapping->private_lock);
841 if (page_has_buffers(page)) {
842 struct buffer_head *head = page_buffers(page);
843 struct buffer_head *bh = head;
844
845 do {
846 set_buffer_dirty(bh);
847 bh = bh->b_this_page;
848 } while (bh != head);
849 }
850 spin_unlock(&mapping->private_lock);
851
852 if (!TestSetPageDirty(page)) {
853 write_lock_irq(&mapping->tree_lock);
854 if (page->mapping) { /* Race with truncate? */
855 if (mapping_cap_account_dirty(mapping))
856 inc_page_state(nr_dirty);
857 radix_tree_tag_set(&mapping->page_tree,
858 page_index(page),
859 PAGECACHE_TAG_DIRTY);
860 }
861 write_unlock_irq(&mapping->tree_lock);
862 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
863 }
864
865 return 0;
866}
867EXPORT_SYMBOL(__set_page_dirty_buffers);
868
869/*
870 * Write out and wait upon a list of buffers.
871 *
872 * We have conflicting pressures: we want to make sure that all
873 * initially dirty buffers get waited on, but that any subsequently
874 * dirtied buffers don't. After all, we don't want fsync to last
875 * forever if somebody is actively writing to the file.
876 *
877 * Do this in two main stages: first we copy dirty buffers to a
878 * temporary inode list, queueing the writes as we go. Then we clean
879 * up, waiting for those writes to complete.
880 *
881 * During this second stage, any subsequent updates to the file may end
882 * up refiling the buffer on the original inode's dirty list again, so
883 * there is a chance we will end up with a buffer queued for write but
884 * not yet completed on that list. So, as a final cleanup we go through
885 * the osync code to catch these locked, dirty buffers without requeuing
886 * any newly dirty buffers for write.
887 */
888static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
889{
890 struct buffer_head *bh;
891 struct list_head tmp;
892 int err = 0, err2;
893
894 INIT_LIST_HEAD(&tmp);
895
896 spin_lock(lock);
897 while (!list_empty(list)) {
898 bh = BH_ENTRY(list->next);
899 list_del_init(&bh->b_assoc_buffers);
900 if (buffer_dirty(bh) || buffer_locked(bh)) {
901 list_add(&bh->b_assoc_buffers, &tmp);
902 if (buffer_dirty(bh)) {
903 get_bh(bh);
904 spin_unlock(lock);
905 /*
906 * Ensure any pending I/O completes so that
907 * ll_rw_block() actually writes the current
908 * contents - it is a noop if I/O is still in
909 * flight on potentially older contents.
910 */
911 wait_on_buffer(bh);
912 ll_rw_block(WRITE, 1, &bh);
913 brelse(bh);
914 spin_lock(lock);
915 }
916 }
917 }
918
919 while (!list_empty(&tmp)) {
920 bh = BH_ENTRY(tmp.prev);
921 __remove_assoc_queue(bh);
922 get_bh(bh);
923 spin_unlock(lock);
924 wait_on_buffer(bh);
925 if (!buffer_uptodate(bh))
926 err = -EIO;
927 brelse(bh);
928 spin_lock(lock);
929 }
930
931 spin_unlock(lock);
932 err2 = osync_buffers_list(lock, list);
933 if (err)
934 return err;
935 else
936 return err2;
937}
938
939/*
940 * Invalidate any and all dirty buffers on a given inode. We are
941 * probably unmounting the fs, but that doesn't mean we have already
942 * done a sync(). Just drop the buffers from the inode list.
943 *
944 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
945 * assumes that all the buffers are against the blockdev. Not true
946 * for reiserfs.
947 */
948void invalidate_inode_buffers(struct inode *inode)
949{
950 if (inode_has_buffers(inode)) {
951 struct address_space *mapping = &inode->i_data;
952 struct list_head *list = &mapping->private_list;
953 struct address_space *buffer_mapping = mapping->assoc_mapping;
954
955 spin_lock(&buffer_mapping->private_lock);
956 while (!list_empty(list))
957 __remove_assoc_queue(BH_ENTRY(list->next));
958 spin_unlock(&buffer_mapping->private_lock);
959 }
960}
961
962/*
963 * Remove any clean buffers from the inode's buffer list. This is called
964 * when we're trying to free the inode itself. Those buffers can pin it.
965 *
966 * Returns true if all buffers were removed.
967 */
968int remove_inode_buffers(struct inode *inode)
969{
970 int ret = 1;
971
972 if (inode_has_buffers(inode)) {
973 struct address_space *mapping = &inode->i_data;
974 struct list_head *list = &mapping->private_list;
975 struct address_space *buffer_mapping = mapping->assoc_mapping;
976
977 spin_lock(&buffer_mapping->private_lock);
978 while (!list_empty(list)) {
979 struct buffer_head *bh = BH_ENTRY(list->next);
980 if (buffer_dirty(bh)) {
981 ret = 0;
982 break;
983 }
984 __remove_assoc_queue(bh);
985 }
986 spin_unlock(&buffer_mapping->private_lock);
987 }
988 return ret;
989}
990
991/*
992 * Create the appropriate buffers when given a page for data area and
993 * the size of each buffer.. Use the bh->b_this_page linked list to
994 * follow the buffers created. Return NULL if unable to create more
995 * buffers.
996 *
997 * The retry flag is used to differentiate async IO (paging, swapping)
998 * which may not fail from ordinary buffer allocations.
999 */
1000struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1001 int retry)
1002{
1003 struct buffer_head *bh, *head;
1004 long offset;
1005
1006try_again:
1007 head = NULL;
1008 offset = PAGE_SIZE;
1009 while ((offset -= size) >= 0) {
1010 bh = alloc_buffer_head(GFP_NOFS);
1011 if (!bh)
1012 goto no_grow;
1013
1014 bh->b_bdev = NULL;
1015 bh->b_this_page = head;
1016 bh->b_blocknr = -1;
1017 head = bh;
1018
1019 bh->b_state = 0;
1020 atomic_set(&bh->b_count, 0);
1021 bh->b_size = size;
1022
1023 /* Link the buffer to its page */
1024 set_bh_page(bh, page, offset);
1025
1026 bh->b_end_io = NULL;
1027 }
1028 return head;
1029/*
1030 * In case anything failed, we just free everything we got.
1031 */
1032no_grow:
1033 if (head) {
1034 do {
1035 bh = head;
1036 head = head->b_this_page;
1037 free_buffer_head(bh);
1038 } while (head);
1039 }
1040
1041 /*
1042 * Return failure for non-async IO requests. Async IO requests
1043 * are not allowed to fail, so we have to wait until buffer heads
1044 * become available. But we don't want tasks sleeping with
1045 * partially complete buffers, so all were released above.
1046 */
1047 if (!retry)
1048 return NULL;
1049
1050 /* We're _really_ low on memory. Now we just
1051 * wait for old buffer heads to become free due to
1052 * finishing IO. Since this is an async request and
1053 * the reserve list is empty, we're sure there are
1054 * async buffer heads in use.
1055 */
1056 free_more_memory();
1057 goto try_again;
1058}
1059EXPORT_SYMBOL_GPL(alloc_page_buffers);
1060
1061static inline void
1062link_dev_buffers(struct page *page, struct buffer_head *head)
1063{
1064 struct buffer_head *bh, *tail;
1065
1066 bh = head;
1067 do {
1068 tail = bh;
1069 bh = bh->b_this_page;
1070 } while (bh);
1071 tail->b_this_page = head;
1072 attach_page_buffers(page, head);
1073}
1074
1075/*
1076 * Initialise the state of a blockdev page's buffers.
1077 */
1078static void
1079init_page_buffers(struct page *page, struct block_device *bdev,
1080 sector_t block, int size)
1081{
1082 struct buffer_head *head = page_buffers(page);
1083 struct buffer_head *bh = head;
1084 int uptodate = PageUptodate(page);
1085
1086 do {
1087 if (!buffer_mapped(bh)) {
1088 init_buffer(bh, NULL, NULL);
1089 bh->b_bdev = bdev;
1090 bh->b_blocknr = block;
1091 if (uptodate)
1092 set_buffer_uptodate(bh);
1093 set_buffer_mapped(bh);
1094 }
1095 block++;
1096 bh = bh->b_this_page;
1097 } while (bh != head);
1098}
1099
1100/*
1101 * Create the page-cache page that contains the requested block.
1102 *
1103 * This is user purely for blockdev mappings.
1104 */
1105static struct page *
1106grow_dev_page(struct block_device *bdev, sector_t block,
1107 pgoff_t index, int size)
1108{
1109 struct inode *inode = bdev->bd_inode;
1110 struct page *page;
1111 struct buffer_head *bh;
1112
1113 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1114 if (!page)
1115 return NULL;
1116
1117 if (!PageLocked(page))
1118 BUG();
1119
1120 if (page_has_buffers(page)) {
1121 bh = page_buffers(page);
1122 if (bh->b_size == size) {
1123 init_page_buffers(page, bdev, block, size);
1124 return page;
1125 }
1126 if (!try_to_free_buffers(page))
1127 goto failed;
1128 }
1129
1130 /*
1131 * Allocate some buffers for this page
1132 */
1133 bh = alloc_page_buffers(page, size, 0);
1134 if (!bh)
1135 goto failed;
1136
1137 /*
1138 * Link the page to the buffers and initialise them. Take the
1139 * lock to be atomic wrt __find_get_block(), which does not
1140 * run under the page lock.
1141 */
1142 spin_lock(&inode->i_mapping->private_lock);
1143 link_dev_buffers(page, bh);
1144 init_page_buffers(page, bdev, block, size);
1145 spin_unlock(&inode->i_mapping->private_lock);
1146 return page;
1147
1148failed:
1149 BUG();
1150 unlock_page(page);
1151 page_cache_release(page);
1152 return NULL;
1153}
1154
1155/*
1156 * Create buffers for the specified block device block's page. If
1157 * that page was dirty, the buffers are set dirty also.
1158 *
1159 * Except that's a bug. Attaching dirty buffers to a dirty
1160 * blockdev's page can result in filesystem corruption, because
1161 * some of those buffers may be aliases of filesystem data.
1162 * grow_dev_page() will go BUG() if this happens.
1163 */
1164static inline int
1165grow_buffers(struct block_device *bdev, sector_t block, int size)
1166{
1167 struct page *page;
1168 pgoff_t index;
1169 int sizebits;
1170
1171 sizebits = -1;
1172 do {
1173 sizebits++;
1174 } while ((size << sizebits) < PAGE_SIZE);
1175
1176 index = block >> sizebits;
1177 block = index << sizebits;
1178
1179 /* Create a page with the proper size buffers.. */
1180 page = grow_dev_page(bdev, block, index, size);
1181 if (!page)
1182 return 0;
1183 unlock_page(page);
1184 page_cache_release(page);
1185 return 1;
1186}
1187
Adrian Bunk75c96f82005-05-05 16:16:09 -07001188static struct buffer_head *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189__getblk_slow(struct block_device *bdev, sector_t block, int size)
1190{
1191 /* Size must be multiple of hard sectorsize */
1192 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1193 (size < 512 || size > PAGE_SIZE))) {
1194 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1195 size);
1196 printk(KERN_ERR "hardsect size: %d\n",
1197 bdev_hardsect_size(bdev));
1198
1199 dump_stack();
1200 return NULL;
1201 }
1202
1203 for (;;) {
1204 struct buffer_head * bh;
1205
1206 bh = __find_get_block(bdev, block, size);
1207 if (bh)
1208 return bh;
1209
1210 if (!grow_buffers(bdev, block, size))
1211 free_more_memory();
1212 }
1213}
1214
1215/*
1216 * The relationship between dirty buffers and dirty pages:
1217 *
1218 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1219 * the page is tagged dirty in its radix tree.
1220 *
1221 * At all times, the dirtiness of the buffers represents the dirtiness of
1222 * subsections of the page. If the page has buffers, the page dirty bit is
1223 * merely a hint about the true dirty state.
1224 *
1225 * When a page is set dirty in its entirety, all its buffers are marked dirty
1226 * (if the page has buffers).
1227 *
1228 * When a buffer is marked dirty, its page is dirtied, but the page's other
1229 * buffers are not.
1230 *
1231 * Also. When blockdev buffers are explicitly read with bread(), they
1232 * individually become uptodate. But their backing page remains not
1233 * uptodate - even if all of its buffers are uptodate. A subsequent
1234 * block_read_full_page() against that page will discover all the uptodate
1235 * buffers, will set the page uptodate and will perform no I/O.
1236 */
1237
1238/**
1239 * mark_buffer_dirty - mark a buffer_head as needing writeout
Martin Waitz67be2dd2005-05-01 08:59:26 -07001240 * @bh: the buffer_head to mark dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 *
1242 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1243 * backing page dirty, then tag the page as dirty in its address_space's radix
1244 * tree and then attach the address_space's inode to its superblock's dirty
1245 * inode list.
1246 *
1247 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1248 * mapping->tree_lock and the global inode_lock.
1249 */
1250void fastcall mark_buffer_dirty(struct buffer_head *bh)
1251{
1252 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1253 __set_page_dirty_nobuffers(bh->b_page);
1254}
1255
1256/*
1257 * Decrement a buffer_head's reference count. If all buffers against a page
1258 * have zero reference count, are clean and unlocked, and if the page is clean
1259 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1260 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1261 * a page but it ends up not being freed, and buffers may later be reattached).
1262 */
1263void __brelse(struct buffer_head * buf)
1264{
1265 if (atomic_read(&buf->b_count)) {
1266 put_bh(buf);
1267 return;
1268 }
1269 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1270 WARN_ON(1);
1271}
1272
1273/*
1274 * bforget() is like brelse(), except it discards any
1275 * potentially dirty data.
1276 */
1277void __bforget(struct buffer_head *bh)
1278{
1279 clear_buffer_dirty(bh);
1280 if (!list_empty(&bh->b_assoc_buffers)) {
1281 struct address_space *buffer_mapping = bh->b_page->mapping;
1282
1283 spin_lock(&buffer_mapping->private_lock);
1284 list_del_init(&bh->b_assoc_buffers);
1285 spin_unlock(&buffer_mapping->private_lock);
1286 }
1287 __brelse(bh);
1288}
1289
1290static struct buffer_head *__bread_slow(struct buffer_head *bh)
1291{
1292 lock_buffer(bh);
1293 if (buffer_uptodate(bh)) {
1294 unlock_buffer(bh);
1295 return bh;
1296 } else {
1297 get_bh(bh);
1298 bh->b_end_io = end_buffer_read_sync;
1299 submit_bh(READ, bh);
1300 wait_on_buffer(bh);
1301 if (buffer_uptodate(bh))
1302 return bh;
1303 }
1304 brelse(bh);
1305 return NULL;
1306}
1307
1308/*
1309 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1310 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1311 * refcount elevated by one when they're in an LRU. A buffer can only appear
1312 * once in a particular CPU's LRU. A single buffer can be present in multiple
1313 * CPU's LRUs at the same time.
1314 *
1315 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1316 * sb_find_get_block().
1317 *
1318 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1319 * a local interrupt disable for that.
1320 */
1321
1322#define BH_LRU_SIZE 8
1323
1324struct bh_lru {
1325 struct buffer_head *bhs[BH_LRU_SIZE];
1326};
1327
1328static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1329
1330#ifdef CONFIG_SMP
1331#define bh_lru_lock() local_irq_disable()
1332#define bh_lru_unlock() local_irq_enable()
1333#else
1334#define bh_lru_lock() preempt_disable()
1335#define bh_lru_unlock() preempt_enable()
1336#endif
1337
1338static inline void check_irqs_on(void)
1339{
1340#ifdef irqs_disabled
1341 BUG_ON(irqs_disabled());
1342#endif
1343}
1344
1345/*
1346 * The LRU management algorithm is dopey-but-simple. Sorry.
1347 */
1348static void bh_lru_install(struct buffer_head *bh)
1349{
1350 struct buffer_head *evictee = NULL;
1351 struct bh_lru *lru;
1352
1353 check_irqs_on();
1354 bh_lru_lock();
1355 lru = &__get_cpu_var(bh_lrus);
1356 if (lru->bhs[0] != bh) {
1357 struct buffer_head *bhs[BH_LRU_SIZE];
1358 int in;
1359 int out = 0;
1360
1361 get_bh(bh);
1362 bhs[out++] = bh;
1363 for (in = 0; in < BH_LRU_SIZE; in++) {
1364 struct buffer_head *bh2 = lru->bhs[in];
1365
1366 if (bh2 == bh) {
1367 __brelse(bh2);
1368 } else {
1369 if (out >= BH_LRU_SIZE) {
1370 BUG_ON(evictee != NULL);
1371 evictee = bh2;
1372 } else {
1373 bhs[out++] = bh2;
1374 }
1375 }
1376 }
1377 while (out < BH_LRU_SIZE)
1378 bhs[out++] = NULL;
1379 memcpy(lru->bhs, bhs, sizeof(bhs));
1380 }
1381 bh_lru_unlock();
1382
1383 if (evictee)
1384 __brelse(evictee);
1385}
1386
1387/*
1388 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1389 */
1390static inline struct buffer_head *
1391lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1392{
1393 struct buffer_head *ret = NULL;
1394 struct bh_lru *lru;
1395 int i;
1396
1397 check_irqs_on();
1398 bh_lru_lock();
1399 lru = &__get_cpu_var(bh_lrus);
1400 for (i = 0; i < BH_LRU_SIZE; i++) {
1401 struct buffer_head *bh = lru->bhs[i];
1402
1403 if (bh && bh->b_bdev == bdev &&
1404 bh->b_blocknr == block && bh->b_size == size) {
1405 if (i) {
1406 while (i) {
1407 lru->bhs[i] = lru->bhs[i - 1];
1408 i--;
1409 }
1410 lru->bhs[0] = bh;
1411 }
1412 get_bh(bh);
1413 ret = bh;
1414 break;
1415 }
1416 }
1417 bh_lru_unlock();
1418 return ret;
1419}
1420
1421/*
1422 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1423 * it in the LRU and mark it as accessed. If it is not present then return
1424 * NULL
1425 */
1426struct buffer_head *
1427__find_get_block(struct block_device *bdev, sector_t block, int size)
1428{
1429 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1430
1431 if (bh == NULL) {
1432 bh = __find_get_block_slow(bdev, block, size);
1433 if (bh)
1434 bh_lru_install(bh);
1435 }
1436 if (bh)
1437 touch_buffer(bh);
1438 return bh;
1439}
1440EXPORT_SYMBOL(__find_get_block);
1441
1442/*
1443 * __getblk will locate (and, if necessary, create) the buffer_head
1444 * which corresponds to the passed block_device, block and size. The
1445 * returned buffer has its reference count incremented.
1446 *
1447 * __getblk() cannot fail - it just keeps trying. If you pass it an
1448 * illegal block number, __getblk() will happily return a buffer_head
1449 * which represents the non-existent block. Very weird.
1450 *
1451 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1452 * attempt is failing. FIXME, perhaps?
1453 */
1454struct buffer_head *
1455__getblk(struct block_device *bdev, sector_t block, int size)
1456{
1457 struct buffer_head *bh = __find_get_block(bdev, block, size);
1458
1459 might_sleep();
1460 if (bh == NULL)
1461 bh = __getblk_slow(bdev, block, size);
1462 return bh;
1463}
1464EXPORT_SYMBOL(__getblk);
1465
1466/*
1467 * Do async read-ahead on a buffer..
1468 */
1469void __breadahead(struct block_device *bdev, sector_t block, int size)
1470{
1471 struct buffer_head *bh = __getblk(bdev, block, size);
1472 ll_rw_block(READA, 1, &bh);
1473 brelse(bh);
1474}
1475EXPORT_SYMBOL(__breadahead);
1476
1477/**
1478 * __bread() - reads a specified block and returns the bh
Martin Waitz67be2dd2005-05-01 08:59:26 -07001479 * @bdev: the block_device to read from
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480 * @block: number of block
1481 * @size: size (in bytes) to read
1482 *
1483 * Reads a specified block, and returns buffer head that contains it.
1484 * It returns NULL if the block was unreadable.
1485 */
1486struct buffer_head *
1487__bread(struct block_device *bdev, sector_t block, int size)
1488{
1489 struct buffer_head *bh = __getblk(bdev, block, size);
1490
1491 if (!buffer_uptodate(bh))
1492 bh = __bread_slow(bh);
1493 return bh;
1494}
1495EXPORT_SYMBOL(__bread);
1496
1497/*
1498 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1499 * This doesn't race because it runs in each cpu either in irq
1500 * or with preempt disabled.
1501 */
1502static void invalidate_bh_lru(void *arg)
1503{
1504 struct bh_lru *b = &get_cpu_var(bh_lrus);
1505 int i;
1506
1507 for (i = 0; i < BH_LRU_SIZE; i++) {
1508 brelse(b->bhs[i]);
1509 b->bhs[i] = NULL;
1510 }
1511 put_cpu_var(bh_lrus);
1512}
1513
1514static void invalidate_bh_lrus(void)
1515{
1516 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1517}
1518
1519void set_bh_page(struct buffer_head *bh,
1520 struct page *page, unsigned long offset)
1521{
1522 bh->b_page = page;
1523 if (offset >= PAGE_SIZE)
1524 BUG();
1525 if (PageHighMem(page))
1526 /*
1527 * This catches illegal uses and preserves the offset:
1528 */
1529 bh->b_data = (char *)(0 + offset);
1530 else
1531 bh->b_data = page_address(page) + offset;
1532}
1533EXPORT_SYMBOL(set_bh_page);
1534
1535/*
1536 * Called when truncating a buffer on a page completely.
1537 */
1538static inline void discard_buffer(struct buffer_head * bh)
1539{
1540 lock_buffer(bh);
1541 clear_buffer_dirty(bh);
1542 bh->b_bdev = NULL;
1543 clear_buffer_mapped(bh);
1544 clear_buffer_req(bh);
1545 clear_buffer_new(bh);
1546 clear_buffer_delay(bh);
1547 unlock_buffer(bh);
1548}
1549
1550/**
1551 * try_to_release_page() - release old fs-specific metadata on a page
1552 *
1553 * @page: the page which the kernel is trying to free
1554 * @gfp_mask: memory allocation flags (and I/O mode)
1555 *
1556 * The address_space is to try to release any data against the page
1557 * (presumably at page->private). If the release was successful, return `1'.
1558 * Otherwise return zero.
1559 *
1560 * The @gfp_mask argument specifies whether I/O may be performed to release
1561 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1562 *
1563 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1564 */
1565int try_to_release_page(struct page *page, int gfp_mask)
1566{
1567 struct address_space * const mapping = page->mapping;
1568
1569 BUG_ON(!PageLocked(page));
1570 if (PageWriteback(page))
1571 return 0;
1572
1573 if (mapping && mapping->a_ops->releasepage)
1574 return mapping->a_ops->releasepage(page, gfp_mask);
1575 return try_to_free_buffers(page);
1576}
1577EXPORT_SYMBOL(try_to_release_page);
1578
1579/**
1580 * block_invalidatepage - invalidate part of all of a buffer-backed page
1581 *
1582 * @page: the page which is affected
1583 * @offset: the index of the truncation point
1584 *
1585 * block_invalidatepage() is called when all or part of the page has become
1586 * invalidatedby a truncate operation.
1587 *
1588 * block_invalidatepage() does not have to release all buffers, but it must
1589 * ensure that no dirty buffer is left outside @offset and that no I/O
1590 * is underway against any of the blocks which are outside the truncation
1591 * point. Because the caller is about to free (and possibly reuse) those
1592 * blocks on-disk.
1593 */
1594int block_invalidatepage(struct page *page, unsigned long offset)
1595{
1596 struct buffer_head *head, *bh, *next;
1597 unsigned int curr_off = 0;
1598 int ret = 1;
1599
1600 BUG_ON(!PageLocked(page));
1601 if (!page_has_buffers(page))
1602 goto out;
1603
1604 head = page_buffers(page);
1605 bh = head;
1606 do {
1607 unsigned int next_off = curr_off + bh->b_size;
1608 next = bh->b_this_page;
1609
1610 /*
1611 * is this block fully invalidated?
1612 */
1613 if (offset <= curr_off)
1614 discard_buffer(bh);
1615 curr_off = next_off;
1616 bh = next;
1617 } while (bh != head);
1618
1619 /*
1620 * We release buffers only if the entire page is being invalidated.
1621 * The get_block cached value has been unconditionally invalidated,
1622 * so real IO is not possible anymore.
1623 */
1624 if (offset == 0)
1625 ret = try_to_release_page(page, 0);
1626out:
1627 return ret;
1628}
1629EXPORT_SYMBOL(block_invalidatepage);
1630
1631/*
1632 * We attach and possibly dirty the buffers atomically wrt
1633 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1634 * is already excluded via the page lock.
1635 */
1636void create_empty_buffers(struct page *page,
1637 unsigned long blocksize, unsigned long b_state)
1638{
1639 struct buffer_head *bh, *head, *tail;
1640
1641 head = alloc_page_buffers(page, blocksize, 1);
1642 bh = head;
1643 do {
1644 bh->b_state |= b_state;
1645 tail = bh;
1646 bh = bh->b_this_page;
1647 } while (bh);
1648 tail->b_this_page = head;
1649
1650 spin_lock(&page->mapping->private_lock);
1651 if (PageUptodate(page) || PageDirty(page)) {
1652 bh = head;
1653 do {
1654 if (PageDirty(page))
1655 set_buffer_dirty(bh);
1656 if (PageUptodate(page))
1657 set_buffer_uptodate(bh);
1658 bh = bh->b_this_page;
1659 } while (bh != head);
1660 }
1661 attach_page_buffers(page, head);
1662 spin_unlock(&page->mapping->private_lock);
1663}
1664EXPORT_SYMBOL(create_empty_buffers);
1665
1666/*
1667 * We are taking a block for data and we don't want any output from any
1668 * buffer-cache aliases starting from return from that function and
1669 * until the moment when something will explicitly mark the buffer
1670 * dirty (hopefully that will not happen until we will free that block ;-)
1671 * We don't even need to mark it not-uptodate - nobody can expect
1672 * anything from a newly allocated buffer anyway. We used to used
1673 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1674 * don't want to mark the alias unmapped, for example - it would confuse
1675 * anyone who might pick it with bread() afterwards...
1676 *
1677 * Also.. Note that bforget() doesn't lock the buffer. So there can
1678 * be writeout I/O going on against recently-freed buffers. We don't
1679 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1680 * only if we really need to. That happens here.
1681 */
1682void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1683{
1684 struct buffer_head *old_bh;
1685
1686 might_sleep();
1687
1688 old_bh = __find_get_block_slow(bdev, block, 0);
1689 if (old_bh) {
1690 clear_buffer_dirty(old_bh);
1691 wait_on_buffer(old_bh);
1692 clear_buffer_req(old_bh);
1693 __brelse(old_bh);
1694 }
1695}
1696EXPORT_SYMBOL(unmap_underlying_metadata);
1697
1698/*
1699 * NOTE! All mapped/uptodate combinations are valid:
1700 *
1701 * Mapped Uptodate Meaning
1702 *
1703 * No No "unknown" - must do get_block()
1704 * No Yes "hole" - zero-filled
1705 * Yes No "allocated" - allocated on disk, not read in
1706 * Yes Yes "valid" - allocated and up-to-date in memory.
1707 *
1708 * "Dirty" is valid only with the last case (mapped+uptodate).
1709 */
1710
1711/*
1712 * While block_write_full_page is writing back the dirty buffers under
1713 * the page lock, whoever dirtied the buffers may decide to clean them
1714 * again at any time. We handle that by only looking at the buffer
1715 * state inside lock_buffer().
1716 *
1717 * If block_write_full_page() is called for regular writeback
1718 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1719 * locked buffer. This only can happen if someone has written the buffer
1720 * directly, with submit_bh(). At the address_space level PageWriteback
1721 * prevents this contention from occurring.
1722 */
1723static int __block_write_full_page(struct inode *inode, struct page *page,
1724 get_block_t *get_block, struct writeback_control *wbc)
1725{
1726 int err;
1727 sector_t block;
1728 sector_t last_block;
Andrew Mortonf0fbd5f2005-05-05 16:15:48 -07001729 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 int nr_underway = 0;
1731
1732 BUG_ON(!PageLocked(page));
1733
1734 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1735
1736 if (!page_has_buffers(page)) {
1737 create_empty_buffers(page, 1 << inode->i_blkbits,
1738 (1 << BH_Dirty)|(1 << BH_Uptodate));
1739 }
1740
1741 /*
1742 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1743 * here, and the (potentially unmapped) buffers may become dirty at
1744 * any time. If a buffer becomes dirty here after we've inspected it
1745 * then we just miss that fact, and the page stays dirty.
1746 *
1747 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1748 * handle that here by just cleaning them.
1749 */
1750
1751 block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1752 head = page_buffers(page);
1753 bh = head;
1754
1755 /*
1756 * Get all the dirty buffers mapped to disk addresses and
1757 * handle any aliases from the underlying blockdev's mapping.
1758 */
1759 do {
1760 if (block > last_block) {
1761 /*
1762 * mapped buffers outside i_size will occur, because
1763 * this page can be outside i_size when there is a
1764 * truncate in progress.
1765 */
1766 /*
1767 * The buffer was zeroed by block_write_full_page()
1768 */
1769 clear_buffer_dirty(bh);
1770 set_buffer_uptodate(bh);
1771 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1772 err = get_block(inode, block, bh, 1);
1773 if (err)
1774 goto recover;
1775 if (buffer_new(bh)) {
1776 /* blockdev mappings never come here */
1777 clear_buffer_new(bh);
1778 unmap_underlying_metadata(bh->b_bdev,
1779 bh->b_blocknr);
1780 }
1781 }
1782 bh = bh->b_this_page;
1783 block++;
1784 } while (bh != head);
1785
1786 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787 if (!buffer_mapped(bh))
1788 continue;
1789 /*
1790 * If it's a fully non-blocking write attempt and we cannot
1791 * lock the buffer then redirty the page. Note that this can
1792 * potentially cause a busy-wait loop from pdflush and kswapd
1793 * activity, but those code paths have their own higher-level
1794 * throttling.
1795 */
1796 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1797 lock_buffer(bh);
1798 } else if (test_set_buffer_locked(bh)) {
1799 redirty_page_for_writepage(wbc, page);
1800 continue;
1801 }
1802 if (test_clear_buffer_dirty(bh)) {
1803 mark_buffer_async_write(bh);
1804 } else {
1805 unlock_buffer(bh);
1806 }
1807 } while ((bh = bh->b_this_page) != head);
1808
1809 /*
1810 * The page and its buffers are protected by PageWriteback(), so we can
1811 * drop the bh refcounts early.
1812 */
1813 BUG_ON(PageWriteback(page));
1814 set_page_writeback(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815
1816 do {
1817 struct buffer_head *next = bh->b_this_page;
1818 if (buffer_async_write(bh)) {
1819 submit_bh(WRITE, bh);
1820 nr_underway++;
1821 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001822 bh = next;
1823 } while (bh != head);
Andrew Morton05937ba2005-05-05 16:15:47 -07001824 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825
1826 err = 0;
1827done:
1828 if (nr_underway == 0) {
1829 /*
1830 * The page was marked dirty, but the buffers were
1831 * clean. Someone wrote them back by hand with
1832 * ll_rw_block/submit_bh. A rare case.
1833 */
1834 int uptodate = 1;
1835 do {
1836 if (!buffer_uptodate(bh)) {
1837 uptodate = 0;
1838 break;
1839 }
1840 bh = bh->b_this_page;
1841 } while (bh != head);
1842 if (uptodate)
1843 SetPageUptodate(page);
1844 end_page_writeback(page);
1845 /*
1846 * The page and buffer_heads can be released at any time from
1847 * here on.
1848 */
1849 wbc->pages_skipped++; /* We didn't write this page */
1850 }
1851 return err;
1852
1853recover:
1854 /*
1855 * ENOSPC, or some other error. We may already have added some
1856 * blocks to the file, so we need to write these out to avoid
1857 * exposing stale data.
1858 * The page is currently locked and not marked for writeback
1859 */
1860 bh = head;
1861 /* Recovery: lock and submit the mapped buffers */
1862 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1864 lock_buffer(bh);
1865 mark_buffer_async_write(bh);
1866 } else {
1867 /*
1868 * The buffer may have been set dirty during
1869 * attachment to a dirty page.
1870 */
1871 clear_buffer_dirty(bh);
1872 }
1873 } while ((bh = bh->b_this_page) != head);
1874 SetPageError(page);
1875 BUG_ON(PageWriteback(page));
1876 set_page_writeback(page);
1877 unlock_page(page);
1878 do {
1879 struct buffer_head *next = bh->b_this_page;
1880 if (buffer_async_write(bh)) {
1881 clear_buffer_dirty(bh);
1882 submit_bh(WRITE, bh);
1883 nr_underway++;
1884 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001885 bh = next;
1886 } while (bh != head);
1887 goto done;
1888}
1889
1890static int __block_prepare_write(struct inode *inode, struct page *page,
1891 unsigned from, unsigned to, get_block_t *get_block)
1892{
1893 unsigned block_start, block_end;
1894 sector_t block;
1895 int err = 0;
1896 unsigned blocksize, bbits;
1897 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1898
1899 BUG_ON(!PageLocked(page));
1900 BUG_ON(from > PAGE_CACHE_SIZE);
1901 BUG_ON(to > PAGE_CACHE_SIZE);
1902 BUG_ON(from > to);
1903
1904 blocksize = 1 << inode->i_blkbits;
1905 if (!page_has_buffers(page))
1906 create_empty_buffers(page, blocksize, 0);
1907 head = page_buffers(page);
1908
1909 bbits = inode->i_blkbits;
1910 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1911
1912 for(bh = head, block_start = 0; bh != head || !block_start;
1913 block++, block_start=block_end, bh = bh->b_this_page) {
1914 block_end = block_start + blocksize;
1915 if (block_end <= from || block_start >= to) {
1916 if (PageUptodate(page)) {
1917 if (!buffer_uptodate(bh))
1918 set_buffer_uptodate(bh);
1919 }
1920 continue;
1921 }
1922 if (buffer_new(bh))
1923 clear_buffer_new(bh);
1924 if (!buffer_mapped(bh)) {
1925 err = get_block(inode, block, bh, 1);
1926 if (err)
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001927 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 if (buffer_new(bh)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001929 unmap_underlying_metadata(bh->b_bdev,
1930 bh->b_blocknr);
1931 if (PageUptodate(page)) {
1932 set_buffer_uptodate(bh);
1933 continue;
1934 }
1935 if (block_end > to || block_start < from) {
1936 void *kaddr;
1937
1938 kaddr = kmap_atomic(page, KM_USER0);
1939 if (block_end > to)
1940 memset(kaddr+to, 0,
1941 block_end-to);
1942 if (block_start < from)
1943 memset(kaddr+block_start,
1944 0, from-block_start);
1945 flush_dcache_page(page);
1946 kunmap_atomic(kaddr, KM_USER0);
1947 }
1948 continue;
1949 }
1950 }
1951 if (PageUptodate(page)) {
1952 if (!buffer_uptodate(bh))
1953 set_buffer_uptodate(bh);
1954 continue;
1955 }
1956 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1957 (block_start < from || block_end > to)) {
1958 ll_rw_block(READ, 1, &bh);
1959 *wait_bh++=bh;
1960 }
1961 }
1962 /*
1963 * If we issued read requests - let them complete.
1964 */
1965 while(wait_bh > wait) {
1966 wait_on_buffer(*--wait_bh);
1967 if (!buffer_uptodate(*wait_bh))
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001968 err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969 }
Anton Altaparmakov152becd2005-06-23 00:10:21 -07001970 if (!err) {
1971 bh = head;
1972 do {
1973 if (buffer_new(bh))
1974 clear_buffer_new(bh);
1975 } while ((bh = bh->b_this_page) != head);
1976 return 0;
1977 }
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001978 /* Error case: */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 /*
1980 * Zero out any newly allocated blocks to avoid exposing stale
1981 * data. If BH_New is set, we know that the block was newly
1982 * allocated in the above loop.
1983 */
1984 bh = head;
1985 block_start = 0;
1986 do {
1987 block_end = block_start+blocksize;
1988 if (block_end <= from)
1989 goto next_bh;
1990 if (block_start >= to)
1991 break;
1992 if (buffer_new(bh)) {
1993 void *kaddr;
1994
1995 clear_buffer_new(bh);
1996 kaddr = kmap_atomic(page, KM_USER0);
1997 memset(kaddr+block_start, 0, bh->b_size);
1998 kunmap_atomic(kaddr, KM_USER0);
1999 set_buffer_uptodate(bh);
2000 mark_buffer_dirty(bh);
2001 }
2002next_bh:
2003 block_start = block_end;
2004 bh = bh->b_this_page;
2005 } while (bh != head);
2006 return err;
2007}
2008
2009static int __block_commit_write(struct inode *inode, struct page *page,
2010 unsigned from, unsigned to)
2011{
2012 unsigned block_start, block_end;
2013 int partial = 0;
2014 unsigned blocksize;
2015 struct buffer_head *bh, *head;
2016
2017 blocksize = 1 << inode->i_blkbits;
2018
2019 for(bh = head = page_buffers(page), block_start = 0;
2020 bh != head || !block_start;
2021 block_start=block_end, bh = bh->b_this_page) {
2022 block_end = block_start + blocksize;
2023 if (block_end <= from || block_start >= to) {
2024 if (!buffer_uptodate(bh))
2025 partial = 1;
2026 } else {
2027 set_buffer_uptodate(bh);
2028 mark_buffer_dirty(bh);
2029 }
2030 }
2031
2032 /*
2033 * If this is a partial write which happened to make all buffers
2034 * uptodate then we can optimize away a bogus readpage() for
2035 * the next read(). Here we 'discover' whether the page went
2036 * uptodate as a result of this (potentially partial) write.
2037 */
2038 if (!partial)
2039 SetPageUptodate(page);
2040 return 0;
2041}
2042
2043/*
2044 * Generic "read page" function for block devices that have the normal
2045 * get_block functionality. This is most of the block device filesystems.
2046 * Reads the page asynchronously --- the unlock_buffer() and
2047 * set/clear_buffer_uptodate() functions propagate buffer state into the
2048 * page struct once IO has completed.
2049 */
2050int block_read_full_page(struct page *page, get_block_t *get_block)
2051{
2052 struct inode *inode = page->mapping->host;
2053 sector_t iblock, lblock;
2054 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2055 unsigned int blocksize;
2056 int nr, i;
2057 int fully_mapped = 1;
2058
Matt Mackallcd7619d2005-05-01 08:59:01 -07002059 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060 blocksize = 1 << inode->i_blkbits;
2061 if (!page_has_buffers(page))
2062 create_empty_buffers(page, blocksize, 0);
2063 head = page_buffers(page);
2064
2065 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2066 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2067 bh = head;
2068 nr = 0;
2069 i = 0;
2070
2071 do {
2072 if (buffer_uptodate(bh))
2073 continue;
2074
2075 if (!buffer_mapped(bh)) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002076 int err = 0;
2077
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078 fully_mapped = 0;
2079 if (iblock < lblock) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002080 err = get_block(inode, iblock, bh, 0);
2081 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082 SetPageError(page);
2083 }
2084 if (!buffer_mapped(bh)) {
2085 void *kaddr = kmap_atomic(page, KM_USER0);
2086 memset(kaddr + i * blocksize, 0, blocksize);
2087 flush_dcache_page(page);
2088 kunmap_atomic(kaddr, KM_USER0);
Andrew Mortonc64610b2005-05-16 21:53:49 -07002089 if (!err)
2090 set_buffer_uptodate(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 continue;
2092 }
2093 /*
2094 * get_block() might have updated the buffer
2095 * synchronously
2096 */
2097 if (buffer_uptodate(bh))
2098 continue;
2099 }
2100 arr[nr++] = bh;
2101 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2102
2103 if (fully_mapped)
2104 SetPageMappedToDisk(page);
2105
2106 if (!nr) {
2107 /*
2108 * All buffers are uptodate - we can set the page uptodate
2109 * as well. But not if get_block() returned an error.
2110 */
2111 if (!PageError(page))
2112 SetPageUptodate(page);
2113 unlock_page(page);
2114 return 0;
2115 }
2116
2117 /* Stage two: lock the buffers */
2118 for (i = 0; i < nr; i++) {
2119 bh = arr[i];
2120 lock_buffer(bh);
2121 mark_buffer_async_read(bh);
2122 }
2123
2124 /*
2125 * Stage 3: start the IO. Check for uptodateness
2126 * inside the buffer lock in case another process reading
2127 * the underlying blockdev brought it uptodate (the sct fix).
2128 */
2129 for (i = 0; i < nr; i++) {
2130 bh = arr[i];
2131 if (buffer_uptodate(bh))
2132 end_buffer_async_read(bh, 1);
2133 else
2134 submit_bh(READ, bh);
2135 }
2136 return 0;
2137}
2138
2139/* utility function for filesystems that need to do work on expanding
2140 * truncates. Uses prepare/commit_write to allow the filesystem to
2141 * deal with the hole.
2142 */
2143int generic_cont_expand(struct inode *inode, loff_t size)
2144{
2145 struct address_space *mapping = inode->i_mapping;
2146 struct page *page;
2147 unsigned long index, offset, limit;
2148 int err;
2149
2150 err = -EFBIG;
2151 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2152 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2153 send_sig(SIGXFSZ, current, 0);
2154 goto out;
2155 }
2156 if (size > inode->i_sb->s_maxbytes)
2157 goto out;
2158
2159 offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2160
2161 /* ugh. in prepare/commit_write, if from==to==start of block, we
2162 ** skip the prepare. make sure we never send an offset for the start
2163 ** of a block
2164 */
2165 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2166 offset++;
2167 }
2168 index = size >> PAGE_CACHE_SHIFT;
2169 err = -ENOMEM;
2170 page = grab_cache_page(mapping, index);
2171 if (!page)
2172 goto out;
2173 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2174 if (!err) {
2175 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2176 }
2177 unlock_page(page);
2178 page_cache_release(page);
2179 if (err > 0)
2180 err = 0;
2181out:
2182 return err;
2183}
2184
2185/*
2186 * For moronic filesystems that do not allow holes in file.
2187 * We may have to extend the file.
2188 */
2189
2190int cont_prepare_write(struct page *page, unsigned offset,
2191 unsigned to, get_block_t *get_block, loff_t *bytes)
2192{
2193 struct address_space *mapping = page->mapping;
2194 struct inode *inode = mapping->host;
2195 struct page *new_page;
2196 pgoff_t pgpos;
2197 long status;
2198 unsigned zerofrom;
2199 unsigned blocksize = 1 << inode->i_blkbits;
2200 void *kaddr;
2201
2202 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2203 status = -ENOMEM;
2204 new_page = grab_cache_page(mapping, pgpos);
2205 if (!new_page)
2206 goto out;
2207 /* we might sleep */
2208 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2209 unlock_page(new_page);
2210 page_cache_release(new_page);
2211 continue;
2212 }
2213 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2214 if (zerofrom & (blocksize-1)) {
2215 *bytes |= (blocksize-1);
2216 (*bytes)++;
2217 }
2218 status = __block_prepare_write(inode, new_page, zerofrom,
2219 PAGE_CACHE_SIZE, get_block);
2220 if (status)
2221 goto out_unmap;
2222 kaddr = kmap_atomic(new_page, KM_USER0);
2223 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2224 flush_dcache_page(new_page);
2225 kunmap_atomic(kaddr, KM_USER0);
2226 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2227 unlock_page(new_page);
2228 page_cache_release(new_page);
2229 }
2230
2231 if (page->index < pgpos) {
2232 /* completely inside the area */
2233 zerofrom = offset;
2234 } else {
2235 /* page covers the boundary, find the boundary offset */
2236 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2237
2238 /* if we will expand the thing last block will be filled */
2239 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2240 *bytes |= (blocksize-1);
2241 (*bytes)++;
2242 }
2243
2244 /* starting below the boundary? Nothing to zero out */
2245 if (offset <= zerofrom)
2246 zerofrom = offset;
2247 }
2248 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2249 if (status)
2250 goto out1;
2251 if (zerofrom < offset) {
2252 kaddr = kmap_atomic(page, KM_USER0);
2253 memset(kaddr+zerofrom, 0, offset-zerofrom);
2254 flush_dcache_page(page);
2255 kunmap_atomic(kaddr, KM_USER0);
2256 __block_commit_write(inode, page, zerofrom, offset);
2257 }
2258 return 0;
2259out1:
2260 ClearPageUptodate(page);
2261 return status;
2262
2263out_unmap:
2264 ClearPageUptodate(new_page);
2265 unlock_page(new_page);
2266 page_cache_release(new_page);
2267out:
2268 return status;
2269}
2270
2271int block_prepare_write(struct page *page, unsigned from, unsigned to,
2272 get_block_t *get_block)
2273{
2274 struct inode *inode = page->mapping->host;
2275 int err = __block_prepare_write(inode, page, from, to, get_block);
2276 if (err)
2277 ClearPageUptodate(page);
2278 return err;
2279}
2280
2281int block_commit_write(struct page *page, unsigned from, unsigned to)
2282{
2283 struct inode *inode = page->mapping->host;
2284 __block_commit_write(inode,page,from,to);
2285 return 0;
2286}
2287
2288int generic_commit_write(struct file *file, struct page *page,
2289 unsigned from, unsigned to)
2290{
2291 struct inode *inode = page->mapping->host;
2292 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2293 __block_commit_write(inode,page,from,to);
2294 /*
2295 * No need to use i_size_read() here, the i_size
2296 * cannot change under us because we hold i_sem.
2297 */
2298 if (pos > inode->i_size) {
2299 i_size_write(inode, pos);
2300 mark_inode_dirty(inode);
2301 }
2302 return 0;
2303}
2304
2305
2306/*
2307 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2308 * immediately, while under the page lock. So it needs a special end_io
2309 * handler which does not touch the bh after unlocking it.
2310 *
2311 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2312 * a race there is benign: unlock_buffer() only use the bh's address for
2313 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2314 * itself.
2315 */
2316static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2317{
2318 if (uptodate) {
2319 set_buffer_uptodate(bh);
2320 } else {
2321 /* This happens, due to failed READA attempts. */
2322 clear_buffer_uptodate(bh);
2323 }
2324 unlock_buffer(bh);
2325}
2326
2327/*
2328 * On entry, the page is fully not uptodate.
2329 * On exit the page is fully uptodate in the areas outside (from,to)
2330 */
2331int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2332 get_block_t *get_block)
2333{
2334 struct inode *inode = page->mapping->host;
2335 const unsigned blkbits = inode->i_blkbits;
2336 const unsigned blocksize = 1 << blkbits;
2337 struct buffer_head map_bh;
2338 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2339 unsigned block_in_page;
2340 unsigned block_start;
2341 sector_t block_in_file;
2342 char *kaddr;
2343 int nr_reads = 0;
2344 int i;
2345 int ret = 0;
2346 int is_mapped_to_disk = 1;
2347 int dirtied_it = 0;
2348
2349 if (PageMappedToDisk(page))
2350 return 0;
2351
2352 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2353 map_bh.b_page = page;
2354
2355 /*
2356 * We loop across all blocks in the page, whether or not they are
2357 * part of the affected region. This is so we can discover if the
2358 * page is fully mapped-to-disk.
2359 */
2360 for (block_start = 0, block_in_page = 0;
2361 block_start < PAGE_CACHE_SIZE;
2362 block_in_page++, block_start += blocksize) {
2363 unsigned block_end = block_start + blocksize;
2364 int create;
2365
2366 map_bh.b_state = 0;
2367 create = 1;
2368 if (block_start >= to)
2369 create = 0;
2370 ret = get_block(inode, block_in_file + block_in_page,
2371 &map_bh, create);
2372 if (ret)
2373 goto failed;
2374 if (!buffer_mapped(&map_bh))
2375 is_mapped_to_disk = 0;
2376 if (buffer_new(&map_bh))
2377 unmap_underlying_metadata(map_bh.b_bdev,
2378 map_bh.b_blocknr);
2379 if (PageUptodate(page))
2380 continue;
2381 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2382 kaddr = kmap_atomic(page, KM_USER0);
2383 if (block_start < from) {
2384 memset(kaddr+block_start, 0, from-block_start);
2385 dirtied_it = 1;
2386 }
2387 if (block_end > to) {
2388 memset(kaddr + to, 0, block_end - to);
2389 dirtied_it = 1;
2390 }
2391 flush_dcache_page(page);
2392 kunmap_atomic(kaddr, KM_USER0);
2393 continue;
2394 }
2395 if (buffer_uptodate(&map_bh))
2396 continue; /* reiserfs does this */
2397 if (block_start < from || block_end > to) {
2398 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2399
2400 if (!bh) {
2401 ret = -ENOMEM;
2402 goto failed;
2403 }
2404 bh->b_state = map_bh.b_state;
2405 atomic_set(&bh->b_count, 0);
2406 bh->b_this_page = NULL;
2407 bh->b_page = page;
2408 bh->b_blocknr = map_bh.b_blocknr;
2409 bh->b_size = blocksize;
2410 bh->b_data = (char *)(long)block_start;
2411 bh->b_bdev = map_bh.b_bdev;
2412 bh->b_private = NULL;
2413 read_bh[nr_reads++] = bh;
2414 }
2415 }
2416
2417 if (nr_reads) {
2418 struct buffer_head *bh;
2419
2420 /*
2421 * The page is locked, so these buffers are protected from
2422 * any VM or truncate activity. Hence we don't need to care
2423 * for the buffer_head refcounts.
2424 */
2425 for (i = 0; i < nr_reads; i++) {
2426 bh = read_bh[i];
2427 lock_buffer(bh);
2428 bh->b_end_io = end_buffer_read_nobh;
2429 submit_bh(READ, bh);
2430 }
2431 for (i = 0; i < nr_reads; i++) {
2432 bh = read_bh[i];
2433 wait_on_buffer(bh);
2434 if (!buffer_uptodate(bh))
2435 ret = -EIO;
2436 free_buffer_head(bh);
2437 read_bh[i] = NULL;
2438 }
2439 if (ret)
2440 goto failed;
2441 }
2442
2443 if (is_mapped_to_disk)
2444 SetPageMappedToDisk(page);
2445 SetPageUptodate(page);
2446
2447 /*
2448 * Setting the page dirty here isn't necessary for the prepare_write
2449 * function - commit_write will do that. But if/when this function is
2450 * used within the pagefault handler to ensure that all mmapped pages
2451 * have backing space in the filesystem, we will need to dirty the page
2452 * if its contents were altered.
2453 */
2454 if (dirtied_it)
2455 set_page_dirty(page);
2456
2457 return 0;
2458
2459failed:
2460 for (i = 0; i < nr_reads; i++) {
2461 if (read_bh[i])
2462 free_buffer_head(read_bh[i]);
2463 }
2464
2465 /*
2466 * Error recovery is pretty slack. Clear the page and mark it dirty
2467 * so we'll later zero out any blocks which _were_ allocated.
2468 */
2469 kaddr = kmap_atomic(page, KM_USER0);
2470 memset(kaddr, 0, PAGE_CACHE_SIZE);
2471 kunmap_atomic(kaddr, KM_USER0);
2472 SetPageUptodate(page);
2473 set_page_dirty(page);
2474 return ret;
2475}
2476EXPORT_SYMBOL(nobh_prepare_write);
2477
2478int nobh_commit_write(struct file *file, struct page *page,
2479 unsigned from, unsigned to)
2480{
2481 struct inode *inode = page->mapping->host;
2482 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2483
2484 set_page_dirty(page);
2485 if (pos > inode->i_size) {
2486 i_size_write(inode, pos);
2487 mark_inode_dirty(inode);
2488 }
2489 return 0;
2490}
2491EXPORT_SYMBOL(nobh_commit_write);
2492
2493/*
2494 * nobh_writepage() - based on block_full_write_page() except
2495 * that it tries to operate without attaching bufferheads to
2496 * the page.
2497 */
2498int nobh_writepage(struct page *page, get_block_t *get_block,
2499 struct writeback_control *wbc)
2500{
2501 struct inode * const inode = page->mapping->host;
2502 loff_t i_size = i_size_read(inode);
2503 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2504 unsigned offset;
2505 void *kaddr;
2506 int ret;
2507
2508 /* Is the page fully inside i_size? */
2509 if (page->index < end_index)
2510 goto out;
2511
2512 /* Is the page fully outside i_size? (truncate in progress) */
2513 offset = i_size & (PAGE_CACHE_SIZE-1);
2514 if (page->index >= end_index+1 || !offset) {
2515 /*
2516 * The page may have dirty, unmapped buffers. For example,
2517 * they may have been added in ext3_writepage(). Make them
2518 * freeable here, so the page does not leak.
2519 */
2520#if 0
2521 /* Not really sure about this - do we need this ? */
2522 if (page->mapping->a_ops->invalidatepage)
2523 page->mapping->a_ops->invalidatepage(page, offset);
2524#endif
2525 unlock_page(page);
2526 return 0; /* don't care */
2527 }
2528
2529 /*
2530 * The page straddles i_size. It must be zeroed out on each and every
2531 * writepage invocation because it may be mmapped. "A file is mapped
2532 * in multiples of the page size. For a file that is not a multiple of
2533 * the page size, the remaining memory is zeroed when mapped, and
2534 * writes to that region are not written out to the file."
2535 */
2536 kaddr = kmap_atomic(page, KM_USER0);
2537 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2538 flush_dcache_page(page);
2539 kunmap_atomic(kaddr, KM_USER0);
2540out:
2541 ret = mpage_writepage(page, get_block, wbc);
2542 if (ret == -EAGAIN)
2543 ret = __block_write_full_page(inode, page, get_block, wbc);
2544 return ret;
2545}
2546EXPORT_SYMBOL(nobh_writepage);
2547
2548/*
2549 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2550 */
2551int nobh_truncate_page(struct address_space *mapping, loff_t from)
2552{
2553 struct inode *inode = mapping->host;
2554 unsigned blocksize = 1 << inode->i_blkbits;
2555 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2556 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2557 unsigned to;
2558 struct page *page;
2559 struct address_space_operations *a_ops = mapping->a_ops;
2560 char *kaddr;
2561 int ret = 0;
2562
2563 if ((offset & (blocksize - 1)) == 0)
2564 goto out;
2565
2566 ret = -ENOMEM;
2567 page = grab_cache_page(mapping, index);
2568 if (!page)
2569 goto out;
2570
2571 to = (offset + blocksize) & ~(blocksize - 1);
2572 ret = a_ops->prepare_write(NULL, page, offset, to);
2573 if (ret == 0) {
2574 kaddr = kmap_atomic(page, KM_USER0);
2575 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2576 flush_dcache_page(page);
2577 kunmap_atomic(kaddr, KM_USER0);
2578 set_page_dirty(page);
2579 }
2580 unlock_page(page);
2581 page_cache_release(page);
2582out:
2583 return ret;
2584}
2585EXPORT_SYMBOL(nobh_truncate_page);
2586
2587int block_truncate_page(struct address_space *mapping,
2588 loff_t from, get_block_t *get_block)
2589{
2590 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2591 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2592 unsigned blocksize;
2593 pgoff_t iblock;
2594 unsigned length, pos;
2595 struct inode *inode = mapping->host;
2596 struct page *page;
2597 struct buffer_head *bh;
2598 void *kaddr;
2599 int err;
2600
2601 blocksize = 1 << inode->i_blkbits;
2602 length = offset & (blocksize - 1);
2603
2604 /* Block boundary? Nothing to do */
2605 if (!length)
2606 return 0;
2607
2608 length = blocksize - length;
2609 iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2610
2611 page = grab_cache_page(mapping, index);
2612 err = -ENOMEM;
2613 if (!page)
2614 goto out;
2615
2616 if (!page_has_buffers(page))
2617 create_empty_buffers(page, blocksize, 0);
2618
2619 /* Find the buffer that contains "offset" */
2620 bh = page_buffers(page);
2621 pos = blocksize;
2622 while (offset >= pos) {
2623 bh = bh->b_this_page;
2624 iblock++;
2625 pos += blocksize;
2626 }
2627
2628 err = 0;
2629 if (!buffer_mapped(bh)) {
2630 err = get_block(inode, iblock, bh, 0);
2631 if (err)
2632 goto unlock;
2633 /* unmapped? It's a hole - nothing to do */
2634 if (!buffer_mapped(bh))
2635 goto unlock;
2636 }
2637
2638 /* Ok, it's mapped. Make sure it's up-to-date */
2639 if (PageUptodate(page))
2640 set_buffer_uptodate(bh);
2641
2642 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2643 err = -EIO;
2644 ll_rw_block(READ, 1, &bh);
2645 wait_on_buffer(bh);
2646 /* Uhhuh. Read error. Complain and punt. */
2647 if (!buffer_uptodate(bh))
2648 goto unlock;
2649 }
2650
2651 kaddr = kmap_atomic(page, KM_USER0);
2652 memset(kaddr + offset, 0, length);
2653 flush_dcache_page(page);
2654 kunmap_atomic(kaddr, KM_USER0);
2655
2656 mark_buffer_dirty(bh);
2657 err = 0;
2658
2659unlock:
2660 unlock_page(page);
2661 page_cache_release(page);
2662out:
2663 return err;
2664}
2665
2666/*
2667 * The generic ->writepage function for buffer-backed address_spaces
2668 */
2669int block_write_full_page(struct page *page, get_block_t *get_block,
2670 struct writeback_control *wbc)
2671{
2672 struct inode * const inode = page->mapping->host;
2673 loff_t i_size = i_size_read(inode);
2674 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2675 unsigned offset;
2676 void *kaddr;
2677
2678 /* Is the page fully inside i_size? */
2679 if (page->index < end_index)
2680 return __block_write_full_page(inode, page, get_block, wbc);
2681
2682 /* Is the page fully outside i_size? (truncate in progress) */
2683 offset = i_size & (PAGE_CACHE_SIZE-1);
2684 if (page->index >= end_index+1 || !offset) {
2685 /*
2686 * The page may have dirty, unmapped buffers. For example,
2687 * they may have been added in ext3_writepage(). Make them
2688 * freeable here, so the page does not leak.
2689 */
2690 block_invalidatepage(page, 0);
2691 unlock_page(page);
2692 return 0; /* don't care */
2693 }
2694
2695 /*
2696 * The page straddles i_size. It must be zeroed out on each and every
2697 * writepage invokation because it may be mmapped. "A file is mapped
2698 * in multiples of the page size. For a file that is not a multiple of
2699 * the page size, the remaining memory is zeroed when mapped, and
2700 * writes to that region are not written out to the file."
2701 */
2702 kaddr = kmap_atomic(page, KM_USER0);
2703 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2704 flush_dcache_page(page);
2705 kunmap_atomic(kaddr, KM_USER0);
2706 return __block_write_full_page(inode, page, get_block, wbc);
2707}
2708
2709sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2710 get_block_t *get_block)
2711{
2712 struct buffer_head tmp;
2713 struct inode *inode = mapping->host;
2714 tmp.b_state = 0;
2715 tmp.b_blocknr = 0;
2716 get_block(inode, block, &tmp, 0);
2717 return tmp.b_blocknr;
2718}
2719
2720static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2721{
2722 struct buffer_head *bh = bio->bi_private;
2723
2724 if (bio->bi_size)
2725 return 1;
2726
2727 if (err == -EOPNOTSUPP) {
2728 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2729 set_bit(BH_Eopnotsupp, &bh->b_state);
2730 }
2731
2732 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2733 bio_put(bio);
2734 return 0;
2735}
2736
2737int submit_bh(int rw, struct buffer_head * bh)
2738{
2739 struct bio *bio;
2740 int ret = 0;
2741
2742 BUG_ON(!buffer_locked(bh));
2743 BUG_ON(!buffer_mapped(bh));
2744 BUG_ON(!bh->b_end_io);
2745
2746 if (buffer_ordered(bh) && (rw == WRITE))
2747 rw = WRITE_BARRIER;
2748
2749 /*
2750 * Only clear out a write error when rewriting, should this
2751 * include WRITE_SYNC as well?
2752 */
2753 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2754 clear_buffer_write_io_error(bh);
2755
2756 /*
2757 * from here on down, it's all bio -- do the initial mapping,
2758 * submit_bio -> generic_make_request may further map this bio around
2759 */
2760 bio = bio_alloc(GFP_NOIO, 1);
2761
2762 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2763 bio->bi_bdev = bh->b_bdev;
2764 bio->bi_io_vec[0].bv_page = bh->b_page;
2765 bio->bi_io_vec[0].bv_len = bh->b_size;
2766 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2767
2768 bio->bi_vcnt = 1;
2769 bio->bi_idx = 0;
2770 bio->bi_size = bh->b_size;
2771
2772 bio->bi_end_io = end_bio_bh_io_sync;
2773 bio->bi_private = bh;
2774
2775 bio_get(bio);
2776 submit_bio(rw, bio);
2777
2778 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2779 ret = -EOPNOTSUPP;
2780
2781 bio_put(bio);
2782 return ret;
2783}
2784
2785/**
2786 * ll_rw_block: low-level access to block devices (DEPRECATED)
2787 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
2788 * @nr: number of &struct buffer_heads in the array
2789 * @bhs: array of pointers to &struct buffer_head
2790 *
2791 * ll_rw_block() takes an array of pointers to &struct buffer_heads,
2792 * and requests an I/O operation on them, either a %READ or a %WRITE.
2793 * The third %READA option is described in the documentation for
2794 * generic_make_request() which ll_rw_block() calls.
2795 *
2796 * This function drops any buffer that it cannot get a lock on (with the
2797 * BH_Lock state bit), any buffer that appears to be clean when doing a
2798 * write request, and any buffer that appears to be up-to-date when doing
2799 * read request. Further it marks as clean buffers that are processed for
2800 * writing (the buffer cache won't assume that they are actually clean until
2801 * the buffer gets unlocked).
2802 *
2803 * ll_rw_block sets b_end_io to simple completion handler that marks
2804 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2805 * any waiters.
2806 *
2807 * All of the buffers must be for the same device, and must also be a
2808 * multiple of the current approved size for the device.
2809 */
2810void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2811{
2812 int i;
2813
2814 for (i = 0; i < nr; i++) {
2815 struct buffer_head *bh = bhs[i];
2816
2817 if (test_set_buffer_locked(bh))
2818 continue;
2819
2820 get_bh(bh);
2821 if (rw == WRITE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002822 if (test_clear_buffer_dirty(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002823 bh->b_end_io = end_buffer_write_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002824 submit_bh(WRITE, bh);
2825 continue;
2826 }
2827 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002828 if (!buffer_uptodate(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002829 bh->b_end_io = end_buffer_read_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 submit_bh(rw, bh);
2831 continue;
2832 }
2833 }
2834 unlock_buffer(bh);
2835 put_bh(bh);
2836 }
2837}
2838
2839/*
2840 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2841 * and then start new I/O and then wait upon it. The caller must have a ref on
2842 * the buffer_head.
2843 */
2844int sync_dirty_buffer(struct buffer_head *bh)
2845{
2846 int ret = 0;
2847
2848 WARN_ON(atomic_read(&bh->b_count) < 1);
2849 lock_buffer(bh);
2850 if (test_clear_buffer_dirty(bh)) {
2851 get_bh(bh);
2852 bh->b_end_io = end_buffer_write_sync;
2853 ret = submit_bh(WRITE, bh);
2854 wait_on_buffer(bh);
2855 if (buffer_eopnotsupp(bh)) {
2856 clear_buffer_eopnotsupp(bh);
2857 ret = -EOPNOTSUPP;
2858 }
2859 if (!ret && !buffer_uptodate(bh))
2860 ret = -EIO;
2861 } else {
2862 unlock_buffer(bh);
2863 }
2864 return ret;
2865}
2866
2867/*
2868 * try_to_free_buffers() checks if all the buffers on this particular page
2869 * are unused, and releases them if so.
2870 *
2871 * Exclusion against try_to_free_buffers may be obtained by either
2872 * locking the page or by holding its mapping's private_lock.
2873 *
2874 * If the page is dirty but all the buffers are clean then we need to
2875 * be sure to mark the page clean as well. This is because the page
2876 * may be against a block device, and a later reattachment of buffers
2877 * to a dirty page will set *all* buffers dirty. Which would corrupt
2878 * filesystem data on the same device.
2879 *
2880 * The same applies to regular filesystem pages: if all the buffers are
2881 * clean then we set the page clean and proceed. To do that, we require
2882 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2883 * private_lock.
2884 *
2885 * try_to_free_buffers() is non-blocking.
2886 */
2887static inline int buffer_busy(struct buffer_head *bh)
2888{
2889 return atomic_read(&bh->b_count) |
2890 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2891}
2892
2893static int
2894drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2895{
2896 struct buffer_head *head = page_buffers(page);
2897 struct buffer_head *bh;
2898
2899 bh = head;
2900 do {
akpm@osdl.orgde7d5a32005-05-01 08:58:39 -07002901 if (buffer_write_io_error(bh) && page->mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002902 set_bit(AS_EIO, &page->mapping->flags);
2903 if (buffer_busy(bh))
2904 goto failed;
2905 bh = bh->b_this_page;
2906 } while (bh != head);
2907
2908 do {
2909 struct buffer_head *next = bh->b_this_page;
2910
2911 if (!list_empty(&bh->b_assoc_buffers))
2912 __remove_assoc_queue(bh);
2913 bh = next;
2914 } while (bh != head);
2915 *buffers_to_free = head;
2916 __clear_page_buffers(page);
2917 return 1;
2918failed:
2919 return 0;
2920}
2921
2922int try_to_free_buffers(struct page *page)
2923{
2924 struct address_space * const mapping = page->mapping;
2925 struct buffer_head *buffers_to_free = NULL;
2926 int ret = 0;
2927
2928 BUG_ON(!PageLocked(page));
2929 if (PageWriteback(page))
2930 return 0;
2931
2932 if (mapping == NULL) { /* can this still happen? */
2933 ret = drop_buffers(page, &buffers_to_free);
2934 goto out;
2935 }
2936
2937 spin_lock(&mapping->private_lock);
2938 ret = drop_buffers(page, &buffers_to_free);
2939 if (ret) {
2940 /*
2941 * If the filesystem writes its buffers by hand (eg ext3)
2942 * then we can have clean buffers against a dirty page. We
2943 * clean the page here; otherwise later reattachment of buffers
2944 * could encounter a non-uptodate page, which is unresolvable.
2945 * This only applies in the rare case where try_to_free_buffers
2946 * succeeds but the page is not freed.
2947 */
2948 clear_page_dirty(page);
2949 }
2950 spin_unlock(&mapping->private_lock);
2951out:
2952 if (buffers_to_free) {
2953 struct buffer_head *bh = buffers_to_free;
2954
2955 do {
2956 struct buffer_head *next = bh->b_this_page;
2957 free_buffer_head(bh);
2958 bh = next;
2959 } while (bh != buffers_to_free);
2960 }
2961 return ret;
2962}
2963EXPORT_SYMBOL(try_to_free_buffers);
2964
2965int block_sync_page(struct page *page)
2966{
2967 struct address_space *mapping;
2968
2969 smp_mb();
2970 mapping = page_mapping(page);
2971 if (mapping)
2972 blk_run_backing_dev(mapping->backing_dev_info, page);
2973 return 0;
2974}
2975
2976/*
2977 * There are no bdflush tunables left. But distributions are
2978 * still running obsolete flush daemons, so we terminate them here.
2979 *
2980 * Use of bdflush() is deprecated and will be removed in a future kernel.
2981 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2982 */
2983asmlinkage long sys_bdflush(int func, long data)
2984{
2985 static int msg_count;
2986
2987 if (!capable(CAP_SYS_ADMIN))
2988 return -EPERM;
2989
2990 if (msg_count < 5) {
2991 msg_count++;
2992 printk(KERN_INFO
2993 "warning: process `%s' used the obsolete bdflush"
2994 " system call\n", current->comm);
2995 printk(KERN_INFO "Fix your initscripts?\n");
2996 }
2997
2998 if (func == 1)
2999 do_exit(0);
3000 return 0;
3001}
3002
3003/*
3004 * Buffer-head allocation
3005 */
3006static kmem_cache_t *bh_cachep;
3007
3008/*
3009 * Once the number of bh's in the machine exceeds this level, we start
3010 * stripping them in writeback.
3011 */
3012static int max_buffer_heads;
3013
3014int buffer_heads_over_limit;
3015
3016struct bh_accounting {
3017 int nr; /* Number of live bh's */
3018 int ratelimit; /* Limit cacheline bouncing */
3019};
3020
3021static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3022
3023static void recalc_bh_state(void)
3024{
3025 int i;
3026 int tot = 0;
3027
3028 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3029 return;
3030 __get_cpu_var(bh_accounting).ratelimit = 0;
3031 for_each_cpu(i)
3032 tot += per_cpu(bh_accounting, i).nr;
3033 buffer_heads_over_limit = (tot > max_buffer_heads);
3034}
3035
3036struct buffer_head *alloc_buffer_head(unsigned int __nocast gfp_flags)
3037{
3038 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3039 if (ret) {
3040 preempt_disable();
3041 __get_cpu_var(bh_accounting).nr++;
3042 recalc_bh_state();
3043 preempt_enable();
3044 }
3045 return ret;
3046}
3047EXPORT_SYMBOL(alloc_buffer_head);
3048
3049void free_buffer_head(struct buffer_head *bh)
3050{
3051 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3052 kmem_cache_free(bh_cachep, bh);
3053 preempt_disable();
3054 __get_cpu_var(bh_accounting).nr--;
3055 recalc_bh_state();
3056 preempt_enable();
3057}
3058EXPORT_SYMBOL(free_buffer_head);
3059
3060static void
3061init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3062{
3063 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3064 SLAB_CTOR_CONSTRUCTOR) {
3065 struct buffer_head * bh = (struct buffer_head *)data;
3066
3067 memset(bh, 0, sizeof(*bh));
3068 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3069 }
3070}
3071
3072#ifdef CONFIG_HOTPLUG_CPU
3073static void buffer_exit_cpu(int cpu)
3074{
3075 int i;
3076 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3077
3078 for (i = 0; i < BH_LRU_SIZE; i++) {
3079 brelse(b->bhs[i]);
3080 b->bhs[i] = NULL;
3081 }
3082}
3083
3084static int buffer_cpu_notify(struct notifier_block *self,
3085 unsigned long action, void *hcpu)
3086{
3087 if (action == CPU_DEAD)
3088 buffer_exit_cpu((unsigned long)hcpu);
3089 return NOTIFY_OK;
3090}
3091#endif /* CONFIG_HOTPLUG_CPU */
3092
3093void __init buffer_init(void)
3094{
3095 int nrpages;
3096
3097 bh_cachep = kmem_cache_create("buffer_head",
3098 sizeof(struct buffer_head), 0,
Andrea Arcangelie422fd2c2005-05-05 16:15:04 -07003099 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003100
3101 /*
3102 * Limit the bh occupancy to 10% of ZONE_NORMAL
3103 */
3104 nrpages = (nr_free_buffer_pages() * 10) / 100;
3105 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3106 hotcpu_notifier(buffer_cpu_notify, 0);
3107}
3108
3109EXPORT_SYMBOL(__bforget);
3110EXPORT_SYMBOL(__brelse);
3111EXPORT_SYMBOL(__wait_on_buffer);
3112EXPORT_SYMBOL(block_commit_write);
3113EXPORT_SYMBOL(block_prepare_write);
3114EXPORT_SYMBOL(block_read_full_page);
3115EXPORT_SYMBOL(block_sync_page);
3116EXPORT_SYMBOL(block_truncate_page);
3117EXPORT_SYMBOL(block_write_full_page);
3118EXPORT_SYMBOL(cont_prepare_write);
3119EXPORT_SYMBOL(end_buffer_async_write);
3120EXPORT_SYMBOL(end_buffer_read_sync);
3121EXPORT_SYMBOL(end_buffer_write_sync);
3122EXPORT_SYMBOL(file_fsync);
3123EXPORT_SYMBOL(fsync_bdev);
3124EXPORT_SYMBOL(generic_block_bmap);
3125EXPORT_SYMBOL(generic_commit_write);
3126EXPORT_SYMBOL(generic_cont_expand);
3127EXPORT_SYMBOL(init_buffer);
3128EXPORT_SYMBOL(invalidate_bdev);
3129EXPORT_SYMBOL(ll_rw_block);
3130EXPORT_SYMBOL(mark_buffer_dirty);
3131EXPORT_SYMBOL(submit_bh);
3132EXPORT_SYMBOL(sync_dirty_buffer);
3133EXPORT_SYMBOL(unlock_buffer);