blob: 6a25d7df89b176db6bb7163517538476398a9af4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
21#include <linux/config.h>
22#include <linux/kernel.h>
23#include <linux/syscalls.h>
24#include <linux/fs.h>
25#include <linux/mm.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/smp_lock.h>
29#include <linux/blkdev.h>
30#include <linux/file.h>
31#include <linux/quotaops.h>
32#include <linux/highmem.h>
33#include <linux/module.h>
34#include <linux/writeback.h>
35#include <linux/hash.h>
36#include <linux/suspend.h>
37#include <linux/buffer_head.h>
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
43
44static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
45static void invalidate_bh_lrus(void);
46
47#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48
49inline void
50init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51{
52 bh->b_end_io = handler;
53 bh->b_private = private;
54}
55
56static int sync_buffer(void *word)
57{
58 struct block_device *bd;
59 struct buffer_head *bh
60 = container_of(word, struct buffer_head, b_state);
61
62 smp_mb();
63 bd = bh->b_bdev;
64 if (bd)
65 blk_run_address_space(bd->bd_inode->i_mapping);
66 io_schedule();
67 return 0;
68}
69
70void fastcall __lock_buffer(struct buffer_head *bh)
71{
72 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
73 TASK_UNINTERRUPTIBLE);
74}
75EXPORT_SYMBOL(__lock_buffer);
76
77void fastcall unlock_buffer(struct buffer_head *bh)
78{
79 clear_buffer_locked(bh);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh->b_state, BH_Lock);
82}
83
84/*
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
88 */
89void __wait_on_buffer(struct buffer_head * bh)
90{
91 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
92}
93
94static void
95__clear_page_buffers(struct page *page)
96{
97 ClearPagePrivate(page);
98 page->private = 0;
99 page_cache_release(page);
100}
101
102static void buffer_io_error(struct buffer_head *bh)
103{
104 char b[BDEVNAME_SIZE];
105
106 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
107 bdevname(bh->b_bdev, b),
108 (unsigned long long)bh->b_blocknr);
109}
110
111/*
112 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
113 * unlock the buffer. This is what ll_rw_block uses too.
114 */
115void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
116{
117 if (uptodate) {
118 set_buffer_uptodate(bh);
119 } else {
120 /* This happens, due to failed READA attempts. */
121 clear_buffer_uptodate(bh);
122 }
123 unlock_buffer(bh);
124 put_bh(bh);
125}
126
127void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
128{
129 char b[BDEVNAME_SIZE];
130
131 if (uptodate) {
132 set_buffer_uptodate(bh);
133 } else {
134 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
135 buffer_io_error(bh);
136 printk(KERN_WARNING "lost page write due to "
137 "I/O error on %s\n",
138 bdevname(bh->b_bdev, b));
139 }
140 set_buffer_write_io_error(bh);
141 clear_buffer_uptodate(bh);
142 }
143 unlock_buffer(bh);
144 put_bh(bh);
145}
146
147/*
148 * Write out and wait upon all the dirty data associated with a block
149 * device via its mapping. Does not take the superblock lock.
150 */
151int sync_blockdev(struct block_device *bdev)
152{
153 int ret = 0;
154
155 if (bdev) {
156 int err;
157
158 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
159 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
160 if (!ret)
161 ret = err;
162 }
163 return ret;
164}
165EXPORT_SYMBOL(sync_blockdev);
166
167/*
168 * Write out and wait upon all dirty data associated with this
169 * superblock. Filesystem data as well as the underlying block
170 * device. Takes the superblock lock.
171 */
172int fsync_super(struct super_block *sb)
173{
174 sync_inodes_sb(sb, 0);
175 DQUOT_SYNC(sb);
176 lock_super(sb);
177 if (sb->s_dirt && sb->s_op->write_super)
178 sb->s_op->write_super(sb);
179 unlock_super(sb);
180 if (sb->s_op->sync_fs)
181 sb->s_op->sync_fs(sb, 1);
182 sync_blockdev(sb->s_bdev);
183 sync_inodes_sb(sb, 1);
184
185 return sync_blockdev(sb->s_bdev);
186}
187
188/*
189 * Write out and wait upon all dirty data associated with this
190 * device. Filesystem data as well as the underlying block
191 * device. Takes the superblock lock.
192 */
193int fsync_bdev(struct block_device *bdev)
194{
195 struct super_block *sb = get_super(bdev);
196 if (sb) {
197 int res = fsync_super(sb);
198 drop_super(sb);
199 return res;
200 }
201 return sync_blockdev(bdev);
202}
203
204/**
205 * freeze_bdev -- lock a filesystem and force it into a consistent state
206 * @bdev: blockdevice to lock
207 *
208 * This takes the block device bd_mount_sem to make sure no new mounts
209 * happen on bdev until thaw_bdev() is called.
210 * If a superblock is found on this device, we take the s_umount semaphore
211 * on it to make sure nobody unmounts until the snapshot creation is done.
212 */
213struct super_block *freeze_bdev(struct block_device *bdev)
214{
215 struct super_block *sb;
216
217 down(&bdev->bd_mount_sem);
218 sb = get_super(bdev);
219 if (sb && !(sb->s_flags & MS_RDONLY)) {
220 sb->s_frozen = SB_FREEZE_WRITE;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700221 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
223 sync_inodes_sb(sb, 0);
224 DQUOT_SYNC(sb);
225
226 lock_super(sb);
227 if (sb->s_dirt && sb->s_op->write_super)
228 sb->s_op->write_super(sb);
229 unlock_super(sb);
230
231 if (sb->s_op->sync_fs)
232 sb->s_op->sync_fs(sb, 1);
233
234 sync_blockdev(sb->s_bdev);
235 sync_inodes_sb(sb, 1);
236
237 sb->s_frozen = SB_FREEZE_TRANS;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700238 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
240 sync_blockdev(sb->s_bdev);
241
242 if (sb->s_op->write_super_lockfs)
243 sb->s_op->write_super_lockfs(sb);
244 }
245
246 sync_blockdev(bdev);
247 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
248}
249EXPORT_SYMBOL(freeze_bdev);
250
251/**
252 * thaw_bdev -- unlock filesystem
253 * @bdev: blockdevice to unlock
254 * @sb: associated superblock
255 *
256 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
257 */
258void thaw_bdev(struct block_device *bdev, struct super_block *sb)
259{
260 if (sb) {
261 BUG_ON(sb->s_bdev != bdev);
262
263 if (sb->s_op->unlockfs)
264 sb->s_op->unlockfs(sb);
265 sb->s_frozen = SB_UNFROZEN;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700266 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 wake_up(&sb->s_wait_unfrozen);
268 drop_super(sb);
269 }
270
271 up(&bdev->bd_mount_sem);
272}
273EXPORT_SYMBOL(thaw_bdev);
274
275/*
276 * sync everything. Start out by waking pdflush, because that writes back
277 * all queues in parallel.
278 */
279static void do_sync(unsigned long wait)
280{
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700281 wakeup_pdflush(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 sync_inodes(0); /* All mappings, inodes and their blockdevs */
283 DQUOT_SYNC(NULL);
284 sync_supers(); /* Write the superblocks */
285 sync_filesystems(0); /* Start syncing the filesystems */
286 sync_filesystems(wait); /* Waitingly sync the filesystems */
287 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
288 if (!wait)
289 printk("Emergency Sync complete\n");
290 if (unlikely(laptop_mode))
291 laptop_sync_completion();
292}
293
294asmlinkage long sys_sync(void)
295{
296 do_sync(1);
297 return 0;
298}
299
300void emergency_sync(void)
301{
302 pdflush_operation(do_sync, 0);
303}
304
305/*
306 * Generic function to fsync a file.
307 *
308 * filp may be NULL if called via the msync of a vma.
309 */
310
311int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
312{
313 struct inode * inode = dentry->d_inode;
314 struct super_block * sb;
315 int ret, err;
316
317 /* sync the inode to buffers */
318 ret = write_inode_now(inode, 0);
319
320 /* sync the superblock to buffers */
321 sb = inode->i_sb;
322 lock_super(sb);
323 if (sb->s_op->write_super)
324 sb->s_op->write_super(sb);
325 unlock_super(sb);
326
327 /* .. finally sync the buffers to disk */
328 err = sync_blockdev(sb->s_bdev);
329 if (!ret)
330 ret = err;
331 return ret;
332}
333
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700334static long do_fsync(unsigned int fd, int datasync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
336 struct file * file;
337 struct address_space *mapping;
338 int ret, err;
339
340 ret = -EBADF;
341 file = fget(fd);
342 if (!file)
343 goto out;
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 ret = -EINVAL;
346 if (!file->f_op || !file->f_op->fsync) {
347 /* Why? We can still call filemap_fdatawrite */
348 goto out_putf;
349 }
350
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700351 mapping = file->f_mapping;
352
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 current->flags |= PF_SYNCWRITE;
354 ret = filemap_fdatawrite(mapping);
355
356 /*
357 * We need to protect against concurrent writers,
358 * which could cause livelocks in fsync_buffers_list
359 */
360 down(&mapping->host->i_sem);
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700361 err = file->f_op->fsync(file, file->f_dentry, datasync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 if (!ret)
363 ret = err;
364 up(&mapping->host->i_sem);
365 err = filemap_fdatawait(mapping);
366 if (!ret)
367 ret = err;
368 current->flags &= ~PF_SYNCWRITE;
369
370out_putf:
371 fput(file);
372out:
373 return ret;
374}
375
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700376asmlinkage long sys_fsync(unsigned int fd)
377{
378 return do_fsync(fd, 0);
379}
380
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381asmlinkage long sys_fdatasync(unsigned int fd)
382{
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700383 return do_fsync(fd, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385
386/*
387 * Various filesystems appear to want __find_get_block to be non-blocking.
388 * But it's the page lock which protects the buffers. To get around this,
389 * we get exclusion from try_to_free_buffers with the blockdev mapping's
390 * private_lock.
391 *
392 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
393 * may be quite high. This code could TryLock the page, and if that
394 * succeeds, there is no need to take private_lock. (But if
395 * private_lock is contended then so is mapping->tree_lock).
396 */
397static struct buffer_head *
398__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
399{
400 struct inode *bd_inode = bdev->bd_inode;
401 struct address_space *bd_mapping = bd_inode->i_mapping;
402 struct buffer_head *ret = NULL;
403 pgoff_t index;
404 struct buffer_head *bh;
405 struct buffer_head *head;
406 struct page *page;
407 int all_mapped = 1;
408
409 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
410 page = find_get_page(bd_mapping, index);
411 if (!page)
412 goto out;
413
414 spin_lock(&bd_mapping->private_lock);
415 if (!page_has_buffers(page))
416 goto out_unlock;
417 head = page_buffers(page);
418 bh = head;
419 do {
420 if (bh->b_blocknr == block) {
421 ret = bh;
422 get_bh(bh);
423 goto out_unlock;
424 }
425 if (!buffer_mapped(bh))
426 all_mapped = 0;
427 bh = bh->b_this_page;
428 } while (bh != head);
429
430 /* we might be here because some of the buffers on this page are
431 * not mapped. This is due to various races between
432 * file io on the block device and getblk. It gets dealt with
433 * elsewhere, don't buffer_error if we had some unmapped buffers
434 */
435 if (all_mapped) {
436 printk("__find_get_block_slow() failed. "
437 "block=%llu, b_blocknr=%llu\n",
438 (unsigned long long)block, (unsigned long long)bh->b_blocknr);
439 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
440 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
441 }
442out_unlock:
443 spin_unlock(&bd_mapping->private_lock);
444 page_cache_release(page);
445out:
446 return ret;
447}
448
449/* If invalidate_buffers() will trash dirty buffers, it means some kind
450 of fs corruption is going on. Trashing dirty data always imply losing
451 information that was supposed to be just stored on the physical layer
452 by the user.
453
454 Thus invalidate_buffers in general usage is not allwowed to trash
455 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
456 be preserved. These buffers are simply skipped.
457
458 We also skip buffers which are still in use. For example this can
459 happen if a userspace program is reading the block device.
460
461 NOTE: In the case where the user removed a removable-media-disk even if
462 there's still dirty data not synced on disk (due a bug in the device driver
463 or due an error of the user), by not destroying the dirty buffers we could
464 generate corruption also on the next media inserted, thus a parameter is
465 necessary to handle this case in the most safe way possible (trying
466 to not corrupt also the new disk inserted with the data belonging to
467 the old now corrupted disk). Also for the ramdisk the natural thing
468 to do in order to release the ramdisk memory is to destroy dirty buffers.
469
470 These are two special cases. Normal usage imply the device driver
471 to issue a sync on the device (without waiting I/O completion) and
472 then an invalidate_buffers call that doesn't trash dirty buffers.
473
474 For handling cache coherency with the blkdev pagecache the 'update' case
475 is been introduced. It is needed to re-read from disk any pinned
476 buffer. NOTE: re-reading from disk is destructive so we can do it only
477 when we assume nobody is changing the buffercache under our I/O and when
478 we think the disk contains more recent information than the buffercache.
479 The update == 1 pass marks the buffers we need to update, the update == 2
480 pass does the actual I/O. */
481void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
482{
483 invalidate_bh_lrus();
484 /*
485 * FIXME: what about destroy_dirty_buffers?
486 * We really want to use invalidate_inode_pages2() for
487 * that, but not until that's cleaned up.
488 */
489 invalidate_inode_pages(bdev->bd_inode->i_mapping);
490}
491
492/*
493 * Kick pdflush then try to free up some ZONE_NORMAL memory.
494 */
495static void free_more_memory(void)
496{
497 struct zone **zones;
498 pg_data_t *pgdat;
499
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700500 wakeup_pdflush(1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 yield();
502
503 for_each_pgdat(pgdat) {
504 zones = pgdat->node_zonelists[GFP_NOFS&GFP_ZONEMASK].zones;
505 if (*zones)
Darren Hart1ad539b2005-06-21 17:14:53 -0700506 try_to_free_pages(zones, GFP_NOFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 }
508}
509
510/*
511 * I/O completion handler for block_read_full_page() - pages
512 * which come unlocked at the end of I/O.
513 */
514static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
515{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700517 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 struct buffer_head *tmp;
519 struct page *page;
520 int page_uptodate = 1;
521
522 BUG_ON(!buffer_async_read(bh));
523
524 page = bh->b_page;
525 if (uptodate) {
526 set_buffer_uptodate(bh);
527 } else {
528 clear_buffer_uptodate(bh);
529 if (printk_ratelimit())
530 buffer_io_error(bh);
531 SetPageError(page);
532 }
533
534 /*
535 * Be _very_ careful from here on. Bad things can happen if
536 * two buffer heads end IO at almost the same time and both
537 * decide that the page is now completely done.
538 */
Nick Piggina3972202005-07-07 17:56:56 -0700539 first = page_buffers(page);
540 local_irq_save(flags);
541 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 clear_buffer_async_read(bh);
543 unlock_buffer(bh);
544 tmp = bh;
545 do {
546 if (!buffer_uptodate(tmp))
547 page_uptodate = 0;
548 if (buffer_async_read(tmp)) {
549 BUG_ON(!buffer_locked(tmp));
550 goto still_busy;
551 }
552 tmp = tmp->b_this_page;
553 } while (tmp != bh);
Nick Piggina3972202005-07-07 17:56:56 -0700554 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
555 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
557 /*
558 * If none of the buffers had errors and they are all
559 * uptodate then we can set the page uptodate.
560 */
561 if (page_uptodate && !PageError(page))
562 SetPageUptodate(page);
563 unlock_page(page);
564 return;
565
566still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700567 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
568 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 return;
570}
571
572/*
573 * Completion handler for block_write_full_page() - pages which are unlocked
574 * during I/O, and which have PageWriteback cleared upon I/O completion.
575 */
576void end_buffer_async_write(struct buffer_head *bh, int uptodate)
577{
578 char b[BDEVNAME_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700580 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 struct buffer_head *tmp;
582 struct page *page;
583
584 BUG_ON(!buffer_async_write(bh));
585
586 page = bh->b_page;
587 if (uptodate) {
588 set_buffer_uptodate(bh);
589 } else {
590 if (printk_ratelimit()) {
591 buffer_io_error(bh);
592 printk(KERN_WARNING "lost page write due to "
593 "I/O error on %s\n",
594 bdevname(bh->b_bdev, b));
595 }
596 set_bit(AS_EIO, &page->mapping->flags);
597 clear_buffer_uptodate(bh);
598 SetPageError(page);
599 }
600
Nick Piggina3972202005-07-07 17:56:56 -0700601 first = page_buffers(page);
602 local_irq_save(flags);
603 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
604
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 clear_buffer_async_write(bh);
606 unlock_buffer(bh);
607 tmp = bh->b_this_page;
608 while (tmp != bh) {
609 if (buffer_async_write(tmp)) {
610 BUG_ON(!buffer_locked(tmp));
611 goto still_busy;
612 }
613 tmp = tmp->b_this_page;
614 }
Nick Piggina3972202005-07-07 17:56:56 -0700615 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
616 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 end_page_writeback(page);
618 return;
619
620still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700621 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
622 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 return;
624}
625
626/*
627 * If a page's buffers are under async readin (end_buffer_async_read
628 * completion) then there is a possibility that another thread of
629 * control could lock one of the buffers after it has completed
630 * but while some of the other buffers have not completed. This
631 * locked buffer would confuse end_buffer_async_read() into not unlocking
632 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
633 * that this buffer is not under async I/O.
634 *
635 * The page comes unlocked when it has no locked buffer_async buffers
636 * left.
637 *
638 * PageLocked prevents anyone starting new async I/O reads any of
639 * the buffers.
640 *
641 * PageWriteback is used to prevent simultaneous writeout of the same
642 * page.
643 *
644 * PageLocked prevents anyone from starting writeback of a page which is
645 * under read I/O (PageWriteback is only ever set against a locked page).
646 */
647static void mark_buffer_async_read(struct buffer_head *bh)
648{
649 bh->b_end_io = end_buffer_async_read;
650 set_buffer_async_read(bh);
651}
652
653void mark_buffer_async_write(struct buffer_head *bh)
654{
655 bh->b_end_io = end_buffer_async_write;
656 set_buffer_async_write(bh);
657}
658EXPORT_SYMBOL(mark_buffer_async_write);
659
660
661/*
662 * fs/buffer.c contains helper functions for buffer-backed address space's
663 * fsync functions. A common requirement for buffer-based filesystems is
664 * that certain data from the backing blockdev needs to be written out for
665 * a successful fsync(). For example, ext2 indirect blocks need to be
666 * written back and waited upon before fsync() returns.
667 *
668 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
669 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
670 * management of a list of dependent buffers at ->i_mapping->private_list.
671 *
672 * Locking is a little subtle: try_to_free_buffers() will remove buffers
673 * from their controlling inode's queue when they are being freed. But
674 * try_to_free_buffers() will be operating against the *blockdev* mapping
675 * at the time, not against the S_ISREG file which depends on those buffers.
676 * So the locking for private_list is via the private_lock in the address_space
677 * which backs the buffers. Which is different from the address_space
678 * against which the buffers are listed. So for a particular address_space,
679 * mapping->private_lock does *not* protect mapping->private_list! In fact,
680 * mapping->private_list will always be protected by the backing blockdev's
681 * ->private_lock.
682 *
683 * Which introduces a requirement: all buffers on an address_space's
684 * ->private_list must be from the same address_space: the blockdev's.
685 *
686 * address_spaces which do not place buffers at ->private_list via these
687 * utility functions are free to use private_lock and private_list for
688 * whatever they want. The only requirement is that list_empty(private_list)
689 * be true at clear_inode() time.
690 *
691 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
692 * filesystems should do that. invalidate_inode_buffers() should just go
693 * BUG_ON(!list_empty).
694 *
695 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
696 * take an address_space, not an inode. And it should be called
697 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
698 * queued up.
699 *
700 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
701 * list if it is already on a list. Because if the buffer is on a list,
702 * it *must* already be on the right one. If not, the filesystem is being
703 * silly. This will save a ton of locking. But first we have to ensure
704 * that buffers are taken *off* the old inode's list when they are freed
705 * (presumably in truncate). That requires careful auditing of all
706 * filesystems (do it inside bforget()). It could also be done by bringing
707 * b_inode back.
708 */
709
710/*
711 * The buffer's backing address_space's private_lock must be held
712 */
713static inline void __remove_assoc_queue(struct buffer_head *bh)
714{
715 list_del_init(&bh->b_assoc_buffers);
716}
717
718int inode_has_buffers(struct inode *inode)
719{
720 return !list_empty(&inode->i_data.private_list);
721}
722
723/*
724 * osync is designed to support O_SYNC io. It waits synchronously for
725 * all already-submitted IO to complete, but does not queue any new
726 * writes to the disk.
727 *
728 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
729 * you dirty the buffers, and then use osync_inode_buffers to wait for
730 * completion. Any other dirty buffers which are not yet queued for
731 * write will not be flushed to disk by the osync.
732 */
733static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
734{
735 struct buffer_head *bh;
736 struct list_head *p;
737 int err = 0;
738
739 spin_lock(lock);
740repeat:
741 list_for_each_prev(p, list) {
742 bh = BH_ENTRY(p);
743 if (buffer_locked(bh)) {
744 get_bh(bh);
745 spin_unlock(lock);
746 wait_on_buffer(bh);
747 if (!buffer_uptodate(bh))
748 err = -EIO;
749 brelse(bh);
750 spin_lock(lock);
751 goto repeat;
752 }
753 }
754 spin_unlock(lock);
755 return err;
756}
757
758/**
759 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
760 * buffers
Martin Waitz67be2dd2005-05-01 08:59:26 -0700761 * @mapping: the mapping which wants those buffers written
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 *
763 * Starts I/O against the buffers at mapping->private_list, and waits upon
764 * that I/O.
765 *
Martin Waitz67be2dd2005-05-01 08:59:26 -0700766 * Basically, this is a convenience function for fsync().
767 * @mapping is a file or directory which needs those buffers to be written for
768 * a successful fsync().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 */
770int sync_mapping_buffers(struct address_space *mapping)
771{
772 struct address_space *buffer_mapping = mapping->assoc_mapping;
773
774 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
775 return 0;
776
777 return fsync_buffers_list(&buffer_mapping->private_lock,
778 &mapping->private_list);
779}
780EXPORT_SYMBOL(sync_mapping_buffers);
781
782/*
783 * Called when we've recently written block `bblock', and it is known that
784 * `bblock' was for a buffer_boundary() buffer. This means that the block at
785 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
786 * dirty, schedule it for IO. So that indirects merge nicely with their data.
787 */
788void write_boundary_block(struct block_device *bdev,
789 sector_t bblock, unsigned blocksize)
790{
791 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
792 if (bh) {
793 if (buffer_dirty(bh))
794 ll_rw_block(WRITE, 1, &bh);
795 put_bh(bh);
796 }
797}
798
799void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
800{
801 struct address_space *mapping = inode->i_mapping;
802 struct address_space *buffer_mapping = bh->b_page->mapping;
803
804 mark_buffer_dirty(bh);
805 if (!mapping->assoc_mapping) {
806 mapping->assoc_mapping = buffer_mapping;
807 } else {
808 if (mapping->assoc_mapping != buffer_mapping)
809 BUG();
810 }
811 if (list_empty(&bh->b_assoc_buffers)) {
812 spin_lock(&buffer_mapping->private_lock);
813 list_move_tail(&bh->b_assoc_buffers,
814 &mapping->private_list);
815 spin_unlock(&buffer_mapping->private_lock);
816 }
817}
818EXPORT_SYMBOL(mark_buffer_dirty_inode);
819
820/*
821 * Add a page to the dirty page list.
822 *
823 * It is a sad fact of life that this function is called from several places
824 * deeply under spinlocking. It may not sleep.
825 *
826 * If the page has buffers, the uptodate buffers are set dirty, to preserve
827 * dirty-state coherency between the page and the buffers. It the page does
828 * not have buffers then when they are later attached they will all be set
829 * dirty.
830 *
831 * The buffers are dirtied before the page is dirtied. There's a small race
832 * window in which a writepage caller may see the page cleanness but not the
833 * buffer dirtiness. That's fine. If this code were to set the page dirty
834 * before the buffers, a concurrent writepage caller could clear the page dirty
835 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
836 * page on the dirty page list.
837 *
838 * We use private_lock to lock against try_to_free_buffers while using the
839 * page's buffer list. Also use this to protect against clean buffers being
840 * added to the page after it was set dirty.
841 *
842 * FIXME: may need to call ->reservepage here as well. That's rather up to the
843 * address_space though.
844 */
845int __set_page_dirty_buffers(struct page *page)
846{
847 struct address_space * const mapping = page->mapping;
848
849 spin_lock(&mapping->private_lock);
850 if (page_has_buffers(page)) {
851 struct buffer_head *head = page_buffers(page);
852 struct buffer_head *bh = head;
853
854 do {
855 set_buffer_dirty(bh);
856 bh = bh->b_this_page;
857 } while (bh != head);
858 }
859 spin_unlock(&mapping->private_lock);
860
861 if (!TestSetPageDirty(page)) {
862 write_lock_irq(&mapping->tree_lock);
863 if (page->mapping) { /* Race with truncate? */
864 if (mapping_cap_account_dirty(mapping))
865 inc_page_state(nr_dirty);
866 radix_tree_tag_set(&mapping->page_tree,
867 page_index(page),
868 PAGECACHE_TAG_DIRTY);
869 }
870 write_unlock_irq(&mapping->tree_lock);
871 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
872 }
873
874 return 0;
875}
876EXPORT_SYMBOL(__set_page_dirty_buffers);
877
878/*
879 * Write out and wait upon a list of buffers.
880 *
881 * We have conflicting pressures: we want to make sure that all
882 * initially dirty buffers get waited on, but that any subsequently
883 * dirtied buffers don't. After all, we don't want fsync to last
884 * forever if somebody is actively writing to the file.
885 *
886 * Do this in two main stages: first we copy dirty buffers to a
887 * temporary inode list, queueing the writes as we go. Then we clean
888 * up, waiting for those writes to complete.
889 *
890 * During this second stage, any subsequent updates to the file may end
891 * up refiling the buffer on the original inode's dirty list again, so
892 * there is a chance we will end up with a buffer queued for write but
893 * not yet completed on that list. So, as a final cleanup we go through
894 * the osync code to catch these locked, dirty buffers without requeuing
895 * any newly dirty buffers for write.
896 */
897static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
898{
899 struct buffer_head *bh;
900 struct list_head tmp;
901 int err = 0, err2;
902
903 INIT_LIST_HEAD(&tmp);
904
905 spin_lock(lock);
906 while (!list_empty(list)) {
907 bh = BH_ENTRY(list->next);
908 list_del_init(&bh->b_assoc_buffers);
909 if (buffer_dirty(bh) || buffer_locked(bh)) {
910 list_add(&bh->b_assoc_buffers, &tmp);
911 if (buffer_dirty(bh)) {
912 get_bh(bh);
913 spin_unlock(lock);
914 /*
915 * Ensure any pending I/O completes so that
916 * ll_rw_block() actually writes the current
917 * contents - it is a noop if I/O is still in
918 * flight on potentially older contents.
919 */
920 wait_on_buffer(bh);
921 ll_rw_block(WRITE, 1, &bh);
922 brelse(bh);
923 spin_lock(lock);
924 }
925 }
926 }
927
928 while (!list_empty(&tmp)) {
929 bh = BH_ENTRY(tmp.prev);
930 __remove_assoc_queue(bh);
931 get_bh(bh);
932 spin_unlock(lock);
933 wait_on_buffer(bh);
934 if (!buffer_uptodate(bh))
935 err = -EIO;
936 brelse(bh);
937 spin_lock(lock);
938 }
939
940 spin_unlock(lock);
941 err2 = osync_buffers_list(lock, list);
942 if (err)
943 return err;
944 else
945 return err2;
946}
947
948/*
949 * Invalidate any and all dirty buffers on a given inode. We are
950 * probably unmounting the fs, but that doesn't mean we have already
951 * done a sync(). Just drop the buffers from the inode list.
952 *
953 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
954 * assumes that all the buffers are against the blockdev. Not true
955 * for reiserfs.
956 */
957void invalidate_inode_buffers(struct inode *inode)
958{
959 if (inode_has_buffers(inode)) {
960 struct address_space *mapping = &inode->i_data;
961 struct list_head *list = &mapping->private_list;
962 struct address_space *buffer_mapping = mapping->assoc_mapping;
963
964 spin_lock(&buffer_mapping->private_lock);
965 while (!list_empty(list))
966 __remove_assoc_queue(BH_ENTRY(list->next));
967 spin_unlock(&buffer_mapping->private_lock);
968 }
969}
970
971/*
972 * Remove any clean buffers from the inode's buffer list. This is called
973 * when we're trying to free the inode itself. Those buffers can pin it.
974 *
975 * Returns true if all buffers were removed.
976 */
977int remove_inode_buffers(struct inode *inode)
978{
979 int ret = 1;
980
981 if (inode_has_buffers(inode)) {
982 struct address_space *mapping = &inode->i_data;
983 struct list_head *list = &mapping->private_list;
984 struct address_space *buffer_mapping = mapping->assoc_mapping;
985
986 spin_lock(&buffer_mapping->private_lock);
987 while (!list_empty(list)) {
988 struct buffer_head *bh = BH_ENTRY(list->next);
989 if (buffer_dirty(bh)) {
990 ret = 0;
991 break;
992 }
993 __remove_assoc_queue(bh);
994 }
995 spin_unlock(&buffer_mapping->private_lock);
996 }
997 return ret;
998}
999
1000/*
1001 * Create the appropriate buffers when given a page for data area and
1002 * the size of each buffer.. Use the bh->b_this_page linked list to
1003 * follow the buffers created. Return NULL if unable to create more
1004 * buffers.
1005 *
1006 * The retry flag is used to differentiate async IO (paging, swapping)
1007 * which may not fail from ordinary buffer allocations.
1008 */
1009struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1010 int retry)
1011{
1012 struct buffer_head *bh, *head;
1013 long offset;
1014
1015try_again:
1016 head = NULL;
1017 offset = PAGE_SIZE;
1018 while ((offset -= size) >= 0) {
1019 bh = alloc_buffer_head(GFP_NOFS);
1020 if (!bh)
1021 goto no_grow;
1022
1023 bh->b_bdev = NULL;
1024 bh->b_this_page = head;
1025 bh->b_blocknr = -1;
1026 head = bh;
1027
1028 bh->b_state = 0;
1029 atomic_set(&bh->b_count, 0);
1030 bh->b_size = size;
1031
1032 /* Link the buffer to its page */
1033 set_bh_page(bh, page, offset);
1034
1035 bh->b_end_io = NULL;
1036 }
1037 return head;
1038/*
1039 * In case anything failed, we just free everything we got.
1040 */
1041no_grow:
1042 if (head) {
1043 do {
1044 bh = head;
1045 head = head->b_this_page;
1046 free_buffer_head(bh);
1047 } while (head);
1048 }
1049
1050 /*
1051 * Return failure for non-async IO requests. Async IO requests
1052 * are not allowed to fail, so we have to wait until buffer heads
1053 * become available. But we don't want tasks sleeping with
1054 * partially complete buffers, so all were released above.
1055 */
1056 if (!retry)
1057 return NULL;
1058
1059 /* We're _really_ low on memory. Now we just
1060 * wait for old buffer heads to become free due to
1061 * finishing IO. Since this is an async request and
1062 * the reserve list is empty, we're sure there are
1063 * async buffer heads in use.
1064 */
1065 free_more_memory();
1066 goto try_again;
1067}
1068EXPORT_SYMBOL_GPL(alloc_page_buffers);
1069
1070static inline void
1071link_dev_buffers(struct page *page, struct buffer_head *head)
1072{
1073 struct buffer_head *bh, *tail;
1074
1075 bh = head;
1076 do {
1077 tail = bh;
1078 bh = bh->b_this_page;
1079 } while (bh);
1080 tail->b_this_page = head;
1081 attach_page_buffers(page, head);
1082}
1083
1084/*
1085 * Initialise the state of a blockdev page's buffers.
1086 */
1087static void
1088init_page_buffers(struct page *page, struct block_device *bdev,
1089 sector_t block, int size)
1090{
1091 struct buffer_head *head = page_buffers(page);
1092 struct buffer_head *bh = head;
1093 int uptodate = PageUptodate(page);
1094
1095 do {
1096 if (!buffer_mapped(bh)) {
1097 init_buffer(bh, NULL, NULL);
1098 bh->b_bdev = bdev;
1099 bh->b_blocknr = block;
1100 if (uptodate)
1101 set_buffer_uptodate(bh);
1102 set_buffer_mapped(bh);
1103 }
1104 block++;
1105 bh = bh->b_this_page;
1106 } while (bh != head);
1107}
1108
1109/*
1110 * Create the page-cache page that contains the requested block.
1111 *
1112 * This is user purely for blockdev mappings.
1113 */
1114static struct page *
1115grow_dev_page(struct block_device *bdev, sector_t block,
1116 pgoff_t index, int size)
1117{
1118 struct inode *inode = bdev->bd_inode;
1119 struct page *page;
1120 struct buffer_head *bh;
1121
1122 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1123 if (!page)
1124 return NULL;
1125
1126 if (!PageLocked(page))
1127 BUG();
1128
1129 if (page_has_buffers(page)) {
1130 bh = page_buffers(page);
1131 if (bh->b_size == size) {
1132 init_page_buffers(page, bdev, block, size);
1133 return page;
1134 }
1135 if (!try_to_free_buffers(page))
1136 goto failed;
1137 }
1138
1139 /*
1140 * Allocate some buffers for this page
1141 */
1142 bh = alloc_page_buffers(page, size, 0);
1143 if (!bh)
1144 goto failed;
1145
1146 /*
1147 * Link the page to the buffers and initialise them. Take the
1148 * lock to be atomic wrt __find_get_block(), which does not
1149 * run under the page lock.
1150 */
1151 spin_lock(&inode->i_mapping->private_lock);
1152 link_dev_buffers(page, bh);
1153 init_page_buffers(page, bdev, block, size);
1154 spin_unlock(&inode->i_mapping->private_lock);
1155 return page;
1156
1157failed:
1158 BUG();
1159 unlock_page(page);
1160 page_cache_release(page);
1161 return NULL;
1162}
1163
1164/*
1165 * Create buffers for the specified block device block's page. If
1166 * that page was dirty, the buffers are set dirty also.
1167 *
1168 * Except that's a bug. Attaching dirty buffers to a dirty
1169 * blockdev's page can result in filesystem corruption, because
1170 * some of those buffers may be aliases of filesystem data.
1171 * grow_dev_page() will go BUG() if this happens.
1172 */
1173static inline int
1174grow_buffers(struct block_device *bdev, sector_t block, int size)
1175{
1176 struct page *page;
1177 pgoff_t index;
1178 int sizebits;
1179
1180 sizebits = -1;
1181 do {
1182 sizebits++;
1183 } while ((size << sizebits) < PAGE_SIZE);
1184
1185 index = block >> sizebits;
1186 block = index << sizebits;
1187
1188 /* Create a page with the proper size buffers.. */
1189 page = grow_dev_page(bdev, block, index, size);
1190 if (!page)
1191 return 0;
1192 unlock_page(page);
1193 page_cache_release(page);
1194 return 1;
1195}
1196
Adrian Bunk75c96f82005-05-05 16:16:09 -07001197static struct buffer_head *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198__getblk_slow(struct block_device *bdev, sector_t block, int size)
1199{
1200 /* Size must be multiple of hard sectorsize */
1201 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1202 (size < 512 || size > PAGE_SIZE))) {
1203 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1204 size);
1205 printk(KERN_ERR "hardsect size: %d\n",
1206 bdev_hardsect_size(bdev));
1207
1208 dump_stack();
1209 return NULL;
1210 }
1211
1212 for (;;) {
1213 struct buffer_head * bh;
1214
1215 bh = __find_get_block(bdev, block, size);
1216 if (bh)
1217 return bh;
1218
1219 if (!grow_buffers(bdev, block, size))
1220 free_more_memory();
1221 }
1222}
1223
1224/*
1225 * The relationship between dirty buffers and dirty pages:
1226 *
1227 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1228 * the page is tagged dirty in its radix tree.
1229 *
1230 * At all times, the dirtiness of the buffers represents the dirtiness of
1231 * subsections of the page. If the page has buffers, the page dirty bit is
1232 * merely a hint about the true dirty state.
1233 *
1234 * When a page is set dirty in its entirety, all its buffers are marked dirty
1235 * (if the page has buffers).
1236 *
1237 * When a buffer is marked dirty, its page is dirtied, but the page's other
1238 * buffers are not.
1239 *
1240 * Also. When blockdev buffers are explicitly read with bread(), they
1241 * individually become uptodate. But their backing page remains not
1242 * uptodate - even if all of its buffers are uptodate. A subsequent
1243 * block_read_full_page() against that page will discover all the uptodate
1244 * buffers, will set the page uptodate and will perform no I/O.
1245 */
1246
1247/**
1248 * mark_buffer_dirty - mark a buffer_head as needing writeout
Martin Waitz67be2dd2005-05-01 08:59:26 -07001249 * @bh: the buffer_head to mark dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 *
1251 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1252 * backing page dirty, then tag the page as dirty in its address_space's radix
1253 * tree and then attach the address_space's inode to its superblock's dirty
1254 * inode list.
1255 *
1256 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1257 * mapping->tree_lock and the global inode_lock.
1258 */
1259void fastcall mark_buffer_dirty(struct buffer_head *bh)
1260{
1261 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1262 __set_page_dirty_nobuffers(bh->b_page);
1263}
1264
1265/*
1266 * Decrement a buffer_head's reference count. If all buffers against a page
1267 * have zero reference count, are clean and unlocked, and if the page is clean
1268 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1269 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1270 * a page but it ends up not being freed, and buffers may later be reattached).
1271 */
1272void __brelse(struct buffer_head * buf)
1273{
1274 if (atomic_read(&buf->b_count)) {
1275 put_bh(buf);
1276 return;
1277 }
1278 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1279 WARN_ON(1);
1280}
1281
1282/*
1283 * bforget() is like brelse(), except it discards any
1284 * potentially dirty data.
1285 */
1286void __bforget(struct buffer_head *bh)
1287{
1288 clear_buffer_dirty(bh);
1289 if (!list_empty(&bh->b_assoc_buffers)) {
1290 struct address_space *buffer_mapping = bh->b_page->mapping;
1291
1292 spin_lock(&buffer_mapping->private_lock);
1293 list_del_init(&bh->b_assoc_buffers);
1294 spin_unlock(&buffer_mapping->private_lock);
1295 }
1296 __brelse(bh);
1297}
1298
1299static struct buffer_head *__bread_slow(struct buffer_head *bh)
1300{
1301 lock_buffer(bh);
1302 if (buffer_uptodate(bh)) {
1303 unlock_buffer(bh);
1304 return bh;
1305 } else {
1306 get_bh(bh);
1307 bh->b_end_io = end_buffer_read_sync;
1308 submit_bh(READ, bh);
1309 wait_on_buffer(bh);
1310 if (buffer_uptodate(bh))
1311 return bh;
1312 }
1313 brelse(bh);
1314 return NULL;
1315}
1316
1317/*
1318 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1319 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1320 * refcount elevated by one when they're in an LRU. A buffer can only appear
1321 * once in a particular CPU's LRU. A single buffer can be present in multiple
1322 * CPU's LRUs at the same time.
1323 *
1324 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1325 * sb_find_get_block().
1326 *
1327 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1328 * a local interrupt disable for that.
1329 */
1330
1331#define BH_LRU_SIZE 8
1332
1333struct bh_lru {
1334 struct buffer_head *bhs[BH_LRU_SIZE];
1335};
1336
1337static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1338
1339#ifdef CONFIG_SMP
1340#define bh_lru_lock() local_irq_disable()
1341#define bh_lru_unlock() local_irq_enable()
1342#else
1343#define bh_lru_lock() preempt_disable()
1344#define bh_lru_unlock() preempt_enable()
1345#endif
1346
1347static inline void check_irqs_on(void)
1348{
1349#ifdef irqs_disabled
1350 BUG_ON(irqs_disabled());
1351#endif
1352}
1353
1354/*
1355 * The LRU management algorithm is dopey-but-simple. Sorry.
1356 */
1357static void bh_lru_install(struct buffer_head *bh)
1358{
1359 struct buffer_head *evictee = NULL;
1360 struct bh_lru *lru;
1361
1362 check_irqs_on();
1363 bh_lru_lock();
1364 lru = &__get_cpu_var(bh_lrus);
1365 if (lru->bhs[0] != bh) {
1366 struct buffer_head *bhs[BH_LRU_SIZE];
1367 int in;
1368 int out = 0;
1369
1370 get_bh(bh);
1371 bhs[out++] = bh;
1372 for (in = 0; in < BH_LRU_SIZE; in++) {
1373 struct buffer_head *bh2 = lru->bhs[in];
1374
1375 if (bh2 == bh) {
1376 __brelse(bh2);
1377 } else {
1378 if (out >= BH_LRU_SIZE) {
1379 BUG_ON(evictee != NULL);
1380 evictee = bh2;
1381 } else {
1382 bhs[out++] = bh2;
1383 }
1384 }
1385 }
1386 while (out < BH_LRU_SIZE)
1387 bhs[out++] = NULL;
1388 memcpy(lru->bhs, bhs, sizeof(bhs));
1389 }
1390 bh_lru_unlock();
1391
1392 if (evictee)
1393 __brelse(evictee);
1394}
1395
1396/*
1397 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1398 */
1399static inline struct buffer_head *
1400lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1401{
1402 struct buffer_head *ret = NULL;
1403 struct bh_lru *lru;
1404 int i;
1405
1406 check_irqs_on();
1407 bh_lru_lock();
1408 lru = &__get_cpu_var(bh_lrus);
1409 for (i = 0; i < BH_LRU_SIZE; i++) {
1410 struct buffer_head *bh = lru->bhs[i];
1411
1412 if (bh && bh->b_bdev == bdev &&
1413 bh->b_blocknr == block && bh->b_size == size) {
1414 if (i) {
1415 while (i) {
1416 lru->bhs[i] = lru->bhs[i - 1];
1417 i--;
1418 }
1419 lru->bhs[0] = bh;
1420 }
1421 get_bh(bh);
1422 ret = bh;
1423 break;
1424 }
1425 }
1426 bh_lru_unlock();
1427 return ret;
1428}
1429
1430/*
1431 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1432 * it in the LRU and mark it as accessed. If it is not present then return
1433 * NULL
1434 */
1435struct buffer_head *
1436__find_get_block(struct block_device *bdev, sector_t block, int size)
1437{
1438 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1439
1440 if (bh == NULL) {
1441 bh = __find_get_block_slow(bdev, block, size);
1442 if (bh)
1443 bh_lru_install(bh);
1444 }
1445 if (bh)
1446 touch_buffer(bh);
1447 return bh;
1448}
1449EXPORT_SYMBOL(__find_get_block);
1450
1451/*
1452 * __getblk will locate (and, if necessary, create) the buffer_head
1453 * which corresponds to the passed block_device, block and size. The
1454 * returned buffer has its reference count incremented.
1455 *
1456 * __getblk() cannot fail - it just keeps trying. If you pass it an
1457 * illegal block number, __getblk() will happily return a buffer_head
1458 * which represents the non-existent block. Very weird.
1459 *
1460 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1461 * attempt is failing. FIXME, perhaps?
1462 */
1463struct buffer_head *
1464__getblk(struct block_device *bdev, sector_t block, int size)
1465{
1466 struct buffer_head *bh = __find_get_block(bdev, block, size);
1467
1468 might_sleep();
1469 if (bh == NULL)
1470 bh = __getblk_slow(bdev, block, size);
1471 return bh;
1472}
1473EXPORT_SYMBOL(__getblk);
1474
1475/*
1476 * Do async read-ahead on a buffer..
1477 */
1478void __breadahead(struct block_device *bdev, sector_t block, int size)
1479{
1480 struct buffer_head *bh = __getblk(bdev, block, size);
1481 ll_rw_block(READA, 1, &bh);
1482 brelse(bh);
1483}
1484EXPORT_SYMBOL(__breadahead);
1485
1486/**
1487 * __bread() - reads a specified block and returns the bh
Martin Waitz67be2dd2005-05-01 08:59:26 -07001488 * @bdev: the block_device to read from
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 * @block: number of block
1490 * @size: size (in bytes) to read
1491 *
1492 * Reads a specified block, and returns buffer head that contains it.
1493 * It returns NULL if the block was unreadable.
1494 */
1495struct buffer_head *
1496__bread(struct block_device *bdev, sector_t block, int size)
1497{
1498 struct buffer_head *bh = __getblk(bdev, block, size);
1499
1500 if (!buffer_uptodate(bh))
1501 bh = __bread_slow(bh);
1502 return bh;
1503}
1504EXPORT_SYMBOL(__bread);
1505
1506/*
1507 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1508 * This doesn't race because it runs in each cpu either in irq
1509 * or with preempt disabled.
1510 */
1511static void invalidate_bh_lru(void *arg)
1512{
1513 struct bh_lru *b = &get_cpu_var(bh_lrus);
1514 int i;
1515
1516 for (i = 0; i < BH_LRU_SIZE; i++) {
1517 brelse(b->bhs[i]);
1518 b->bhs[i] = NULL;
1519 }
1520 put_cpu_var(bh_lrus);
1521}
1522
1523static void invalidate_bh_lrus(void)
1524{
1525 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1526}
1527
1528void set_bh_page(struct buffer_head *bh,
1529 struct page *page, unsigned long offset)
1530{
1531 bh->b_page = page;
1532 if (offset >= PAGE_SIZE)
1533 BUG();
1534 if (PageHighMem(page))
1535 /*
1536 * This catches illegal uses and preserves the offset:
1537 */
1538 bh->b_data = (char *)(0 + offset);
1539 else
1540 bh->b_data = page_address(page) + offset;
1541}
1542EXPORT_SYMBOL(set_bh_page);
1543
1544/*
1545 * Called when truncating a buffer on a page completely.
1546 */
1547static inline void discard_buffer(struct buffer_head * bh)
1548{
1549 lock_buffer(bh);
1550 clear_buffer_dirty(bh);
1551 bh->b_bdev = NULL;
1552 clear_buffer_mapped(bh);
1553 clear_buffer_req(bh);
1554 clear_buffer_new(bh);
1555 clear_buffer_delay(bh);
1556 unlock_buffer(bh);
1557}
1558
1559/**
1560 * try_to_release_page() - release old fs-specific metadata on a page
1561 *
1562 * @page: the page which the kernel is trying to free
1563 * @gfp_mask: memory allocation flags (and I/O mode)
1564 *
1565 * The address_space is to try to release any data against the page
1566 * (presumably at page->private). If the release was successful, return `1'.
1567 * Otherwise return zero.
1568 *
1569 * The @gfp_mask argument specifies whether I/O may be performed to release
1570 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1571 *
1572 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1573 */
1574int try_to_release_page(struct page *page, int gfp_mask)
1575{
1576 struct address_space * const mapping = page->mapping;
1577
1578 BUG_ON(!PageLocked(page));
1579 if (PageWriteback(page))
1580 return 0;
1581
1582 if (mapping && mapping->a_ops->releasepage)
1583 return mapping->a_ops->releasepage(page, gfp_mask);
1584 return try_to_free_buffers(page);
1585}
1586EXPORT_SYMBOL(try_to_release_page);
1587
1588/**
1589 * block_invalidatepage - invalidate part of all of a buffer-backed page
1590 *
1591 * @page: the page which is affected
1592 * @offset: the index of the truncation point
1593 *
1594 * block_invalidatepage() is called when all or part of the page has become
1595 * invalidatedby a truncate operation.
1596 *
1597 * block_invalidatepage() does not have to release all buffers, but it must
1598 * ensure that no dirty buffer is left outside @offset and that no I/O
1599 * is underway against any of the blocks which are outside the truncation
1600 * point. Because the caller is about to free (and possibly reuse) those
1601 * blocks on-disk.
1602 */
1603int block_invalidatepage(struct page *page, unsigned long offset)
1604{
1605 struct buffer_head *head, *bh, *next;
1606 unsigned int curr_off = 0;
1607 int ret = 1;
1608
1609 BUG_ON(!PageLocked(page));
1610 if (!page_has_buffers(page))
1611 goto out;
1612
1613 head = page_buffers(page);
1614 bh = head;
1615 do {
1616 unsigned int next_off = curr_off + bh->b_size;
1617 next = bh->b_this_page;
1618
1619 /*
1620 * is this block fully invalidated?
1621 */
1622 if (offset <= curr_off)
1623 discard_buffer(bh);
1624 curr_off = next_off;
1625 bh = next;
1626 } while (bh != head);
1627
1628 /*
1629 * We release buffers only if the entire page is being invalidated.
1630 * The get_block cached value has been unconditionally invalidated,
1631 * so real IO is not possible anymore.
1632 */
1633 if (offset == 0)
1634 ret = try_to_release_page(page, 0);
1635out:
1636 return ret;
1637}
1638EXPORT_SYMBOL(block_invalidatepage);
1639
1640/*
1641 * We attach and possibly dirty the buffers atomically wrt
1642 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1643 * is already excluded via the page lock.
1644 */
1645void create_empty_buffers(struct page *page,
1646 unsigned long blocksize, unsigned long b_state)
1647{
1648 struct buffer_head *bh, *head, *tail;
1649
1650 head = alloc_page_buffers(page, blocksize, 1);
1651 bh = head;
1652 do {
1653 bh->b_state |= b_state;
1654 tail = bh;
1655 bh = bh->b_this_page;
1656 } while (bh);
1657 tail->b_this_page = head;
1658
1659 spin_lock(&page->mapping->private_lock);
1660 if (PageUptodate(page) || PageDirty(page)) {
1661 bh = head;
1662 do {
1663 if (PageDirty(page))
1664 set_buffer_dirty(bh);
1665 if (PageUptodate(page))
1666 set_buffer_uptodate(bh);
1667 bh = bh->b_this_page;
1668 } while (bh != head);
1669 }
1670 attach_page_buffers(page, head);
1671 spin_unlock(&page->mapping->private_lock);
1672}
1673EXPORT_SYMBOL(create_empty_buffers);
1674
1675/*
1676 * We are taking a block for data and we don't want any output from any
1677 * buffer-cache aliases starting from return from that function and
1678 * until the moment when something will explicitly mark the buffer
1679 * dirty (hopefully that will not happen until we will free that block ;-)
1680 * We don't even need to mark it not-uptodate - nobody can expect
1681 * anything from a newly allocated buffer anyway. We used to used
1682 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1683 * don't want to mark the alias unmapped, for example - it would confuse
1684 * anyone who might pick it with bread() afterwards...
1685 *
1686 * Also.. Note that bforget() doesn't lock the buffer. So there can
1687 * be writeout I/O going on against recently-freed buffers. We don't
1688 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1689 * only if we really need to. That happens here.
1690 */
1691void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1692{
1693 struct buffer_head *old_bh;
1694
1695 might_sleep();
1696
1697 old_bh = __find_get_block_slow(bdev, block, 0);
1698 if (old_bh) {
1699 clear_buffer_dirty(old_bh);
1700 wait_on_buffer(old_bh);
1701 clear_buffer_req(old_bh);
1702 __brelse(old_bh);
1703 }
1704}
1705EXPORT_SYMBOL(unmap_underlying_metadata);
1706
1707/*
1708 * NOTE! All mapped/uptodate combinations are valid:
1709 *
1710 * Mapped Uptodate Meaning
1711 *
1712 * No No "unknown" - must do get_block()
1713 * No Yes "hole" - zero-filled
1714 * Yes No "allocated" - allocated on disk, not read in
1715 * Yes Yes "valid" - allocated and up-to-date in memory.
1716 *
1717 * "Dirty" is valid only with the last case (mapped+uptodate).
1718 */
1719
1720/*
1721 * While block_write_full_page is writing back the dirty buffers under
1722 * the page lock, whoever dirtied the buffers may decide to clean them
1723 * again at any time. We handle that by only looking at the buffer
1724 * state inside lock_buffer().
1725 *
1726 * If block_write_full_page() is called for regular writeback
1727 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1728 * locked buffer. This only can happen if someone has written the buffer
1729 * directly, with submit_bh(). At the address_space level PageWriteback
1730 * prevents this contention from occurring.
1731 */
1732static int __block_write_full_page(struct inode *inode, struct page *page,
1733 get_block_t *get_block, struct writeback_control *wbc)
1734{
1735 int err;
1736 sector_t block;
1737 sector_t last_block;
Andrew Mortonf0fbd5f2005-05-05 16:15:48 -07001738 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739 int nr_underway = 0;
1740
1741 BUG_ON(!PageLocked(page));
1742
1743 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1744
1745 if (!page_has_buffers(page)) {
1746 create_empty_buffers(page, 1 << inode->i_blkbits,
1747 (1 << BH_Dirty)|(1 << BH_Uptodate));
1748 }
1749
1750 /*
1751 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1752 * here, and the (potentially unmapped) buffers may become dirty at
1753 * any time. If a buffer becomes dirty here after we've inspected it
1754 * then we just miss that fact, and the page stays dirty.
1755 *
1756 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1757 * handle that here by just cleaning them.
1758 */
1759
1760 block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1761 head = page_buffers(page);
1762 bh = head;
1763
1764 /*
1765 * Get all the dirty buffers mapped to disk addresses and
1766 * handle any aliases from the underlying blockdev's mapping.
1767 */
1768 do {
1769 if (block > last_block) {
1770 /*
1771 * mapped buffers outside i_size will occur, because
1772 * this page can be outside i_size when there is a
1773 * truncate in progress.
1774 */
1775 /*
1776 * The buffer was zeroed by block_write_full_page()
1777 */
1778 clear_buffer_dirty(bh);
1779 set_buffer_uptodate(bh);
1780 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1781 err = get_block(inode, block, bh, 1);
1782 if (err)
1783 goto recover;
1784 if (buffer_new(bh)) {
1785 /* blockdev mappings never come here */
1786 clear_buffer_new(bh);
1787 unmap_underlying_metadata(bh->b_bdev,
1788 bh->b_blocknr);
1789 }
1790 }
1791 bh = bh->b_this_page;
1792 block++;
1793 } while (bh != head);
1794
1795 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 if (!buffer_mapped(bh))
1797 continue;
1798 /*
1799 * If it's a fully non-blocking write attempt and we cannot
1800 * lock the buffer then redirty the page. Note that this can
1801 * potentially cause a busy-wait loop from pdflush and kswapd
1802 * activity, but those code paths have their own higher-level
1803 * throttling.
1804 */
1805 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1806 lock_buffer(bh);
1807 } else if (test_set_buffer_locked(bh)) {
1808 redirty_page_for_writepage(wbc, page);
1809 continue;
1810 }
1811 if (test_clear_buffer_dirty(bh)) {
1812 mark_buffer_async_write(bh);
1813 } else {
1814 unlock_buffer(bh);
1815 }
1816 } while ((bh = bh->b_this_page) != head);
1817
1818 /*
1819 * The page and its buffers are protected by PageWriteback(), so we can
1820 * drop the bh refcounts early.
1821 */
1822 BUG_ON(PageWriteback(page));
1823 set_page_writeback(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824
1825 do {
1826 struct buffer_head *next = bh->b_this_page;
1827 if (buffer_async_write(bh)) {
1828 submit_bh(WRITE, bh);
1829 nr_underway++;
1830 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831 bh = next;
1832 } while (bh != head);
Andrew Morton05937ba2005-05-05 16:15:47 -07001833 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834
1835 err = 0;
1836done:
1837 if (nr_underway == 0) {
1838 /*
1839 * The page was marked dirty, but the buffers were
1840 * clean. Someone wrote them back by hand with
1841 * ll_rw_block/submit_bh. A rare case.
1842 */
1843 int uptodate = 1;
1844 do {
1845 if (!buffer_uptodate(bh)) {
1846 uptodate = 0;
1847 break;
1848 }
1849 bh = bh->b_this_page;
1850 } while (bh != head);
1851 if (uptodate)
1852 SetPageUptodate(page);
1853 end_page_writeback(page);
1854 /*
1855 * The page and buffer_heads can be released at any time from
1856 * here on.
1857 */
1858 wbc->pages_skipped++; /* We didn't write this page */
1859 }
1860 return err;
1861
1862recover:
1863 /*
1864 * ENOSPC, or some other error. We may already have added some
1865 * blocks to the file, so we need to write these out to avoid
1866 * exposing stale data.
1867 * The page is currently locked and not marked for writeback
1868 */
1869 bh = head;
1870 /* Recovery: lock and submit the mapped buffers */
1871 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1873 lock_buffer(bh);
1874 mark_buffer_async_write(bh);
1875 } else {
1876 /*
1877 * The buffer may have been set dirty during
1878 * attachment to a dirty page.
1879 */
1880 clear_buffer_dirty(bh);
1881 }
1882 } while ((bh = bh->b_this_page) != head);
1883 SetPageError(page);
1884 BUG_ON(PageWriteback(page));
1885 set_page_writeback(page);
1886 unlock_page(page);
1887 do {
1888 struct buffer_head *next = bh->b_this_page;
1889 if (buffer_async_write(bh)) {
1890 clear_buffer_dirty(bh);
1891 submit_bh(WRITE, bh);
1892 nr_underway++;
1893 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894 bh = next;
1895 } while (bh != head);
1896 goto done;
1897}
1898
1899static int __block_prepare_write(struct inode *inode, struct page *page,
1900 unsigned from, unsigned to, get_block_t *get_block)
1901{
1902 unsigned block_start, block_end;
1903 sector_t block;
1904 int err = 0;
1905 unsigned blocksize, bbits;
1906 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1907
1908 BUG_ON(!PageLocked(page));
1909 BUG_ON(from > PAGE_CACHE_SIZE);
1910 BUG_ON(to > PAGE_CACHE_SIZE);
1911 BUG_ON(from > to);
1912
1913 blocksize = 1 << inode->i_blkbits;
1914 if (!page_has_buffers(page))
1915 create_empty_buffers(page, blocksize, 0);
1916 head = page_buffers(page);
1917
1918 bbits = inode->i_blkbits;
1919 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1920
1921 for(bh = head, block_start = 0; bh != head || !block_start;
1922 block++, block_start=block_end, bh = bh->b_this_page) {
1923 block_end = block_start + blocksize;
1924 if (block_end <= from || block_start >= to) {
1925 if (PageUptodate(page)) {
1926 if (!buffer_uptodate(bh))
1927 set_buffer_uptodate(bh);
1928 }
1929 continue;
1930 }
1931 if (buffer_new(bh))
1932 clear_buffer_new(bh);
1933 if (!buffer_mapped(bh)) {
1934 err = get_block(inode, block, bh, 1);
1935 if (err)
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001936 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 if (buffer_new(bh)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 unmap_underlying_metadata(bh->b_bdev,
1939 bh->b_blocknr);
1940 if (PageUptodate(page)) {
1941 set_buffer_uptodate(bh);
1942 continue;
1943 }
1944 if (block_end > to || block_start < from) {
1945 void *kaddr;
1946
1947 kaddr = kmap_atomic(page, KM_USER0);
1948 if (block_end > to)
1949 memset(kaddr+to, 0,
1950 block_end-to);
1951 if (block_start < from)
1952 memset(kaddr+block_start,
1953 0, from-block_start);
1954 flush_dcache_page(page);
1955 kunmap_atomic(kaddr, KM_USER0);
1956 }
1957 continue;
1958 }
1959 }
1960 if (PageUptodate(page)) {
1961 if (!buffer_uptodate(bh))
1962 set_buffer_uptodate(bh);
1963 continue;
1964 }
1965 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1966 (block_start < from || block_end > to)) {
1967 ll_rw_block(READ, 1, &bh);
1968 *wait_bh++=bh;
1969 }
1970 }
1971 /*
1972 * If we issued read requests - let them complete.
1973 */
1974 while(wait_bh > wait) {
1975 wait_on_buffer(*--wait_bh);
1976 if (!buffer_uptodate(*wait_bh))
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001977 err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978 }
Anton Altaparmakov152becd2005-06-23 00:10:21 -07001979 if (!err) {
1980 bh = head;
1981 do {
1982 if (buffer_new(bh))
1983 clear_buffer_new(bh);
1984 } while ((bh = bh->b_this_page) != head);
1985 return 0;
1986 }
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001987 /* Error case: */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001988 /*
1989 * Zero out any newly allocated blocks to avoid exposing stale
1990 * data. If BH_New is set, we know that the block was newly
1991 * allocated in the above loop.
1992 */
1993 bh = head;
1994 block_start = 0;
1995 do {
1996 block_end = block_start+blocksize;
1997 if (block_end <= from)
1998 goto next_bh;
1999 if (block_start >= to)
2000 break;
2001 if (buffer_new(bh)) {
2002 void *kaddr;
2003
2004 clear_buffer_new(bh);
2005 kaddr = kmap_atomic(page, KM_USER0);
2006 memset(kaddr+block_start, 0, bh->b_size);
2007 kunmap_atomic(kaddr, KM_USER0);
2008 set_buffer_uptodate(bh);
2009 mark_buffer_dirty(bh);
2010 }
2011next_bh:
2012 block_start = block_end;
2013 bh = bh->b_this_page;
2014 } while (bh != head);
2015 return err;
2016}
2017
2018static int __block_commit_write(struct inode *inode, struct page *page,
2019 unsigned from, unsigned to)
2020{
2021 unsigned block_start, block_end;
2022 int partial = 0;
2023 unsigned blocksize;
2024 struct buffer_head *bh, *head;
2025
2026 blocksize = 1 << inode->i_blkbits;
2027
2028 for(bh = head = page_buffers(page), block_start = 0;
2029 bh != head || !block_start;
2030 block_start=block_end, bh = bh->b_this_page) {
2031 block_end = block_start + blocksize;
2032 if (block_end <= from || block_start >= to) {
2033 if (!buffer_uptodate(bh))
2034 partial = 1;
2035 } else {
2036 set_buffer_uptodate(bh);
2037 mark_buffer_dirty(bh);
2038 }
2039 }
2040
2041 /*
2042 * If this is a partial write which happened to make all buffers
2043 * uptodate then we can optimize away a bogus readpage() for
2044 * the next read(). Here we 'discover' whether the page went
2045 * uptodate as a result of this (potentially partial) write.
2046 */
2047 if (!partial)
2048 SetPageUptodate(page);
2049 return 0;
2050}
2051
2052/*
2053 * Generic "read page" function for block devices that have the normal
2054 * get_block functionality. This is most of the block device filesystems.
2055 * Reads the page asynchronously --- the unlock_buffer() and
2056 * set/clear_buffer_uptodate() functions propagate buffer state into the
2057 * page struct once IO has completed.
2058 */
2059int block_read_full_page(struct page *page, get_block_t *get_block)
2060{
2061 struct inode *inode = page->mapping->host;
2062 sector_t iblock, lblock;
2063 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2064 unsigned int blocksize;
2065 int nr, i;
2066 int fully_mapped = 1;
2067
Matt Mackallcd7619d2005-05-01 08:59:01 -07002068 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 blocksize = 1 << inode->i_blkbits;
2070 if (!page_has_buffers(page))
2071 create_empty_buffers(page, blocksize, 0);
2072 head = page_buffers(page);
2073
2074 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2075 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2076 bh = head;
2077 nr = 0;
2078 i = 0;
2079
2080 do {
2081 if (buffer_uptodate(bh))
2082 continue;
2083
2084 if (!buffer_mapped(bh)) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002085 int err = 0;
2086
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087 fully_mapped = 0;
2088 if (iblock < lblock) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002089 err = get_block(inode, iblock, bh, 0);
2090 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 SetPageError(page);
2092 }
2093 if (!buffer_mapped(bh)) {
2094 void *kaddr = kmap_atomic(page, KM_USER0);
2095 memset(kaddr + i * blocksize, 0, blocksize);
2096 flush_dcache_page(page);
2097 kunmap_atomic(kaddr, KM_USER0);
Andrew Mortonc64610b2005-05-16 21:53:49 -07002098 if (!err)
2099 set_buffer_uptodate(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 continue;
2101 }
2102 /*
2103 * get_block() might have updated the buffer
2104 * synchronously
2105 */
2106 if (buffer_uptodate(bh))
2107 continue;
2108 }
2109 arr[nr++] = bh;
2110 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2111
2112 if (fully_mapped)
2113 SetPageMappedToDisk(page);
2114
2115 if (!nr) {
2116 /*
2117 * All buffers are uptodate - we can set the page uptodate
2118 * as well. But not if get_block() returned an error.
2119 */
2120 if (!PageError(page))
2121 SetPageUptodate(page);
2122 unlock_page(page);
2123 return 0;
2124 }
2125
2126 /* Stage two: lock the buffers */
2127 for (i = 0; i < nr; i++) {
2128 bh = arr[i];
2129 lock_buffer(bh);
2130 mark_buffer_async_read(bh);
2131 }
2132
2133 /*
2134 * Stage 3: start the IO. Check for uptodateness
2135 * inside the buffer lock in case another process reading
2136 * the underlying blockdev brought it uptodate (the sct fix).
2137 */
2138 for (i = 0; i < nr; i++) {
2139 bh = arr[i];
2140 if (buffer_uptodate(bh))
2141 end_buffer_async_read(bh, 1);
2142 else
2143 submit_bh(READ, bh);
2144 }
2145 return 0;
2146}
2147
2148/* utility function for filesystems that need to do work on expanding
2149 * truncates. Uses prepare/commit_write to allow the filesystem to
2150 * deal with the hole.
2151 */
2152int generic_cont_expand(struct inode *inode, loff_t size)
2153{
2154 struct address_space *mapping = inode->i_mapping;
2155 struct page *page;
2156 unsigned long index, offset, limit;
2157 int err;
2158
2159 err = -EFBIG;
2160 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2161 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2162 send_sig(SIGXFSZ, current, 0);
2163 goto out;
2164 }
2165 if (size > inode->i_sb->s_maxbytes)
2166 goto out;
2167
2168 offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2169
2170 /* ugh. in prepare/commit_write, if from==to==start of block, we
2171 ** skip the prepare. make sure we never send an offset for the start
2172 ** of a block
2173 */
2174 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2175 offset++;
2176 }
2177 index = size >> PAGE_CACHE_SHIFT;
2178 err = -ENOMEM;
2179 page = grab_cache_page(mapping, index);
2180 if (!page)
2181 goto out;
2182 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2183 if (!err) {
2184 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2185 }
2186 unlock_page(page);
2187 page_cache_release(page);
2188 if (err > 0)
2189 err = 0;
2190out:
2191 return err;
2192}
2193
2194/*
2195 * For moronic filesystems that do not allow holes in file.
2196 * We may have to extend the file.
2197 */
2198
2199int cont_prepare_write(struct page *page, unsigned offset,
2200 unsigned to, get_block_t *get_block, loff_t *bytes)
2201{
2202 struct address_space *mapping = page->mapping;
2203 struct inode *inode = mapping->host;
2204 struct page *new_page;
2205 pgoff_t pgpos;
2206 long status;
2207 unsigned zerofrom;
2208 unsigned blocksize = 1 << inode->i_blkbits;
2209 void *kaddr;
2210
2211 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2212 status = -ENOMEM;
2213 new_page = grab_cache_page(mapping, pgpos);
2214 if (!new_page)
2215 goto out;
2216 /* we might sleep */
2217 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2218 unlock_page(new_page);
2219 page_cache_release(new_page);
2220 continue;
2221 }
2222 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2223 if (zerofrom & (blocksize-1)) {
2224 *bytes |= (blocksize-1);
2225 (*bytes)++;
2226 }
2227 status = __block_prepare_write(inode, new_page, zerofrom,
2228 PAGE_CACHE_SIZE, get_block);
2229 if (status)
2230 goto out_unmap;
2231 kaddr = kmap_atomic(new_page, KM_USER0);
2232 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2233 flush_dcache_page(new_page);
2234 kunmap_atomic(kaddr, KM_USER0);
2235 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2236 unlock_page(new_page);
2237 page_cache_release(new_page);
2238 }
2239
2240 if (page->index < pgpos) {
2241 /* completely inside the area */
2242 zerofrom = offset;
2243 } else {
2244 /* page covers the boundary, find the boundary offset */
2245 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2246
2247 /* if we will expand the thing last block will be filled */
2248 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2249 *bytes |= (blocksize-1);
2250 (*bytes)++;
2251 }
2252
2253 /* starting below the boundary? Nothing to zero out */
2254 if (offset <= zerofrom)
2255 zerofrom = offset;
2256 }
2257 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2258 if (status)
2259 goto out1;
2260 if (zerofrom < offset) {
2261 kaddr = kmap_atomic(page, KM_USER0);
2262 memset(kaddr+zerofrom, 0, offset-zerofrom);
2263 flush_dcache_page(page);
2264 kunmap_atomic(kaddr, KM_USER0);
2265 __block_commit_write(inode, page, zerofrom, offset);
2266 }
2267 return 0;
2268out1:
2269 ClearPageUptodate(page);
2270 return status;
2271
2272out_unmap:
2273 ClearPageUptodate(new_page);
2274 unlock_page(new_page);
2275 page_cache_release(new_page);
2276out:
2277 return status;
2278}
2279
2280int block_prepare_write(struct page *page, unsigned from, unsigned to,
2281 get_block_t *get_block)
2282{
2283 struct inode *inode = page->mapping->host;
2284 int err = __block_prepare_write(inode, page, from, to, get_block);
2285 if (err)
2286 ClearPageUptodate(page);
2287 return err;
2288}
2289
2290int block_commit_write(struct page *page, unsigned from, unsigned to)
2291{
2292 struct inode *inode = page->mapping->host;
2293 __block_commit_write(inode,page,from,to);
2294 return 0;
2295}
2296
2297int generic_commit_write(struct file *file, struct page *page,
2298 unsigned from, unsigned to)
2299{
2300 struct inode *inode = page->mapping->host;
2301 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2302 __block_commit_write(inode,page,from,to);
2303 /*
2304 * No need to use i_size_read() here, the i_size
2305 * cannot change under us because we hold i_sem.
2306 */
2307 if (pos > inode->i_size) {
2308 i_size_write(inode, pos);
2309 mark_inode_dirty(inode);
2310 }
2311 return 0;
2312}
2313
2314
2315/*
2316 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2317 * immediately, while under the page lock. So it needs a special end_io
2318 * handler which does not touch the bh after unlocking it.
2319 *
2320 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2321 * a race there is benign: unlock_buffer() only use the bh's address for
2322 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2323 * itself.
2324 */
2325static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2326{
2327 if (uptodate) {
2328 set_buffer_uptodate(bh);
2329 } else {
2330 /* This happens, due to failed READA attempts. */
2331 clear_buffer_uptodate(bh);
2332 }
2333 unlock_buffer(bh);
2334}
2335
2336/*
2337 * On entry, the page is fully not uptodate.
2338 * On exit the page is fully uptodate in the areas outside (from,to)
2339 */
2340int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2341 get_block_t *get_block)
2342{
2343 struct inode *inode = page->mapping->host;
2344 const unsigned blkbits = inode->i_blkbits;
2345 const unsigned blocksize = 1 << blkbits;
2346 struct buffer_head map_bh;
2347 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2348 unsigned block_in_page;
2349 unsigned block_start;
2350 sector_t block_in_file;
2351 char *kaddr;
2352 int nr_reads = 0;
2353 int i;
2354 int ret = 0;
2355 int is_mapped_to_disk = 1;
2356 int dirtied_it = 0;
2357
2358 if (PageMappedToDisk(page))
2359 return 0;
2360
2361 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2362 map_bh.b_page = page;
2363
2364 /*
2365 * We loop across all blocks in the page, whether or not they are
2366 * part of the affected region. This is so we can discover if the
2367 * page is fully mapped-to-disk.
2368 */
2369 for (block_start = 0, block_in_page = 0;
2370 block_start < PAGE_CACHE_SIZE;
2371 block_in_page++, block_start += blocksize) {
2372 unsigned block_end = block_start + blocksize;
2373 int create;
2374
2375 map_bh.b_state = 0;
2376 create = 1;
2377 if (block_start >= to)
2378 create = 0;
2379 ret = get_block(inode, block_in_file + block_in_page,
2380 &map_bh, create);
2381 if (ret)
2382 goto failed;
2383 if (!buffer_mapped(&map_bh))
2384 is_mapped_to_disk = 0;
2385 if (buffer_new(&map_bh))
2386 unmap_underlying_metadata(map_bh.b_bdev,
2387 map_bh.b_blocknr);
2388 if (PageUptodate(page))
2389 continue;
2390 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2391 kaddr = kmap_atomic(page, KM_USER0);
2392 if (block_start < from) {
2393 memset(kaddr+block_start, 0, from-block_start);
2394 dirtied_it = 1;
2395 }
2396 if (block_end > to) {
2397 memset(kaddr + to, 0, block_end - to);
2398 dirtied_it = 1;
2399 }
2400 flush_dcache_page(page);
2401 kunmap_atomic(kaddr, KM_USER0);
2402 continue;
2403 }
2404 if (buffer_uptodate(&map_bh))
2405 continue; /* reiserfs does this */
2406 if (block_start < from || block_end > to) {
2407 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2408
2409 if (!bh) {
2410 ret = -ENOMEM;
2411 goto failed;
2412 }
2413 bh->b_state = map_bh.b_state;
2414 atomic_set(&bh->b_count, 0);
2415 bh->b_this_page = NULL;
2416 bh->b_page = page;
2417 bh->b_blocknr = map_bh.b_blocknr;
2418 bh->b_size = blocksize;
2419 bh->b_data = (char *)(long)block_start;
2420 bh->b_bdev = map_bh.b_bdev;
2421 bh->b_private = NULL;
2422 read_bh[nr_reads++] = bh;
2423 }
2424 }
2425
2426 if (nr_reads) {
2427 struct buffer_head *bh;
2428
2429 /*
2430 * The page is locked, so these buffers are protected from
2431 * any VM or truncate activity. Hence we don't need to care
2432 * for the buffer_head refcounts.
2433 */
2434 for (i = 0; i < nr_reads; i++) {
2435 bh = read_bh[i];
2436 lock_buffer(bh);
2437 bh->b_end_io = end_buffer_read_nobh;
2438 submit_bh(READ, bh);
2439 }
2440 for (i = 0; i < nr_reads; i++) {
2441 bh = read_bh[i];
2442 wait_on_buffer(bh);
2443 if (!buffer_uptodate(bh))
2444 ret = -EIO;
2445 free_buffer_head(bh);
2446 read_bh[i] = NULL;
2447 }
2448 if (ret)
2449 goto failed;
2450 }
2451
2452 if (is_mapped_to_disk)
2453 SetPageMappedToDisk(page);
2454 SetPageUptodate(page);
2455
2456 /*
2457 * Setting the page dirty here isn't necessary for the prepare_write
2458 * function - commit_write will do that. But if/when this function is
2459 * used within the pagefault handler to ensure that all mmapped pages
2460 * have backing space in the filesystem, we will need to dirty the page
2461 * if its contents were altered.
2462 */
2463 if (dirtied_it)
2464 set_page_dirty(page);
2465
2466 return 0;
2467
2468failed:
2469 for (i = 0; i < nr_reads; i++) {
2470 if (read_bh[i])
2471 free_buffer_head(read_bh[i]);
2472 }
2473
2474 /*
2475 * Error recovery is pretty slack. Clear the page and mark it dirty
2476 * so we'll later zero out any blocks which _were_ allocated.
2477 */
2478 kaddr = kmap_atomic(page, KM_USER0);
2479 memset(kaddr, 0, PAGE_CACHE_SIZE);
2480 kunmap_atomic(kaddr, KM_USER0);
2481 SetPageUptodate(page);
2482 set_page_dirty(page);
2483 return ret;
2484}
2485EXPORT_SYMBOL(nobh_prepare_write);
2486
2487int nobh_commit_write(struct file *file, struct page *page,
2488 unsigned from, unsigned to)
2489{
2490 struct inode *inode = page->mapping->host;
2491 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2492
2493 set_page_dirty(page);
2494 if (pos > inode->i_size) {
2495 i_size_write(inode, pos);
2496 mark_inode_dirty(inode);
2497 }
2498 return 0;
2499}
2500EXPORT_SYMBOL(nobh_commit_write);
2501
2502/*
2503 * nobh_writepage() - based on block_full_write_page() except
2504 * that it tries to operate without attaching bufferheads to
2505 * the page.
2506 */
2507int nobh_writepage(struct page *page, get_block_t *get_block,
2508 struct writeback_control *wbc)
2509{
2510 struct inode * const inode = page->mapping->host;
2511 loff_t i_size = i_size_read(inode);
2512 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2513 unsigned offset;
2514 void *kaddr;
2515 int ret;
2516
2517 /* Is the page fully inside i_size? */
2518 if (page->index < end_index)
2519 goto out;
2520
2521 /* Is the page fully outside i_size? (truncate in progress) */
2522 offset = i_size & (PAGE_CACHE_SIZE-1);
2523 if (page->index >= end_index+1 || !offset) {
2524 /*
2525 * The page may have dirty, unmapped buffers. For example,
2526 * they may have been added in ext3_writepage(). Make them
2527 * freeable here, so the page does not leak.
2528 */
2529#if 0
2530 /* Not really sure about this - do we need this ? */
2531 if (page->mapping->a_ops->invalidatepage)
2532 page->mapping->a_ops->invalidatepage(page, offset);
2533#endif
2534 unlock_page(page);
2535 return 0; /* don't care */
2536 }
2537
2538 /*
2539 * The page straddles i_size. It must be zeroed out on each and every
2540 * writepage invocation because it may be mmapped. "A file is mapped
2541 * in multiples of the page size. For a file that is not a multiple of
2542 * the page size, the remaining memory is zeroed when mapped, and
2543 * writes to that region are not written out to the file."
2544 */
2545 kaddr = kmap_atomic(page, KM_USER0);
2546 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2547 flush_dcache_page(page);
2548 kunmap_atomic(kaddr, KM_USER0);
2549out:
2550 ret = mpage_writepage(page, get_block, wbc);
2551 if (ret == -EAGAIN)
2552 ret = __block_write_full_page(inode, page, get_block, wbc);
2553 return ret;
2554}
2555EXPORT_SYMBOL(nobh_writepage);
2556
2557/*
2558 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2559 */
2560int nobh_truncate_page(struct address_space *mapping, loff_t from)
2561{
2562 struct inode *inode = mapping->host;
2563 unsigned blocksize = 1 << inode->i_blkbits;
2564 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2565 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2566 unsigned to;
2567 struct page *page;
2568 struct address_space_operations *a_ops = mapping->a_ops;
2569 char *kaddr;
2570 int ret = 0;
2571
2572 if ((offset & (blocksize - 1)) == 0)
2573 goto out;
2574
2575 ret = -ENOMEM;
2576 page = grab_cache_page(mapping, index);
2577 if (!page)
2578 goto out;
2579
2580 to = (offset + blocksize) & ~(blocksize - 1);
2581 ret = a_ops->prepare_write(NULL, page, offset, to);
2582 if (ret == 0) {
2583 kaddr = kmap_atomic(page, KM_USER0);
2584 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2585 flush_dcache_page(page);
2586 kunmap_atomic(kaddr, KM_USER0);
2587 set_page_dirty(page);
2588 }
2589 unlock_page(page);
2590 page_cache_release(page);
2591out:
2592 return ret;
2593}
2594EXPORT_SYMBOL(nobh_truncate_page);
2595
2596int block_truncate_page(struct address_space *mapping,
2597 loff_t from, get_block_t *get_block)
2598{
2599 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2600 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2601 unsigned blocksize;
2602 pgoff_t iblock;
2603 unsigned length, pos;
2604 struct inode *inode = mapping->host;
2605 struct page *page;
2606 struct buffer_head *bh;
2607 void *kaddr;
2608 int err;
2609
2610 blocksize = 1 << inode->i_blkbits;
2611 length = offset & (blocksize - 1);
2612
2613 /* Block boundary? Nothing to do */
2614 if (!length)
2615 return 0;
2616
2617 length = blocksize - length;
2618 iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2619
2620 page = grab_cache_page(mapping, index);
2621 err = -ENOMEM;
2622 if (!page)
2623 goto out;
2624
2625 if (!page_has_buffers(page))
2626 create_empty_buffers(page, blocksize, 0);
2627
2628 /* Find the buffer that contains "offset" */
2629 bh = page_buffers(page);
2630 pos = blocksize;
2631 while (offset >= pos) {
2632 bh = bh->b_this_page;
2633 iblock++;
2634 pos += blocksize;
2635 }
2636
2637 err = 0;
2638 if (!buffer_mapped(bh)) {
2639 err = get_block(inode, iblock, bh, 0);
2640 if (err)
2641 goto unlock;
2642 /* unmapped? It's a hole - nothing to do */
2643 if (!buffer_mapped(bh))
2644 goto unlock;
2645 }
2646
2647 /* Ok, it's mapped. Make sure it's up-to-date */
2648 if (PageUptodate(page))
2649 set_buffer_uptodate(bh);
2650
2651 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2652 err = -EIO;
2653 ll_rw_block(READ, 1, &bh);
2654 wait_on_buffer(bh);
2655 /* Uhhuh. Read error. Complain and punt. */
2656 if (!buffer_uptodate(bh))
2657 goto unlock;
2658 }
2659
2660 kaddr = kmap_atomic(page, KM_USER0);
2661 memset(kaddr + offset, 0, length);
2662 flush_dcache_page(page);
2663 kunmap_atomic(kaddr, KM_USER0);
2664
2665 mark_buffer_dirty(bh);
2666 err = 0;
2667
2668unlock:
2669 unlock_page(page);
2670 page_cache_release(page);
2671out:
2672 return err;
2673}
2674
2675/*
2676 * The generic ->writepage function for buffer-backed address_spaces
2677 */
2678int block_write_full_page(struct page *page, get_block_t *get_block,
2679 struct writeback_control *wbc)
2680{
2681 struct inode * const inode = page->mapping->host;
2682 loff_t i_size = i_size_read(inode);
2683 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2684 unsigned offset;
2685 void *kaddr;
2686
2687 /* Is the page fully inside i_size? */
2688 if (page->index < end_index)
2689 return __block_write_full_page(inode, page, get_block, wbc);
2690
2691 /* Is the page fully outside i_size? (truncate in progress) */
2692 offset = i_size & (PAGE_CACHE_SIZE-1);
2693 if (page->index >= end_index+1 || !offset) {
2694 /*
2695 * The page may have dirty, unmapped buffers. For example,
2696 * they may have been added in ext3_writepage(). Make them
2697 * freeable here, so the page does not leak.
2698 */
2699 block_invalidatepage(page, 0);
2700 unlock_page(page);
2701 return 0; /* don't care */
2702 }
2703
2704 /*
2705 * The page straddles i_size. It must be zeroed out on each and every
2706 * writepage invokation because it may be mmapped. "A file is mapped
2707 * in multiples of the page size. For a file that is not a multiple of
2708 * the page size, the remaining memory is zeroed when mapped, and
2709 * writes to that region are not written out to the file."
2710 */
2711 kaddr = kmap_atomic(page, KM_USER0);
2712 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2713 flush_dcache_page(page);
2714 kunmap_atomic(kaddr, KM_USER0);
2715 return __block_write_full_page(inode, page, get_block, wbc);
2716}
2717
2718sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2719 get_block_t *get_block)
2720{
2721 struct buffer_head tmp;
2722 struct inode *inode = mapping->host;
2723 tmp.b_state = 0;
2724 tmp.b_blocknr = 0;
2725 get_block(inode, block, &tmp, 0);
2726 return tmp.b_blocknr;
2727}
2728
2729static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2730{
2731 struct buffer_head *bh = bio->bi_private;
2732
2733 if (bio->bi_size)
2734 return 1;
2735
2736 if (err == -EOPNOTSUPP) {
2737 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2738 set_bit(BH_Eopnotsupp, &bh->b_state);
2739 }
2740
2741 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2742 bio_put(bio);
2743 return 0;
2744}
2745
2746int submit_bh(int rw, struct buffer_head * bh)
2747{
2748 struct bio *bio;
2749 int ret = 0;
2750
2751 BUG_ON(!buffer_locked(bh));
2752 BUG_ON(!buffer_mapped(bh));
2753 BUG_ON(!bh->b_end_io);
2754
2755 if (buffer_ordered(bh) && (rw == WRITE))
2756 rw = WRITE_BARRIER;
2757
2758 /*
2759 * Only clear out a write error when rewriting, should this
2760 * include WRITE_SYNC as well?
2761 */
2762 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2763 clear_buffer_write_io_error(bh);
2764
2765 /*
2766 * from here on down, it's all bio -- do the initial mapping,
2767 * submit_bio -> generic_make_request may further map this bio around
2768 */
2769 bio = bio_alloc(GFP_NOIO, 1);
2770
2771 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2772 bio->bi_bdev = bh->b_bdev;
2773 bio->bi_io_vec[0].bv_page = bh->b_page;
2774 bio->bi_io_vec[0].bv_len = bh->b_size;
2775 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2776
2777 bio->bi_vcnt = 1;
2778 bio->bi_idx = 0;
2779 bio->bi_size = bh->b_size;
2780
2781 bio->bi_end_io = end_bio_bh_io_sync;
2782 bio->bi_private = bh;
2783
2784 bio_get(bio);
2785 submit_bio(rw, bio);
2786
2787 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2788 ret = -EOPNOTSUPP;
2789
2790 bio_put(bio);
2791 return ret;
2792}
2793
2794/**
2795 * ll_rw_block: low-level access to block devices (DEPRECATED)
2796 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
2797 * @nr: number of &struct buffer_heads in the array
2798 * @bhs: array of pointers to &struct buffer_head
2799 *
2800 * ll_rw_block() takes an array of pointers to &struct buffer_heads,
2801 * and requests an I/O operation on them, either a %READ or a %WRITE.
2802 * The third %READA option is described in the documentation for
2803 * generic_make_request() which ll_rw_block() calls.
2804 *
2805 * This function drops any buffer that it cannot get a lock on (with the
2806 * BH_Lock state bit), any buffer that appears to be clean when doing a
2807 * write request, and any buffer that appears to be up-to-date when doing
2808 * read request. Further it marks as clean buffers that are processed for
2809 * writing (the buffer cache won't assume that they are actually clean until
2810 * the buffer gets unlocked).
2811 *
2812 * ll_rw_block sets b_end_io to simple completion handler that marks
2813 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2814 * any waiters.
2815 *
2816 * All of the buffers must be for the same device, and must also be a
2817 * multiple of the current approved size for the device.
2818 */
2819void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2820{
2821 int i;
2822
2823 for (i = 0; i < nr; i++) {
2824 struct buffer_head *bh = bhs[i];
2825
2826 if (test_set_buffer_locked(bh))
2827 continue;
2828
2829 get_bh(bh);
2830 if (rw == WRITE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002831 if (test_clear_buffer_dirty(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002832 bh->b_end_io = end_buffer_write_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833 submit_bh(WRITE, bh);
2834 continue;
2835 }
2836 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002837 if (!buffer_uptodate(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002838 bh->b_end_io = end_buffer_read_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 submit_bh(rw, bh);
2840 continue;
2841 }
2842 }
2843 unlock_buffer(bh);
2844 put_bh(bh);
2845 }
2846}
2847
2848/*
2849 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2850 * and then start new I/O and then wait upon it. The caller must have a ref on
2851 * the buffer_head.
2852 */
2853int sync_dirty_buffer(struct buffer_head *bh)
2854{
2855 int ret = 0;
2856
2857 WARN_ON(atomic_read(&bh->b_count) < 1);
2858 lock_buffer(bh);
2859 if (test_clear_buffer_dirty(bh)) {
2860 get_bh(bh);
2861 bh->b_end_io = end_buffer_write_sync;
2862 ret = submit_bh(WRITE, bh);
2863 wait_on_buffer(bh);
2864 if (buffer_eopnotsupp(bh)) {
2865 clear_buffer_eopnotsupp(bh);
2866 ret = -EOPNOTSUPP;
2867 }
2868 if (!ret && !buffer_uptodate(bh))
2869 ret = -EIO;
2870 } else {
2871 unlock_buffer(bh);
2872 }
2873 return ret;
2874}
2875
2876/*
2877 * try_to_free_buffers() checks if all the buffers on this particular page
2878 * are unused, and releases them if so.
2879 *
2880 * Exclusion against try_to_free_buffers may be obtained by either
2881 * locking the page or by holding its mapping's private_lock.
2882 *
2883 * If the page is dirty but all the buffers are clean then we need to
2884 * be sure to mark the page clean as well. This is because the page
2885 * may be against a block device, and a later reattachment of buffers
2886 * to a dirty page will set *all* buffers dirty. Which would corrupt
2887 * filesystem data on the same device.
2888 *
2889 * The same applies to regular filesystem pages: if all the buffers are
2890 * clean then we set the page clean and proceed. To do that, we require
2891 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2892 * private_lock.
2893 *
2894 * try_to_free_buffers() is non-blocking.
2895 */
2896static inline int buffer_busy(struct buffer_head *bh)
2897{
2898 return atomic_read(&bh->b_count) |
2899 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2900}
2901
2902static int
2903drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2904{
2905 struct buffer_head *head = page_buffers(page);
2906 struct buffer_head *bh;
2907
2908 bh = head;
2909 do {
akpm@osdl.orgde7d5a32005-05-01 08:58:39 -07002910 if (buffer_write_io_error(bh) && page->mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002911 set_bit(AS_EIO, &page->mapping->flags);
2912 if (buffer_busy(bh))
2913 goto failed;
2914 bh = bh->b_this_page;
2915 } while (bh != head);
2916
2917 do {
2918 struct buffer_head *next = bh->b_this_page;
2919
2920 if (!list_empty(&bh->b_assoc_buffers))
2921 __remove_assoc_queue(bh);
2922 bh = next;
2923 } while (bh != head);
2924 *buffers_to_free = head;
2925 __clear_page_buffers(page);
2926 return 1;
2927failed:
2928 return 0;
2929}
2930
2931int try_to_free_buffers(struct page *page)
2932{
2933 struct address_space * const mapping = page->mapping;
2934 struct buffer_head *buffers_to_free = NULL;
2935 int ret = 0;
2936
2937 BUG_ON(!PageLocked(page));
2938 if (PageWriteback(page))
2939 return 0;
2940
2941 if (mapping == NULL) { /* can this still happen? */
2942 ret = drop_buffers(page, &buffers_to_free);
2943 goto out;
2944 }
2945
2946 spin_lock(&mapping->private_lock);
2947 ret = drop_buffers(page, &buffers_to_free);
2948 if (ret) {
2949 /*
2950 * If the filesystem writes its buffers by hand (eg ext3)
2951 * then we can have clean buffers against a dirty page. We
2952 * clean the page here; otherwise later reattachment of buffers
2953 * could encounter a non-uptodate page, which is unresolvable.
2954 * This only applies in the rare case where try_to_free_buffers
2955 * succeeds but the page is not freed.
2956 */
2957 clear_page_dirty(page);
2958 }
2959 spin_unlock(&mapping->private_lock);
2960out:
2961 if (buffers_to_free) {
2962 struct buffer_head *bh = buffers_to_free;
2963
2964 do {
2965 struct buffer_head *next = bh->b_this_page;
2966 free_buffer_head(bh);
2967 bh = next;
2968 } while (bh != buffers_to_free);
2969 }
2970 return ret;
2971}
2972EXPORT_SYMBOL(try_to_free_buffers);
2973
2974int block_sync_page(struct page *page)
2975{
2976 struct address_space *mapping;
2977
2978 smp_mb();
2979 mapping = page_mapping(page);
2980 if (mapping)
2981 blk_run_backing_dev(mapping->backing_dev_info, page);
2982 return 0;
2983}
2984
2985/*
2986 * There are no bdflush tunables left. But distributions are
2987 * still running obsolete flush daemons, so we terminate them here.
2988 *
2989 * Use of bdflush() is deprecated and will be removed in a future kernel.
2990 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2991 */
2992asmlinkage long sys_bdflush(int func, long data)
2993{
2994 static int msg_count;
2995
2996 if (!capable(CAP_SYS_ADMIN))
2997 return -EPERM;
2998
2999 if (msg_count < 5) {
3000 msg_count++;
3001 printk(KERN_INFO
3002 "warning: process `%s' used the obsolete bdflush"
3003 " system call\n", current->comm);
3004 printk(KERN_INFO "Fix your initscripts?\n");
3005 }
3006
3007 if (func == 1)
3008 do_exit(0);
3009 return 0;
3010}
3011
3012/*
3013 * Buffer-head allocation
3014 */
3015static kmem_cache_t *bh_cachep;
3016
3017/*
3018 * Once the number of bh's in the machine exceeds this level, we start
3019 * stripping them in writeback.
3020 */
3021static int max_buffer_heads;
3022
3023int buffer_heads_over_limit;
3024
3025struct bh_accounting {
3026 int nr; /* Number of live bh's */
3027 int ratelimit; /* Limit cacheline bouncing */
3028};
3029
3030static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3031
3032static void recalc_bh_state(void)
3033{
3034 int i;
3035 int tot = 0;
3036
3037 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3038 return;
3039 __get_cpu_var(bh_accounting).ratelimit = 0;
3040 for_each_cpu(i)
3041 tot += per_cpu(bh_accounting, i).nr;
3042 buffer_heads_over_limit = (tot > max_buffer_heads);
3043}
3044
3045struct buffer_head *alloc_buffer_head(unsigned int __nocast gfp_flags)
3046{
3047 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3048 if (ret) {
3049 preempt_disable();
3050 __get_cpu_var(bh_accounting).nr++;
3051 recalc_bh_state();
3052 preempt_enable();
3053 }
3054 return ret;
3055}
3056EXPORT_SYMBOL(alloc_buffer_head);
3057
3058void free_buffer_head(struct buffer_head *bh)
3059{
3060 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3061 kmem_cache_free(bh_cachep, bh);
3062 preempt_disable();
3063 __get_cpu_var(bh_accounting).nr--;
3064 recalc_bh_state();
3065 preempt_enable();
3066}
3067EXPORT_SYMBOL(free_buffer_head);
3068
3069static void
3070init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3071{
3072 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3073 SLAB_CTOR_CONSTRUCTOR) {
3074 struct buffer_head * bh = (struct buffer_head *)data;
3075
3076 memset(bh, 0, sizeof(*bh));
3077 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3078 }
3079}
3080
3081#ifdef CONFIG_HOTPLUG_CPU
3082static void buffer_exit_cpu(int cpu)
3083{
3084 int i;
3085 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3086
3087 for (i = 0; i < BH_LRU_SIZE; i++) {
3088 brelse(b->bhs[i]);
3089 b->bhs[i] = NULL;
3090 }
3091}
3092
3093static int buffer_cpu_notify(struct notifier_block *self,
3094 unsigned long action, void *hcpu)
3095{
3096 if (action == CPU_DEAD)
3097 buffer_exit_cpu((unsigned long)hcpu);
3098 return NOTIFY_OK;
3099}
3100#endif /* CONFIG_HOTPLUG_CPU */
3101
3102void __init buffer_init(void)
3103{
3104 int nrpages;
3105
3106 bh_cachep = kmem_cache_create("buffer_head",
3107 sizeof(struct buffer_head), 0,
Andrea Arcangelie422fd22005-05-05 16:15:04 -07003108 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109
3110 /*
3111 * Limit the bh occupancy to 10% of ZONE_NORMAL
3112 */
3113 nrpages = (nr_free_buffer_pages() * 10) / 100;
3114 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3115 hotcpu_notifier(buffer_cpu_notify, 0);
3116}
3117
3118EXPORT_SYMBOL(__bforget);
3119EXPORT_SYMBOL(__brelse);
3120EXPORT_SYMBOL(__wait_on_buffer);
3121EXPORT_SYMBOL(block_commit_write);
3122EXPORT_SYMBOL(block_prepare_write);
3123EXPORT_SYMBOL(block_read_full_page);
3124EXPORT_SYMBOL(block_sync_page);
3125EXPORT_SYMBOL(block_truncate_page);
3126EXPORT_SYMBOL(block_write_full_page);
3127EXPORT_SYMBOL(cont_prepare_write);
3128EXPORT_SYMBOL(end_buffer_async_write);
3129EXPORT_SYMBOL(end_buffer_read_sync);
3130EXPORT_SYMBOL(end_buffer_write_sync);
3131EXPORT_SYMBOL(file_fsync);
3132EXPORT_SYMBOL(fsync_bdev);
3133EXPORT_SYMBOL(generic_block_bmap);
3134EXPORT_SYMBOL(generic_commit_write);
3135EXPORT_SYMBOL(generic_cont_expand);
3136EXPORT_SYMBOL(init_buffer);
3137EXPORT_SYMBOL(invalidate_bdev);
3138EXPORT_SYMBOL(ll_rw_block);
3139EXPORT_SYMBOL(mark_buffer_dirty);
3140EXPORT_SYMBOL(submit_bh);
3141EXPORT_SYMBOL(sync_dirty_buffer);
3142EXPORT_SYMBOL(unlock_buffer);