blob: 75cac9ada0264e9e3a3351a0c552785288227126 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
21#include <linux/config.h>
22#include <linux/kernel.h>
23#include <linux/syscalls.h>
24#include <linux/fs.h>
25#include <linux/mm.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/smp_lock.h>
29#include <linux/blkdev.h>
30#include <linux/file.h>
31#include <linux/quotaops.h>
32#include <linux/highmem.h>
33#include <linux/module.h>
34#include <linux/writeback.h>
35#include <linux/hash.h>
36#include <linux/suspend.h>
37#include <linux/buffer_head.h>
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070043#include <linux/bit_spinlock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46static void invalidate_bh_lrus(void);
47
48#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50inline void
51init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52{
53 bh->b_end_io = handler;
54 bh->b_private = private;
55}
56
57static int sync_buffer(void *word)
58{
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
62
63 smp_mb();
64 bd = bh->b_bdev;
65 if (bd)
66 blk_run_address_space(bd->bd_inode->i_mapping);
67 io_schedule();
68 return 0;
69}
70
71void fastcall __lock_buffer(struct buffer_head *bh)
72{
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
75}
76EXPORT_SYMBOL(__lock_buffer);
77
78void fastcall unlock_buffer(struct buffer_head *bh)
79{
80 clear_buffer_locked(bh);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
83}
84
85/*
86 * Block until a buffer comes unlocked. This doesn't stop it
87 * from becoming locked again - you have to lock it yourself
88 * if you want to preserve its state.
89 */
90void __wait_on_buffer(struct buffer_head * bh)
91{
92 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93}
94
95static void
96__clear_page_buffers(struct page *page)
97{
98 ClearPagePrivate(page);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -070099 set_page_private(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 page_cache_release(page);
101}
102
103static void buffer_io_error(struct buffer_head *bh)
104{
105 char b[BDEVNAME_SIZE];
106
107 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108 bdevname(bh->b_bdev, b),
109 (unsigned long long)bh->b_blocknr);
110}
111
112/*
113 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
114 * unlock the buffer. This is what ll_rw_block uses too.
115 */
116void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117{
118 if (uptodate) {
119 set_buffer_uptodate(bh);
120 } else {
121 /* This happens, due to failed READA attempts. */
122 clear_buffer_uptodate(bh);
123 }
124 unlock_buffer(bh);
125 put_bh(bh);
126}
127
128void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129{
130 char b[BDEVNAME_SIZE];
131
132 if (uptodate) {
133 set_buffer_uptodate(bh);
134 } else {
135 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136 buffer_io_error(bh);
137 printk(KERN_WARNING "lost page write due to "
138 "I/O error on %s\n",
139 bdevname(bh->b_bdev, b));
140 }
141 set_buffer_write_io_error(bh);
142 clear_buffer_uptodate(bh);
143 }
144 unlock_buffer(bh);
145 put_bh(bh);
146}
147
148/*
149 * Write out and wait upon all the dirty data associated with a block
150 * device via its mapping. Does not take the superblock lock.
151 */
152int sync_blockdev(struct block_device *bdev)
153{
154 int ret = 0;
155
156 if (bdev) {
157 int err;
158
159 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
160 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
161 if (!ret)
162 ret = err;
163 }
164 return ret;
165}
166EXPORT_SYMBOL(sync_blockdev);
167
168/*
169 * Write out and wait upon all dirty data associated with this
170 * superblock. Filesystem data as well as the underlying block
171 * device. Takes the superblock lock.
172 */
173int fsync_super(struct super_block *sb)
174{
175 sync_inodes_sb(sb, 0);
176 DQUOT_SYNC(sb);
177 lock_super(sb);
178 if (sb->s_dirt && sb->s_op->write_super)
179 sb->s_op->write_super(sb);
180 unlock_super(sb);
181 if (sb->s_op->sync_fs)
182 sb->s_op->sync_fs(sb, 1);
183 sync_blockdev(sb->s_bdev);
184 sync_inodes_sb(sb, 1);
185
186 return sync_blockdev(sb->s_bdev);
187}
188
189/*
190 * Write out and wait upon all dirty data associated with this
191 * device. Filesystem data as well as the underlying block
192 * device. Takes the superblock lock.
193 */
194int fsync_bdev(struct block_device *bdev)
195{
196 struct super_block *sb = get_super(bdev);
197 if (sb) {
198 int res = fsync_super(sb);
199 drop_super(sb);
200 return res;
201 }
202 return sync_blockdev(bdev);
203}
204
205/**
206 * freeze_bdev -- lock a filesystem and force it into a consistent state
207 * @bdev: blockdevice to lock
208 *
209 * This takes the block device bd_mount_sem to make sure no new mounts
210 * happen on bdev until thaw_bdev() is called.
211 * If a superblock is found on this device, we take the s_umount semaphore
212 * on it to make sure nobody unmounts until the snapshot creation is done.
213 */
214struct super_block *freeze_bdev(struct block_device *bdev)
215{
216 struct super_block *sb;
217
218 down(&bdev->bd_mount_sem);
219 sb = get_super(bdev);
220 if (sb && !(sb->s_flags & MS_RDONLY)) {
221 sb->s_frozen = SB_FREEZE_WRITE;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700222 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223
224 sync_inodes_sb(sb, 0);
225 DQUOT_SYNC(sb);
226
227 lock_super(sb);
228 if (sb->s_dirt && sb->s_op->write_super)
229 sb->s_op->write_super(sb);
230 unlock_super(sb);
231
232 if (sb->s_op->sync_fs)
233 sb->s_op->sync_fs(sb, 1);
234
235 sync_blockdev(sb->s_bdev);
236 sync_inodes_sb(sb, 1);
237
238 sb->s_frozen = SB_FREEZE_TRANS;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700239 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
241 sync_blockdev(sb->s_bdev);
242
243 if (sb->s_op->write_super_lockfs)
244 sb->s_op->write_super_lockfs(sb);
245 }
246
247 sync_blockdev(bdev);
248 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
249}
250EXPORT_SYMBOL(freeze_bdev);
251
252/**
253 * thaw_bdev -- unlock filesystem
254 * @bdev: blockdevice to unlock
255 * @sb: associated superblock
256 *
257 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258 */
259void thaw_bdev(struct block_device *bdev, struct super_block *sb)
260{
261 if (sb) {
262 BUG_ON(sb->s_bdev != bdev);
263
264 if (sb->s_op->unlockfs)
265 sb->s_op->unlockfs(sb);
266 sb->s_frozen = SB_UNFROZEN;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700267 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 wake_up(&sb->s_wait_unfrozen);
269 drop_super(sb);
270 }
271
272 up(&bdev->bd_mount_sem);
273}
274EXPORT_SYMBOL(thaw_bdev);
275
276/*
277 * sync everything. Start out by waking pdflush, because that writes back
278 * all queues in parallel.
279 */
280static void do_sync(unsigned long wait)
281{
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700282 wakeup_pdflush(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 sync_inodes(0); /* All mappings, inodes and their blockdevs */
284 DQUOT_SYNC(NULL);
285 sync_supers(); /* Write the superblocks */
286 sync_filesystems(0); /* Start syncing the filesystems */
287 sync_filesystems(wait); /* Waitingly sync the filesystems */
288 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
289 if (!wait)
290 printk("Emergency Sync complete\n");
291 if (unlikely(laptop_mode))
292 laptop_sync_completion();
293}
294
295asmlinkage long sys_sync(void)
296{
297 do_sync(1);
298 return 0;
299}
300
301void emergency_sync(void)
302{
303 pdflush_operation(do_sync, 0);
304}
305
306/*
307 * Generic function to fsync a file.
308 *
309 * filp may be NULL if called via the msync of a vma.
310 */
311
312int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
313{
314 struct inode * inode = dentry->d_inode;
315 struct super_block * sb;
316 int ret, err;
317
318 /* sync the inode to buffers */
319 ret = write_inode_now(inode, 0);
320
321 /* sync the superblock to buffers */
322 sb = inode->i_sb;
323 lock_super(sb);
324 if (sb->s_op->write_super)
325 sb->s_op->write_super(sb);
326 unlock_super(sb);
327
328 /* .. finally sync the buffers to disk */
329 err = sync_blockdev(sb->s_bdev);
330 if (!ret)
331 ret = err;
332 return ret;
333}
334
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700335static long do_fsync(unsigned int fd, int datasync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
337 struct file * file;
338 struct address_space *mapping;
339 int ret, err;
340
341 ret = -EBADF;
342 file = fget(fd);
343 if (!file)
344 goto out;
345
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 ret = -EINVAL;
347 if (!file->f_op || !file->f_op->fsync) {
348 /* Why? We can still call filemap_fdatawrite */
349 goto out_putf;
350 }
351
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700352 mapping = file->f_mapping;
353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 current->flags |= PF_SYNCWRITE;
355 ret = filemap_fdatawrite(mapping);
356
357 /*
358 * We need to protect against concurrent writers,
359 * which could cause livelocks in fsync_buffers_list
360 */
361 down(&mapping->host->i_sem);
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700362 err = file->f_op->fsync(file, file->f_dentry, datasync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 if (!ret)
364 ret = err;
365 up(&mapping->host->i_sem);
366 err = filemap_fdatawait(mapping);
367 if (!ret)
368 ret = err;
369 current->flags &= ~PF_SYNCWRITE;
370
371out_putf:
372 fput(file);
373out:
374 return ret;
375}
376
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700377asmlinkage long sys_fsync(unsigned int fd)
378{
379 return do_fsync(fd, 0);
380}
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382asmlinkage long sys_fdatasync(unsigned int fd)
383{
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700384 return do_fsync(fd, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
387/*
388 * Various filesystems appear to want __find_get_block to be non-blocking.
389 * But it's the page lock which protects the buffers. To get around this,
390 * we get exclusion from try_to_free_buffers with the blockdev mapping's
391 * private_lock.
392 *
393 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
394 * may be quite high. This code could TryLock the page, and if that
395 * succeeds, there is no need to take private_lock. (But if
396 * private_lock is contended then so is mapping->tree_lock).
397 */
398static struct buffer_head *
399__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
400{
401 struct inode *bd_inode = bdev->bd_inode;
402 struct address_space *bd_mapping = bd_inode->i_mapping;
403 struct buffer_head *ret = NULL;
404 pgoff_t index;
405 struct buffer_head *bh;
406 struct buffer_head *head;
407 struct page *page;
408 int all_mapped = 1;
409
410 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
411 page = find_get_page(bd_mapping, index);
412 if (!page)
413 goto out;
414
415 spin_lock(&bd_mapping->private_lock);
416 if (!page_has_buffers(page))
417 goto out_unlock;
418 head = page_buffers(page);
419 bh = head;
420 do {
421 if (bh->b_blocknr == block) {
422 ret = bh;
423 get_bh(bh);
424 goto out_unlock;
425 }
426 if (!buffer_mapped(bh))
427 all_mapped = 0;
428 bh = bh->b_this_page;
429 } while (bh != head);
430
431 /* we might be here because some of the buffers on this page are
432 * not mapped. This is due to various races between
433 * file io on the block device and getblk. It gets dealt with
434 * elsewhere, don't buffer_error if we had some unmapped buffers
435 */
436 if (all_mapped) {
437 printk("__find_get_block_slow() failed. "
438 "block=%llu, b_blocknr=%llu\n",
439 (unsigned long long)block, (unsigned long long)bh->b_blocknr);
440 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
441 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
442 }
443out_unlock:
444 spin_unlock(&bd_mapping->private_lock);
445 page_cache_release(page);
446out:
447 return ret;
448}
449
450/* If invalidate_buffers() will trash dirty buffers, it means some kind
451 of fs corruption is going on. Trashing dirty data always imply losing
452 information that was supposed to be just stored on the physical layer
453 by the user.
454
455 Thus invalidate_buffers in general usage is not allwowed to trash
456 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
457 be preserved. These buffers are simply skipped.
458
459 We also skip buffers which are still in use. For example this can
460 happen if a userspace program is reading the block device.
461
462 NOTE: In the case where the user removed a removable-media-disk even if
463 there's still dirty data not synced on disk (due a bug in the device driver
464 or due an error of the user), by not destroying the dirty buffers we could
465 generate corruption also on the next media inserted, thus a parameter is
466 necessary to handle this case in the most safe way possible (trying
467 to not corrupt also the new disk inserted with the data belonging to
468 the old now corrupted disk). Also for the ramdisk the natural thing
469 to do in order to release the ramdisk memory is to destroy dirty buffers.
470
471 These are two special cases. Normal usage imply the device driver
472 to issue a sync on the device (without waiting I/O completion) and
473 then an invalidate_buffers call that doesn't trash dirty buffers.
474
475 For handling cache coherency with the blkdev pagecache the 'update' case
476 is been introduced. It is needed to re-read from disk any pinned
477 buffer. NOTE: re-reading from disk is destructive so we can do it only
478 when we assume nobody is changing the buffercache under our I/O and when
479 we think the disk contains more recent information than the buffercache.
480 The update == 1 pass marks the buffers we need to update, the update == 2
481 pass does the actual I/O. */
482void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
483{
484 invalidate_bh_lrus();
485 /*
486 * FIXME: what about destroy_dirty_buffers?
487 * We really want to use invalidate_inode_pages2() for
488 * that, but not until that's cleaned up.
489 */
490 invalidate_inode_pages(bdev->bd_inode->i_mapping);
491}
492
493/*
494 * Kick pdflush then try to free up some ZONE_NORMAL memory.
495 */
496static void free_more_memory(void)
497{
498 struct zone **zones;
499 pg_data_t *pgdat;
500
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700501 wakeup_pdflush(1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 yield();
503
504 for_each_pgdat(pgdat) {
Al Viroaf4ca452005-10-21 02:55:38 -0400505 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 if (*zones)
Darren Hart1ad539b2005-06-21 17:14:53 -0700507 try_to_free_pages(zones, GFP_NOFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 }
509}
510
511/*
512 * I/O completion handler for block_read_full_page() - pages
513 * which come unlocked at the end of I/O.
514 */
515static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
516{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700518 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 struct buffer_head *tmp;
520 struct page *page;
521 int page_uptodate = 1;
522
523 BUG_ON(!buffer_async_read(bh));
524
525 page = bh->b_page;
526 if (uptodate) {
527 set_buffer_uptodate(bh);
528 } else {
529 clear_buffer_uptodate(bh);
530 if (printk_ratelimit())
531 buffer_io_error(bh);
532 SetPageError(page);
533 }
534
535 /*
536 * Be _very_ careful from here on. Bad things can happen if
537 * two buffer heads end IO at almost the same time and both
538 * decide that the page is now completely done.
539 */
Nick Piggina3972202005-07-07 17:56:56 -0700540 first = page_buffers(page);
541 local_irq_save(flags);
542 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 clear_buffer_async_read(bh);
544 unlock_buffer(bh);
545 tmp = bh;
546 do {
547 if (!buffer_uptodate(tmp))
548 page_uptodate = 0;
549 if (buffer_async_read(tmp)) {
550 BUG_ON(!buffer_locked(tmp));
551 goto still_busy;
552 }
553 tmp = tmp->b_this_page;
554 } while (tmp != bh);
Nick Piggina3972202005-07-07 17:56:56 -0700555 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
556 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
558 /*
559 * If none of the buffers had errors and they are all
560 * uptodate then we can set the page uptodate.
561 */
562 if (page_uptodate && !PageError(page))
563 SetPageUptodate(page);
564 unlock_page(page);
565 return;
566
567still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700568 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
569 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 return;
571}
572
573/*
574 * Completion handler for block_write_full_page() - pages which are unlocked
575 * during I/O, and which have PageWriteback cleared upon I/O completion.
576 */
577void end_buffer_async_write(struct buffer_head *bh, int uptodate)
578{
579 char b[BDEVNAME_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700581 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 struct buffer_head *tmp;
583 struct page *page;
584
585 BUG_ON(!buffer_async_write(bh));
586
587 page = bh->b_page;
588 if (uptodate) {
589 set_buffer_uptodate(bh);
590 } else {
591 if (printk_ratelimit()) {
592 buffer_io_error(bh);
593 printk(KERN_WARNING "lost page write due to "
594 "I/O error on %s\n",
595 bdevname(bh->b_bdev, b));
596 }
597 set_bit(AS_EIO, &page->mapping->flags);
598 clear_buffer_uptodate(bh);
599 SetPageError(page);
600 }
601
Nick Piggina3972202005-07-07 17:56:56 -0700602 first = page_buffers(page);
603 local_irq_save(flags);
604 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
605
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 clear_buffer_async_write(bh);
607 unlock_buffer(bh);
608 tmp = bh->b_this_page;
609 while (tmp != bh) {
610 if (buffer_async_write(tmp)) {
611 BUG_ON(!buffer_locked(tmp));
612 goto still_busy;
613 }
614 tmp = tmp->b_this_page;
615 }
Nick Piggina3972202005-07-07 17:56:56 -0700616 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
617 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 end_page_writeback(page);
619 return;
620
621still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700622 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
623 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 return;
625}
626
627/*
628 * If a page's buffers are under async readin (end_buffer_async_read
629 * completion) then there is a possibility that another thread of
630 * control could lock one of the buffers after it has completed
631 * but while some of the other buffers have not completed. This
632 * locked buffer would confuse end_buffer_async_read() into not unlocking
633 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
634 * that this buffer is not under async I/O.
635 *
636 * The page comes unlocked when it has no locked buffer_async buffers
637 * left.
638 *
639 * PageLocked prevents anyone starting new async I/O reads any of
640 * the buffers.
641 *
642 * PageWriteback is used to prevent simultaneous writeout of the same
643 * page.
644 *
645 * PageLocked prevents anyone from starting writeback of a page which is
646 * under read I/O (PageWriteback is only ever set against a locked page).
647 */
648static void mark_buffer_async_read(struct buffer_head *bh)
649{
650 bh->b_end_io = end_buffer_async_read;
651 set_buffer_async_read(bh);
652}
653
654void mark_buffer_async_write(struct buffer_head *bh)
655{
656 bh->b_end_io = end_buffer_async_write;
657 set_buffer_async_write(bh);
658}
659EXPORT_SYMBOL(mark_buffer_async_write);
660
661
662/*
663 * fs/buffer.c contains helper functions for buffer-backed address space's
664 * fsync functions. A common requirement for buffer-based filesystems is
665 * that certain data from the backing blockdev needs to be written out for
666 * a successful fsync(). For example, ext2 indirect blocks need to be
667 * written back and waited upon before fsync() returns.
668 *
669 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
670 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
671 * management of a list of dependent buffers at ->i_mapping->private_list.
672 *
673 * Locking is a little subtle: try_to_free_buffers() will remove buffers
674 * from their controlling inode's queue when they are being freed. But
675 * try_to_free_buffers() will be operating against the *blockdev* mapping
676 * at the time, not against the S_ISREG file which depends on those buffers.
677 * So the locking for private_list is via the private_lock in the address_space
678 * which backs the buffers. Which is different from the address_space
679 * against which the buffers are listed. So for a particular address_space,
680 * mapping->private_lock does *not* protect mapping->private_list! In fact,
681 * mapping->private_list will always be protected by the backing blockdev's
682 * ->private_lock.
683 *
684 * Which introduces a requirement: all buffers on an address_space's
685 * ->private_list must be from the same address_space: the blockdev's.
686 *
687 * address_spaces which do not place buffers at ->private_list via these
688 * utility functions are free to use private_lock and private_list for
689 * whatever they want. The only requirement is that list_empty(private_list)
690 * be true at clear_inode() time.
691 *
692 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
693 * filesystems should do that. invalidate_inode_buffers() should just go
694 * BUG_ON(!list_empty).
695 *
696 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
697 * take an address_space, not an inode. And it should be called
698 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
699 * queued up.
700 *
701 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
702 * list if it is already on a list. Because if the buffer is on a list,
703 * it *must* already be on the right one. If not, the filesystem is being
704 * silly. This will save a ton of locking. But first we have to ensure
705 * that buffers are taken *off* the old inode's list when they are freed
706 * (presumably in truncate). That requires careful auditing of all
707 * filesystems (do it inside bforget()). It could also be done by bringing
708 * b_inode back.
709 */
710
711/*
712 * The buffer's backing address_space's private_lock must be held
713 */
714static inline void __remove_assoc_queue(struct buffer_head *bh)
715{
716 list_del_init(&bh->b_assoc_buffers);
717}
718
719int inode_has_buffers(struct inode *inode)
720{
721 return !list_empty(&inode->i_data.private_list);
722}
723
724/*
725 * osync is designed to support O_SYNC io. It waits synchronously for
726 * all already-submitted IO to complete, but does not queue any new
727 * writes to the disk.
728 *
729 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
730 * you dirty the buffers, and then use osync_inode_buffers to wait for
731 * completion. Any other dirty buffers which are not yet queued for
732 * write will not be flushed to disk by the osync.
733 */
734static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
735{
736 struct buffer_head *bh;
737 struct list_head *p;
738 int err = 0;
739
740 spin_lock(lock);
741repeat:
742 list_for_each_prev(p, list) {
743 bh = BH_ENTRY(p);
744 if (buffer_locked(bh)) {
745 get_bh(bh);
746 spin_unlock(lock);
747 wait_on_buffer(bh);
748 if (!buffer_uptodate(bh))
749 err = -EIO;
750 brelse(bh);
751 spin_lock(lock);
752 goto repeat;
753 }
754 }
755 spin_unlock(lock);
756 return err;
757}
758
759/**
760 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
761 * buffers
Martin Waitz67be2dd2005-05-01 08:59:26 -0700762 * @mapping: the mapping which wants those buffers written
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 *
764 * Starts I/O against the buffers at mapping->private_list, and waits upon
765 * that I/O.
766 *
Martin Waitz67be2dd2005-05-01 08:59:26 -0700767 * Basically, this is a convenience function for fsync().
768 * @mapping is a file or directory which needs those buffers to be written for
769 * a successful fsync().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 */
771int sync_mapping_buffers(struct address_space *mapping)
772{
773 struct address_space *buffer_mapping = mapping->assoc_mapping;
774
775 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
776 return 0;
777
778 return fsync_buffers_list(&buffer_mapping->private_lock,
779 &mapping->private_list);
780}
781EXPORT_SYMBOL(sync_mapping_buffers);
782
783/*
784 * Called when we've recently written block `bblock', and it is known that
785 * `bblock' was for a buffer_boundary() buffer. This means that the block at
786 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
787 * dirty, schedule it for IO. So that indirects merge nicely with their data.
788 */
789void write_boundary_block(struct block_device *bdev,
790 sector_t bblock, unsigned blocksize)
791{
792 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
793 if (bh) {
794 if (buffer_dirty(bh))
795 ll_rw_block(WRITE, 1, &bh);
796 put_bh(bh);
797 }
798}
799
800void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
801{
802 struct address_space *mapping = inode->i_mapping;
803 struct address_space *buffer_mapping = bh->b_page->mapping;
804
805 mark_buffer_dirty(bh);
806 if (!mapping->assoc_mapping) {
807 mapping->assoc_mapping = buffer_mapping;
808 } else {
809 if (mapping->assoc_mapping != buffer_mapping)
810 BUG();
811 }
812 if (list_empty(&bh->b_assoc_buffers)) {
813 spin_lock(&buffer_mapping->private_lock);
814 list_move_tail(&bh->b_assoc_buffers,
815 &mapping->private_list);
816 spin_unlock(&buffer_mapping->private_lock);
817 }
818}
819EXPORT_SYMBOL(mark_buffer_dirty_inode);
820
821/*
822 * Add a page to the dirty page list.
823 *
824 * It is a sad fact of life that this function is called from several places
825 * deeply under spinlocking. It may not sleep.
826 *
827 * If the page has buffers, the uptodate buffers are set dirty, to preserve
828 * dirty-state coherency between the page and the buffers. It the page does
829 * not have buffers then when they are later attached they will all be set
830 * dirty.
831 *
832 * The buffers are dirtied before the page is dirtied. There's a small race
833 * window in which a writepage caller may see the page cleanness but not the
834 * buffer dirtiness. That's fine. If this code were to set the page dirty
835 * before the buffers, a concurrent writepage caller could clear the page dirty
836 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
837 * page on the dirty page list.
838 *
839 * We use private_lock to lock against try_to_free_buffers while using the
840 * page's buffer list. Also use this to protect against clean buffers being
841 * added to the page after it was set dirty.
842 *
843 * FIXME: may need to call ->reservepage here as well. That's rather up to the
844 * address_space though.
845 */
846int __set_page_dirty_buffers(struct page *page)
847{
848 struct address_space * const mapping = page->mapping;
849
850 spin_lock(&mapping->private_lock);
851 if (page_has_buffers(page)) {
852 struct buffer_head *head = page_buffers(page);
853 struct buffer_head *bh = head;
854
855 do {
856 set_buffer_dirty(bh);
857 bh = bh->b_this_page;
858 } while (bh != head);
859 }
860 spin_unlock(&mapping->private_lock);
861
862 if (!TestSetPageDirty(page)) {
863 write_lock_irq(&mapping->tree_lock);
864 if (page->mapping) { /* Race with truncate? */
865 if (mapping_cap_account_dirty(mapping))
866 inc_page_state(nr_dirty);
867 radix_tree_tag_set(&mapping->page_tree,
868 page_index(page),
869 PAGECACHE_TAG_DIRTY);
870 }
871 write_unlock_irq(&mapping->tree_lock);
872 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
873 }
874
875 return 0;
876}
877EXPORT_SYMBOL(__set_page_dirty_buffers);
878
879/*
880 * Write out and wait upon a list of buffers.
881 *
882 * We have conflicting pressures: we want to make sure that all
883 * initially dirty buffers get waited on, but that any subsequently
884 * dirtied buffers don't. After all, we don't want fsync to last
885 * forever if somebody is actively writing to the file.
886 *
887 * Do this in two main stages: first we copy dirty buffers to a
888 * temporary inode list, queueing the writes as we go. Then we clean
889 * up, waiting for those writes to complete.
890 *
891 * During this second stage, any subsequent updates to the file may end
892 * up refiling the buffer on the original inode's dirty list again, so
893 * there is a chance we will end up with a buffer queued for write but
894 * not yet completed on that list. So, as a final cleanup we go through
895 * the osync code to catch these locked, dirty buffers without requeuing
896 * any newly dirty buffers for write.
897 */
898static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
899{
900 struct buffer_head *bh;
901 struct list_head tmp;
902 int err = 0, err2;
903
904 INIT_LIST_HEAD(&tmp);
905
906 spin_lock(lock);
907 while (!list_empty(list)) {
908 bh = BH_ENTRY(list->next);
909 list_del_init(&bh->b_assoc_buffers);
910 if (buffer_dirty(bh) || buffer_locked(bh)) {
911 list_add(&bh->b_assoc_buffers, &tmp);
912 if (buffer_dirty(bh)) {
913 get_bh(bh);
914 spin_unlock(lock);
915 /*
916 * Ensure any pending I/O completes so that
917 * ll_rw_block() actually writes the current
918 * contents - it is a noop if I/O is still in
919 * flight on potentially older contents.
920 */
Jan Karaa7662232005-09-06 15:19:10 -0700921 ll_rw_block(SWRITE, 1, &bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 brelse(bh);
923 spin_lock(lock);
924 }
925 }
926 }
927
928 while (!list_empty(&tmp)) {
929 bh = BH_ENTRY(tmp.prev);
930 __remove_assoc_queue(bh);
931 get_bh(bh);
932 spin_unlock(lock);
933 wait_on_buffer(bh);
934 if (!buffer_uptodate(bh))
935 err = -EIO;
936 brelse(bh);
937 spin_lock(lock);
938 }
939
940 spin_unlock(lock);
941 err2 = osync_buffers_list(lock, list);
942 if (err)
943 return err;
944 else
945 return err2;
946}
947
948/*
949 * Invalidate any and all dirty buffers on a given inode. We are
950 * probably unmounting the fs, but that doesn't mean we have already
951 * done a sync(). Just drop the buffers from the inode list.
952 *
953 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
954 * assumes that all the buffers are against the blockdev. Not true
955 * for reiserfs.
956 */
957void invalidate_inode_buffers(struct inode *inode)
958{
959 if (inode_has_buffers(inode)) {
960 struct address_space *mapping = &inode->i_data;
961 struct list_head *list = &mapping->private_list;
962 struct address_space *buffer_mapping = mapping->assoc_mapping;
963
964 spin_lock(&buffer_mapping->private_lock);
965 while (!list_empty(list))
966 __remove_assoc_queue(BH_ENTRY(list->next));
967 spin_unlock(&buffer_mapping->private_lock);
968 }
969}
970
971/*
972 * Remove any clean buffers from the inode's buffer list. This is called
973 * when we're trying to free the inode itself. Those buffers can pin it.
974 *
975 * Returns true if all buffers were removed.
976 */
977int remove_inode_buffers(struct inode *inode)
978{
979 int ret = 1;
980
981 if (inode_has_buffers(inode)) {
982 struct address_space *mapping = &inode->i_data;
983 struct list_head *list = &mapping->private_list;
984 struct address_space *buffer_mapping = mapping->assoc_mapping;
985
986 spin_lock(&buffer_mapping->private_lock);
987 while (!list_empty(list)) {
988 struct buffer_head *bh = BH_ENTRY(list->next);
989 if (buffer_dirty(bh)) {
990 ret = 0;
991 break;
992 }
993 __remove_assoc_queue(bh);
994 }
995 spin_unlock(&buffer_mapping->private_lock);
996 }
997 return ret;
998}
999
1000/*
1001 * Create the appropriate buffers when given a page for data area and
1002 * the size of each buffer.. Use the bh->b_this_page linked list to
1003 * follow the buffers created. Return NULL if unable to create more
1004 * buffers.
1005 *
1006 * The retry flag is used to differentiate async IO (paging, swapping)
1007 * which may not fail from ordinary buffer allocations.
1008 */
1009struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1010 int retry)
1011{
1012 struct buffer_head *bh, *head;
1013 long offset;
1014
1015try_again:
1016 head = NULL;
1017 offset = PAGE_SIZE;
1018 while ((offset -= size) >= 0) {
1019 bh = alloc_buffer_head(GFP_NOFS);
1020 if (!bh)
1021 goto no_grow;
1022
1023 bh->b_bdev = NULL;
1024 bh->b_this_page = head;
1025 bh->b_blocknr = -1;
1026 head = bh;
1027
1028 bh->b_state = 0;
1029 atomic_set(&bh->b_count, 0);
1030 bh->b_size = size;
1031
1032 /* Link the buffer to its page */
1033 set_bh_page(bh, page, offset);
1034
1035 bh->b_end_io = NULL;
1036 }
1037 return head;
1038/*
1039 * In case anything failed, we just free everything we got.
1040 */
1041no_grow:
1042 if (head) {
1043 do {
1044 bh = head;
1045 head = head->b_this_page;
1046 free_buffer_head(bh);
1047 } while (head);
1048 }
1049
1050 /*
1051 * Return failure for non-async IO requests. Async IO requests
1052 * are not allowed to fail, so we have to wait until buffer heads
1053 * become available. But we don't want tasks sleeping with
1054 * partially complete buffers, so all were released above.
1055 */
1056 if (!retry)
1057 return NULL;
1058
1059 /* We're _really_ low on memory. Now we just
1060 * wait for old buffer heads to become free due to
1061 * finishing IO. Since this is an async request and
1062 * the reserve list is empty, we're sure there are
1063 * async buffer heads in use.
1064 */
1065 free_more_memory();
1066 goto try_again;
1067}
1068EXPORT_SYMBOL_GPL(alloc_page_buffers);
1069
1070static inline void
1071link_dev_buffers(struct page *page, struct buffer_head *head)
1072{
1073 struct buffer_head *bh, *tail;
1074
1075 bh = head;
1076 do {
1077 tail = bh;
1078 bh = bh->b_this_page;
1079 } while (bh);
1080 tail->b_this_page = head;
1081 attach_page_buffers(page, head);
1082}
1083
1084/*
1085 * Initialise the state of a blockdev page's buffers.
1086 */
1087static void
1088init_page_buffers(struct page *page, struct block_device *bdev,
1089 sector_t block, int size)
1090{
1091 struct buffer_head *head = page_buffers(page);
1092 struct buffer_head *bh = head;
1093 int uptodate = PageUptodate(page);
1094
1095 do {
1096 if (!buffer_mapped(bh)) {
1097 init_buffer(bh, NULL, NULL);
1098 bh->b_bdev = bdev;
1099 bh->b_blocknr = block;
1100 if (uptodate)
1101 set_buffer_uptodate(bh);
1102 set_buffer_mapped(bh);
1103 }
1104 block++;
1105 bh = bh->b_this_page;
1106 } while (bh != head);
1107}
1108
1109/*
1110 * Create the page-cache page that contains the requested block.
1111 *
1112 * This is user purely for blockdev mappings.
1113 */
1114static struct page *
1115grow_dev_page(struct block_device *bdev, sector_t block,
1116 pgoff_t index, int size)
1117{
1118 struct inode *inode = bdev->bd_inode;
1119 struct page *page;
1120 struct buffer_head *bh;
1121
1122 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1123 if (!page)
1124 return NULL;
1125
1126 if (!PageLocked(page))
1127 BUG();
1128
1129 if (page_has_buffers(page)) {
1130 bh = page_buffers(page);
1131 if (bh->b_size == size) {
1132 init_page_buffers(page, bdev, block, size);
1133 return page;
1134 }
1135 if (!try_to_free_buffers(page))
1136 goto failed;
1137 }
1138
1139 /*
1140 * Allocate some buffers for this page
1141 */
1142 bh = alloc_page_buffers(page, size, 0);
1143 if (!bh)
1144 goto failed;
1145
1146 /*
1147 * Link the page to the buffers and initialise them. Take the
1148 * lock to be atomic wrt __find_get_block(), which does not
1149 * run under the page lock.
1150 */
1151 spin_lock(&inode->i_mapping->private_lock);
1152 link_dev_buffers(page, bh);
1153 init_page_buffers(page, bdev, block, size);
1154 spin_unlock(&inode->i_mapping->private_lock);
1155 return page;
1156
1157failed:
1158 BUG();
1159 unlock_page(page);
1160 page_cache_release(page);
1161 return NULL;
1162}
1163
1164/*
1165 * Create buffers for the specified block device block's page. If
1166 * that page was dirty, the buffers are set dirty also.
1167 *
1168 * Except that's a bug. Attaching dirty buffers to a dirty
1169 * blockdev's page can result in filesystem corruption, because
1170 * some of those buffers may be aliases of filesystem data.
1171 * grow_dev_page() will go BUG() if this happens.
1172 */
1173static inline int
1174grow_buffers(struct block_device *bdev, sector_t block, int size)
1175{
1176 struct page *page;
1177 pgoff_t index;
1178 int sizebits;
1179
1180 sizebits = -1;
1181 do {
1182 sizebits++;
1183 } while ((size << sizebits) < PAGE_SIZE);
1184
1185 index = block >> sizebits;
1186 block = index << sizebits;
1187
1188 /* Create a page with the proper size buffers.. */
1189 page = grow_dev_page(bdev, block, index, size);
1190 if (!page)
1191 return 0;
1192 unlock_page(page);
1193 page_cache_release(page);
1194 return 1;
1195}
1196
Adrian Bunk75c96f82005-05-05 16:16:09 -07001197static struct buffer_head *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198__getblk_slow(struct block_device *bdev, sector_t block, int size)
1199{
1200 /* Size must be multiple of hard sectorsize */
1201 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1202 (size < 512 || size > PAGE_SIZE))) {
1203 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1204 size);
1205 printk(KERN_ERR "hardsect size: %d\n",
1206 bdev_hardsect_size(bdev));
1207
1208 dump_stack();
1209 return NULL;
1210 }
1211
1212 for (;;) {
1213 struct buffer_head * bh;
1214
1215 bh = __find_get_block(bdev, block, size);
1216 if (bh)
1217 return bh;
1218
1219 if (!grow_buffers(bdev, block, size))
1220 free_more_memory();
1221 }
1222}
1223
1224/*
1225 * The relationship between dirty buffers and dirty pages:
1226 *
1227 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1228 * the page is tagged dirty in its radix tree.
1229 *
1230 * At all times, the dirtiness of the buffers represents the dirtiness of
1231 * subsections of the page. If the page has buffers, the page dirty bit is
1232 * merely a hint about the true dirty state.
1233 *
1234 * When a page is set dirty in its entirety, all its buffers are marked dirty
1235 * (if the page has buffers).
1236 *
1237 * When a buffer is marked dirty, its page is dirtied, but the page's other
1238 * buffers are not.
1239 *
1240 * Also. When blockdev buffers are explicitly read with bread(), they
1241 * individually become uptodate. But their backing page remains not
1242 * uptodate - even if all of its buffers are uptodate. A subsequent
1243 * block_read_full_page() against that page will discover all the uptodate
1244 * buffers, will set the page uptodate and will perform no I/O.
1245 */
1246
1247/**
1248 * mark_buffer_dirty - mark a buffer_head as needing writeout
Martin Waitz67be2dd2005-05-01 08:59:26 -07001249 * @bh: the buffer_head to mark dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 *
1251 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1252 * backing page dirty, then tag the page as dirty in its address_space's radix
1253 * tree and then attach the address_space's inode to its superblock's dirty
1254 * inode list.
1255 *
1256 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1257 * mapping->tree_lock and the global inode_lock.
1258 */
1259void fastcall mark_buffer_dirty(struct buffer_head *bh)
1260{
1261 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1262 __set_page_dirty_nobuffers(bh->b_page);
1263}
1264
1265/*
1266 * Decrement a buffer_head's reference count. If all buffers against a page
1267 * have zero reference count, are clean and unlocked, and if the page is clean
1268 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1269 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1270 * a page but it ends up not being freed, and buffers may later be reattached).
1271 */
1272void __brelse(struct buffer_head * buf)
1273{
1274 if (atomic_read(&buf->b_count)) {
1275 put_bh(buf);
1276 return;
1277 }
1278 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1279 WARN_ON(1);
1280}
1281
1282/*
1283 * bforget() is like brelse(), except it discards any
1284 * potentially dirty data.
1285 */
1286void __bforget(struct buffer_head *bh)
1287{
1288 clear_buffer_dirty(bh);
1289 if (!list_empty(&bh->b_assoc_buffers)) {
1290 struct address_space *buffer_mapping = bh->b_page->mapping;
1291
1292 spin_lock(&buffer_mapping->private_lock);
1293 list_del_init(&bh->b_assoc_buffers);
1294 spin_unlock(&buffer_mapping->private_lock);
1295 }
1296 __brelse(bh);
1297}
1298
1299static struct buffer_head *__bread_slow(struct buffer_head *bh)
1300{
1301 lock_buffer(bh);
1302 if (buffer_uptodate(bh)) {
1303 unlock_buffer(bh);
1304 return bh;
1305 } else {
1306 get_bh(bh);
1307 bh->b_end_io = end_buffer_read_sync;
1308 submit_bh(READ, bh);
1309 wait_on_buffer(bh);
1310 if (buffer_uptodate(bh))
1311 return bh;
1312 }
1313 brelse(bh);
1314 return NULL;
1315}
1316
1317/*
1318 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1319 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1320 * refcount elevated by one when they're in an LRU. A buffer can only appear
1321 * once in a particular CPU's LRU. A single buffer can be present in multiple
1322 * CPU's LRUs at the same time.
1323 *
1324 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1325 * sb_find_get_block().
1326 *
1327 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1328 * a local interrupt disable for that.
1329 */
1330
1331#define BH_LRU_SIZE 8
1332
1333struct bh_lru {
1334 struct buffer_head *bhs[BH_LRU_SIZE];
1335};
1336
1337static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1338
1339#ifdef CONFIG_SMP
1340#define bh_lru_lock() local_irq_disable()
1341#define bh_lru_unlock() local_irq_enable()
1342#else
1343#define bh_lru_lock() preempt_disable()
1344#define bh_lru_unlock() preempt_enable()
1345#endif
1346
1347static inline void check_irqs_on(void)
1348{
1349#ifdef irqs_disabled
1350 BUG_ON(irqs_disabled());
1351#endif
1352}
1353
1354/*
1355 * The LRU management algorithm is dopey-but-simple. Sorry.
1356 */
1357static void bh_lru_install(struct buffer_head *bh)
1358{
1359 struct buffer_head *evictee = NULL;
1360 struct bh_lru *lru;
1361
1362 check_irqs_on();
1363 bh_lru_lock();
1364 lru = &__get_cpu_var(bh_lrus);
1365 if (lru->bhs[0] != bh) {
1366 struct buffer_head *bhs[BH_LRU_SIZE];
1367 int in;
1368 int out = 0;
1369
1370 get_bh(bh);
1371 bhs[out++] = bh;
1372 for (in = 0; in < BH_LRU_SIZE; in++) {
1373 struct buffer_head *bh2 = lru->bhs[in];
1374
1375 if (bh2 == bh) {
1376 __brelse(bh2);
1377 } else {
1378 if (out >= BH_LRU_SIZE) {
1379 BUG_ON(evictee != NULL);
1380 evictee = bh2;
1381 } else {
1382 bhs[out++] = bh2;
1383 }
1384 }
1385 }
1386 while (out < BH_LRU_SIZE)
1387 bhs[out++] = NULL;
1388 memcpy(lru->bhs, bhs, sizeof(bhs));
1389 }
1390 bh_lru_unlock();
1391
1392 if (evictee)
1393 __brelse(evictee);
1394}
1395
1396/*
1397 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1398 */
1399static inline struct buffer_head *
1400lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1401{
1402 struct buffer_head *ret = NULL;
1403 struct bh_lru *lru;
1404 int i;
1405
1406 check_irqs_on();
1407 bh_lru_lock();
1408 lru = &__get_cpu_var(bh_lrus);
1409 for (i = 0; i < BH_LRU_SIZE; i++) {
1410 struct buffer_head *bh = lru->bhs[i];
1411
1412 if (bh && bh->b_bdev == bdev &&
1413 bh->b_blocknr == block && bh->b_size == size) {
1414 if (i) {
1415 while (i) {
1416 lru->bhs[i] = lru->bhs[i - 1];
1417 i--;
1418 }
1419 lru->bhs[0] = bh;
1420 }
1421 get_bh(bh);
1422 ret = bh;
1423 break;
1424 }
1425 }
1426 bh_lru_unlock();
1427 return ret;
1428}
1429
1430/*
1431 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1432 * it in the LRU and mark it as accessed. If it is not present then return
1433 * NULL
1434 */
1435struct buffer_head *
1436__find_get_block(struct block_device *bdev, sector_t block, int size)
1437{
1438 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1439
1440 if (bh == NULL) {
1441 bh = __find_get_block_slow(bdev, block, size);
1442 if (bh)
1443 bh_lru_install(bh);
1444 }
1445 if (bh)
1446 touch_buffer(bh);
1447 return bh;
1448}
1449EXPORT_SYMBOL(__find_get_block);
1450
1451/*
1452 * __getblk will locate (and, if necessary, create) the buffer_head
1453 * which corresponds to the passed block_device, block and size. The
1454 * returned buffer has its reference count incremented.
1455 *
1456 * __getblk() cannot fail - it just keeps trying. If you pass it an
1457 * illegal block number, __getblk() will happily return a buffer_head
1458 * which represents the non-existent block. Very weird.
1459 *
1460 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1461 * attempt is failing. FIXME, perhaps?
1462 */
1463struct buffer_head *
1464__getblk(struct block_device *bdev, sector_t block, int size)
1465{
1466 struct buffer_head *bh = __find_get_block(bdev, block, size);
1467
1468 might_sleep();
1469 if (bh == NULL)
1470 bh = __getblk_slow(bdev, block, size);
1471 return bh;
1472}
1473EXPORT_SYMBOL(__getblk);
1474
1475/*
1476 * Do async read-ahead on a buffer..
1477 */
1478void __breadahead(struct block_device *bdev, sector_t block, int size)
1479{
1480 struct buffer_head *bh = __getblk(bdev, block, size);
1481 ll_rw_block(READA, 1, &bh);
1482 brelse(bh);
1483}
1484EXPORT_SYMBOL(__breadahead);
1485
1486/**
1487 * __bread() - reads a specified block and returns the bh
Martin Waitz67be2dd2005-05-01 08:59:26 -07001488 * @bdev: the block_device to read from
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 * @block: number of block
1490 * @size: size (in bytes) to read
1491 *
1492 * Reads a specified block, and returns buffer head that contains it.
1493 * It returns NULL if the block was unreadable.
1494 */
1495struct buffer_head *
1496__bread(struct block_device *bdev, sector_t block, int size)
1497{
1498 struct buffer_head *bh = __getblk(bdev, block, size);
1499
1500 if (!buffer_uptodate(bh))
1501 bh = __bread_slow(bh);
1502 return bh;
1503}
1504EXPORT_SYMBOL(__bread);
1505
1506/*
1507 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1508 * This doesn't race because it runs in each cpu either in irq
1509 * or with preempt disabled.
1510 */
1511static void invalidate_bh_lru(void *arg)
1512{
1513 struct bh_lru *b = &get_cpu_var(bh_lrus);
1514 int i;
1515
1516 for (i = 0; i < BH_LRU_SIZE; i++) {
1517 brelse(b->bhs[i]);
1518 b->bhs[i] = NULL;
1519 }
1520 put_cpu_var(bh_lrus);
1521}
1522
1523static void invalidate_bh_lrus(void)
1524{
1525 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1526}
1527
1528void set_bh_page(struct buffer_head *bh,
1529 struct page *page, unsigned long offset)
1530{
1531 bh->b_page = page;
1532 if (offset >= PAGE_SIZE)
1533 BUG();
1534 if (PageHighMem(page))
1535 /*
1536 * This catches illegal uses and preserves the offset:
1537 */
1538 bh->b_data = (char *)(0 + offset);
1539 else
1540 bh->b_data = page_address(page) + offset;
1541}
1542EXPORT_SYMBOL(set_bh_page);
1543
1544/*
1545 * Called when truncating a buffer on a page completely.
1546 */
1547static inline void discard_buffer(struct buffer_head * bh)
1548{
1549 lock_buffer(bh);
1550 clear_buffer_dirty(bh);
1551 bh->b_bdev = NULL;
1552 clear_buffer_mapped(bh);
1553 clear_buffer_req(bh);
1554 clear_buffer_new(bh);
1555 clear_buffer_delay(bh);
1556 unlock_buffer(bh);
1557}
1558
1559/**
1560 * try_to_release_page() - release old fs-specific metadata on a page
1561 *
1562 * @page: the page which the kernel is trying to free
1563 * @gfp_mask: memory allocation flags (and I/O mode)
1564 *
1565 * The address_space is to try to release any data against the page
1566 * (presumably at page->private). If the release was successful, return `1'.
1567 * Otherwise return zero.
1568 *
1569 * The @gfp_mask argument specifies whether I/O may be performed to release
1570 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1571 *
1572 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1573 */
Al Viro27496a82005-10-21 03:20:48 -04001574int try_to_release_page(struct page *page, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575{
1576 struct address_space * const mapping = page->mapping;
1577
1578 BUG_ON(!PageLocked(page));
1579 if (PageWriteback(page))
1580 return 0;
1581
1582 if (mapping && mapping->a_ops->releasepage)
1583 return mapping->a_ops->releasepage(page, gfp_mask);
1584 return try_to_free_buffers(page);
1585}
1586EXPORT_SYMBOL(try_to_release_page);
1587
1588/**
1589 * block_invalidatepage - invalidate part of all of a buffer-backed page
1590 *
1591 * @page: the page which is affected
1592 * @offset: the index of the truncation point
1593 *
1594 * block_invalidatepage() is called when all or part of the page has become
1595 * invalidatedby a truncate operation.
1596 *
1597 * block_invalidatepage() does not have to release all buffers, but it must
1598 * ensure that no dirty buffer is left outside @offset and that no I/O
1599 * is underway against any of the blocks which are outside the truncation
1600 * point. Because the caller is about to free (and possibly reuse) those
1601 * blocks on-disk.
1602 */
1603int block_invalidatepage(struct page *page, unsigned long offset)
1604{
1605 struct buffer_head *head, *bh, *next;
1606 unsigned int curr_off = 0;
1607 int ret = 1;
1608
1609 BUG_ON(!PageLocked(page));
1610 if (!page_has_buffers(page))
1611 goto out;
1612
1613 head = page_buffers(page);
1614 bh = head;
1615 do {
1616 unsigned int next_off = curr_off + bh->b_size;
1617 next = bh->b_this_page;
1618
1619 /*
1620 * is this block fully invalidated?
1621 */
1622 if (offset <= curr_off)
1623 discard_buffer(bh);
1624 curr_off = next_off;
1625 bh = next;
1626 } while (bh != head);
1627
1628 /*
1629 * We release buffers only if the entire page is being invalidated.
1630 * The get_block cached value has been unconditionally invalidated,
1631 * so real IO is not possible anymore.
1632 */
1633 if (offset == 0)
1634 ret = try_to_release_page(page, 0);
1635out:
1636 return ret;
1637}
1638EXPORT_SYMBOL(block_invalidatepage);
1639
Jan Karaaaa40592005-10-30 15:00:16 -08001640int do_invalidatepage(struct page *page, unsigned long offset)
1641{
1642 int (*invalidatepage)(struct page *, unsigned long);
1643 invalidatepage = page->mapping->a_ops->invalidatepage;
1644 if (invalidatepage == NULL)
1645 invalidatepage = block_invalidatepage;
1646 return (*invalidatepage)(page, offset);
1647}
1648
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649/*
1650 * We attach and possibly dirty the buffers atomically wrt
1651 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1652 * is already excluded via the page lock.
1653 */
1654void create_empty_buffers(struct page *page,
1655 unsigned long blocksize, unsigned long b_state)
1656{
1657 struct buffer_head *bh, *head, *tail;
1658
1659 head = alloc_page_buffers(page, blocksize, 1);
1660 bh = head;
1661 do {
1662 bh->b_state |= b_state;
1663 tail = bh;
1664 bh = bh->b_this_page;
1665 } while (bh);
1666 tail->b_this_page = head;
1667
1668 spin_lock(&page->mapping->private_lock);
1669 if (PageUptodate(page) || PageDirty(page)) {
1670 bh = head;
1671 do {
1672 if (PageDirty(page))
1673 set_buffer_dirty(bh);
1674 if (PageUptodate(page))
1675 set_buffer_uptodate(bh);
1676 bh = bh->b_this_page;
1677 } while (bh != head);
1678 }
1679 attach_page_buffers(page, head);
1680 spin_unlock(&page->mapping->private_lock);
1681}
1682EXPORT_SYMBOL(create_empty_buffers);
1683
1684/*
1685 * We are taking a block for data and we don't want any output from any
1686 * buffer-cache aliases starting from return from that function and
1687 * until the moment when something will explicitly mark the buffer
1688 * dirty (hopefully that will not happen until we will free that block ;-)
1689 * We don't even need to mark it not-uptodate - nobody can expect
1690 * anything from a newly allocated buffer anyway. We used to used
1691 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1692 * don't want to mark the alias unmapped, for example - it would confuse
1693 * anyone who might pick it with bread() afterwards...
1694 *
1695 * Also.. Note that bforget() doesn't lock the buffer. So there can
1696 * be writeout I/O going on against recently-freed buffers. We don't
1697 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1698 * only if we really need to. That happens here.
1699 */
1700void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1701{
1702 struct buffer_head *old_bh;
1703
1704 might_sleep();
1705
1706 old_bh = __find_get_block_slow(bdev, block, 0);
1707 if (old_bh) {
1708 clear_buffer_dirty(old_bh);
1709 wait_on_buffer(old_bh);
1710 clear_buffer_req(old_bh);
1711 __brelse(old_bh);
1712 }
1713}
1714EXPORT_SYMBOL(unmap_underlying_metadata);
1715
1716/*
1717 * NOTE! All mapped/uptodate combinations are valid:
1718 *
1719 * Mapped Uptodate Meaning
1720 *
1721 * No No "unknown" - must do get_block()
1722 * No Yes "hole" - zero-filled
1723 * Yes No "allocated" - allocated on disk, not read in
1724 * Yes Yes "valid" - allocated and up-to-date in memory.
1725 *
1726 * "Dirty" is valid only with the last case (mapped+uptodate).
1727 */
1728
1729/*
1730 * While block_write_full_page is writing back the dirty buffers under
1731 * the page lock, whoever dirtied the buffers may decide to clean them
1732 * again at any time. We handle that by only looking at the buffer
1733 * state inside lock_buffer().
1734 *
1735 * If block_write_full_page() is called for regular writeback
1736 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1737 * locked buffer. This only can happen if someone has written the buffer
1738 * directly, with submit_bh(). At the address_space level PageWriteback
1739 * prevents this contention from occurring.
1740 */
1741static int __block_write_full_page(struct inode *inode, struct page *page,
1742 get_block_t *get_block, struct writeback_control *wbc)
1743{
1744 int err;
1745 sector_t block;
1746 sector_t last_block;
Andrew Mortonf0fbd5f2005-05-05 16:15:48 -07001747 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748 int nr_underway = 0;
1749
1750 BUG_ON(!PageLocked(page));
1751
1752 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1753
1754 if (!page_has_buffers(page)) {
1755 create_empty_buffers(page, 1 << inode->i_blkbits,
1756 (1 << BH_Dirty)|(1 << BH_Uptodate));
1757 }
1758
1759 /*
1760 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1761 * here, and the (potentially unmapped) buffers may become dirty at
1762 * any time. If a buffer becomes dirty here after we've inspected it
1763 * then we just miss that fact, and the page stays dirty.
1764 *
1765 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1766 * handle that here by just cleaning them.
1767 */
1768
1769 block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1770 head = page_buffers(page);
1771 bh = head;
1772
1773 /*
1774 * Get all the dirty buffers mapped to disk addresses and
1775 * handle any aliases from the underlying blockdev's mapping.
1776 */
1777 do {
1778 if (block > last_block) {
1779 /*
1780 * mapped buffers outside i_size will occur, because
1781 * this page can be outside i_size when there is a
1782 * truncate in progress.
1783 */
1784 /*
1785 * The buffer was zeroed by block_write_full_page()
1786 */
1787 clear_buffer_dirty(bh);
1788 set_buffer_uptodate(bh);
1789 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1790 err = get_block(inode, block, bh, 1);
1791 if (err)
1792 goto recover;
1793 if (buffer_new(bh)) {
1794 /* blockdev mappings never come here */
1795 clear_buffer_new(bh);
1796 unmap_underlying_metadata(bh->b_bdev,
1797 bh->b_blocknr);
1798 }
1799 }
1800 bh = bh->b_this_page;
1801 block++;
1802 } while (bh != head);
1803
1804 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805 if (!buffer_mapped(bh))
1806 continue;
1807 /*
1808 * If it's a fully non-blocking write attempt and we cannot
1809 * lock the buffer then redirty the page. Note that this can
1810 * potentially cause a busy-wait loop from pdflush and kswapd
1811 * activity, but those code paths have their own higher-level
1812 * throttling.
1813 */
1814 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1815 lock_buffer(bh);
1816 } else if (test_set_buffer_locked(bh)) {
1817 redirty_page_for_writepage(wbc, page);
1818 continue;
1819 }
1820 if (test_clear_buffer_dirty(bh)) {
1821 mark_buffer_async_write(bh);
1822 } else {
1823 unlock_buffer(bh);
1824 }
1825 } while ((bh = bh->b_this_page) != head);
1826
1827 /*
1828 * The page and its buffers are protected by PageWriteback(), so we can
1829 * drop the bh refcounts early.
1830 */
1831 BUG_ON(PageWriteback(page));
1832 set_page_writeback(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833
1834 do {
1835 struct buffer_head *next = bh->b_this_page;
1836 if (buffer_async_write(bh)) {
1837 submit_bh(WRITE, bh);
1838 nr_underway++;
1839 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840 bh = next;
1841 } while (bh != head);
Andrew Morton05937ba2005-05-05 16:15:47 -07001842 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843
1844 err = 0;
1845done:
1846 if (nr_underway == 0) {
1847 /*
1848 * The page was marked dirty, but the buffers were
1849 * clean. Someone wrote them back by hand with
1850 * ll_rw_block/submit_bh. A rare case.
1851 */
1852 int uptodate = 1;
1853 do {
1854 if (!buffer_uptodate(bh)) {
1855 uptodate = 0;
1856 break;
1857 }
1858 bh = bh->b_this_page;
1859 } while (bh != head);
1860 if (uptodate)
1861 SetPageUptodate(page);
1862 end_page_writeback(page);
1863 /*
1864 * The page and buffer_heads can be released at any time from
1865 * here on.
1866 */
1867 wbc->pages_skipped++; /* We didn't write this page */
1868 }
1869 return err;
1870
1871recover:
1872 /*
1873 * ENOSPC, or some other error. We may already have added some
1874 * blocks to the file, so we need to write these out to avoid
1875 * exposing stale data.
1876 * The page is currently locked and not marked for writeback
1877 */
1878 bh = head;
1879 /* Recovery: lock and submit the mapped buffers */
1880 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1882 lock_buffer(bh);
1883 mark_buffer_async_write(bh);
1884 } else {
1885 /*
1886 * The buffer may have been set dirty during
1887 * attachment to a dirty page.
1888 */
1889 clear_buffer_dirty(bh);
1890 }
1891 } while ((bh = bh->b_this_page) != head);
1892 SetPageError(page);
1893 BUG_ON(PageWriteback(page));
1894 set_page_writeback(page);
1895 unlock_page(page);
1896 do {
1897 struct buffer_head *next = bh->b_this_page;
1898 if (buffer_async_write(bh)) {
1899 clear_buffer_dirty(bh);
1900 submit_bh(WRITE, bh);
1901 nr_underway++;
1902 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903 bh = next;
1904 } while (bh != head);
1905 goto done;
1906}
1907
1908static int __block_prepare_write(struct inode *inode, struct page *page,
1909 unsigned from, unsigned to, get_block_t *get_block)
1910{
1911 unsigned block_start, block_end;
1912 sector_t block;
1913 int err = 0;
1914 unsigned blocksize, bbits;
1915 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1916
1917 BUG_ON(!PageLocked(page));
1918 BUG_ON(from > PAGE_CACHE_SIZE);
1919 BUG_ON(to > PAGE_CACHE_SIZE);
1920 BUG_ON(from > to);
1921
1922 blocksize = 1 << inode->i_blkbits;
1923 if (!page_has_buffers(page))
1924 create_empty_buffers(page, blocksize, 0);
1925 head = page_buffers(page);
1926
1927 bbits = inode->i_blkbits;
1928 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1929
1930 for(bh = head, block_start = 0; bh != head || !block_start;
1931 block++, block_start=block_end, bh = bh->b_this_page) {
1932 block_end = block_start + blocksize;
1933 if (block_end <= from || block_start >= to) {
1934 if (PageUptodate(page)) {
1935 if (!buffer_uptodate(bh))
1936 set_buffer_uptodate(bh);
1937 }
1938 continue;
1939 }
1940 if (buffer_new(bh))
1941 clear_buffer_new(bh);
1942 if (!buffer_mapped(bh)) {
1943 err = get_block(inode, block, bh, 1);
1944 if (err)
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001945 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001946 if (buffer_new(bh)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 unmap_underlying_metadata(bh->b_bdev,
1948 bh->b_blocknr);
1949 if (PageUptodate(page)) {
1950 set_buffer_uptodate(bh);
1951 continue;
1952 }
1953 if (block_end > to || block_start < from) {
1954 void *kaddr;
1955
1956 kaddr = kmap_atomic(page, KM_USER0);
1957 if (block_end > to)
1958 memset(kaddr+to, 0,
1959 block_end-to);
1960 if (block_start < from)
1961 memset(kaddr+block_start,
1962 0, from-block_start);
1963 flush_dcache_page(page);
1964 kunmap_atomic(kaddr, KM_USER0);
1965 }
1966 continue;
1967 }
1968 }
1969 if (PageUptodate(page)) {
1970 if (!buffer_uptodate(bh))
1971 set_buffer_uptodate(bh);
1972 continue;
1973 }
1974 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1975 (block_start < from || block_end > to)) {
1976 ll_rw_block(READ, 1, &bh);
1977 *wait_bh++=bh;
1978 }
1979 }
1980 /*
1981 * If we issued read requests - let them complete.
1982 */
1983 while(wait_bh > wait) {
1984 wait_on_buffer(*--wait_bh);
1985 if (!buffer_uptodate(*wait_bh))
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001986 err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987 }
Anton Altaparmakov152becd2005-06-23 00:10:21 -07001988 if (!err) {
1989 bh = head;
1990 do {
1991 if (buffer_new(bh))
1992 clear_buffer_new(bh);
1993 } while ((bh = bh->b_this_page) != head);
1994 return 0;
1995 }
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001996 /* Error case: */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997 /*
1998 * Zero out any newly allocated blocks to avoid exposing stale
1999 * data. If BH_New is set, we know that the block was newly
2000 * allocated in the above loop.
2001 */
2002 bh = head;
2003 block_start = 0;
2004 do {
2005 block_end = block_start+blocksize;
2006 if (block_end <= from)
2007 goto next_bh;
2008 if (block_start >= to)
2009 break;
2010 if (buffer_new(bh)) {
2011 void *kaddr;
2012
2013 clear_buffer_new(bh);
2014 kaddr = kmap_atomic(page, KM_USER0);
2015 memset(kaddr+block_start, 0, bh->b_size);
2016 kunmap_atomic(kaddr, KM_USER0);
2017 set_buffer_uptodate(bh);
2018 mark_buffer_dirty(bh);
2019 }
2020next_bh:
2021 block_start = block_end;
2022 bh = bh->b_this_page;
2023 } while (bh != head);
2024 return err;
2025}
2026
2027static int __block_commit_write(struct inode *inode, struct page *page,
2028 unsigned from, unsigned to)
2029{
2030 unsigned block_start, block_end;
2031 int partial = 0;
2032 unsigned blocksize;
2033 struct buffer_head *bh, *head;
2034
2035 blocksize = 1 << inode->i_blkbits;
2036
2037 for(bh = head = page_buffers(page), block_start = 0;
2038 bh != head || !block_start;
2039 block_start=block_end, bh = bh->b_this_page) {
2040 block_end = block_start + blocksize;
2041 if (block_end <= from || block_start >= to) {
2042 if (!buffer_uptodate(bh))
2043 partial = 1;
2044 } else {
2045 set_buffer_uptodate(bh);
2046 mark_buffer_dirty(bh);
2047 }
2048 }
2049
2050 /*
2051 * If this is a partial write which happened to make all buffers
2052 * uptodate then we can optimize away a bogus readpage() for
2053 * the next read(). Here we 'discover' whether the page went
2054 * uptodate as a result of this (potentially partial) write.
2055 */
2056 if (!partial)
2057 SetPageUptodate(page);
2058 return 0;
2059}
2060
2061/*
2062 * Generic "read page" function for block devices that have the normal
2063 * get_block functionality. This is most of the block device filesystems.
2064 * Reads the page asynchronously --- the unlock_buffer() and
2065 * set/clear_buffer_uptodate() functions propagate buffer state into the
2066 * page struct once IO has completed.
2067 */
2068int block_read_full_page(struct page *page, get_block_t *get_block)
2069{
2070 struct inode *inode = page->mapping->host;
2071 sector_t iblock, lblock;
2072 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2073 unsigned int blocksize;
2074 int nr, i;
2075 int fully_mapped = 1;
2076
Matt Mackallcd7619d2005-05-01 08:59:01 -07002077 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078 blocksize = 1 << inode->i_blkbits;
2079 if (!page_has_buffers(page))
2080 create_empty_buffers(page, blocksize, 0);
2081 head = page_buffers(page);
2082
2083 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2084 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2085 bh = head;
2086 nr = 0;
2087 i = 0;
2088
2089 do {
2090 if (buffer_uptodate(bh))
2091 continue;
2092
2093 if (!buffer_mapped(bh)) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002094 int err = 0;
2095
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096 fully_mapped = 0;
2097 if (iblock < lblock) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002098 err = get_block(inode, iblock, bh, 0);
2099 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 SetPageError(page);
2101 }
2102 if (!buffer_mapped(bh)) {
2103 void *kaddr = kmap_atomic(page, KM_USER0);
2104 memset(kaddr + i * blocksize, 0, blocksize);
2105 flush_dcache_page(page);
2106 kunmap_atomic(kaddr, KM_USER0);
Andrew Mortonc64610b2005-05-16 21:53:49 -07002107 if (!err)
2108 set_buffer_uptodate(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002109 continue;
2110 }
2111 /*
2112 * get_block() might have updated the buffer
2113 * synchronously
2114 */
2115 if (buffer_uptodate(bh))
2116 continue;
2117 }
2118 arr[nr++] = bh;
2119 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2120
2121 if (fully_mapped)
2122 SetPageMappedToDisk(page);
2123
2124 if (!nr) {
2125 /*
2126 * All buffers are uptodate - we can set the page uptodate
2127 * as well. But not if get_block() returned an error.
2128 */
2129 if (!PageError(page))
2130 SetPageUptodate(page);
2131 unlock_page(page);
2132 return 0;
2133 }
2134
2135 /* Stage two: lock the buffers */
2136 for (i = 0; i < nr; i++) {
2137 bh = arr[i];
2138 lock_buffer(bh);
2139 mark_buffer_async_read(bh);
2140 }
2141
2142 /*
2143 * Stage 3: start the IO. Check for uptodateness
2144 * inside the buffer lock in case another process reading
2145 * the underlying blockdev brought it uptodate (the sct fix).
2146 */
2147 for (i = 0; i < nr; i++) {
2148 bh = arr[i];
2149 if (buffer_uptodate(bh))
2150 end_buffer_async_read(bh, 1);
2151 else
2152 submit_bh(READ, bh);
2153 }
2154 return 0;
2155}
2156
2157/* utility function for filesystems that need to do work on expanding
2158 * truncates. Uses prepare/commit_write to allow the filesystem to
2159 * deal with the hole.
2160 */
2161int generic_cont_expand(struct inode *inode, loff_t size)
2162{
2163 struct address_space *mapping = inode->i_mapping;
2164 struct page *page;
2165 unsigned long index, offset, limit;
2166 int err;
2167
2168 err = -EFBIG;
2169 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2170 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2171 send_sig(SIGXFSZ, current, 0);
2172 goto out;
2173 }
2174 if (size > inode->i_sb->s_maxbytes)
2175 goto out;
2176
2177 offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2178
2179 /* ugh. in prepare/commit_write, if from==to==start of block, we
2180 ** skip the prepare. make sure we never send an offset for the start
2181 ** of a block
2182 */
2183 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2184 offset++;
2185 }
2186 index = size >> PAGE_CACHE_SHIFT;
2187 err = -ENOMEM;
2188 page = grab_cache_page(mapping, index);
2189 if (!page)
2190 goto out;
2191 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2192 if (!err) {
2193 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2194 }
2195 unlock_page(page);
2196 page_cache_release(page);
2197 if (err > 0)
2198 err = 0;
2199out:
2200 return err;
2201}
2202
2203/*
2204 * For moronic filesystems that do not allow holes in file.
2205 * We may have to extend the file.
2206 */
2207
2208int cont_prepare_write(struct page *page, unsigned offset,
2209 unsigned to, get_block_t *get_block, loff_t *bytes)
2210{
2211 struct address_space *mapping = page->mapping;
2212 struct inode *inode = mapping->host;
2213 struct page *new_page;
2214 pgoff_t pgpos;
2215 long status;
2216 unsigned zerofrom;
2217 unsigned blocksize = 1 << inode->i_blkbits;
2218 void *kaddr;
2219
2220 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2221 status = -ENOMEM;
2222 new_page = grab_cache_page(mapping, pgpos);
2223 if (!new_page)
2224 goto out;
2225 /* we might sleep */
2226 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2227 unlock_page(new_page);
2228 page_cache_release(new_page);
2229 continue;
2230 }
2231 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2232 if (zerofrom & (blocksize-1)) {
2233 *bytes |= (blocksize-1);
2234 (*bytes)++;
2235 }
2236 status = __block_prepare_write(inode, new_page, zerofrom,
2237 PAGE_CACHE_SIZE, get_block);
2238 if (status)
2239 goto out_unmap;
2240 kaddr = kmap_atomic(new_page, KM_USER0);
2241 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2242 flush_dcache_page(new_page);
2243 kunmap_atomic(kaddr, KM_USER0);
2244 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2245 unlock_page(new_page);
2246 page_cache_release(new_page);
2247 }
2248
2249 if (page->index < pgpos) {
2250 /* completely inside the area */
2251 zerofrom = offset;
2252 } else {
2253 /* page covers the boundary, find the boundary offset */
2254 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2255
2256 /* if we will expand the thing last block will be filled */
2257 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2258 *bytes |= (blocksize-1);
2259 (*bytes)++;
2260 }
2261
2262 /* starting below the boundary? Nothing to zero out */
2263 if (offset <= zerofrom)
2264 zerofrom = offset;
2265 }
2266 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2267 if (status)
2268 goto out1;
2269 if (zerofrom < offset) {
2270 kaddr = kmap_atomic(page, KM_USER0);
2271 memset(kaddr+zerofrom, 0, offset-zerofrom);
2272 flush_dcache_page(page);
2273 kunmap_atomic(kaddr, KM_USER0);
2274 __block_commit_write(inode, page, zerofrom, offset);
2275 }
2276 return 0;
2277out1:
2278 ClearPageUptodate(page);
2279 return status;
2280
2281out_unmap:
2282 ClearPageUptodate(new_page);
2283 unlock_page(new_page);
2284 page_cache_release(new_page);
2285out:
2286 return status;
2287}
2288
2289int block_prepare_write(struct page *page, unsigned from, unsigned to,
2290 get_block_t *get_block)
2291{
2292 struct inode *inode = page->mapping->host;
2293 int err = __block_prepare_write(inode, page, from, to, get_block);
2294 if (err)
2295 ClearPageUptodate(page);
2296 return err;
2297}
2298
2299int block_commit_write(struct page *page, unsigned from, unsigned to)
2300{
2301 struct inode *inode = page->mapping->host;
2302 __block_commit_write(inode,page,from,to);
2303 return 0;
2304}
2305
2306int generic_commit_write(struct file *file, struct page *page,
2307 unsigned from, unsigned to)
2308{
2309 struct inode *inode = page->mapping->host;
2310 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2311 __block_commit_write(inode,page,from,to);
2312 /*
2313 * No need to use i_size_read() here, the i_size
2314 * cannot change under us because we hold i_sem.
2315 */
2316 if (pos > inode->i_size) {
2317 i_size_write(inode, pos);
2318 mark_inode_dirty(inode);
2319 }
2320 return 0;
2321}
2322
2323
2324/*
2325 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2326 * immediately, while under the page lock. So it needs a special end_io
2327 * handler which does not touch the bh after unlocking it.
2328 *
2329 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2330 * a race there is benign: unlock_buffer() only use the bh's address for
2331 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2332 * itself.
2333 */
2334static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2335{
2336 if (uptodate) {
2337 set_buffer_uptodate(bh);
2338 } else {
2339 /* This happens, due to failed READA attempts. */
2340 clear_buffer_uptodate(bh);
2341 }
2342 unlock_buffer(bh);
2343}
2344
2345/*
2346 * On entry, the page is fully not uptodate.
2347 * On exit the page is fully uptodate in the areas outside (from,to)
2348 */
2349int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2350 get_block_t *get_block)
2351{
2352 struct inode *inode = page->mapping->host;
2353 const unsigned blkbits = inode->i_blkbits;
2354 const unsigned blocksize = 1 << blkbits;
2355 struct buffer_head map_bh;
2356 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2357 unsigned block_in_page;
2358 unsigned block_start;
2359 sector_t block_in_file;
2360 char *kaddr;
2361 int nr_reads = 0;
2362 int i;
2363 int ret = 0;
2364 int is_mapped_to_disk = 1;
2365 int dirtied_it = 0;
2366
2367 if (PageMappedToDisk(page))
2368 return 0;
2369
2370 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2371 map_bh.b_page = page;
2372
2373 /*
2374 * We loop across all blocks in the page, whether or not they are
2375 * part of the affected region. This is so we can discover if the
2376 * page is fully mapped-to-disk.
2377 */
2378 for (block_start = 0, block_in_page = 0;
2379 block_start < PAGE_CACHE_SIZE;
2380 block_in_page++, block_start += blocksize) {
2381 unsigned block_end = block_start + blocksize;
2382 int create;
2383
2384 map_bh.b_state = 0;
2385 create = 1;
2386 if (block_start >= to)
2387 create = 0;
2388 ret = get_block(inode, block_in_file + block_in_page,
2389 &map_bh, create);
2390 if (ret)
2391 goto failed;
2392 if (!buffer_mapped(&map_bh))
2393 is_mapped_to_disk = 0;
2394 if (buffer_new(&map_bh))
2395 unmap_underlying_metadata(map_bh.b_bdev,
2396 map_bh.b_blocknr);
2397 if (PageUptodate(page))
2398 continue;
2399 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2400 kaddr = kmap_atomic(page, KM_USER0);
2401 if (block_start < from) {
2402 memset(kaddr+block_start, 0, from-block_start);
2403 dirtied_it = 1;
2404 }
2405 if (block_end > to) {
2406 memset(kaddr + to, 0, block_end - to);
2407 dirtied_it = 1;
2408 }
2409 flush_dcache_page(page);
2410 kunmap_atomic(kaddr, KM_USER0);
2411 continue;
2412 }
2413 if (buffer_uptodate(&map_bh))
2414 continue; /* reiserfs does this */
2415 if (block_start < from || block_end > to) {
2416 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2417
2418 if (!bh) {
2419 ret = -ENOMEM;
2420 goto failed;
2421 }
2422 bh->b_state = map_bh.b_state;
2423 atomic_set(&bh->b_count, 0);
2424 bh->b_this_page = NULL;
2425 bh->b_page = page;
2426 bh->b_blocknr = map_bh.b_blocknr;
2427 bh->b_size = blocksize;
2428 bh->b_data = (char *)(long)block_start;
2429 bh->b_bdev = map_bh.b_bdev;
2430 bh->b_private = NULL;
2431 read_bh[nr_reads++] = bh;
2432 }
2433 }
2434
2435 if (nr_reads) {
2436 struct buffer_head *bh;
2437
2438 /*
2439 * The page is locked, so these buffers are protected from
2440 * any VM or truncate activity. Hence we don't need to care
2441 * for the buffer_head refcounts.
2442 */
2443 for (i = 0; i < nr_reads; i++) {
2444 bh = read_bh[i];
2445 lock_buffer(bh);
2446 bh->b_end_io = end_buffer_read_nobh;
2447 submit_bh(READ, bh);
2448 }
2449 for (i = 0; i < nr_reads; i++) {
2450 bh = read_bh[i];
2451 wait_on_buffer(bh);
2452 if (!buffer_uptodate(bh))
2453 ret = -EIO;
2454 free_buffer_head(bh);
2455 read_bh[i] = NULL;
2456 }
2457 if (ret)
2458 goto failed;
2459 }
2460
2461 if (is_mapped_to_disk)
2462 SetPageMappedToDisk(page);
2463 SetPageUptodate(page);
2464
2465 /*
2466 * Setting the page dirty here isn't necessary for the prepare_write
2467 * function - commit_write will do that. But if/when this function is
2468 * used within the pagefault handler to ensure that all mmapped pages
2469 * have backing space in the filesystem, we will need to dirty the page
2470 * if its contents were altered.
2471 */
2472 if (dirtied_it)
2473 set_page_dirty(page);
2474
2475 return 0;
2476
2477failed:
2478 for (i = 0; i < nr_reads; i++) {
2479 if (read_bh[i])
2480 free_buffer_head(read_bh[i]);
2481 }
2482
2483 /*
2484 * Error recovery is pretty slack. Clear the page and mark it dirty
2485 * so we'll later zero out any blocks which _were_ allocated.
2486 */
2487 kaddr = kmap_atomic(page, KM_USER0);
2488 memset(kaddr, 0, PAGE_CACHE_SIZE);
2489 kunmap_atomic(kaddr, KM_USER0);
2490 SetPageUptodate(page);
2491 set_page_dirty(page);
2492 return ret;
2493}
2494EXPORT_SYMBOL(nobh_prepare_write);
2495
2496int nobh_commit_write(struct file *file, struct page *page,
2497 unsigned from, unsigned to)
2498{
2499 struct inode *inode = page->mapping->host;
2500 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2501
2502 set_page_dirty(page);
2503 if (pos > inode->i_size) {
2504 i_size_write(inode, pos);
2505 mark_inode_dirty(inode);
2506 }
2507 return 0;
2508}
2509EXPORT_SYMBOL(nobh_commit_write);
2510
2511/*
2512 * nobh_writepage() - based on block_full_write_page() except
2513 * that it tries to operate without attaching bufferheads to
2514 * the page.
2515 */
2516int nobh_writepage(struct page *page, get_block_t *get_block,
2517 struct writeback_control *wbc)
2518{
2519 struct inode * const inode = page->mapping->host;
2520 loff_t i_size = i_size_read(inode);
2521 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2522 unsigned offset;
2523 void *kaddr;
2524 int ret;
2525
2526 /* Is the page fully inside i_size? */
2527 if (page->index < end_index)
2528 goto out;
2529
2530 /* Is the page fully outside i_size? (truncate in progress) */
2531 offset = i_size & (PAGE_CACHE_SIZE-1);
2532 if (page->index >= end_index+1 || !offset) {
2533 /*
2534 * The page may have dirty, unmapped buffers. For example,
2535 * they may have been added in ext3_writepage(). Make them
2536 * freeable here, so the page does not leak.
2537 */
2538#if 0
2539 /* Not really sure about this - do we need this ? */
2540 if (page->mapping->a_ops->invalidatepage)
2541 page->mapping->a_ops->invalidatepage(page, offset);
2542#endif
2543 unlock_page(page);
2544 return 0; /* don't care */
2545 }
2546
2547 /*
2548 * The page straddles i_size. It must be zeroed out on each and every
2549 * writepage invocation because it may be mmapped. "A file is mapped
2550 * in multiples of the page size. For a file that is not a multiple of
2551 * the page size, the remaining memory is zeroed when mapped, and
2552 * writes to that region are not written out to the file."
2553 */
2554 kaddr = kmap_atomic(page, KM_USER0);
2555 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2556 flush_dcache_page(page);
2557 kunmap_atomic(kaddr, KM_USER0);
2558out:
2559 ret = mpage_writepage(page, get_block, wbc);
2560 if (ret == -EAGAIN)
2561 ret = __block_write_full_page(inode, page, get_block, wbc);
2562 return ret;
2563}
2564EXPORT_SYMBOL(nobh_writepage);
2565
2566/*
2567 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2568 */
2569int nobh_truncate_page(struct address_space *mapping, loff_t from)
2570{
2571 struct inode *inode = mapping->host;
2572 unsigned blocksize = 1 << inode->i_blkbits;
2573 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2574 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2575 unsigned to;
2576 struct page *page;
2577 struct address_space_operations *a_ops = mapping->a_ops;
2578 char *kaddr;
2579 int ret = 0;
2580
2581 if ((offset & (blocksize - 1)) == 0)
2582 goto out;
2583
2584 ret = -ENOMEM;
2585 page = grab_cache_page(mapping, index);
2586 if (!page)
2587 goto out;
2588
2589 to = (offset + blocksize) & ~(blocksize - 1);
2590 ret = a_ops->prepare_write(NULL, page, offset, to);
2591 if (ret == 0) {
2592 kaddr = kmap_atomic(page, KM_USER0);
2593 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2594 flush_dcache_page(page);
2595 kunmap_atomic(kaddr, KM_USER0);
2596 set_page_dirty(page);
2597 }
2598 unlock_page(page);
2599 page_cache_release(page);
2600out:
2601 return ret;
2602}
2603EXPORT_SYMBOL(nobh_truncate_page);
2604
2605int block_truncate_page(struct address_space *mapping,
2606 loff_t from, get_block_t *get_block)
2607{
2608 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2609 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2610 unsigned blocksize;
2611 pgoff_t iblock;
2612 unsigned length, pos;
2613 struct inode *inode = mapping->host;
2614 struct page *page;
2615 struct buffer_head *bh;
2616 void *kaddr;
2617 int err;
2618
2619 blocksize = 1 << inode->i_blkbits;
2620 length = offset & (blocksize - 1);
2621
2622 /* Block boundary? Nothing to do */
2623 if (!length)
2624 return 0;
2625
2626 length = blocksize - length;
2627 iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2628
2629 page = grab_cache_page(mapping, index);
2630 err = -ENOMEM;
2631 if (!page)
2632 goto out;
2633
2634 if (!page_has_buffers(page))
2635 create_empty_buffers(page, blocksize, 0);
2636
2637 /* Find the buffer that contains "offset" */
2638 bh = page_buffers(page);
2639 pos = blocksize;
2640 while (offset >= pos) {
2641 bh = bh->b_this_page;
2642 iblock++;
2643 pos += blocksize;
2644 }
2645
2646 err = 0;
2647 if (!buffer_mapped(bh)) {
2648 err = get_block(inode, iblock, bh, 0);
2649 if (err)
2650 goto unlock;
2651 /* unmapped? It's a hole - nothing to do */
2652 if (!buffer_mapped(bh))
2653 goto unlock;
2654 }
2655
2656 /* Ok, it's mapped. Make sure it's up-to-date */
2657 if (PageUptodate(page))
2658 set_buffer_uptodate(bh);
2659
2660 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2661 err = -EIO;
2662 ll_rw_block(READ, 1, &bh);
2663 wait_on_buffer(bh);
2664 /* Uhhuh. Read error. Complain and punt. */
2665 if (!buffer_uptodate(bh))
2666 goto unlock;
2667 }
2668
2669 kaddr = kmap_atomic(page, KM_USER0);
2670 memset(kaddr + offset, 0, length);
2671 flush_dcache_page(page);
2672 kunmap_atomic(kaddr, KM_USER0);
2673
2674 mark_buffer_dirty(bh);
2675 err = 0;
2676
2677unlock:
2678 unlock_page(page);
2679 page_cache_release(page);
2680out:
2681 return err;
2682}
2683
2684/*
2685 * The generic ->writepage function for buffer-backed address_spaces
2686 */
2687int block_write_full_page(struct page *page, get_block_t *get_block,
2688 struct writeback_control *wbc)
2689{
2690 struct inode * const inode = page->mapping->host;
2691 loff_t i_size = i_size_read(inode);
2692 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2693 unsigned offset;
2694 void *kaddr;
2695
2696 /* Is the page fully inside i_size? */
2697 if (page->index < end_index)
2698 return __block_write_full_page(inode, page, get_block, wbc);
2699
2700 /* Is the page fully outside i_size? (truncate in progress) */
2701 offset = i_size & (PAGE_CACHE_SIZE-1);
2702 if (page->index >= end_index+1 || !offset) {
2703 /*
2704 * The page may have dirty, unmapped buffers. For example,
2705 * they may have been added in ext3_writepage(). Make them
2706 * freeable here, so the page does not leak.
2707 */
Jan Karaaaa40592005-10-30 15:00:16 -08002708 do_invalidatepage(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002709 unlock_page(page);
2710 return 0; /* don't care */
2711 }
2712
2713 /*
2714 * The page straddles i_size. It must be zeroed out on each and every
2715 * writepage invokation because it may be mmapped. "A file is mapped
2716 * in multiples of the page size. For a file that is not a multiple of
2717 * the page size, the remaining memory is zeroed when mapped, and
2718 * writes to that region are not written out to the file."
2719 */
2720 kaddr = kmap_atomic(page, KM_USER0);
2721 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2722 flush_dcache_page(page);
2723 kunmap_atomic(kaddr, KM_USER0);
2724 return __block_write_full_page(inode, page, get_block, wbc);
2725}
2726
2727sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2728 get_block_t *get_block)
2729{
2730 struct buffer_head tmp;
2731 struct inode *inode = mapping->host;
2732 tmp.b_state = 0;
2733 tmp.b_blocknr = 0;
2734 get_block(inode, block, &tmp, 0);
2735 return tmp.b_blocknr;
2736}
2737
2738static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2739{
2740 struct buffer_head *bh = bio->bi_private;
2741
2742 if (bio->bi_size)
2743 return 1;
2744
2745 if (err == -EOPNOTSUPP) {
2746 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2747 set_bit(BH_Eopnotsupp, &bh->b_state);
2748 }
2749
2750 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2751 bio_put(bio);
2752 return 0;
2753}
2754
2755int submit_bh(int rw, struct buffer_head * bh)
2756{
2757 struct bio *bio;
2758 int ret = 0;
2759
2760 BUG_ON(!buffer_locked(bh));
2761 BUG_ON(!buffer_mapped(bh));
2762 BUG_ON(!bh->b_end_io);
2763
2764 if (buffer_ordered(bh) && (rw == WRITE))
2765 rw = WRITE_BARRIER;
2766
2767 /*
2768 * Only clear out a write error when rewriting, should this
2769 * include WRITE_SYNC as well?
2770 */
2771 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2772 clear_buffer_write_io_error(bh);
2773
2774 /*
2775 * from here on down, it's all bio -- do the initial mapping,
2776 * submit_bio -> generic_make_request may further map this bio around
2777 */
2778 bio = bio_alloc(GFP_NOIO, 1);
2779
2780 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2781 bio->bi_bdev = bh->b_bdev;
2782 bio->bi_io_vec[0].bv_page = bh->b_page;
2783 bio->bi_io_vec[0].bv_len = bh->b_size;
2784 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2785
2786 bio->bi_vcnt = 1;
2787 bio->bi_idx = 0;
2788 bio->bi_size = bh->b_size;
2789
2790 bio->bi_end_io = end_bio_bh_io_sync;
2791 bio->bi_private = bh;
2792
2793 bio_get(bio);
2794 submit_bio(rw, bio);
2795
2796 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2797 ret = -EOPNOTSUPP;
2798
2799 bio_put(bio);
2800 return ret;
2801}
2802
2803/**
2804 * ll_rw_block: low-level access to block devices (DEPRECATED)
Jan Karaa7662232005-09-06 15:19:10 -07002805 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002806 * @nr: number of &struct buffer_heads in the array
2807 * @bhs: array of pointers to &struct buffer_head
2808 *
Jan Karaa7662232005-09-06 15:19:10 -07002809 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2810 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2811 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2812 * are sent to disk. The fourth %READA option is described in the documentation
2813 * for generic_make_request() which ll_rw_block() calls.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002814 *
2815 * This function drops any buffer that it cannot get a lock on (with the
Jan Karaa7662232005-09-06 15:19:10 -07002816 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2817 * clean when doing a write request, and any buffer that appears to be
2818 * up-to-date when doing read request. Further it marks as clean buffers that
2819 * are processed for writing (the buffer cache won't assume that they are
2820 * actually clean until the buffer gets unlocked).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002821 *
2822 * ll_rw_block sets b_end_io to simple completion handler that marks
2823 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2824 * any waiters.
2825 *
2826 * All of the buffers must be for the same device, and must also be a
2827 * multiple of the current approved size for the device.
2828 */
2829void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2830{
2831 int i;
2832
2833 for (i = 0; i < nr; i++) {
2834 struct buffer_head *bh = bhs[i];
2835
Jan Karaa7662232005-09-06 15:19:10 -07002836 if (rw == SWRITE)
2837 lock_buffer(bh);
2838 else if (test_set_buffer_locked(bh))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 continue;
2840
2841 get_bh(bh);
Jan Karaa7662232005-09-06 15:19:10 -07002842 if (rw == WRITE || rw == SWRITE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843 if (test_clear_buffer_dirty(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002844 bh->b_end_io = end_buffer_write_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002845 submit_bh(WRITE, bh);
2846 continue;
2847 }
2848 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002849 if (!buffer_uptodate(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002850 bh->b_end_io = end_buffer_read_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 submit_bh(rw, bh);
2852 continue;
2853 }
2854 }
2855 unlock_buffer(bh);
2856 put_bh(bh);
2857 }
2858}
2859
2860/*
2861 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2862 * and then start new I/O and then wait upon it. The caller must have a ref on
2863 * the buffer_head.
2864 */
2865int sync_dirty_buffer(struct buffer_head *bh)
2866{
2867 int ret = 0;
2868
2869 WARN_ON(atomic_read(&bh->b_count) < 1);
2870 lock_buffer(bh);
2871 if (test_clear_buffer_dirty(bh)) {
2872 get_bh(bh);
2873 bh->b_end_io = end_buffer_write_sync;
2874 ret = submit_bh(WRITE, bh);
2875 wait_on_buffer(bh);
2876 if (buffer_eopnotsupp(bh)) {
2877 clear_buffer_eopnotsupp(bh);
2878 ret = -EOPNOTSUPP;
2879 }
2880 if (!ret && !buffer_uptodate(bh))
2881 ret = -EIO;
2882 } else {
2883 unlock_buffer(bh);
2884 }
2885 return ret;
2886}
2887
2888/*
2889 * try_to_free_buffers() checks if all the buffers on this particular page
2890 * are unused, and releases them if so.
2891 *
2892 * Exclusion against try_to_free_buffers may be obtained by either
2893 * locking the page or by holding its mapping's private_lock.
2894 *
2895 * If the page is dirty but all the buffers are clean then we need to
2896 * be sure to mark the page clean as well. This is because the page
2897 * may be against a block device, and a later reattachment of buffers
2898 * to a dirty page will set *all* buffers dirty. Which would corrupt
2899 * filesystem data on the same device.
2900 *
2901 * The same applies to regular filesystem pages: if all the buffers are
2902 * clean then we set the page clean and proceed. To do that, we require
2903 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2904 * private_lock.
2905 *
2906 * try_to_free_buffers() is non-blocking.
2907 */
2908static inline int buffer_busy(struct buffer_head *bh)
2909{
2910 return atomic_read(&bh->b_count) |
2911 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2912}
2913
2914static int
2915drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2916{
2917 struct buffer_head *head = page_buffers(page);
2918 struct buffer_head *bh;
2919
2920 bh = head;
2921 do {
akpm@osdl.orgde7d5a32005-05-01 08:58:39 -07002922 if (buffer_write_io_error(bh) && page->mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923 set_bit(AS_EIO, &page->mapping->flags);
2924 if (buffer_busy(bh))
2925 goto failed;
2926 bh = bh->b_this_page;
2927 } while (bh != head);
2928
2929 do {
2930 struct buffer_head *next = bh->b_this_page;
2931
2932 if (!list_empty(&bh->b_assoc_buffers))
2933 __remove_assoc_queue(bh);
2934 bh = next;
2935 } while (bh != head);
2936 *buffers_to_free = head;
2937 __clear_page_buffers(page);
2938 return 1;
2939failed:
2940 return 0;
2941}
2942
2943int try_to_free_buffers(struct page *page)
2944{
2945 struct address_space * const mapping = page->mapping;
2946 struct buffer_head *buffers_to_free = NULL;
2947 int ret = 0;
2948
2949 BUG_ON(!PageLocked(page));
2950 if (PageWriteback(page))
2951 return 0;
2952
2953 if (mapping == NULL) { /* can this still happen? */
2954 ret = drop_buffers(page, &buffers_to_free);
2955 goto out;
2956 }
2957
2958 spin_lock(&mapping->private_lock);
2959 ret = drop_buffers(page, &buffers_to_free);
2960 if (ret) {
2961 /*
2962 * If the filesystem writes its buffers by hand (eg ext3)
2963 * then we can have clean buffers against a dirty page. We
2964 * clean the page here; otherwise later reattachment of buffers
2965 * could encounter a non-uptodate page, which is unresolvable.
2966 * This only applies in the rare case where try_to_free_buffers
2967 * succeeds but the page is not freed.
2968 */
2969 clear_page_dirty(page);
2970 }
2971 spin_unlock(&mapping->private_lock);
2972out:
2973 if (buffers_to_free) {
2974 struct buffer_head *bh = buffers_to_free;
2975
2976 do {
2977 struct buffer_head *next = bh->b_this_page;
2978 free_buffer_head(bh);
2979 bh = next;
2980 } while (bh != buffers_to_free);
2981 }
2982 return ret;
2983}
2984EXPORT_SYMBOL(try_to_free_buffers);
2985
2986int block_sync_page(struct page *page)
2987{
2988 struct address_space *mapping;
2989
2990 smp_mb();
2991 mapping = page_mapping(page);
2992 if (mapping)
2993 blk_run_backing_dev(mapping->backing_dev_info, page);
2994 return 0;
2995}
2996
2997/*
2998 * There are no bdflush tunables left. But distributions are
2999 * still running obsolete flush daemons, so we terminate them here.
3000 *
3001 * Use of bdflush() is deprecated and will be removed in a future kernel.
3002 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3003 */
3004asmlinkage long sys_bdflush(int func, long data)
3005{
3006 static int msg_count;
3007
3008 if (!capable(CAP_SYS_ADMIN))
3009 return -EPERM;
3010
3011 if (msg_count < 5) {
3012 msg_count++;
3013 printk(KERN_INFO
3014 "warning: process `%s' used the obsolete bdflush"
3015 " system call\n", current->comm);
3016 printk(KERN_INFO "Fix your initscripts?\n");
3017 }
3018
3019 if (func == 1)
3020 do_exit(0);
3021 return 0;
3022}
3023
3024/*
3025 * Buffer-head allocation
3026 */
3027static kmem_cache_t *bh_cachep;
3028
3029/*
3030 * Once the number of bh's in the machine exceeds this level, we start
3031 * stripping them in writeback.
3032 */
3033static int max_buffer_heads;
3034
3035int buffer_heads_over_limit;
3036
3037struct bh_accounting {
3038 int nr; /* Number of live bh's */
3039 int ratelimit; /* Limit cacheline bouncing */
3040};
3041
3042static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3043
3044static void recalc_bh_state(void)
3045{
3046 int i;
3047 int tot = 0;
3048
3049 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3050 return;
3051 __get_cpu_var(bh_accounting).ratelimit = 0;
3052 for_each_cpu(i)
3053 tot += per_cpu(bh_accounting, i).nr;
3054 buffer_heads_over_limit = (tot > max_buffer_heads);
3055}
3056
Al Virodd0fc662005-10-07 07:46:04 +01003057struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058{
3059 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3060 if (ret) {
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003061 get_cpu_var(bh_accounting).nr++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003063 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064 }
3065 return ret;
3066}
3067EXPORT_SYMBOL(alloc_buffer_head);
3068
3069void free_buffer_head(struct buffer_head *bh)
3070{
3071 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3072 kmem_cache_free(bh_cachep, bh);
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003073 get_cpu_var(bh_accounting).nr--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003075 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003076}
3077EXPORT_SYMBOL(free_buffer_head);
3078
3079static void
3080init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3081{
3082 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3083 SLAB_CTOR_CONSTRUCTOR) {
3084 struct buffer_head * bh = (struct buffer_head *)data;
3085
3086 memset(bh, 0, sizeof(*bh));
3087 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3088 }
3089}
3090
3091#ifdef CONFIG_HOTPLUG_CPU
3092static void buffer_exit_cpu(int cpu)
3093{
3094 int i;
3095 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3096
3097 for (i = 0; i < BH_LRU_SIZE; i++) {
3098 brelse(b->bhs[i]);
3099 b->bhs[i] = NULL;
3100 }
3101}
3102
3103static int buffer_cpu_notify(struct notifier_block *self,
3104 unsigned long action, void *hcpu)
3105{
3106 if (action == CPU_DEAD)
3107 buffer_exit_cpu((unsigned long)hcpu);
3108 return NOTIFY_OK;
3109}
3110#endif /* CONFIG_HOTPLUG_CPU */
3111
3112void __init buffer_init(void)
3113{
3114 int nrpages;
3115
3116 bh_cachep = kmem_cache_create("buffer_head",
3117 sizeof(struct buffer_head), 0,
Andrea Arcangelie422fd22005-05-05 16:15:04 -07003118 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119
3120 /*
3121 * Limit the bh occupancy to 10% of ZONE_NORMAL
3122 */
3123 nrpages = (nr_free_buffer_pages() * 10) / 100;
3124 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3125 hotcpu_notifier(buffer_cpu_notify, 0);
3126}
3127
3128EXPORT_SYMBOL(__bforget);
3129EXPORT_SYMBOL(__brelse);
3130EXPORT_SYMBOL(__wait_on_buffer);
3131EXPORT_SYMBOL(block_commit_write);
3132EXPORT_SYMBOL(block_prepare_write);
3133EXPORT_SYMBOL(block_read_full_page);
3134EXPORT_SYMBOL(block_sync_page);
3135EXPORT_SYMBOL(block_truncate_page);
3136EXPORT_SYMBOL(block_write_full_page);
3137EXPORT_SYMBOL(cont_prepare_write);
3138EXPORT_SYMBOL(end_buffer_async_write);
3139EXPORT_SYMBOL(end_buffer_read_sync);
3140EXPORT_SYMBOL(end_buffer_write_sync);
3141EXPORT_SYMBOL(file_fsync);
3142EXPORT_SYMBOL(fsync_bdev);
3143EXPORT_SYMBOL(generic_block_bmap);
3144EXPORT_SYMBOL(generic_commit_write);
3145EXPORT_SYMBOL(generic_cont_expand);
3146EXPORT_SYMBOL(init_buffer);
3147EXPORT_SYMBOL(invalidate_bdev);
3148EXPORT_SYMBOL(ll_rw_block);
3149EXPORT_SYMBOL(mark_buffer_dirty);
3150EXPORT_SYMBOL(submit_bh);
3151EXPORT_SYMBOL(sync_dirty_buffer);
3152EXPORT_SYMBOL(unlock_buffer);