blob: 23a7668e07dacafbb799a1fad94f807acf534963 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scott7b718762005-11-02 14:58:39 +11002 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include "xfs.h"
Nathan Scotta844f452005-11-02 14:38:42 +110019#include "xfs_bit.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include "xfs_log.h"
Nathan Scotta844f452005-11-02 14:38:42 +110021#include "xfs_inum.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include "xfs_sb.h"
Nathan Scotta844f452005-11-02 14:38:42 +110023#include "xfs_ag.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include "xfs_trans.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include "xfs_mount.h"
26#include "xfs_bmap_btree.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include "xfs_dinode.h"
28#include "xfs_inode.h"
Nathan Scotta844f452005-11-02 14:38:42 +110029#include "xfs_alloc.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include "xfs_error.h"
31#include "xfs_rw.h"
32#include "xfs_iomap.h"
Christoph Hellwig739bfb22007-08-29 10:58:01 +100033#include "xfs_vnodeops.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000034#include "xfs_trace.h"
Dave Chinner3ed3a432010-03-05 02:00:42 +000035#include "xfs_bmap.h"
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <linux/mpage.h>
Christoph Hellwig10ce4442006-01-11 20:48:14 +110038#include <linux/pagevec.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/writeback.h>
40
Christoph Hellwig34a52c62010-04-28 12:28:57 +000041/*
42 * Types of I/O for bmap clustering and I/O completion tracking.
43 */
44enum {
45 IO_READ, /* mapping for a read */
46 IO_DELAY, /* mapping covers delalloc region */
47 IO_UNWRITTEN, /* mapping covers allocated but uninitialized data */
48 IO_NEW /* just allocated */
49};
Christoph Hellwig25e41b32008-12-03 12:20:39 +010050
51/*
52 * Prime number of hash buckets since address is used as the key.
53 */
54#define NVSYNC 37
55#define to_ioend_wq(v) (&xfs_ioend_wq[((unsigned long)v) % NVSYNC])
56static wait_queue_head_t xfs_ioend_wq[NVSYNC];
57
58void __init
59xfs_ioend_init(void)
60{
61 int i;
62
63 for (i = 0; i < NVSYNC; i++)
64 init_waitqueue_head(&xfs_ioend_wq[i]);
65}
66
67void
68xfs_ioend_wait(
69 xfs_inode_t *ip)
70{
71 wait_queue_head_t *wq = to_ioend_wq(ip);
72
73 wait_event(*wq, (atomic_read(&ip->i_iocount) == 0));
74}
75
76STATIC void
77xfs_ioend_wake(
78 xfs_inode_t *ip)
79{
80 if (atomic_dec_and_test(&ip->i_iocount))
81 wake_up(to_ioend_wq(ip));
82}
83
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000084void
Nathan Scottf51623b2006-03-14 13:26:27 +110085xfs_count_page_state(
86 struct page *page,
87 int *delalloc,
Nathan Scottf51623b2006-03-14 13:26:27 +110088 int *unwritten)
89{
90 struct buffer_head *bh, *head;
91
Christoph Hellwig20cb52e2010-06-24 09:46:01 +100092 *delalloc = *unwritten = 0;
Nathan Scottf51623b2006-03-14 13:26:27 +110093
94 bh = head = page_buffers(page);
95 do {
Christoph Hellwig20cb52e2010-06-24 09:46:01 +100096 if (buffer_unwritten(bh))
Nathan Scottf51623b2006-03-14 13:26:27 +110097 (*unwritten) = 1;
98 else if (buffer_delay(bh))
99 (*delalloc) = 1;
100 } while ((bh = bh->b_this_page) != head);
101}
102
Christoph Hellwig6214ed42007-09-14 15:23:17 +1000103STATIC struct block_device *
104xfs_find_bdev_for_inode(
Christoph Hellwig046f1682010-04-28 12:28:52 +0000105 struct inode *inode)
Christoph Hellwig6214ed42007-09-14 15:23:17 +1000106{
Christoph Hellwig046f1682010-04-28 12:28:52 +0000107 struct xfs_inode *ip = XFS_I(inode);
Christoph Hellwig6214ed42007-09-14 15:23:17 +1000108 struct xfs_mount *mp = ip->i_mount;
109
Eric Sandeen71ddabb2007-11-23 16:29:42 +1100110 if (XFS_IS_REALTIME_INODE(ip))
Christoph Hellwig6214ed42007-09-14 15:23:17 +1000111 return mp->m_rtdev_targp->bt_bdev;
112 else
113 return mp->m_ddev_targp->bt_bdev;
114}
115
Christoph Hellwig0829c362005-09-02 16:58:49 +1000116/*
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100117 * We're now finished for good with this ioend structure.
118 * Update the page state via the associated buffer_heads,
119 * release holds on the inode and bio, and finally free
120 * up memory. Do not use the ioend after this.
121 */
Christoph Hellwig0829c362005-09-02 16:58:49 +1000122STATIC void
123xfs_destroy_ioend(
124 xfs_ioend_t *ioend)
125{
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100126 struct buffer_head *bh, *next;
Christoph Hellwig583fa582008-12-03 12:20:38 +0100127 struct xfs_inode *ip = XFS_I(ioend->io_inode);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100128
129 for (bh = ioend->io_buffer_head; bh; bh = next) {
130 next = bh->b_private;
Nathan Scott7d04a332006-06-09 14:58:38 +1000131 bh->b_end_io(bh, !ioend->io_error);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100132 }
Christoph Hellwig583fa582008-12-03 12:20:38 +0100133
134 /*
135 * Volume managers supporting multiple paths can send back ENODEV
136 * when the final path disappears. In this case continuing to fill
137 * the page cache with dirty data which cannot be written out is
138 * evil, so prevent that.
139 */
140 if (unlikely(ioend->io_error == -ENODEV)) {
141 xfs_do_force_shutdown(ip->i_mount, SHUTDOWN_DEVICE_REQ,
142 __FILE__, __LINE__);
Christoph Hellwigb677c212007-08-29 11:46:28 +1000143 }
Christoph Hellwig583fa582008-12-03 12:20:38 +0100144
Christoph Hellwig25e41b32008-12-03 12:20:39 +0100145 xfs_ioend_wake(ip);
Christoph Hellwig0829c362005-09-02 16:58:49 +1000146 mempool_free(ioend, xfs_ioend_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147}
148
149/*
Dave Chinner932640e2009-10-06 20:29:29 +0000150 * If the end of the current ioend is beyond the current EOF,
151 * return the new EOF value, otherwise zero.
152 */
153STATIC xfs_fsize_t
154xfs_ioend_new_eof(
155 xfs_ioend_t *ioend)
156{
157 xfs_inode_t *ip = XFS_I(ioend->io_inode);
158 xfs_fsize_t isize;
159 xfs_fsize_t bsize;
160
161 bsize = ioend->io_offset + ioend->io_size;
162 isize = MAX(ip->i_size, ip->i_new_size);
163 isize = MIN(isize, bsize);
164 return isize > ip->i_d.di_size ? isize : 0;
165}
166
167/*
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000168 * Update on-disk file size now that data has been written to disk. The
169 * current in-memory file size is i_size. If a write is beyond eof i_new_size
170 * will be the intended file size until i_size is updated. If this write does
171 * not extend all the way to the valid file size then restrict this update to
172 * the end of the write.
173 *
174 * This function does not block as blocking on the inode lock in IO completion
175 * can lead to IO completion order dependency deadlocks.. If it can't get the
176 * inode ilock it will return EAGAIN. Callers must handle this.
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000177 */
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000178STATIC int
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000179xfs_setfilesize(
180 xfs_ioend_t *ioend)
181{
Christoph Hellwigb677c212007-08-29 11:46:28 +1000182 xfs_inode_t *ip = XFS_I(ioend->io_inode);
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000183 xfs_fsize_t isize;
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000184
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000185 ASSERT((ip->i_d.di_mode & S_IFMT) == S_IFREG);
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000186 ASSERT(ioend->io_type != IO_READ);
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000187
188 if (unlikely(ioend->io_error))
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000189 return 0;
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000190
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000191 if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
192 return EAGAIN;
193
Dave Chinner932640e2009-10-06 20:29:29 +0000194 isize = xfs_ioend_new_eof(ioend);
195 if (isize) {
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000196 ip->i_d.di_size = isize;
Christoph Hellwig66d834e2010-02-15 09:44:49 +0000197 xfs_mark_inode_dirty(ip);
Lachlan McIlroyba87ea62007-05-08 13:49:46 +1000198 }
199
200 xfs_iunlock(ip, XFS_ILOCK_EXCL);
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000201 return 0;
Christoph Hellwig0829c362005-09-02 16:58:49 +1000202}
203
204/*
Christoph Hellwig209fb872010-07-18 21:17:11 +0000205 * Schedule IO completion handling on the final put of an ioend.
Dave Chinnerc626d172009-04-06 18:42:11 +0200206 */
207STATIC void
208xfs_finish_ioend(
Christoph Hellwig209fb872010-07-18 21:17:11 +0000209 struct xfs_ioend *ioend)
Dave Chinnerc626d172009-04-06 18:42:11 +0200210{
211 if (atomic_dec_and_test(&ioend->io_remaining)) {
Christoph Hellwig209fb872010-07-18 21:17:11 +0000212 if (ioend->io_type == IO_UNWRITTEN)
213 queue_work(xfsconvertd_workqueue, &ioend->io_work);
214 else
215 queue_work(xfsdatad_workqueue, &ioend->io_work);
Dave Chinnerc626d172009-04-06 18:42:11 +0200216 }
217}
218
219/*
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000220 * IO write completion.
221 */
222STATIC void
223xfs_end_io(
224 struct work_struct *work)
225{
226 xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work);
227 struct xfs_inode *ip = XFS_I(ioend->io_inode);
Dave Chinner69418932010-03-04 00:57:09 +0000228 int error = 0;
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000229
230 /*
231 * For unwritten extents we need to issue transactions to convert a
232 * range to normal written extens after the data I/O has finished.
233 */
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000234 if (ioend->io_type == IO_UNWRITTEN &&
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000235 likely(!ioend->io_error && !XFS_FORCED_SHUTDOWN(ip->i_mount))) {
236
237 error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
238 ioend->io_size);
239 if (error)
240 ioend->io_error = error;
241 }
242
243 /*
244 * We might have to update the on-disk file size after extending
245 * writes.
246 */
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000247 if (ioend->io_type != IO_READ) {
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000248 error = xfs_setfilesize(ioend);
249 ASSERT(!error || error == EAGAIN);
250 }
251
252 /*
253 * If we didn't complete processing of the ioend, requeue it to the
254 * tail of the workqueue for another attempt later. Otherwise destroy
255 * it.
256 */
257 if (error == EAGAIN) {
258 atomic_inc(&ioend->io_remaining);
Christoph Hellwig209fb872010-07-18 21:17:11 +0000259 xfs_finish_ioend(ioend);
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000260 /* ensure we don't spin on blocked ioends */
261 delay(1);
Christoph Hellwigfb511f22010-07-18 21:17:10 +0000262 } else {
263 if (ioend->io_iocb)
264 aio_complete(ioend->io_iocb, ioend->io_result, 0);
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000265 xfs_destroy_ioend(ioend);
Christoph Hellwigfb511f22010-07-18 21:17:10 +0000266 }
Dave Chinner77d7a0c2010-02-17 05:36:29 +0000267}
268
269/*
Christoph Hellwig209fb872010-07-18 21:17:11 +0000270 * Call IO completion handling in caller context on the final put of an ioend.
271 */
272STATIC void
273xfs_finish_ioend_sync(
274 struct xfs_ioend *ioend)
275{
276 if (atomic_dec_and_test(&ioend->io_remaining))
277 xfs_end_io(&ioend->io_work);
278}
279
280/*
Christoph Hellwig0829c362005-09-02 16:58:49 +1000281 * Allocate and initialise an IO completion structure.
282 * We need to track unwritten extent write completion here initially.
283 * We'll need to extend this for updating the ondisk inode size later
284 * (vs. incore size).
285 */
286STATIC xfs_ioend_t *
287xfs_alloc_ioend(
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100288 struct inode *inode,
289 unsigned int type)
Christoph Hellwig0829c362005-09-02 16:58:49 +1000290{
291 xfs_ioend_t *ioend;
292
293 ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
294
295 /*
296 * Set the count to 1 initially, which will prevent an I/O
297 * completion callback from happening before we have started
298 * all the I/O from calling the completion routine too early.
299 */
300 atomic_set(&ioend->io_remaining, 1);
Nathan Scott7d04a332006-06-09 14:58:38 +1000301 ioend->io_error = 0;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100302 ioend->io_list = NULL;
303 ioend->io_type = type;
Christoph Hellwigb677c212007-08-29 11:46:28 +1000304 ioend->io_inode = inode;
Christoph Hellwigc1a073b2005-09-05 08:23:35 +1000305 ioend->io_buffer_head = NULL;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100306 ioend->io_buffer_tail = NULL;
Christoph Hellwigb677c212007-08-29 11:46:28 +1000307 atomic_inc(&XFS_I(ioend->io_inode)->i_iocount);
Christoph Hellwig0829c362005-09-02 16:58:49 +1000308 ioend->io_offset = 0;
309 ioend->io_size = 0;
Christoph Hellwigfb511f22010-07-18 21:17:10 +0000310 ioend->io_iocb = NULL;
311 ioend->io_result = 0;
Christoph Hellwig0829c362005-09-02 16:58:49 +1000312
Christoph Hellwig5ec4fab2009-10-30 09:11:47 +0000313 INIT_WORK(&ioend->io_work, xfs_end_io);
Christoph Hellwig0829c362005-09-02 16:58:49 +1000314 return ioend;
315}
316
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317STATIC int
318xfs_map_blocks(
319 struct inode *inode,
320 loff_t offset,
321 ssize_t count,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000322 struct xfs_bmbt_irec *imap,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 int flags)
324{
Christoph Hellwig6bd16ff2008-12-03 12:20:32 +0100325 int nmaps = 1;
Christoph Hellwig207d0412010-04-28 12:28:56 +0000326 int new = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Christoph Hellwig207d0412010-04-28 12:28:56 +0000328 return -xfs_iomap(XFS_I(inode), offset, count, flags, imap, &nmaps, &new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329}
330
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000331STATIC int
Christoph Hellwig558e6892010-04-28 12:28:58 +0000332xfs_imap_valid(
Christoph Hellwig8699bb02010-04-28 12:28:54 +0000333 struct inode *inode,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000334 struct xfs_bmbt_irec *imap,
Christoph Hellwig558e6892010-04-28 12:28:58 +0000335 xfs_off_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
Christoph Hellwig558e6892010-04-28 12:28:58 +0000337 offset >>= inode->i_blkbits;
Christoph Hellwig8699bb02010-04-28 12:28:54 +0000338
Christoph Hellwig558e6892010-04-28 12:28:58 +0000339 return offset >= imap->br_startoff &&
340 offset < imap->br_startoff + imap->br_blockcount;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341}
342
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100343/*
344 * BIO completion handler for buffered IO.
345 */
Al Viro782e3b32007-10-12 07:17:47 +0100346STATIC void
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100347xfs_end_bio(
348 struct bio *bio,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100349 int error)
350{
351 xfs_ioend_t *ioend = bio->bi_private;
352
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100353 ASSERT(atomic_read(&bio->bi_cnt) >= 1);
Nathan Scott7d04a332006-06-09 14:58:38 +1000354 ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100355
356 /* Toss bio and pass work off to an xfsdatad thread */
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100357 bio->bi_private = NULL;
358 bio->bi_end_io = NULL;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100359 bio_put(bio);
Nathan Scott7d04a332006-06-09 14:58:38 +1000360
Christoph Hellwig209fb872010-07-18 21:17:11 +0000361 xfs_finish_ioend(ioend);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100362}
363
364STATIC void
365xfs_submit_ioend_bio(
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000366 struct writeback_control *wbc,
367 xfs_ioend_t *ioend,
368 struct bio *bio)
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100369{
370 atomic_inc(&ioend->io_remaining);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100371 bio->bi_private = ioend;
372 bio->bi_end_io = xfs_end_bio;
373
Dave Chinner932640e2009-10-06 20:29:29 +0000374 /*
375 * If the I/O is beyond EOF we mark the inode dirty immediately
376 * but don't update the inode size until I/O completion.
377 */
378 if (xfs_ioend_new_eof(ioend))
Christoph Hellwig66d834e2010-02-15 09:44:49 +0000379 xfs_mark_inode_dirty(XFS_I(ioend->io_inode));
Dave Chinner932640e2009-10-06 20:29:29 +0000380
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000381 submit_bio(wbc->sync_mode == WB_SYNC_ALL ?
382 WRITE_SYNC_PLUG : WRITE, bio);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100383 ASSERT(!bio_flagged(bio, BIO_EOPNOTSUPP));
384 bio_put(bio);
385}
386
387STATIC struct bio *
388xfs_alloc_ioend_bio(
389 struct buffer_head *bh)
390{
391 struct bio *bio;
392 int nvecs = bio_get_nr_vecs(bh->b_bdev);
393
394 do {
395 bio = bio_alloc(GFP_NOIO, nvecs);
396 nvecs >>= 1;
397 } while (!bio);
398
399 ASSERT(bio->bi_private == NULL);
400 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
401 bio->bi_bdev = bh->b_bdev;
402 bio_get(bio);
403 return bio;
404}
405
406STATIC void
407xfs_start_buffer_writeback(
408 struct buffer_head *bh)
409{
410 ASSERT(buffer_mapped(bh));
411 ASSERT(buffer_locked(bh));
412 ASSERT(!buffer_delay(bh));
413 ASSERT(!buffer_unwritten(bh));
414
415 mark_buffer_async_write(bh);
416 set_buffer_uptodate(bh);
417 clear_buffer_dirty(bh);
418}
419
420STATIC void
421xfs_start_page_writeback(
422 struct page *page,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100423 int clear_dirty,
424 int buffers)
425{
426 ASSERT(PageLocked(page));
427 ASSERT(!PageWriteback(page));
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100428 if (clear_dirty)
David Chinner92132022006-12-21 10:24:01 +1100429 clear_page_dirty_for_io(page);
430 set_page_writeback(page);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100431 unlock_page(page);
Fengguang Wu1f7decf2007-10-16 23:30:42 -0700432 /* If no buffers on the page are to be written, finish it here */
433 if (!buffers)
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100434 end_page_writeback(page);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100435}
436
437static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
438{
439 return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
440}
441
442/*
David Chinnerd88992f2006-01-18 13:38:12 +1100443 * Submit all of the bios for all of the ioends we have saved up, covering the
444 * initial writepage page and also any probed pages.
445 *
446 * Because we may have multiple ioends spanning a page, we need to start
447 * writeback on all the buffers before we submit them for I/O. If we mark the
448 * buffers as we got, then we can end up with a page that only has buffers
449 * marked async write and I/O complete on can occur before we mark the other
450 * buffers async write.
451 *
452 * The end result of this is that we trip a bug in end_page_writeback() because
453 * we call it twice for the one page as the code in end_buffer_async_write()
454 * assumes that all buffers on the page are started at the same time.
455 *
456 * The fix is two passes across the ioend list - one to start writeback on the
Nathan Scottc41564b2006-03-29 08:55:14 +1000457 * buffer_heads, and then submit them for I/O on the second pass.
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100458 */
459STATIC void
460xfs_submit_ioend(
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000461 struct writeback_control *wbc,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100462 xfs_ioend_t *ioend)
463{
David Chinnerd88992f2006-01-18 13:38:12 +1100464 xfs_ioend_t *head = ioend;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100465 xfs_ioend_t *next;
466 struct buffer_head *bh;
467 struct bio *bio;
468 sector_t lastblock = 0;
469
David Chinnerd88992f2006-01-18 13:38:12 +1100470 /* Pass 1 - start writeback */
471 do {
472 next = ioend->io_list;
473 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
474 xfs_start_buffer_writeback(bh);
475 }
476 } while ((ioend = next) != NULL);
477
478 /* Pass 2 - submit I/O */
479 ioend = head;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100480 do {
481 next = ioend->io_list;
482 bio = NULL;
483
484 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100485
486 if (!bio) {
487 retry:
488 bio = xfs_alloc_ioend_bio(bh);
489 } else if (bh->b_blocknr != lastblock + 1) {
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000490 xfs_submit_ioend_bio(wbc, ioend, bio);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100491 goto retry;
492 }
493
494 if (bio_add_buffer(bio, bh) != bh->b_size) {
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000495 xfs_submit_ioend_bio(wbc, ioend, bio);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100496 goto retry;
497 }
498
499 lastblock = bh->b_blocknr;
500 }
501 if (bio)
Christoph Hellwig06342cf2009-10-30 09:09:15 +0000502 xfs_submit_ioend_bio(wbc, ioend, bio);
Christoph Hellwig209fb872010-07-18 21:17:11 +0000503 xfs_finish_ioend(ioend);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100504 } while ((ioend = next) != NULL);
505}
506
507/*
508 * Cancel submission of all buffer_heads so far in this endio.
509 * Toss the endio too. Only ever called for the initial page
510 * in a writepage request, so only ever one page.
511 */
512STATIC void
513xfs_cancel_ioend(
514 xfs_ioend_t *ioend)
515{
516 xfs_ioend_t *next;
517 struct buffer_head *bh, *next_bh;
518
519 do {
520 next = ioend->io_list;
521 bh = ioend->io_buffer_head;
522 do {
523 next_bh = bh->b_private;
524 clear_buffer_async_write(bh);
525 unlock_buffer(bh);
526 } while ((bh = next_bh) != NULL);
527
Christoph Hellwig25e41b32008-12-03 12:20:39 +0100528 xfs_ioend_wake(XFS_I(ioend->io_inode));
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100529 mempool_free(ioend, xfs_ioend_pool);
530 } while ((ioend = next) != NULL);
531}
532
533/*
534 * Test to see if we've been building up a completion structure for
535 * earlier buffers -- if so, we try to append to this ioend if we
536 * can, otherwise we finish off any current ioend and start another.
537 * Return true if we've finished the given ioend.
538 */
539STATIC void
540xfs_add_to_ioend(
541 struct inode *inode,
542 struct buffer_head *bh,
Christoph Hellwig7336cea2006-01-11 20:49:16 +1100543 xfs_off_t offset,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100544 unsigned int type,
545 xfs_ioend_t **result,
546 int need_ioend)
547{
548 xfs_ioend_t *ioend = *result;
549
550 if (!ioend || need_ioend || type != ioend->io_type) {
551 xfs_ioend_t *previous = *result;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100552
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100553 ioend = xfs_alloc_ioend(inode, type);
554 ioend->io_offset = offset;
555 ioend->io_buffer_head = bh;
556 ioend->io_buffer_tail = bh;
557 if (previous)
558 previous->io_list = ioend;
559 *result = ioend;
560 } else {
561 ioend->io_buffer_tail->b_private = bh;
562 ioend->io_buffer_tail = bh;
563 }
564
565 bh->b_private = NULL;
566 ioend->io_size += bh->b_size;
567}
568
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569STATIC void
Nathan Scott87cbc492006-03-14 13:26:43 +1100570xfs_map_buffer(
Christoph Hellwig046f1682010-04-28 12:28:52 +0000571 struct inode *inode,
Nathan Scott87cbc492006-03-14 13:26:43 +1100572 struct buffer_head *bh,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000573 struct xfs_bmbt_irec *imap,
Christoph Hellwig046f1682010-04-28 12:28:52 +0000574 xfs_off_t offset)
Nathan Scott87cbc492006-03-14 13:26:43 +1100575{
576 sector_t bn;
Christoph Hellwig8699bb02010-04-28 12:28:54 +0000577 struct xfs_mount *m = XFS_I(inode)->i_mount;
Christoph Hellwig207d0412010-04-28 12:28:56 +0000578 xfs_off_t iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
579 xfs_daddr_t iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
Nathan Scott87cbc492006-03-14 13:26:43 +1100580
Christoph Hellwig207d0412010-04-28 12:28:56 +0000581 ASSERT(imap->br_startblock != HOLESTARTBLOCK);
582 ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
Nathan Scott87cbc492006-03-14 13:26:43 +1100583
Christoph Hellwige5131822010-04-28 12:28:55 +0000584 bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
Christoph Hellwig8699bb02010-04-28 12:28:54 +0000585 ((offset - iomap_offset) >> inode->i_blkbits);
Nathan Scott87cbc492006-03-14 13:26:43 +1100586
Christoph Hellwig046f1682010-04-28 12:28:52 +0000587 ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
Nathan Scott87cbc492006-03-14 13:26:43 +1100588
589 bh->b_blocknr = bn;
590 set_buffer_mapped(bh);
591}
592
593STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594xfs_map_at_offset(
Christoph Hellwig046f1682010-04-28 12:28:52 +0000595 struct inode *inode,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 struct buffer_head *bh,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000597 struct xfs_bmbt_irec *imap,
Christoph Hellwig046f1682010-04-28 12:28:52 +0000598 xfs_off_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599{
Christoph Hellwig207d0412010-04-28 12:28:56 +0000600 ASSERT(imap->br_startblock != HOLESTARTBLOCK);
601 ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
603 lock_buffer(bh);
Christoph Hellwig207d0412010-04-28 12:28:56 +0000604 xfs_map_buffer(inode, bh, imap, offset);
Christoph Hellwig046f1682010-04-28 12:28:52 +0000605 bh->b_bdev = xfs_find_bdev_for_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 set_buffer_mapped(bh);
607 clear_buffer_delay(bh);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100608 clear_buffer_unwritten(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609}
610
611/*
Christoph Hellwig6c4fe192006-01-11 20:49:28 +1100612 * Look for a page at index that is suitable for clustering.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 */
614STATIC unsigned int
Christoph Hellwig6c4fe192006-01-11 20:49:28 +1100615xfs_probe_page(
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100616 struct page *page,
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000617 unsigned int pg_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618{
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000619 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 int ret = 0;
621
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 if (PageWriteback(page))
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100623 return 0;
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000624 if (!PageDirty(page))
625 return 0;
626 if (!page->mapping)
627 return 0;
628 if (!page_has_buffers(page))
629 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000631 bh = head = page_buffers(page);
632 do {
633 if (!buffer_uptodate(bh))
634 break;
635 if (!buffer_mapped(bh))
636 break;
637 ret += bh->b_size;
638 if (ret >= pg_offset)
639 break;
640 } while ((bh = bh->b_this_page) != head);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 return ret;
643}
644
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100645STATIC size_t
Christoph Hellwig6c4fe192006-01-11 20:49:28 +1100646xfs_probe_cluster(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 struct inode *inode,
648 struct page *startpage,
649 struct buffer_head *bh,
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000650 struct buffer_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651{
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100652 struct pagevec pvec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 pgoff_t tindex, tlast, tloff;
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100654 size_t total = 0;
655 int done = 0, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656
657 /* First sum forwards in this page */
658 do {
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000659 if (!buffer_uptodate(bh) || !buffer_mapped(bh))
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100660 return total;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 total += bh->b_size;
662 } while ((bh = bh->b_this_page) != head);
663
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100664 /* if we reached the end of the page, sum forwards in following pages */
665 tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
666 tindex = startpage->index + 1;
667
668 /* Prune this back to avoid pathological behavior */
669 tloff = min(tlast, startpage->index + 64);
670
671 pagevec_init(&pvec, 0);
672 while (!done && tindex <= tloff) {
673 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
674
675 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
676 break;
677
678 for (i = 0; i < pagevec_count(&pvec); i++) {
679 struct page *page = pvec.pages[i];
Christoph Hellwig265c1fa2007-08-16 15:38:19 +1000680 size_t pg_offset, pg_len = 0;
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100681
682 if (tindex == tlast) {
683 pg_offset =
684 i_size_read(inode) & (PAGE_CACHE_SIZE - 1);
Christoph Hellwig1defeac2006-01-11 20:48:33 +1100685 if (!pg_offset) {
686 done = 1;
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100687 break;
Christoph Hellwig1defeac2006-01-11 20:48:33 +1100688 }
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100689 } else
690 pg_offset = PAGE_CACHE_SIZE;
691
Nick Piggin529ae9a2008-08-02 12:01:03 +0200692 if (page->index == tindex && trylock_page(page)) {
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000693 pg_len = xfs_probe_page(page, pg_offset);
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100694 unlock_page(page);
695 }
696
Christoph Hellwig265c1fa2007-08-16 15:38:19 +1000697 if (!pg_len) {
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100698 done = 1;
699 break;
700 }
701
Christoph Hellwig265c1fa2007-08-16 15:38:19 +1000702 total += pg_len;
Christoph Hellwig1defeac2006-01-11 20:48:33 +1100703 tindex++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 }
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100705
706 pagevec_release(&pvec);
707 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 }
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 return total;
711}
712
713/*
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100714 * Test if a given page is suitable for writing as part of an unwritten
715 * or delayed allocate extent.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 */
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100717STATIC int
718xfs_is_delayed_page(
719 struct page *page,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100720 unsigned int type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 if (PageWriteback(page))
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100723 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
725 if (page->mapping && page_has_buffers(page)) {
726 struct buffer_head *bh, *head;
727 int acceptable = 0;
728
729 bh = head = page_buffers(page);
730 do {
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100731 if (buffer_unwritten(bh))
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000732 acceptable = (type == IO_UNWRITTEN);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100733 else if (buffer_delay(bh))
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000734 acceptable = (type == IO_DELAY);
David Chinner2ddee842006-03-22 12:47:40 +1100735 else if (buffer_dirty(bh) && buffer_mapped(bh))
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000736 acceptable = (type == IO_NEW);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100737 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 } while ((bh = bh->b_this_page) != head);
740
741 if (acceptable)
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100742 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 }
744
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100745 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746}
747
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748/*
749 * Allocate & map buffers for page given the extent map. Write it out.
750 * except for the original page of a writepage, this is called on
751 * delalloc/unwritten pages only, for the original page it is possible
752 * that the page has no mapping at all.
753 */
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100754STATIC int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755xfs_convert_page(
756 struct inode *inode,
757 struct page *page,
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100758 loff_t tindex,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000759 struct xfs_bmbt_irec *imap,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100760 xfs_ioend_t **ioendp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 struct writeback_control *wbc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 int all_bh)
763{
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100764 struct buffer_head *bh, *head;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100765 xfs_off_t end_offset;
766 unsigned long p_offset;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100767 unsigned int type;
Nathan Scott24e17b52005-05-05 13:33:20 -0700768 int len, page_dirty;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100769 int count = 0, done = 0, uptodate = 1;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100770 xfs_off_t offset = page_offset(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100772 if (page->index != tindex)
773 goto fail;
Nick Piggin529ae9a2008-08-02 12:01:03 +0200774 if (!trylock_page(page))
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100775 goto fail;
776 if (PageWriteback(page))
777 goto fail_unlock_page;
778 if (page->mapping != inode->i_mapping)
779 goto fail_unlock_page;
780 if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
781 goto fail_unlock_page;
782
Nathan Scott24e17b52005-05-05 13:33:20 -0700783 /*
784 * page_dirty is initially a count of buffers on the page before
Nathan Scottc41564b2006-03-29 08:55:14 +1000785 * EOF and is decremented as we move each into a cleanable state.
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100786 *
787 * Derivation:
788 *
789 * End offset is the highest offset that this page should represent.
790 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
791 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
792 * hence give us the correct page_dirty count. On any other page,
793 * it will be zero and in that case we need page_dirty to be the
794 * count of buffers on the page.
Nathan Scott24e17b52005-05-05 13:33:20 -0700795 */
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100796 end_offset = min_t(unsigned long long,
797 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
798 i_size_read(inode));
799
Nathan Scott24e17b52005-05-05 13:33:20 -0700800 len = 1 << inode->i_blkbits;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100801 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
802 PAGE_CACHE_SIZE);
803 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
804 page_dirty = p_offset / len;
Nathan Scott24e17b52005-05-05 13:33:20 -0700805
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 bh = head = page_buffers(page);
807 do {
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100808 if (offset >= end_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 break;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100810 if (!buffer_uptodate(bh))
811 uptodate = 0;
812 if (!(PageUptodate(page) || buffer_uptodate(bh))) {
813 done = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814 continue;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100815 }
816
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100817 if (buffer_unwritten(bh) || buffer_delay(bh)) {
818 if (buffer_unwritten(bh))
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000819 type = IO_UNWRITTEN;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100820 else
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000821 type = IO_DELAY;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100822
Christoph Hellwig558e6892010-04-28 12:28:58 +0000823 if (!xfs_imap_valid(inode, imap, offset)) {
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100824 done = 1;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100825 continue;
826 }
827
Christoph Hellwig207d0412010-04-28 12:28:56 +0000828 ASSERT(imap->br_startblock != HOLESTARTBLOCK);
829 ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100830
Christoph Hellwig207d0412010-04-28 12:28:56 +0000831 xfs_map_at_offset(inode, bh, imap, offset);
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000832 xfs_add_to_ioend(inode, bh, offset, type,
833 ioendp, done);
834
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100835 page_dirty--;
836 count++;
837 } else {
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000838 type = IO_NEW;
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000839 if (buffer_mapped(bh) && all_bh) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 lock_buffer(bh);
Christoph Hellwig7336cea2006-01-11 20:49:16 +1100841 xfs_add_to_ioend(inode, bh, offset,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100842 type, ioendp, done);
843 count++;
Nathan Scott24e17b52005-05-05 13:33:20 -0700844 page_dirty--;
Christoph Hellwig9260dc62006-01-11 20:48:47 +1100845 } else {
846 done = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 }
Christoph Hellwig7336cea2006-01-11 20:49:16 +1100849 } while (offset += len, (bh = bh->b_this_page) != head);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100851 if (uptodate && bh == head)
852 SetPageUptodate(page);
853
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000854 if (count) {
Dave Chinnerefceab12010-08-24 11:44:56 +1000855 if (--wbc->nr_to_write <= 0 &&
856 wbc->sync_mode == WB_SYNC_NONE)
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000857 done = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 }
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000859 xfs_start_page_writeback(page, !page_dirty, count);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100860
861 return done;
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100862 fail_unlock_page:
863 unlock_page(page);
864 fail:
865 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866}
867
868/*
869 * Convert & write out a cluster of pages in the same extent as defined
870 * by mp and following the start page.
871 */
872STATIC void
873xfs_cluster_write(
874 struct inode *inode,
875 pgoff_t tindex,
Christoph Hellwig207d0412010-04-28 12:28:56 +0000876 struct xfs_bmbt_irec *imap,
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100877 xfs_ioend_t **ioendp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 struct writeback_control *wbc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879 int all_bh,
880 pgoff_t tlast)
881{
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100882 struct pagevec pvec;
883 int done = 0, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100885 pagevec_init(&pvec, 0);
886 while (!done && tindex <= tlast) {
887 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
888
889 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890 break;
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100891
892 for (i = 0; i < pagevec_count(&pvec); i++) {
893 done = xfs_convert_page(inode, pvec.pages[i], tindex++,
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000894 imap, ioendp, wbc, all_bh);
Christoph Hellwig10ce4442006-01-11 20:48:14 +1100895 if (done)
896 break;
897 }
898
899 pagevec_release(&pvec);
900 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 }
902}
903
Dave Chinner3ed3a432010-03-05 02:00:42 +0000904STATIC void
905xfs_vm_invalidatepage(
906 struct page *page,
907 unsigned long offset)
908{
909 trace_xfs_invalidatepage(page->mapping->host, page, offset);
910 block_invalidatepage(page, offset);
911}
912
913/*
914 * If the page has delalloc buffers on it, we need to punch them out before we
915 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
916 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
917 * is done on that same region - the delalloc extent is returned when none is
918 * supposed to be there.
919 *
920 * We prevent this by truncating away the delalloc regions on the page before
921 * invalidating it. Because they are delalloc, we can do this without needing a
922 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
923 * truncation without a transaction as there is no space left for block
924 * reservation (typically why we see a ENOSPC in writeback).
925 *
926 * This is not a performance critical path, so for now just do the punching a
927 * buffer head at a time.
928 */
929STATIC void
930xfs_aops_discard_page(
931 struct page *page)
932{
933 struct inode *inode = page->mapping->host;
934 struct xfs_inode *ip = XFS_I(inode);
935 struct buffer_head *bh, *head;
936 loff_t offset = page_offset(page);
Dave Chinner3ed3a432010-03-05 02:00:42 +0000937
Christoph Hellwig34a52c62010-04-28 12:28:57 +0000938 if (!xfs_is_delayed_page(page, IO_DELAY))
Dave Chinner3ed3a432010-03-05 02:00:42 +0000939 goto out_invalidate;
940
Dave Chinnere8c37532010-03-15 02:36:35 +0000941 if (XFS_FORCED_SHUTDOWN(ip->i_mount))
942 goto out_invalidate;
943
Dave Chinner3ed3a432010-03-05 02:00:42 +0000944 xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
945 "page discard on page %p, inode 0x%llx, offset %llu.",
946 page, ip->i_ino, offset);
947
948 xfs_ilock(ip, XFS_ILOCK_EXCL);
949 bh = head = page_buffers(page);
950 do {
Dave Chinner3ed3a432010-03-05 02:00:42 +0000951 int error;
Dave Chinnerc726de42010-11-30 15:14:39 +1100952 xfs_fileoff_t start_fsb;
Dave Chinner3ed3a432010-03-05 02:00:42 +0000953
954 if (!buffer_delay(bh))
955 goto next_buffer;
956
Dave Chinnerc726de42010-11-30 15:14:39 +1100957 start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
958 error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
Dave Chinner3ed3a432010-03-05 02:00:42 +0000959 if (error) {
960 /* something screwed, just bail */
Dave Chinnere8c37532010-03-15 02:36:35 +0000961 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
962 xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
Dave Chinner3ed3a432010-03-05 02:00:42 +0000963 "page discard unable to remove delalloc mapping.");
Dave Chinnere8c37532010-03-15 02:36:35 +0000964 }
Dave Chinner3ed3a432010-03-05 02:00:42 +0000965 break;
966 }
967next_buffer:
Dave Chinnerc726de42010-11-30 15:14:39 +1100968 offset += 1 << inode->i_blkbits;
Dave Chinner3ed3a432010-03-05 02:00:42 +0000969
970 } while ((bh = bh->b_this_page) != head);
971
972 xfs_iunlock(ip, XFS_ILOCK_EXCL);
973out_invalidate:
974 xfs_vm_invalidatepage(page, 0);
975 return;
976}
977
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978/*
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000979 * Write out a dirty page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 *
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000981 * For delalloc space on the page we need to allocate space and flush it.
982 * For unwritten space on the page we need to start the conversion to
983 * regular allocated space.
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000984 * For any other dirty buffer heads on the page we should flush them.
985 *
986 * If we detect that a transaction would be required to flush the page, we
987 * have to check the process flags first, if we are already in a transaction
988 * or disk I/O during allocations is off, we need to fail the writepage and
989 * redirty the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991STATIC int
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000992xfs_vm_writepage(
993 struct page *page,
994 struct writeback_control *wbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
Christoph Hellwig89f3b3632010-06-24 09:45:48 +1000996 struct inode *inode = page->mapping->host;
Christoph Hellwig20cb52e2010-06-24 09:46:01 +1000997 int delalloc, unwritten;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +1100998 struct buffer_head *bh, *head;
Christoph Hellwig207d0412010-04-28 12:28:56 +0000999 struct xfs_bmbt_irec imap;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001000 xfs_ioend_t *ioend = NULL, *iohead = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 loff_t offset;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001002 unsigned int type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003 __uint64_t end_offset;
Christoph Hellwigbd1556a2010-04-28 12:29:00 +00001004 pgoff_t end_index, last_index;
Christoph Hellwigd5cb48a2006-01-11 20:49:02 +11001005 ssize_t size, len;
Christoph Hellwig558e6892010-04-28 12:28:58 +00001006 int flags, err, imap_valid = 0, uptodate = 1;
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001007 int count = 0;
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001008 int all_bh = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001010 trace_xfs_writepage(inode, page, 0);
1011
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001012 ASSERT(page_has_buffers(page));
1013
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001014 /*
1015 * Refuse to write the page out if we are called from reclaim context.
1016 *
Christoph Hellwigd4f7a5c2010-06-28 10:34:44 -04001017 * This avoids stack overflows when called from deeply used stacks in
1018 * random callers for direct reclaim or memcg reclaim. We explicitly
1019 * allow reclaim from kswapd as the stack usage there is relatively low.
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001020 *
1021 * This should really be done by the core VM, but until that happens
1022 * filesystems like XFS, btrfs and ext4 have to take care of this
1023 * by themselves.
1024 */
Christoph Hellwigd4f7a5c2010-06-28 10:34:44 -04001025 if ((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == PF_MEMALLOC)
Christoph Hellwigb5420f22010-08-24 11:47:51 +10001026 goto redirty;
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001027
1028 /*
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001029 * We need a transaction if there are delalloc or unwritten buffers
1030 * on the page.
1031 *
1032 * If we need a transaction and the process flags say we are already
1033 * in a transaction, or no IO is allowed then mark the page dirty
1034 * again and leave the page as is.
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001035 */
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001036 xfs_count_page_state(page, &delalloc, &unwritten);
1037 if ((current->flags & PF_FSTRANS) && (delalloc || unwritten))
Christoph Hellwigb5420f22010-08-24 11:47:51 +10001038 goto redirty;
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001039
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 /* Is this page beyond the end of the file? */
1041 offset = i_size_read(inode);
1042 end_index = offset >> PAGE_CACHE_SHIFT;
1043 last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
1044 if (page->index >= end_index) {
1045 if ((page->index >= end_index + 1) ||
1046 !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001047 unlock_page(page);
Nathan Scott19d5bcf2005-11-02 15:14:09 +11001048 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 }
1050 }
1051
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001052 end_offset = min_t(unsigned long long,
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001053 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
1054 offset);
Nathan Scott24e17b52005-05-05 13:33:20 -07001055 len = 1 << inode->i_blkbits;
Nathan Scott24e17b52005-05-05 13:33:20 -07001056
Nathan Scott24e17b52005-05-05 13:33:20 -07001057 bh = head = page_buffers(page);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001058 offset = page_offset(page);
David Chinnerdf3c7242007-05-24 15:27:03 +10001059 flags = BMAPI_READ;
Christoph Hellwig34a52c62010-04-28 12:28:57 +00001060 type = IO_NEW;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001061
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 do {
1063 if (offset >= end_offset)
1064 break;
1065 if (!buffer_uptodate(bh))
1066 uptodate = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
Eric Sandeen3d9b02e2010-06-24 09:45:30 +10001068 /*
Christoph Hellwigece413f2010-11-10 21:39:11 +00001069 * set_page_dirty dirties all buffers in a page, independent
1070 * of their state. The dirty state however is entirely
1071 * meaningless for holes (!mapped && uptodate), so skip
1072 * buffers covering holes here.
Eric Sandeen3d9b02e2010-06-24 09:45:30 +10001073 */
1074 if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
Eric Sandeen3d9b02e2010-06-24 09:45:30 +10001075 imap_valid = 0;
1076 continue;
1077 }
1078
Christoph Hellwig558e6892010-04-28 12:28:58 +00001079 if (imap_valid)
1080 imap_valid = xfs_imap_valid(inode, &imap, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001082 if (buffer_unwritten(bh) || buffer_delay(bh)) {
David Chinnereffd1202007-06-18 16:49:58 +10001083 int new_ioend = 0;
1084
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001085 if (buffer_unwritten(bh)) {
Christoph Hellwig85da94c2010-12-10 08:42:16 +00001086 if (type != IO_UNWRITTEN) {
1087 type = IO_UNWRITTEN;
1088 imap_valid = 0;
1089 }
Nathan Scott82721452006-04-11 15:10:55 +10001090 flags = BMAPI_WRITE | BMAPI_IGNSTATE;
Christoph Hellwigd5cb48a2006-01-11 20:49:02 +11001091 } else if (buffer_delay(bh)) {
Christoph Hellwig85da94c2010-12-10 08:42:16 +00001092 if (type != IO_DELAY) {
1093 type = IO_DELAY;
1094 imap_valid = 0;
1095 }
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001096 flags = BMAPI_ALLOCATE;
1097
Wu Fengguang1b430be2010-10-26 14:21:26 -07001098 if (wbc->sync_mode == WB_SYNC_NONE)
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001099 flags |= BMAPI_TRYLOCK;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001100 }
1101
Christoph Hellwig558e6892010-04-28 12:28:58 +00001102 if (!imap_valid) {
David Chinnereffd1202007-06-18 16:49:58 +10001103 /*
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001104 * If we didn't have a valid mapping then we
David Chinnereffd1202007-06-18 16:49:58 +10001105 * need to ensure that we put the new mapping
1106 * in a new ioend structure. This needs to be
1107 * done to ensure that the ioends correctly
1108 * reflect the block mappings at io completion
1109 * for unwritten extent conversion.
1110 */
1111 new_ioend = 1;
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001112 err = xfs_map_blocks(inode, offset, len,
Christoph Hellwig207d0412010-04-28 12:28:56 +00001113 &imap, flags);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001114 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 goto error;
Christoph Hellwig558e6892010-04-28 12:28:58 +00001116 imap_valid = xfs_imap_valid(inode, &imap,
1117 offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 }
Christoph Hellwig558e6892010-04-28 12:28:58 +00001119 if (imap_valid) {
Christoph Hellwig207d0412010-04-28 12:28:56 +00001120 xfs_map_at_offset(inode, bh, &imap, offset);
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001121 xfs_add_to_ioend(inode, bh, offset, type,
1122 &ioend, new_ioend);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001123 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 }
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001125 } else if (buffer_uptodate(bh)) {
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001126 /*
1127 * we got here because the buffer is already mapped.
1128 * That means it must already have extents allocated
1129 * underneath it. Map the extent by reading it.
1130 */
Christoph Hellwig85da94c2010-12-10 08:42:16 +00001131 if (flags != BMAPI_READ) {
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001132 flags = BMAPI_READ;
Christoph Hellwig85da94c2010-12-10 08:42:16 +00001133 imap_valid = 0;
1134 }
1135 if (!imap_valid) {
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001136 size = xfs_probe_cluster(inode, page, bh, head);
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001137 err = xfs_map_blocks(inode, offset, size,
Christoph Hellwig207d0412010-04-28 12:28:56 +00001138 &imap, flags);
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001139 if (err)
1140 goto error;
Christoph Hellwig558e6892010-04-28 12:28:58 +00001141 imap_valid = xfs_imap_valid(inode, &imap,
1142 offset);
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001143 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144
David Chinnerdf3c7242007-05-24 15:27:03 +10001145 /*
Christoph Hellwig34a52c62010-04-28 12:28:57 +00001146 * We set the type to IO_NEW in case we are doing a
David Chinnerdf3c7242007-05-24 15:27:03 +10001147 * small write at EOF that is extending the file but
1148 * without needing an allocation. We need to update the
1149 * file size on I/O completion in this case so it is
1150 * the same case as having just allocated a new extent
1151 * that we are writing into for the first time.
1152 */
Christoph Hellwig34a52c62010-04-28 12:28:57 +00001153 type = IO_NEW;
Nick Pigginca5de402008-08-02 12:02:13 +02001154 if (trylock_buffer(bh)) {
Christoph Hellwig558e6892010-04-28 12:28:58 +00001155 if (imap_valid)
Christoph Hellwig6c4fe192006-01-11 20:49:28 +11001156 all_bh = 1;
Christoph Hellwig7336cea2006-01-11 20:49:16 +11001157 xfs_add_to_ioend(inode, bh, offset, type,
Christoph Hellwig558e6892010-04-28 12:28:58 +00001158 &ioend, !imap_valid);
Christoph Hellwigd5cb48a2006-01-11 20:49:02 +11001159 count++;
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001160 } else {
Christoph Hellwig558e6892010-04-28 12:28:58 +00001161 imap_valid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162 }
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001163 } else if (PageUptodate(page)) {
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001164 ASSERT(buffer_mapped(bh));
Christoph Hellwig558e6892010-04-28 12:28:58 +00001165 imap_valid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 }
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001167
1168 if (!iohead)
1169 iohead = ioend;
1170
1171 } while (offset += len, ((bh = bh->b_this_page) != head));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172
1173 if (uptodate && bh == head)
1174 SetPageUptodate(page);
1175
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001176 xfs_start_page_writeback(page, 1, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177
Christoph Hellwig558e6892010-04-28 12:28:58 +00001178 if (ioend && imap_valid) {
Christoph Hellwigbd1556a2010-04-28 12:29:00 +00001179 xfs_off_t end_index;
Christoph Hellwig8699bb02010-04-28 12:28:54 +00001180
Christoph Hellwigbd1556a2010-04-28 12:29:00 +00001181 end_index = imap.br_startoff + imap.br_blockcount;
1182
1183 /* to bytes */
1184 end_index <<= inode->i_blkbits;
1185
1186 /* to pages */
1187 end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;
1188
1189 /* check against file size */
1190 if (end_index > last_index)
1191 end_index = last_index;
1192
Christoph Hellwig207d0412010-04-28 12:28:56 +00001193 xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001194 wbc, all_bh, end_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 }
1196
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001197 if (iohead)
Christoph Hellwig06342cf2009-10-30 09:09:15 +00001198 xfs_submit_ioend(wbc, iohead);
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001199
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001200 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001201
1202error:
Christoph Hellwigf6d6d4f2006-01-11 15:40:13 +11001203 if (iohead)
1204 xfs_cancel_ioend(iohead);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205
Christoph Hellwigb5420f22010-08-24 11:47:51 +10001206 if (err == -EAGAIN)
1207 goto redirty;
1208
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001209 xfs_aops_discard_page(page);
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001210 ClearPageUptodate(page);
1211 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 return err;
Nathan Scottf51623b2006-03-14 13:26:27 +11001213
Christoph Hellwigb5420f22010-08-24 11:47:51 +10001214redirty:
Nathan Scottf51623b2006-03-14 13:26:27 +11001215 redirty_page_for_writepage(wbc, page);
1216 unlock_page(page);
1217 return 0;
Nathan Scottf51623b2006-03-14 13:26:27 +11001218}
1219
Nathan Scott7d4fb402006-06-09 15:27:16 +10001220STATIC int
1221xfs_vm_writepages(
1222 struct address_space *mapping,
1223 struct writeback_control *wbc)
1224{
Christoph Hellwigb3aea4e2007-08-29 11:44:37 +10001225 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
Nathan Scott7d4fb402006-06-09 15:27:16 +10001226 return generic_writepages(mapping, wbc);
1227}
1228
Nathan Scottf51623b2006-03-14 13:26:27 +11001229/*
1230 * Called to move a page into cleanable state - and from there
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001231 * to be released. The page should already be clean. We always
Nathan Scottf51623b2006-03-14 13:26:27 +11001232 * have buffer heads in this call.
1233 *
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001234 * Returns 1 if the page is ok to release, 0 otherwise.
Nathan Scottf51623b2006-03-14 13:26:27 +11001235 */
1236STATIC int
Nathan Scott238f4c52006-03-17 17:26:25 +11001237xfs_vm_releasepage(
Nathan Scottf51623b2006-03-14 13:26:27 +11001238 struct page *page,
1239 gfp_t gfp_mask)
1240{
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001241 int delalloc, unwritten;
Nathan Scottf51623b2006-03-14 13:26:27 +11001242
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001243 trace_xfs_releasepage(page->mapping->host, page, 0);
Nathan Scott238f4c52006-03-17 17:26:25 +11001244
Christoph Hellwig20cb52e2010-06-24 09:46:01 +10001245 xfs_count_page_state(page, &delalloc, &unwritten);
Nathan Scottf51623b2006-03-14 13:26:27 +11001246
Christoph Hellwig89f3b3632010-06-24 09:45:48 +10001247 if (WARN_ON(delalloc))
1248 return 0;
1249 if (WARN_ON(unwritten))
Nathan Scottf51623b2006-03-14 13:26:27 +11001250 return 0;
1251
Nathan Scottf51623b2006-03-14 13:26:27 +11001252 return try_to_free_buffers(page);
1253}
1254
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255STATIC int
Nathan Scottc2536662006-03-29 10:44:40 +10001256__xfs_get_blocks(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 struct inode *inode,
1258 sector_t iblock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 struct buffer_head *bh_result,
1260 int create,
Christoph Hellwigf2bde9b2010-06-24 11:44:35 +10001261 int direct)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262{
Christoph Hellwigf2bde9b2010-06-24 11:44:35 +10001263 int flags = create ? BMAPI_WRITE : BMAPI_READ;
Christoph Hellwig207d0412010-04-28 12:28:56 +00001264 struct xfs_bmbt_irec imap;
Nathan Scottfdc7ed72005-11-02 15:13:13 +11001265 xfs_off_t offset;
1266 ssize_t size;
Christoph Hellwig207d0412010-04-28 12:28:56 +00001267 int nimap = 1;
1268 int new = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270
Nathan Scottfdc7ed72005-11-02 15:13:13 +11001271 offset = (xfs_off_t)iblock << inode->i_blkbits;
Nathan Scottc2536662006-03-29 10:44:40 +10001272 ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
1273 size = bh_result->b_size;
Lachlan McIlroy364f3582008-09-17 16:50:14 +10001274
1275 if (!create && direct && offset >= i_size_read(inode))
1276 return 0;
1277
Christoph Hellwigf2bde9b2010-06-24 11:44:35 +10001278 if (direct && create)
1279 flags |= BMAPI_DIRECT;
1280
1281 error = xfs_iomap(XFS_I(inode), offset, size, flags, &imap, &nimap,
1282 &new);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 if (error)
1284 return -error;
Christoph Hellwig207d0412010-04-28 12:28:56 +00001285 if (nimap == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 return 0;
1287
Christoph Hellwig207d0412010-04-28 12:28:56 +00001288 if (imap.br_startblock != HOLESTARTBLOCK &&
1289 imap.br_startblock != DELAYSTARTBLOCK) {
Nathan Scott87cbc492006-03-14 13:26:43 +11001290 /*
1291 * For unwritten extents do not report a disk address on
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 * the read case (treat as if we're reading into a hole).
1293 */
Christoph Hellwig207d0412010-04-28 12:28:56 +00001294 if (create || !ISUNWRITTEN(&imap))
1295 xfs_map_buffer(inode, bh_result, &imap, offset);
1296 if (create && ISUNWRITTEN(&imap)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297 if (direct)
1298 bh_result->b_private = inode;
1299 set_buffer_unwritten(bh_result);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 }
1301 }
1302
Nathan Scottc2536662006-03-29 10:44:40 +10001303 /*
1304 * If this is a realtime file, data may be on a different device.
1305 * to that pointed to from the buffer_head b_bdev currently.
1306 */
Christoph Hellwig046f1682010-04-28 12:28:52 +00001307 bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001308
Nathan Scottc2536662006-03-29 10:44:40 +10001309 /*
David Chinner549054a2007-02-10 18:36:35 +11001310 * If we previously allocated a block out beyond eof and we are now
1311 * coming back to use it then we will need to flag it as new even if it
1312 * has a disk address.
1313 *
1314 * With sub-block writes into unwritten extents we also need to mark
1315 * the buffer as new so that the unwritten parts of the buffer gets
1316 * correctly zeroed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 */
1318 if (create &&
1319 ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
David Chinner549054a2007-02-10 18:36:35 +11001320 (offset >= i_size_read(inode)) ||
Christoph Hellwig207d0412010-04-28 12:28:56 +00001321 (new || ISUNWRITTEN(&imap))))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 set_buffer_new(bh_result);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323
Christoph Hellwig207d0412010-04-28 12:28:56 +00001324 if (imap.br_startblock == DELAYSTARTBLOCK) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 BUG_ON(direct);
1326 if (create) {
1327 set_buffer_uptodate(bh_result);
1328 set_buffer_mapped(bh_result);
1329 set_buffer_delay(bh_result);
1330 }
1331 }
1332
Christoph Hellwig2b8f12b2010-04-28 12:28:59 +00001333 /*
1334 * If this is O_DIRECT or the mpage code calling tell them how large
1335 * the mapping is, so that we can avoid repeated get_blocks calls.
1336 */
Nathan Scottc2536662006-03-29 10:44:40 +10001337 if (direct || size > (1 << inode->i_blkbits)) {
Christoph Hellwig2b8f12b2010-04-28 12:28:59 +00001338 xfs_off_t mapping_size;
Christoph Hellwig9563b3d2010-04-28 12:28:53 +00001339
Christoph Hellwig2b8f12b2010-04-28 12:28:59 +00001340 mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
1341 mapping_size <<= inode->i_blkbits;
1342
1343 ASSERT(mapping_size > 0);
1344 if (mapping_size > size)
1345 mapping_size = size;
1346 if (mapping_size > LONG_MAX)
1347 mapping_size = LONG_MAX;
1348
1349 bh_result->b_size = mapping_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 }
1351
1352 return 0;
1353}
1354
1355int
Nathan Scottc2536662006-03-29 10:44:40 +10001356xfs_get_blocks(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 struct inode *inode,
1358 sector_t iblock,
1359 struct buffer_head *bh_result,
1360 int create)
1361{
Christoph Hellwigf2bde9b2010-06-24 11:44:35 +10001362 return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363}
1364
1365STATIC int
Nathan Scotte4c573b2006-03-14 13:54:26 +11001366xfs_get_blocks_direct(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 struct inode *inode,
1368 sector_t iblock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 struct buffer_head *bh_result,
1370 int create)
1371{
Christoph Hellwigf2bde9b2010-06-24 11:44:35 +10001372 return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373}
1374
Christoph Hellwig209fb872010-07-18 21:17:11 +00001375/*
1376 * Complete a direct I/O write request.
1377 *
1378 * If the private argument is non-NULL __xfs_get_blocks signals us that we
1379 * need to issue a transaction to convert the range from unwritten to written
1380 * extents. In case this is regular synchronous I/O we just call xfs_end_io
1381 * to do this and we are done. But in case this was a successfull AIO
1382 * request this handler is called from interrupt context, from which we
1383 * can't start transactions. In that case offload the I/O completion to
1384 * the workqueues we also use for buffered I/O completion.
1385 */
Christoph Hellwigf0973862005-09-05 08:22:52 +10001386STATIC void
Christoph Hellwig209fb872010-07-18 21:17:11 +00001387xfs_end_io_direct_write(
1388 struct kiocb *iocb,
1389 loff_t offset,
1390 ssize_t size,
1391 void *private,
1392 int ret,
1393 bool is_async)
Christoph Hellwigf0973862005-09-05 08:22:52 +10001394{
Christoph Hellwig209fb872010-07-18 21:17:11 +00001395 struct xfs_ioend *ioend = iocb->private;
Christoph Hellwigf0973862005-09-05 08:22:52 +10001396
1397 /*
Nathan Scottc41564b2006-03-29 08:55:14 +10001398 * blockdev_direct_IO can return an error even after the I/O
Christoph Hellwigf0973862005-09-05 08:22:52 +10001399 * completion handler was called. Thus we need to protect
1400 * against double-freeing.
1401 */
1402 iocb->private = NULL;
Christoph Hellwig40e2e972010-07-18 21:17:09 +00001403
Christoph Hellwig209fb872010-07-18 21:17:11 +00001404 ioend->io_offset = offset;
1405 ioend->io_size = size;
1406 if (private && size > 0)
1407 ioend->io_type = IO_UNWRITTEN;
1408
1409 if (is_async) {
1410 /*
1411 * If we are converting an unwritten extent we need to delay
1412 * the AIO completion until after the unwrittent extent
1413 * conversion has completed, otherwise do it ASAP.
1414 */
1415 if (ioend->io_type == IO_UNWRITTEN) {
1416 ioend->io_iocb = iocb;
1417 ioend->io_result = ret;
1418 } else {
1419 aio_complete(iocb, ret, 0);
1420 }
1421 xfs_finish_ioend(ioend);
1422 } else {
1423 xfs_finish_ioend_sync(ioend);
1424 }
Christoph Hellwigf0973862005-09-05 08:22:52 +10001425}
1426
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427STATIC ssize_t
Nathan Scotte4c573b2006-03-14 13:54:26 +11001428xfs_vm_direct_IO(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429 int rw,
1430 struct kiocb *iocb,
1431 const struct iovec *iov,
1432 loff_t offset,
1433 unsigned long nr_segs)
1434{
Christoph Hellwig209fb872010-07-18 21:17:11 +00001435 struct inode *inode = iocb->ki_filp->f_mapping->host;
1436 struct block_device *bdev = xfs_find_bdev_for_inode(inode);
1437 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438
Christoph Hellwig209fb872010-07-18 21:17:11 +00001439 if (rw & WRITE) {
1440 iocb->private = xfs_alloc_ioend(inode, IO_NEW);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441
Christoph Hellwigeafdc7d2010-06-04 11:29:53 +02001442 ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
1443 offset, nr_segs,
1444 xfs_get_blocks_direct,
1445 xfs_end_io_direct_write, NULL, 0);
Christoph Hellwig209fb872010-07-18 21:17:11 +00001446 if (ret != -EIOCBQUEUED && iocb->private)
1447 xfs_destroy_ioend(iocb->private);
1448 } else {
Christoph Hellwigeafdc7d2010-06-04 11:29:53 +02001449 ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
1450 offset, nr_segs,
1451 xfs_get_blocks_direct,
1452 NULL, NULL, 0);
Christoph Hellwig209fb872010-07-18 21:17:11 +00001453 }
Christoph Hellwig5fe878ae2009-12-15 16:47:50 -08001454
Christoph Hellwigf0973862005-09-05 08:22:52 +10001455 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456}
1457
Christoph Hellwigfa9b2272010-06-14 05:17:31 -04001458STATIC void
1459xfs_vm_write_failed(
1460 struct address_space *mapping,
1461 loff_t to)
1462{
1463 struct inode *inode = mapping->host;
1464
1465 if (to > inode->i_size) {
Dave Chinnerc726de42010-11-30 15:14:39 +11001466 /*
1467 * punch out the delalloc blocks we have already allocated. We
1468 * don't call xfs_setattr() to do this as we may be in the
1469 * middle of a multi-iovec write and so the vfs inode->i_size
1470 * will not match the xfs ip->i_size and so it will zero too
1471 * much. Hence we jus truncate the page cache to zero what is
1472 * necessary and punch the delalloc blocks directly.
1473 */
1474 struct xfs_inode *ip = XFS_I(inode);
1475 xfs_fileoff_t start_fsb;
1476 xfs_fileoff_t end_fsb;
1477 int error;
1478
1479 truncate_pagecache(inode, to, inode->i_size);
1480
1481 /*
1482 * Check if there are any blocks that are outside of i_size
1483 * that need to be trimmed back.
1484 */
1485 start_fsb = XFS_B_TO_FSB(ip->i_mount, inode->i_size) + 1;
1486 end_fsb = XFS_B_TO_FSB(ip->i_mount, to);
1487 if (end_fsb <= start_fsb)
1488 return;
1489
1490 xfs_ilock(ip, XFS_ILOCK_EXCL);
1491 error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
1492 end_fsb - start_fsb);
1493 if (error) {
1494 /* something screwed, just bail */
1495 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
1496 xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
1497 "xfs_vm_write_failed: unable to clean up ino %lld",
1498 ip->i_ino);
1499 }
1500 }
1501 xfs_iunlock(ip, XFS_ILOCK_EXCL);
Christoph Hellwigfa9b2272010-06-14 05:17:31 -04001502 }
1503}
1504
Nathan Scottf51623b2006-03-14 13:26:27 +11001505STATIC int
Nick Piggind79689c2007-10-16 01:25:06 -07001506xfs_vm_write_begin(
Nathan Scottf51623b2006-03-14 13:26:27 +11001507 struct file *file,
Nick Piggind79689c2007-10-16 01:25:06 -07001508 struct address_space *mapping,
1509 loff_t pos,
1510 unsigned len,
1511 unsigned flags,
1512 struct page **pagep,
1513 void **fsdata)
Nathan Scottf51623b2006-03-14 13:26:27 +11001514{
Christoph Hellwig155130a2010-06-04 11:29:58 +02001515 int ret;
1516
1517 ret = block_write_begin(mapping, pos, len, flags | AOP_FLAG_NOFS,
1518 pagep, xfs_get_blocks);
Christoph Hellwigfa9b2272010-06-14 05:17:31 -04001519 if (unlikely(ret))
1520 xfs_vm_write_failed(mapping, pos + len);
1521 return ret;
1522}
Christoph Hellwig155130a2010-06-04 11:29:58 +02001523
Christoph Hellwigfa9b2272010-06-14 05:17:31 -04001524STATIC int
1525xfs_vm_write_end(
1526 struct file *file,
1527 struct address_space *mapping,
1528 loff_t pos,
1529 unsigned len,
1530 unsigned copied,
1531 struct page *page,
1532 void *fsdata)
1533{
1534 int ret;
1535
1536 ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
1537 if (unlikely(ret < len))
1538 xfs_vm_write_failed(mapping, pos + len);
Christoph Hellwig155130a2010-06-04 11:29:58 +02001539 return ret;
Nathan Scottf51623b2006-03-14 13:26:27 +11001540}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541
1542STATIC sector_t
Nathan Scotte4c573b2006-03-14 13:54:26 +11001543xfs_vm_bmap(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 struct address_space *mapping,
1545 sector_t block)
1546{
1547 struct inode *inode = (struct inode *)mapping->host;
Christoph Hellwig739bfb22007-08-29 10:58:01 +10001548 struct xfs_inode *ip = XFS_I(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
Christoph Hellwigcca28fb2010-06-24 11:57:09 +10001550 trace_xfs_vm_bmap(XFS_I(inode));
Christoph Hellwig126468b2008-03-06 13:44:57 +11001551 xfs_ilock(ip, XFS_IOLOCK_SHARED);
Christoph Hellwig739bfb22007-08-29 10:58:01 +10001552 xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
Christoph Hellwig126468b2008-03-06 13:44:57 +11001553 xfs_iunlock(ip, XFS_IOLOCK_SHARED);
Nathan Scottc2536662006-03-29 10:44:40 +10001554 return generic_block_bmap(mapping, block, xfs_get_blocks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555}
1556
1557STATIC int
Nathan Scotte4c573b2006-03-14 13:54:26 +11001558xfs_vm_readpage(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 struct file *unused,
1560 struct page *page)
1561{
Nathan Scottc2536662006-03-29 10:44:40 +10001562 return mpage_readpage(page, xfs_get_blocks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563}
1564
1565STATIC int
Nathan Scotte4c573b2006-03-14 13:54:26 +11001566xfs_vm_readpages(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 struct file *unused,
1568 struct address_space *mapping,
1569 struct list_head *pages,
1570 unsigned nr_pages)
1571{
Nathan Scottc2536662006-03-29 10:44:40 +10001572 return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573}
1574
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001575const struct address_space_operations xfs_address_space_operations = {
Nathan Scotte4c573b2006-03-14 13:54:26 +11001576 .readpage = xfs_vm_readpage,
1577 .readpages = xfs_vm_readpages,
1578 .writepage = xfs_vm_writepage,
Nathan Scott7d4fb402006-06-09 15:27:16 +10001579 .writepages = xfs_vm_writepages,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 .sync_page = block_sync_page,
Nathan Scott238f4c52006-03-17 17:26:25 +11001581 .releasepage = xfs_vm_releasepage,
1582 .invalidatepage = xfs_vm_invalidatepage,
Nick Piggind79689c2007-10-16 01:25:06 -07001583 .write_begin = xfs_vm_write_begin,
Christoph Hellwigfa9b2272010-06-14 05:17:31 -04001584 .write_end = xfs_vm_write_end,
Nathan Scotte4c573b2006-03-14 13:54:26 +11001585 .bmap = xfs_vm_bmap,
1586 .direct_IO = xfs_vm_direct_IO,
Christoph Lametere965f962006-02-01 03:05:41 -08001587 .migratepage = buffer_migrate_page,
Hisashi Hifumibddaafa2009-03-29 09:53:38 +02001588 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02001589 .error_remove_page = generic_error_remove_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590};