blob: 5765f88b39049adbd023e1de0719b5bba04e3204 [file] [log] [blame]
Theodore Ts'obd2d0212010-10-27 21:30:10 -04001/*
2 * linux/fs/ext4/page-io.c
3 *
4 * This contains the new page_io functions for ext4
5 *
6 * Written by Theodore Ts'o, 2010.
7 */
8
Theodore Ts'obd2d0212010-10-27 21:30:10 -04009#include <linux/fs.h>
10#include <linux/time.h>
Theodore Ts'obd2d0212010-10-27 21:30:10 -040011#include <linux/highuid.h>
12#include <linux/pagemap.h>
13#include <linux/quotaops.h>
14#include <linux/string.h>
15#include <linux/buffer_head.h>
16#include <linux/writeback.h>
17#include <linux/pagevec.h>
18#include <linux/mpage.h>
19#include <linux/namei.h>
20#include <linux/uio.h>
21#include <linux/bio.h>
22#include <linux/workqueue.h>
23#include <linux/kernel.h>
24#include <linux/slab.h>
Jan Kara1ae48a62013-01-28 09:32:54 -050025#include <linux/mm.h>
Theodore Ts'obd2d0212010-10-27 21:30:10 -040026
27#include "ext4_jbd2.h"
28#include "xattr.h"
29#include "acl.h"
Theodore Ts'obd2d0212010-10-27 21:30:10 -040030
Jan Kara0058f962013-04-11 23:48:32 -040031static struct kmem_cache *io_end_cachep;
Theodore Ts'obd2d0212010-10-27 21:30:10 -040032
Theodore Ts'o5dabfc72010-10-27 21:30:14 -040033int __init ext4_init_pageio(void)
Theodore Ts'obd2d0212010-10-27 21:30:10 -040034{
Theodore Ts'obd2d0212010-10-27 21:30:10 -040035 io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT);
Jan Kara0058f962013-04-11 23:48:32 -040036 if (io_end_cachep == NULL)
Theodore Ts'obd2d0212010-10-27 21:30:10 -040037 return -ENOMEM;
Theodore Ts'obd2d0212010-10-27 21:30:10 -040038 return 0;
39}
40
Theodore Ts'o5dabfc72010-10-27 21:30:14 -040041void ext4_exit_pageio(void)
Theodore Ts'obd2d0212010-10-27 21:30:10 -040042{
43 kmem_cache_destroy(io_end_cachep);
Theodore Ts'obd2d0212010-10-27 21:30:10 -040044}
45
Theodore Ts'o1ada47d2013-03-20 09:39:42 -040046/*
Jan Karab0857d32013-06-04 14:23:41 -040047 * Print an buffer I/O error compatible with the fs/buffer.c. This
48 * provides compatibility with dmesg scrapers that look for a specific
49 * buffer I/O error message. We really need a unified error reporting
50 * structure to userspace ala Digital Unix's uerf system, but it's
51 * probably not going to happen in my lifetime, due to LKML politics...
52 */
53static void buffer_io_error(struct buffer_head *bh)
54{
55 char b[BDEVNAME_SIZE];
Anatol Pomozove8974c32013-07-11 22:42:42 -040056 printk_ratelimited(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n",
Jan Karab0857d32013-06-04 14:23:41 -040057 bdevname(bh->b_bdev, b),
58 (unsigned long long)bh->b_blocknr);
59}
60
61static void ext4_finish_bio(struct bio *bio)
62{
63 int i;
64 int error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
Kent Overstreet2c30c712013-11-07 12:20:26 -080065 struct bio_vec *bvec;
Jan Karab0857d32013-06-04 14:23:41 -040066
Kent Overstreet2c30c712013-11-07 12:20:26 -080067 bio_for_each_segment_all(bvec, bio, i) {
Jan Karab0857d32013-06-04 14:23:41 -040068 struct page *page = bvec->bv_page;
Michael Halcrow2058f832015-04-12 00:55:10 -040069#ifdef CONFIG_EXT4_FS_ENCRYPTION
70 struct page *data_page = NULL;
71 struct ext4_crypto_ctx *ctx = NULL;
72#endif
Jan Karab0857d32013-06-04 14:23:41 -040073 struct buffer_head *bh, *head;
74 unsigned bio_start = bvec->bv_offset;
75 unsigned bio_end = bio_start + bvec->bv_len;
76 unsigned under_io = 0;
77 unsigned long flags;
78
79 if (!page)
80 continue;
81
Michael Halcrow2058f832015-04-12 00:55:10 -040082#ifdef CONFIG_EXT4_FS_ENCRYPTION
83 if (!page->mapping) {
84 /* The bounce data pages are unmapped. */
85 data_page = page;
86 ctx = (struct ext4_crypto_ctx *)page_private(data_page);
87 page = ctx->control_page;
88 }
89#endif
90
Jan Karab0857d32013-06-04 14:23:41 -040091 if (error) {
92 SetPageError(page);
93 set_bit(AS_EIO, &page->mapping->flags);
94 }
95 bh = head = page_buffers(page);
96 /*
97 * We check all buffers in the page under BH_Uptodate_Lock
98 * to avoid races with other end io clearing async_write flags
99 */
100 local_irq_save(flags);
101 bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
102 do {
103 if (bh_offset(bh) < bio_start ||
104 bh_offset(bh) + bh->b_size > bio_end) {
105 if (buffer_async_write(bh))
106 under_io++;
107 continue;
108 }
109 clear_buffer_async_write(bh);
110 if (error)
111 buffer_io_error(bh);
112 } while ((bh = bh->b_this_page) != head);
113 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
114 local_irq_restore(flags);
Michael Halcrow2058f832015-04-12 00:55:10 -0400115 if (!under_io) {
116#ifdef CONFIG_EXT4_FS_ENCRYPTION
117 if (ctx)
118 ext4_restore_control_page(data_page);
119#endif
Jan Karab0857d32013-06-04 14:23:41 -0400120 end_page_writeback(page);
Michael Halcrow2058f832015-04-12 00:55:10 -0400121 }
Jan Karab0857d32013-06-04 14:23:41 -0400122 }
123}
124
Jan Kara97a851e2013-06-04 11:58:58 -0400125static void ext4_release_io_end(ext4_io_end_t *io_end)
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400126{
Jan Karab0857d32013-06-04 14:23:41 -0400127 struct bio *bio, *next_bio;
128
Jan Kara97a851e2013-06-04 11:58:58 -0400129 BUG_ON(!list_empty(&io_end->list));
130 BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
Jan Kara6b523df2013-06-04 13:21:11 -0400131 WARN_ON(io_end->handle);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400132
Jan Kara97a851e2013-06-04 11:58:58 -0400133 if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count))
134 wake_up_all(ext4_ioend_wq(io_end->inode));
Jan Karab0857d32013-06-04 14:23:41 -0400135
136 for (bio = io_end->bio; bio; bio = next_bio) {
137 next_bio = bio->bi_private;
138 ext4_finish_bio(bio);
139 bio_put(bio);
140 }
Jan Kara97a851e2013-06-04 11:58:58 -0400141 kmem_cache_free(io_end_cachep, io_end);
142}
143
144static void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end)
145{
146 struct inode *inode = io_end->inode;
147
148 io_end->flag &= ~EXT4_IO_END_UNWRITTEN;
149 /* Wake up anyone waiting on unwritten extent conversion */
150 if (atomic_dec_and_test(&EXT4_I(inode)->i_unwritten))
151 wake_up_all(ext4_ioend_wq(inode));
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400152}
153
Jan Karaa115f742013-06-04 14:30:00 -0400154/*
155 * Check a range of space and convert unwritten extents to written. Note that
156 * we are protected from truncate touching same part of extent tree by the
157 * fact that truncate code waits for all DIO to finish (thus exclusion from
158 * direct IO is achieved) and also waits for PageWriteback bits. Thus we
159 * cannot get to ext4_ext_truncate() before all IOs overlapping that range are
160 * completed (happens from ext4_free_ioend()).
161 */
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400162static int ext4_end_io(ext4_io_end_t *io)
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400163{
164 struct inode *inode = io->inode;
165 loff_t offset = io->offset;
166 ssize_t size = io->size;
Jan Kara6b523df2013-06-04 13:21:11 -0400167 handle_t *handle = io->handle;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400168 int ret = 0;
169
170 ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p,"
171 "list->prev 0x%p\n",
172 io, inode->i_ino, io->list.next, io->list.prev);
173
Jan Kara6b523df2013-06-04 13:21:11 -0400174 io->handle = NULL; /* Following call will use up the handle */
175 ret = ext4_convert_unwritten_extents(handle, inode, offset, size);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400176 if (ret < 0) {
Theodore Ts'ob82e3842011-10-31 10:56:32 -0400177 ext4_msg(inode->i_sb, KERN_EMERG,
178 "failed to convert unwritten extents to written "
179 "extents -- potential data loss! "
180 "(inode %lu, offset %llu, size %zd, error %d)",
181 inode->i_ino, offset, size, ret);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400182 }
Jan Kara97a851e2013-06-04 11:58:58 -0400183 ext4_clear_io_unwritten_flag(io);
184 ext4_release_io_end(io);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400185 return ret;
186}
187
Jan Kara2e8fa542013-06-04 14:21:02 -0400188static void dump_completed_IO(struct inode *inode, struct list_head *head)
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400189{
190#ifdef EXT4FS_DEBUG
191 struct list_head *cur, *before, *after;
192 ext4_io_end_t *io, *io0, *io1;
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400193
Jan Kara2e8fa542013-06-04 14:21:02 -0400194 if (list_empty(head))
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400195 return;
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400196
Jan Kara2e8fa542013-06-04 14:21:02 -0400197 ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
198 list_for_each_entry(io, head, list) {
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400199 cur = &io->list;
200 before = cur->prev;
201 io0 = container_of(before, ext4_io_end_t, list);
202 after = cur->next;
203 io1 = container_of(after, ext4_io_end_t, list);
204
205 ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
206 io, inode->i_ino, io0, io1);
207 }
208#endif
209}
210
211/* Add the io_end to per-inode completed end_io list. */
Jan Kara97a851e2013-06-04 11:58:58 -0400212static void ext4_add_complete_io(ext4_io_end_t *io_end)
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400213{
214 struct ext4_inode_info *ei = EXT4_I(io_end->inode);
Jan Kara78371a42013-10-16 08:25:11 -0400215 struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400216 struct workqueue_struct *wq;
217 unsigned long flags;
218
Christoph Hellwig7b7a8662013-09-04 15:04:39 +0200219 /* Only reserved conversions from writeback should enter here */
220 WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
Jan Kara78371a42013-10-16 08:25:11 -0400221 WARN_ON(!io_end->handle && sbi->s_journal);
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400222 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
Jan Kara78371a42013-10-16 08:25:11 -0400223 wq = sbi->rsv_conversion_wq;
Christoph Hellwig7b7a8662013-09-04 15:04:39 +0200224 if (list_empty(&ei->i_rsv_conversion_list))
225 queue_work(wq, &ei->i_rsv_conversion_work);
226 list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400227 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
228}
229
Jan Kara2e8fa542013-06-04 14:21:02 -0400230static int ext4_do_flush_completed_IO(struct inode *inode,
231 struct list_head *head)
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400232{
233 ext4_io_end_t *io;
Jan Kara002bd7f2013-01-28 09:49:15 -0500234 struct list_head unwritten;
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400235 unsigned long flags;
236 struct ext4_inode_info *ei = EXT4_I(inode);
237 int err, ret = 0;
238
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400239 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
Jan Kara2e8fa542013-06-04 14:21:02 -0400240 dump_completed_IO(inode, head);
241 list_replace_init(head, &unwritten);
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400242 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
243
244 while (!list_empty(&unwritten)) {
245 io = list_entry(unwritten.next, ext4_io_end_t, list);
246 BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN));
247 list_del_init(&io->list);
248
249 err = ext4_end_io(io);
250 if (unlikely(!ret && err))
251 ret = err;
Dmitry Monakhov28a535f2012-09-29 00:14:55 -0400252 }
253 return ret;
254}
255
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400256/*
Jan Kara2e8fa542013-06-04 14:21:02 -0400257 * work on completed IO, to convert unwritten extents to extents
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400258 */
Jan Kara2e8fa542013-06-04 14:21:02 -0400259void ext4_end_io_rsv_work(struct work_struct *work)
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400260{
Jan Kara84c17542013-01-28 09:43:46 -0500261 struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
Jan Kara2e8fa542013-06-04 14:21:02 -0400262 i_rsv_conversion_work);
263 ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
264}
265
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400266ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags)
267{
Jesper Juhlb17b35e2010-12-19 21:41:55 -0500268 ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400269 if (io) {
Theodore Ts'of7ad6d22010-11-08 13:43:33 -0500270 atomic_inc(&EXT4_I(inode)->i_ioend_count);
271 io->inode = inode;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400272 INIT_LIST_HEAD(&io->list);
Jan Kara97a851e2013-06-04 11:58:58 -0400273 atomic_set(&io->count, 1);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400274 }
275 return io;
276}
277
Jan Kara97a851e2013-06-04 11:58:58 -0400278void ext4_put_io_end_defer(ext4_io_end_t *io_end)
279{
280 if (atomic_dec_and_test(&io_end->count)) {
281 if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) {
282 ext4_release_io_end(io_end);
283 return;
284 }
285 ext4_add_complete_io(io_end);
286 }
287}
288
289int ext4_put_io_end(ext4_io_end_t *io_end)
290{
291 int err = 0;
292
293 if (atomic_dec_and_test(&io_end->count)) {
294 if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
Jan Kara6b523df2013-06-04 13:21:11 -0400295 err = ext4_convert_unwritten_extents(io_end->handle,
296 io_end->inode, io_end->offset,
297 io_end->size);
298 io_end->handle = NULL;
Jan Kara97a851e2013-06-04 11:58:58 -0400299 ext4_clear_io_unwritten_flag(io_end);
300 }
301 ext4_release_io_end(io_end);
302 }
303 return err;
304}
305
306ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
307{
308 atomic_inc(&io_end->count);
309 return io_end;
310}
311
Jan Kara822dbba2013-07-10 21:31:04 -0400312/* BIO completion function for page writeback */
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400313static void ext4_end_bio(struct bio *bio, int error)
314{
315 ext4_io_end_t *io_end = bio->bi_private;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700316 sector_t bi_sector = bio->bi_iter.bi_sector;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400317
318 BUG_ON(!io_end);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400319 bio->bi_end_io = NULL;
320 if (test_bit(BIO_UPTODATE, &bio->bi_flags))
321 error = 0;
Jan Kara0058f962013-04-11 23:48:32 -0400322
Theodore Ts'of7ad6d22010-11-08 13:43:33 -0500323 if (error) {
Jan Karab0857d32013-06-04 14:23:41 -0400324 struct inode *inode = io_end->inode;
325
Matthew Wilcox9503c672014-04-07 10:54:20 -0400326 ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
Theodore Ts'of7ad6d22010-11-08 13:43:33 -0500327 "(offset %llu size %ld starting block %llu)",
Matthew Wilcox9503c672014-04-07 10:54:20 -0400328 error, inode->i_ino,
Theodore Ts'of7ad6d22010-11-08 13:43:33 -0500329 (unsigned long long) io_end->offset,
330 (long) io_end->size,
331 (unsigned long long)
Curt Wohlgemuthd50bdd52011-02-07 12:46:14 -0500332 bi_sector >> (inode->i_blkbits - 9));
Matthew Wilcox9503c672014-04-07 10:54:20 -0400333 mapping_set_error(inode->i_mapping, error);
Theodore Ts'of7ad6d22010-11-08 13:43:33 -0500334 }
Jan Kara822dbba2013-07-10 21:31:04 -0400335
336 if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
337 /*
338 * Link bio into list hanging from io_end. We have to do it
339 * atomically as bio completions can be racing against each
340 * other.
341 */
342 bio->bi_private = xchg(&io_end->bio, bio);
343 ext4_put_io_end_defer(io_end);
344 } else {
345 /*
346 * Drop io_end reference early. Inode can get freed once
347 * we finish the bio.
348 */
349 ext4_put_io_end_defer(io_end);
350 ext4_finish_bio(bio);
351 bio_put(bio);
352 }
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400353}
354
355void ext4_io_submit(struct ext4_io_submit *io)
356{
357 struct bio *bio = io->io_bio;
358
359 if (bio) {
360 bio_get(io->io_bio);
361 submit_bio(io->io_op, io->io_bio);
362 BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP));
363 bio_put(io->io_bio);
364 }
Peter Huewe7dc57612011-02-21 21:01:42 -0500365 io->io_bio = NULL;
Jan Kara97a851e2013-06-04 11:58:58 -0400366}
367
368void ext4_io_submit_init(struct ext4_io_submit *io,
369 struct writeback_control *wbc)
370{
371 io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE);
372 io->io_bio = NULL;
Peter Huewe7dc57612011-02-21 21:01:42 -0500373 io->io_end = NULL;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400374}
375
Jan Kara97a851e2013-06-04 11:58:58 -0400376static int io_submit_init_bio(struct ext4_io_submit *io,
377 struct buffer_head *bh)
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400378{
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400379 int nvecs = bio_get_nr_vecs(bh->b_bdev);
380 struct bio *bio;
381
Theodore Ts'o275d3ba2011-06-29 21:44:45 -0400382 bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES));
Theodore Ts'oa1d8d9a72013-06-06 10:18:22 -0400383 if (!bio)
384 return -ENOMEM;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700385 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400386 bio->bi_bdev = bh->b_bdev;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400387 bio->bi_end_io = ext4_end_bio;
Jan Kara97a851e2013-06-04 11:58:58 -0400388 bio->bi_private = ext4_get_io_end(io->io_end);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400389 io->io_bio = bio;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400390 io->io_next_block = bh->b_blocknr;
391 return 0;
392}
393
394static int io_submit_add_bh(struct ext4_io_submit *io,
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400395 struct inode *inode,
Michael Halcrow2058f832015-04-12 00:55:10 -0400396 struct page *page,
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400397 struct buffer_head *bh)
398{
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400399 int ret;
400
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400401 if (io->io_bio && bh->b_blocknr != io->io_next_block) {
402submit_and_retry:
403 ext4_io_submit(io);
404 }
405 if (io->io_bio == NULL) {
Jan Kara97a851e2013-06-04 11:58:58 -0400406 ret = io_submit_init_bio(io, bh);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400407 if (ret)
408 return ret;
409 }
Michael Halcrow2058f832015-04-12 00:55:10 -0400410 ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh));
Theodore Ts'oa5499842013-05-11 19:07:42 -0400411 if (ret != bh->b_size)
412 goto submit_and_retry;
Jan Kara97a851e2013-06-04 11:58:58 -0400413 io->io_next_block++;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400414 return 0;
415}
416
417int ext4_bio_write_page(struct ext4_io_submit *io,
418 struct page *page,
419 int len,
Namjae Jeon1c8349a2014-05-12 08:12:25 -0400420 struct writeback_control *wbc,
421 bool keep_towrite)
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400422{
Michael Halcrow2058f832015-04-12 00:55:10 -0400423 struct page *data_page = NULL;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400424 struct inode *inode = page->mapping->host;
Jan Kara0058f962013-04-11 23:48:32 -0400425 unsigned block_start, blocksize;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400426 struct buffer_head *bh, *head;
427 int ret = 0;
Jan Kara0058f962013-04-11 23:48:32 -0400428 int nr_submitted = 0;
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400429
430 blocksize = 1 << inode->i_blkbits;
431
Curt Wohlgemuthd50bdd52011-02-07 12:46:14 -0500432 BUG_ON(!PageLocked(page));
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400433 BUG_ON(PageWriteback(page));
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400434
Namjae Jeon1c8349a2014-05-12 08:12:25 -0400435 if (keep_towrite)
436 set_page_writeback_keepwrite(page);
437 else
438 set_page_writeback(page);
Theodore Ts'oa54aa762011-02-27 16:43:24 -0500439 ClearPageError(page);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400440
Jan Kara0058f962013-04-11 23:48:32 -0400441 /*
Linus Torvaldsf8409ab2014-06-08 13:03:35 -0700442 * Comments copied from block_write_full_page:
Jan Karaeeece462014-05-27 12:48:55 -0400443 *
444 * The page straddles i_size. It must be zeroed out on each and every
445 * writepage invocation because it may be mmapped. "A file is mapped
446 * in multiples of the page size. For a file that is not a multiple of
447 * the page size, the remaining memory is zeroed when mapped, and
448 * writes to that region are not written out to the file."
449 */
450 if (len < PAGE_CACHE_SIZE)
451 zero_user_segment(page, len, PAGE_CACHE_SIZE);
452 /*
Jan Kara0058f962013-04-11 23:48:32 -0400453 * In the first loop we prepare and mark buffers to submit. We have to
454 * mark all buffers in the page before submitting so that
455 * end_page_writeback() cannot be called from ext4_bio_end_io() when IO
456 * on the first buffer finishes and we are still working on submitting
457 * the second buffer.
458 */
459 bh = head = page_buffers(page);
460 do {
461 block_start = bh_offset(bh);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400462 if (block_start >= len) {
463 clear_buffer_dirty(bh);
464 set_buffer_uptodate(bh);
465 continue;
466 }
Jan Kara8a850c32013-01-28 20:53:28 -0500467 if (!buffer_dirty(bh) || buffer_delay(bh) ||
468 !buffer_mapped(bh) || buffer_unwritten(bh)) {
469 /* A hole? We can safely clear the dirty bit */
470 if (!buffer_mapped(bh))
471 clear_buffer_dirty(bh);
472 if (io->io_bio)
473 ext4_io_submit(io);
474 continue;
475 }
Jan Kara0058f962013-04-11 23:48:32 -0400476 if (buffer_new(bh)) {
477 clear_buffer_new(bh);
478 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
479 }
480 set_buffer_async_write(bh);
481 } while ((bh = bh->b_this_page) != head);
482
Jan Kara0058f962013-04-11 23:48:32 -0400483 bh = head = page_buffers(page);
Michael Halcrow2058f832015-04-12 00:55:10 -0400484
485 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
486 data_page = ext4_encrypt(inode, page);
487 if (IS_ERR(data_page)) {
488 ret = PTR_ERR(data_page);
489 data_page = NULL;
490 goto out;
491 }
492 }
493
494 /* Now submit buffers to write */
Jan Kara0058f962013-04-11 23:48:32 -0400495 do {
496 if (!buffer_async_write(bh))
497 continue;
Michael Halcrow2058f832015-04-12 00:55:10 -0400498 ret = io_submit_add_bh(io, inode,
499 data_page ? data_page : page, bh);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400500 if (ret) {
501 /*
502 * We only get here on ENOMEM. Not much else
503 * we can do but mark the page as dirty, and
504 * better luck next time.
505 */
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400506 break;
507 }
Jan Kara0058f962013-04-11 23:48:32 -0400508 nr_submitted++;
Jan Kara1ae48a62013-01-28 09:32:54 -0500509 clear_buffer_dirty(bh);
Jan Kara0058f962013-04-11 23:48:32 -0400510 } while ((bh = bh->b_this_page) != head);
511
512 /* Error stopped previous loop? Clean up buffers... */
513 if (ret) {
Michael Halcrow2058f832015-04-12 00:55:10 -0400514 out:
515 if (data_page)
516 ext4_restore_control_page(data_page);
517 printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
518 redirty_page_for_writepage(wbc, page);
Jan Kara0058f962013-04-11 23:48:32 -0400519 do {
520 clear_buffer_async_write(bh);
521 bh = bh->b_this_page;
522 } while (bh != head);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400523 }
524 unlock_page(page);
Jan Kara0058f962013-04-11 23:48:32 -0400525 /* Nothing submitted - we have to end page writeback */
526 if (!nr_submitted)
527 end_page_writeback(page);
Theodore Ts'obd2d0212010-10-27 21:30:10 -0400528 return ret;
529}