blob: 3131747199e160f5feca6c322a2b4190201108bf [file] [log] [blame]
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001/*
Mingming Cao617ba132006-10-11 01:20:53 -07002 * linux/fs/ext4/inode.c
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003 *
4 * Copyright (C) 1992, 1993, 1994, 1995
5 * Remy Card (card@masi.ibp.fr)
6 * Laboratoire MASI - Institut Blaise Pascal
7 * Universite Pierre et Marie Curie (Paris VI)
8 *
9 * from
10 *
11 * linux/fs/minix/inode.c
12 *
13 * Copyright (C) 1991, 1992 Linus Torvalds
14 *
Dave Kleikampac27a0e2006-10-11 01:20:50 -070015 * 64-bit file support on 64-bit platforms by Jakub Jelinek
16 * (jj@sunsite.ms.mff.cuni.cz)
17 *
Mingming Cao617ba132006-10-11 01:20:53 -070018 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
Dave Kleikampac27a0e2006-10-11 01:20:50 -070019 */
20
Dave Kleikampac27a0e2006-10-11 01:20:50 -070021#include <linux/fs.h>
22#include <linux/time.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070023#include <linux/highuid.h>
24#include <linux/pagemap.h>
Matthew Wilcoxc94c2ac2015-09-08 14:58:40 -070025#include <linux/dax.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070026#include <linux/quotaops.h>
27#include <linux/string.h>
28#include <linux/buffer_head.h>
29#include <linux/writeback.h>
Alex Tomas64769242008-07-11 19:27:31 -040030#include <linux/pagevec.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070031#include <linux/mpage.h>
Duane Griffine83c1392008-12-19 20:47:15 +000032#include <linux/namei.h>
Dave Kleikampac27a0e2006-10-11 01:20:50 -070033#include <linux/uio.h>
34#include <linux/bio.h>
Mingming Cao4c0425f2009-09-28 15:48:41 -040035#include <linux/workqueue.h>
Jiaying Zhang744692d2010-03-04 16:14:02 -050036#include <linux/kernel.h>
Andrew Morton6db26ff2011-01-12 16:59:13 -080037#include <linux/printk.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090038#include <linux/slab.h>
Theodore Ts'o00a1a052014-03-30 10:20:01 -040039#include <linux/bitops.h>
Theodore Ts'o9bffad12009-06-17 11:48:11 -040040
Christoph Hellwig3dcf5452008-04-29 18:13:32 -040041#include "ext4_jbd2.h"
Dave Kleikampac27a0e2006-10-11 01:20:50 -070042#include "xattr.h"
43#include "acl.h"
Theodore Ts'o9f125d62011-06-27 19:16:04 -040044#include "truncate.h"
Dave Kleikampac27a0e2006-10-11 01:20:50 -070045
Theodore Ts'o9bffad12009-06-17 11:48:11 -040046#include <trace/events/ext4.h>
47
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -040048#define MPAGE_DA_EXTENT_TAIL 0x01
49
Darrick J. Wong814525f2012-04-29 18:31:10 -040050static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
51 struct ext4_inode_info *ei)
52{
53 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Darrick J. Wong814525f2012-04-29 18:31:10 -040054 __u32 csum;
Daeho Jeongb47820e2016-07-03 17:51:39 -040055 __u16 dummy_csum = 0;
56 int offset = offsetof(struct ext4_inode, i_checksum_lo);
57 unsigned int csum_size = sizeof(dummy_csum);
Darrick J. Wong814525f2012-04-29 18:31:10 -040058
Daeho Jeongb47820e2016-07-03 17:51:39 -040059 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
60 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
61 offset += csum_size;
62 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
63 EXT4_GOOD_OLD_INODE_SIZE - offset);
64
65 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
66 offset = offsetof(struct ext4_inode, i_checksum_hi);
67 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
68 EXT4_GOOD_OLD_INODE_SIZE,
69 offset - EXT4_GOOD_OLD_INODE_SIZE);
70 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
71 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
72 csum_size);
73 offset += csum_size;
74 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
75 EXT4_INODE_SIZE(inode->i_sb) -
76 offset);
77 }
Darrick J. Wong814525f2012-04-29 18:31:10 -040078 }
79
Darrick J. Wong814525f2012-04-29 18:31:10 -040080 return csum;
81}
82
83static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
84 struct ext4_inode_info *ei)
85{
86 __u32 provided, calculated;
87
88 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
89 cpu_to_le32(EXT4_OS_LINUX) ||
Dmitry Monakhov9aa5d322014-10-13 03:36:16 -040090 !ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong814525f2012-04-29 18:31:10 -040091 return 1;
92
93 provided = le16_to_cpu(raw->i_checksum_lo);
94 calculated = ext4_inode_csum(inode, raw, ei);
95 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
96 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
97 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
98 else
99 calculated &= 0xFFFF;
100
101 return provided == calculated;
102}
103
104static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
105 struct ext4_inode_info *ei)
106{
107 __u32 csum;
108
109 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
110 cpu_to_le32(EXT4_OS_LINUX) ||
Dmitry Monakhov9aa5d322014-10-13 03:36:16 -0400111 !ext4_has_metadata_csum(inode->i_sb))
Darrick J. Wong814525f2012-04-29 18:31:10 -0400112 return;
113
114 csum = ext4_inode_csum(inode, raw, ei);
115 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
116 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
117 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
118 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
119}
120
Jan Kara678aaf42008-07-11 19:27:31 -0400121static inline int ext4_begin_ordered_truncate(struct inode *inode,
122 loff_t new_size)
123{
Theodore Ts'o7ff9c072010-11-08 13:51:33 -0500124 trace_ext4_begin_ordered_truncate(inode, new_size);
Theodore Ts'o8aefcd52011-01-10 12:29:43 -0500125 /*
126 * If jinode is zero, then we never opened the file for
127 * writing, so there's no need to call
128 * jbd2_journal_begin_ordered_truncate() since there's no
129 * outstanding writes we need to flush.
130 */
131 if (!EXT4_I(inode)->jinode)
132 return 0;
133 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
134 EXT4_I(inode)->jinode,
135 new_size);
Jan Kara678aaf42008-07-11 19:27:31 -0400136}
137
Lukas Czernerd47992f2013-05-21 23:17:23 -0400138static void ext4_invalidatepage(struct page *page, unsigned int offset,
139 unsigned int length);
Theodore Ts'ocb20d512010-10-27 21:30:09 -0400140static int __ext4_journalled_writepage(struct page *page, unsigned int len);
141static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
Jan Karafffb2732013-06-04 13:01:11 -0400142static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
143 int pextents);
Alex Tomas64769242008-07-11 19:27:31 -0400144
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700145/*
146 * Test whether an inode is a fast symlink.
147 */
Theodore Ts'of348c252015-04-16 01:55:00 -0400148int ext4_inode_is_fast_symlink(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700149{
Yongqiang Yang65eddb52014-01-06 14:06:18 -0500150 int ea_blocks = EXT4_I(inode)->i_file_acl ?
151 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700152
Zheng Liubd9db172014-06-02 10:48:22 -0400153 if (ext4_has_inline_data(inode))
154 return 0;
155
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700156 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
157}
158
159/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700160 * Restart the transaction associated with *handle. This does a commit,
161 * so before we call here everything must be consistently dirtied against
162 * this transaction.
163 */
Aneesh Kumar K.Vfa5d1112009-11-02 18:50:49 -0500164int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode,
Jan Kara487caee2009-08-17 22:17:20 -0400165 int nblocks)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700166{
Jan Kara487caee2009-08-17 22:17:20 -0400167 int ret;
168
169 /*
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400170 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
Jan Kara487caee2009-08-17 22:17:20 -0400171 * moment, get_block can be called only for blocks inside i_size since
172 * page cache has been already dropped and writes are blocked by
173 * i_mutex. So we can safely drop the i_data_sem here.
174 */
Frank Mayhar03901312009-01-07 00:06:22 -0500175 BUG_ON(EXT4_JOURNAL(inode) == NULL);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700176 jbd_debug(2, "restarting handle %p\n", handle);
Jan Kara487caee2009-08-17 22:17:20 -0400177 up_write(&EXT4_I(inode)->i_data_sem);
Amir Goldstein8e8eaab2011-02-27 23:32:12 -0500178 ret = ext4_journal_restart(handle, nblocks);
Jan Kara487caee2009-08-17 22:17:20 -0400179 down_write(&EXT4_I(inode)->i_data_sem);
Aneesh Kumar K.Vfa5d1112009-11-02 18:50:49 -0500180 ext4_discard_preallocations(inode);
Jan Kara487caee2009-08-17 22:17:20 -0400181
182 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700183}
184
185/*
186 * Called at the last iput() if i_nlink is zero.
187 */
Al Viro0930fcc2010-06-07 13:16:22 -0400188void ext4_evict_inode(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700189{
190 handle_t *handle;
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400191 int err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700192
Theodore Ts'o7ff9c072010-11-08 13:51:33 -0500193 trace_ext4_evict_inode(inode);
Jiaying Zhang2581fdc2011-08-13 12:17:13 -0400194
Al Viro0930fcc2010-06-07 13:16:22 -0400195 if (inode->i_nlink) {
Jan Kara2d859db2011-07-26 09:07:11 -0400196 /*
197 * When journalling data dirty buffers are tracked only in the
198 * journal. So although mm thinks everything is clean and
199 * ready for reaping the inode might still have some pages to
200 * write in the running transaction or waiting to be
201 * checkpointed. Thus calling jbd2_journal_invalidatepage()
202 * (via truncate_inode_pages()) to discard these buffers can
203 * cause data loss. Also even if we did not discard these
204 * buffers, we would have no way to find them after the inode
205 * is reaped and thus user could see stale data if he tries to
206 * read them before the transaction is checkpointed. So be
207 * careful and force everything to disk here... We use
208 * ei->i_datasync_tid to store the newest transaction
209 * containing inode's data.
210 *
211 * Note that directories do not have this problem because they
212 * don't use page cache.
213 */
Vegard Nossum6a7fd522016-07-04 11:03:00 -0400214 if (inode->i_ino != EXT4_JOURNAL_INO &&
215 ext4_should_journal_data(inode) &&
216 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode))) {
Jan Kara2d859db2011-07-26 09:07:11 -0400217 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
218 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
219
Theodore Ts'od76a3a772013-04-03 22:02:52 -0400220 jbd2_complete_transaction(journal, commit_tid);
Jan Kara2d859db2011-07-26 09:07:11 -0400221 filemap_write_and_wait(&inode->i_data);
222 }
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700223 truncate_inode_pages_final(&inode->i_data);
Jan Kara5dc23bd2013-06-04 14:46:12 -0400224
Al Viro0930fcc2010-06-07 13:16:22 -0400225 goto no_delete;
226 }
227
Theodore Ts'oe2bfb082014-10-05 22:47:07 -0400228 if (is_bad_inode(inode))
229 goto no_delete;
230 dquot_initialize(inode);
Christoph Hellwig907f4552010-03-03 09:05:06 -0500231
Jan Kara678aaf42008-07-11 19:27:31 -0400232 if (ext4_should_order_data(inode))
233 ext4_begin_ordered_truncate(inode, 0);
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700234 truncate_inode_pages_final(&inode->i_data);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700235
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200236 /*
237 * Protect us against freezing - iput() caller didn't have to have any
238 * protection against it
239 */
240 sb_start_intwrite(inode->i_sb);
Theodore Ts'o9924a922013-02-08 21:59:22 -0500241 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
242 ext4_blocks_for_truncate(inode)+3);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700243 if (IS_ERR(handle)) {
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400244 ext4_std_error(inode->i_sb, PTR_ERR(handle));
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700245 /*
246 * If we're going to skip the normal cleanup, we still need to
247 * make sure that the in-core orphan linked list is properly
248 * cleaned up.
249 */
Mingming Cao617ba132006-10-11 01:20:53 -0700250 ext4_orphan_del(NULL, inode);
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200251 sb_end_intwrite(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700252 goto no_delete;
253 }
254
255 if (IS_SYNC(inode))
Frank Mayhar03901312009-01-07 00:06:22 -0500256 ext4_handle_sync(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700257 inode->i_size = 0;
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400258 err = ext4_mark_inode_dirty(handle, inode);
259 if (err) {
Eric Sandeen12062dd2010-02-15 14:19:27 -0500260 ext4_warning(inode->i_sb,
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400261 "couldn't mark inode dirty (err %d)", err);
262 goto stop_handle;
263 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700264 if (inode->i_blocks)
Mingming Cao617ba132006-10-11 01:20:53 -0700265 ext4_truncate(inode);
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400266
267 /*
268 * ext4_ext_truncate() doesn't reserve any slop when it
269 * restarts journal transactions; therefore there may not be
270 * enough credits left in the handle to remove the inode from
271 * the orphan list and set the dtime field.
272 */
Frank Mayhar03901312009-01-07 00:06:22 -0500273 if (!ext4_handle_has_enough_credits(handle, 3)) {
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400274 err = ext4_journal_extend(handle, 3);
275 if (err > 0)
276 err = ext4_journal_restart(handle, 3);
277 if (err != 0) {
Eric Sandeen12062dd2010-02-15 14:19:27 -0500278 ext4_warning(inode->i_sb,
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400279 "couldn't extend journal (err %d)", err);
280 stop_handle:
281 ext4_journal_stop(handle);
Theodore Ts'o45388212010-07-29 15:06:10 -0400282 ext4_orphan_del(NULL, inode);
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200283 sb_end_intwrite(inode->i_sb);
Theodore Ts'obc965ab2008-08-02 21:10:38 -0400284 goto no_delete;
285 }
286 }
287
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700288 /*
Mingming Cao617ba132006-10-11 01:20:53 -0700289 * Kill off the orphan record which ext4_truncate created.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700290 * AKPM: I think this can be inside the above `if'.
Mingming Cao617ba132006-10-11 01:20:53 -0700291 * Note that ext4_orphan_del() has to be able to cope with the
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700292 * deletion of a non-existent orphan - this is because we don't
Mingming Cao617ba132006-10-11 01:20:53 -0700293 * know if ext4_truncate() actually created an orphan record.
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700294 * (Well, we could do this if we need to, but heck - it works)
295 */
Mingming Cao617ba132006-10-11 01:20:53 -0700296 ext4_orphan_del(handle, inode);
297 EXT4_I(inode)->i_dtime = get_seconds();
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700298
299 /*
300 * One subtle ordering requirement: if anything has gone wrong
301 * (transaction abort, IO errors, whatever), then we can still
302 * do these next steps (the fs will already have been marked as
303 * having errors), but we can't free the inode if the mark_dirty
304 * fails.
305 */
Mingming Cao617ba132006-10-11 01:20:53 -0700306 if (ext4_mark_inode_dirty(handle, inode))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700307 /* If that failed, just do the required in-core inode clear. */
Al Viro0930fcc2010-06-07 13:16:22 -0400308 ext4_clear_inode(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700309 else
Mingming Cao617ba132006-10-11 01:20:53 -0700310 ext4_free_inode(handle, inode);
311 ext4_journal_stop(handle);
Jan Kara8e8ad8a2012-06-12 16:20:38 +0200312 sb_end_intwrite(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700313 return;
314no_delete:
Al Viro0930fcc2010-06-07 13:16:22 -0400315 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700316}
317
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300318#ifdef CONFIG_QUOTA
319qsize_t *ext4_get_reserved_space(struct inode *inode)
Mingming Cao60e58e02009-01-22 18:13:05 +0100320{
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300321 return &EXT4_I(inode)->i_reserved_quota;
Mingming Cao60e58e02009-01-22 18:13:05 +0100322}
Dmitry Monakhova9e7f442009-12-14 15:21:14 +0300323#endif
Theodore Ts'o9d0be502010-01-01 02:41:30 -0500324
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400325/*
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500326 * Called with i_data_sem down, which is important since we can call
327 * ext4_discard_preallocations() from here.
328 */
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500329void ext4_da_update_reserve_space(struct inode *inode,
330 int used, int quota_claim)
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400331{
332 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500333 struct ext4_inode_info *ei = EXT4_I(inode);
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400334
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500335 spin_lock(&ei->i_block_reservation_lock);
Aditya Kalid8990242011-09-09 19:18:51 -0400336 trace_ext4_da_update_reserve_space(inode, used, quota_claim);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500337 if (unlikely(used > ei->i_reserved_data_blocks)) {
Theodore Ts'o8de5c322013-02-14 15:11:41 -0500338 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
Theodore Ts'o1084f252012-03-19 23:13:43 -0400339 "with only %d reserved data blocks",
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500340 __func__, inode->i_ino, used,
341 ei->i_reserved_data_blocks);
342 WARN_ON(1);
343 used = ei->i_reserved_data_blocks;
Aneesh Kumar K.V6bc6e632008-10-10 09:39:00 -0400344 }
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400345
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500346 /* Update per-inode reservations */
347 ei->i_reserved_data_blocks -= used;
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -0400348 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500349
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400350 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Cao60e58e02009-01-22 18:13:05 +0100351
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400352 /* Update quota subsystem for data blocks */
353 if (quota_claim)
Aditya Kali7b415bf2011-09-09 19:04:51 -0400354 dquot_claim_block(inode, EXT4_C2B(sbi, used));
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400355 else {
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500356 /*
357 * We did fallocate with an offset that is already delayed
358 * allocated. So on delayed allocated writeback we should
Eric Sandeen72b8ab92010-05-16 11:00:00 -0400359 * not re-claim the quota for fallocated blocks.
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500360 */
Aditya Kali7b415bf2011-09-09 19:04:51 -0400361 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500362 }
Aneesh Kumar K.Vd6014302009-03-27 22:36:43 -0400363
364 /*
365 * If we have done all the pending block allocations and if
366 * there aren't any writers on the inode, we can discard the
367 * inode's preallocations.
368 */
Theodore Ts'o0637c6f2009-12-30 14:20:45 -0500369 if ((ei->i_reserved_data_blocks == 0) &&
370 (atomic_read(&inode->i_writecount) == 0))
Aneesh Kumar K.Vd6014302009-03-27 22:36:43 -0400371 ext4_discard_preallocations(inode);
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -0400372}
373
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400374static int __check_block_validity(struct inode *inode, const char *func,
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400375 unsigned int line,
376 struct ext4_map_blocks *map)
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400377{
Theodore Ts'o24676da2010-05-16 21:00:00 -0400378 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk,
379 map->m_len)) {
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400380 ext4_error_inode(inode, func, line, map->m_pblk,
381 "lblock %lu mapped to illegal pblock "
382 "(length %d)", (unsigned long) map->m_lblk,
383 map->m_len);
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400384 return -EFSCORRUPTED;
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400385 }
386 return 0;
387}
388
Jan Kara53085fa2015-12-07 15:09:35 -0500389int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
390 ext4_lblk_t len)
391{
392 int ret;
393
394 if (ext4_encrypted_inode(inode))
Jaegeuk Kima7550b32016-07-10 14:01:03 -0400395 return fscrypt_zeroout_range(inode, lblk, pblk, len);
Jan Kara53085fa2015-12-07 15:09:35 -0500396
397 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
398 if (ret > 0)
399 ret = 0;
400
401 return ret;
402}
403
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400404#define check_block_validity(inode, map) \
Theodore Ts'oc398eda2010-07-27 11:56:40 -0400405 __check_block_validity((inode), __func__, __LINE__, (map))
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400406
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400407#ifdef ES_AGGRESSIVE_TEST
408static void ext4_map_blocks_es_recheck(handle_t *handle,
409 struct inode *inode,
410 struct ext4_map_blocks *es_map,
411 struct ext4_map_blocks *map,
412 int flags)
413{
414 int retval;
415
416 map->m_flags = 0;
417 /*
418 * There is a race window that the result is not the same.
419 * e.g. xfstests #223 when dioread_nolock enables. The reason
420 * is that we lookup a block mapping in extent status tree with
421 * out taking i_data_sem. So at the time the unwritten extent
422 * could be converted.
423 */
Jan Kara2dcba472015-12-07 15:04:57 -0500424 down_read(&EXT4_I(inode)->i_data_sem);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400425 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
426 retval = ext4_ext_map_blocks(handle, inode, map, flags &
427 EXT4_GET_BLOCKS_KEEP_SIZE);
428 } else {
429 retval = ext4_ind_map_blocks(handle, inode, map, flags &
430 EXT4_GET_BLOCKS_KEEP_SIZE);
431 }
Jan Kara2dcba472015-12-07 15:04:57 -0500432 up_read((&EXT4_I(inode)->i_data_sem));
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400433
434 /*
435 * We don't check m_len because extent will be collpased in status
436 * tree. So the m_len might not equal.
437 */
438 if (es_map->m_lblk != map->m_lblk ||
439 es_map->m_flags != map->m_flags ||
440 es_map->m_pblk != map->m_pblk) {
Theodore Ts'obdafe422013-07-13 00:40:31 -0400441 printk("ES cache assertion failed for inode: %lu "
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400442 "es_cached ex [%d/%d/%llu/%x] != "
443 "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
444 inode->i_ino, es_map->m_lblk, es_map->m_len,
445 es_map->m_pblk, es_map->m_flags, map->m_lblk,
446 map->m_len, map->m_pblk, map->m_flags,
447 retval, flags);
448 }
449}
450#endif /* ES_AGGRESSIVE_TEST */
451
Theodore Ts'o55138e02009-09-29 13:31:31 -0400452/*
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400453 * The ext4_map_blocks() function tries to look up the requested blocks,
Theodore Ts'o2b2d6d02008-07-26 16:15:44 -0400454 * and returns if the blocks are already mapped.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500455 *
Mingming Caof5ab0d12008-02-25 15:29:55 -0500456 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
457 * and store the allocated blocks in the result buffer head and mark it
458 * mapped.
459 *
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400460 * If file type is extents based, it will call ext4_ext_map_blocks(),
461 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
Mingming Caof5ab0d12008-02-25 15:29:55 -0500462 * based files
463 *
Jan Karafacab4d2016-03-09 22:54:00 -0500464 * On success, it returns the number of blocks being mapped or allocated. if
465 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
466 * is marked as unwritten. If the create == 1, it will mark @map as mapped.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500467 *
468 * It returns 0 if plain look up failed (blocks have not been allocated), in
Jan Karafacab4d2016-03-09 22:54:00 -0500469 * that case, @map is returned as unmapped but we still do fill map->m_len to
470 * indicate the length of a hole starting at map->m_lblk.
Mingming Caof5ab0d12008-02-25 15:29:55 -0500471 *
472 * It returns the error in case of allocation failure.
473 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400474int ext4_map_blocks(handle_t *handle, struct inode *inode,
475 struct ext4_map_blocks *map, int flags)
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500476{
Zheng Liud100eef2013-02-18 00:29:59 -0500477 struct extent_status es;
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500478 int retval;
Lukas Czernerb8a86842014-03-18 18:05:35 -0400479 int ret = 0;
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400480#ifdef ES_AGGRESSIVE_TEST
481 struct ext4_map_blocks orig_map;
482
483 memcpy(&orig_map, map, sizeof(*map));
484#endif
Mingming Caof5ab0d12008-02-25 15:29:55 -0500485
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400486 map->m_flags = 0;
487 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u,"
488 "logical block %lu\n", inode->i_ino, flags, map->m_len,
489 (unsigned long) map->m_lblk);
Zheng Liud100eef2013-02-18 00:29:59 -0500490
Theodore Ts'oe861b5e2014-02-20 12:54:05 -0500491 /*
492 * ext4_map_blocks returns an int, and m_len is an unsigned int
493 */
494 if (unlikely(map->m_len > INT_MAX))
495 map->m_len = INT_MAX;
496
Kazuya Mio4adb6ab2014-04-07 10:53:28 -0400497 /* We can handle the block number less than EXT_MAX_BLOCKS */
498 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
Darrick J. Wong6a797d22015-10-17 16:16:04 -0400499 return -EFSCORRUPTED;
Kazuya Mio4adb6ab2014-04-07 10:53:28 -0400500
Zheng Liud100eef2013-02-18 00:29:59 -0500501 /* Lookup extent status tree firstly */
502 if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
503 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
504 map->m_pblk = ext4_es_pblock(&es) +
505 map->m_lblk - es.es_lblk;
506 map->m_flags |= ext4_es_is_written(&es) ?
507 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
508 retval = es.es_len - (map->m_lblk - es.es_lblk);
509 if (retval > map->m_len)
510 retval = map->m_len;
511 map->m_len = retval;
512 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
Jan Karafacab4d2016-03-09 22:54:00 -0500513 map->m_pblk = 0;
514 retval = es.es_len - (map->m_lblk - es.es_lblk);
515 if (retval > map->m_len)
516 retval = map->m_len;
517 map->m_len = retval;
Zheng Liud100eef2013-02-18 00:29:59 -0500518 retval = 0;
519 } else {
520 BUG_ON(1);
521 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400522#ifdef ES_AGGRESSIVE_TEST
523 ext4_map_blocks_es_recheck(handle, inode, map,
524 &orig_map, flags);
525#endif
Zheng Liud100eef2013-02-18 00:29:59 -0500526 goto found;
527 }
528
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500529 /*
Theodore Ts'ob920c752009-05-14 00:54:29 -0400530 * Try to see if we can get the block without requesting a new
531 * file system block.
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500532 */
Jan Kara2dcba472015-12-07 15:04:57 -0500533 down_read(&EXT4_I(inode)->i_data_sem);
Dmitry Monakhov12e9b892010-05-16 22:00:00 -0400534 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
Dmitry Monakhova4e5d882011-10-25 08:15:12 -0400535 retval = ext4_ext_map_blocks(handle, inode, map, flags &
536 EXT4_GET_BLOCKS_KEEP_SIZE);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500537 } else {
Dmitry Monakhova4e5d882011-10-25 08:15:12 -0400538 retval = ext4_ind_map_blocks(handle, inode, map, flags &
539 EXT4_GET_BLOCKS_KEEP_SIZE);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500540 }
Zheng Liuf7fec032013-02-18 00:28:47 -0500541 if (retval > 0) {
Theodore Ts'o3be78c72013-08-16 21:22:41 -0400542 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -0500543
Zheng Liu44fb851d2013-07-29 12:51:42 -0400544 if (unlikely(retval != map->m_len)) {
545 ext4_warning(inode->i_sb,
546 "ES len assertion failed for inode "
547 "%lu: retval %d != map->m_len %d",
548 inode->i_ino, retval, map->m_len);
549 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400550 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400551
Zheng Liuf7fec032013-02-18 00:28:47 -0500552 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
553 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
554 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
Lukas Czernerd2dc3172015-05-02 21:36:55 -0400555 !(status & EXTENT_STATUS_WRITTEN) &&
Zheng Liuf7fec032013-02-18 00:28:47 -0500556 ext4_find_delalloc_range(inode, map->m_lblk,
557 map->m_lblk + map->m_len - 1))
558 status |= EXTENT_STATUS_DELAYED;
559 ret = ext4_es_insert_extent(inode, map->m_lblk,
560 map->m_len, map->m_pblk, status);
561 if (ret < 0)
562 retval = ret;
563 }
Jan Kara2dcba472015-12-07 15:04:57 -0500564 up_read((&EXT4_I(inode)->i_data_sem));
Mingming Caof5ab0d12008-02-25 15:29:55 -0500565
Zheng Liud100eef2013-02-18 00:29:59 -0500566found:
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400567 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
Lukas Czernerb8a86842014-03-18 18:05:35 -0400568 ret = check_block_validity(inode, map);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400569 if (ret != 0)
570 return ret;
571 }
572
Mingming Caof5ab0d12008-02-25 15:29:55 -0500573 /* If it is only a block(s) look up */
Theodore Ts'oc2177052009-05-14 00:58:52 -0400574 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500575 return retval;
576
577 /*
Mingming Caof5ab0d12008-02-25 15:29:55 -0500578 * Returns if the blocks have already allocated
579 *
580 * Note that if blocks have been preallocated
Tao Madf3ab172011-10-08 15:53:49 -0400581 * ext4_ext_get_block() returns the create = 0
Mingming Caof5ab0d12008-02-25 15:29:55 -0500582 * with buffer head unmapped.
583 */
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400584 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
Lukas Czernerb8a86842014-03-18 18:05:35 -0400585 /*
586 * If we need to convert extent to unwritten
587 * we continue and do the actual work in
588 * ext4_ext_map_blocks()
589 */
590 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
591 return retval;
Mingming Caof5ab0d12008-02-25 15:29:55 -0500592
593 /*
Zheng Liua25a4e12013-02-18 00:28:04 -0500594 * Here we clear m_flags because after allocating an new extent,
595 * it will be set again.
Aneesh Kumar K.V2a8964d2009-05-14 17:05:39 -0400596 */
Zheng Liua25a4e12013-02-18 00:28:04 -0500597 map->m_flags &= ~EXT4_MAP_FLAGS;
Aneesh Kumar K.V2a8964d2009-05-14 17:05:39 -0400598
599 /*
Lukas Czerner556615d2014-04-20 23:45:47 -0400600 * New blocks allocate and/or writing to unwritten extent
Mingming Caof5ab0d12008-02-25 15:29:55 -0500601 * will possibly result in updating i_data, so we take
Seunghun Leed91bd2c2014-09-01 22:15:30 -0400602 * the write lock of i_data_sem, and call get_block()
Mingming Caof5ab0d12008-02-25 15:29:55 -0500603 * with create == 1 flag.
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500604 */
Lukas Czernerc8b459f2014-05-12 12:55:07 -0400605 down_write(&EXT4_I(inode)->i_data_sem);
Mingming Caod2a17632008-07-14 17:52:37 -0400606
607 /*
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500608 * We need to check for EXT4 here because migrate
609 * could have changed the inode type in between
610 */
Dmitry Monakhov12e9b892010-05-16 22:00:00 -0400611 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400612 retval = ext4_ext_map_blocks(handle, inode, map, flags);
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500613 } else {
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400614 retval = ext4_ind_map_blocks(handle, inode, map, flags);
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400615
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400616 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400617 /*
618 * We allocated new blocks which will result in
619 * i_data's format changing. Force the migrate
620 * to fail by clearing migrate flags
621 */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -0500622 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
Aneesh Kumar K.V267e4db2008-04-29 08:11:12 -0400623 }
Mingming Caod2a17632008-07-14 17:52:37 -0400624
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500625 /*
626 * Update reserved blocks/metadata blocks after successful
627 * block allocation which had been deferred till now. We don't
628 * support fallocate for non extent files. So we can update
629 * reserve space here.
630 */
631 if ((retval > 0) &&
Aneesh Kumar K.V1296cc82010-01-15 01:27:59 -0500632 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
Aneesh Kumar K.V5f634d02010-01-25 04:00:31 -0500633 ext4_da_update_reserve_space(inode, retval, 1);
634 }
Theodore Ts'o2ac3b6e2009-05-14 13:57:08 -0400635
Zheng Liuf7fec032013-02-18 00:28:47 -0500636 if (retval > 0) {
Theodore Ts'o3be78c72013-08-16 21:22:41 -0400637 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -0500638
Zheng Liu44fb851d2013-07-29 12:51:42 -0400639 if (unlikely(retval != map->m_len)) {
640 ext4_warning(inode->i_sb,
641 "ES len assertion failed for inode "
642 "%lu: retval %d != map->m_len %d",
643 inode->i_ino, retval, map->m_len);
644 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400645 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -0400646
Zheng Liuadb23552013-03-10 21:13:05 -0400647 /*
Jan Karac86d8db2015-12-07 15:10:26 -0500648 * We have to zeroout blocks before inserting them into extent
649 * status tree. Otherwise someone could look them up there and
650 * use them before they are really zeroed.
651 */
652 if (flags & EXT4_GET_BLOCKS_ZERO &&
653 map->m_flags & EXT4_MAP_MAPPED &&
654 map->m_flags & EXT4_MAP_NEW) {
655 ret = ext4_issue_zeroout(inode, map->m_lblk,
656 map->m_pblk, map->m_len);
657 if (ret) {
658 retval = ret;
659 goto out_sem;
660 }
661 }
662
663 /*
Zheng Liuadb23552013-03-10 21:13:05 -0400664 * If the extent has been zeroed out, we don't need to update
665 * extent status tree.
666 */
667 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
668 ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
669 if (ext4_es_is_written(&es))
Jan Karac86d8db2015-12-07 15:10:26 -0500670 goto out_sem;
Zheng Liuadb23552013-03-10 21:13:05 -0400671 }
Zheng Liuf7fec032013-02-18 00:28:47 -0500672 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
673 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
674 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
Lukas Czernerd2dc3172015-05-02 21:36:55 -0400675 !(status & EXTENT_STATUS_WRITTEN) &&
Zheng Liuf7fec032013-02-18 00:28:47 -0500676 ext4_find_delalloc_range(inode, map->m_lblk,
677 map->m_lblk + map->m_len - 1))
678 status |= EXTENT_STATUS_DELAYED;
679 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
680 map->m_pblk, status);
Jan Karac86d8db2015-12-07 15:10:26 -0500681 if (ret < 0) {
Zheng Liuf7fec032013-02-18 00:28:47 -0500682 retval = ret;
Jan Karac86d8db2015-12-07 15:10:26 -0500683 goto out_sem;
684 }
Aditya Kali5356f262011-09-09 19:20:51 -0400685 }
686
Jan Karac86d8db2015-12-07 15:10:26 -0500687out_sem:
Aneesh Kumar K.V4df3d262008-01-28 23:58:29 -0500688 up_write((&EXT4_I(inode)->i_data_sem));
Theodore Ts'oe35fd662010-05-16 19:00:00 -0400689 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
Lukas Czernerb8a86842014-03-18 18:05:35 -0400690 ret = check_block_validity(inode, map);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400691 if (ret != 0)
692 return ret;
Jan Kara06bd3c32016-04-24 00:56:03 -0400693
694 /*
695 * Inodes with freshly allocated blocks where contents will be
696 * visible after transaction commit must be on transaction's
697 * ordered data list.
698 */
699 if (map->m_flags & EXT4_MAP_NEW &&
700 !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
701 !(flags & EXT4_GET_BLOCKS_ZERO) &&
702 !IS_NOQUOTA(inode) &&
703 ext4_should_order_data(inode)) {
Jan Karaee0876b2016-04-24 00:56:08 -0400704 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
705 ret = ext4_jbd2_inode_add_wait(handle, inode);
706 else
707 ret = ext4_jbd2_inode_add_write(handle, inode);
Jan Kara06bd3c32016-04-24 00:56:03 -0400708 if (ret)
709 return ret;
710 }
Theodore Ts'o6fd058f2009-05-17 15:38:01 -0400711 }
Aneesh Kumar K.V0e855ac2008-01-28 23:58:26 -0500712 return retval;
713}
714
Jan Karaed8ad832016-02-19 00:18:25 -0500715/*
716 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
717 * we have to be careful as someone else may be manipulating b_state as well.
718 */
719static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
720{
721 unsigned long old_state;
722 unsigned long new_state;
723
724 flags &= EXT4_MAP_FLAGS;
725
726 /* Dummy buffer_head? Set non-atomically. */
727 if (!bh->b_page) {
728 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
729 return;
730 }
731 /*
732 * Someone else may be modifying b_state. Be careful! This is ugly but
733 * once we get rid of using bh as a container for mapping information
734 * to pass to / from get_block functions, this can go away.
735 */
736 do {
737 old_state = READ_ONCE(bh->b_state);
738 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
739 } while (unlikely(
740 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
741}
742
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400743static int _ext4_get_block(struct inode *inode, sector_t iblock,
744 struct buffer_head *bh, int flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700745{
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400746 struct ext4_map_blocks map;
Jan Karaefe70c22016-03-08 23:35:46 -0500747 int ret = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700748
Tao Ma46c7f252012-12-10 14:04:52 -0500749 if (ext4_has_inline_data(inode))
750 return -ERANGE;
751
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400752 map.m_lblk = iblock;
753 map.m_len = bh->b_size >> inode->i_blkbits;
754
Jan Karaefe70c22016-03-08 23:35:46 -0500755 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
756 flags);
Jan Kara7fb54092008-02-10 01:08:38 -0500757 if (ret > 0) {
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400758 map_bh(bh, inode->i_sb, map.m_pblk);
Jan Karaed8ad832016-02-19 00:18:25 -0500759 ext4_update_bh_state(bh, map.m_flags);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400760 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
Jan Kara7fb54092008-02-10 01:08:38 -0500761 ret = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700762 }
763 return ret;
764}
765
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400766int ext4_get_block(struct inode *inode, sector_t iblock,
767 struct buffer_head *bh, int create)
768{
769 return _ext4_get_block(inode, iblock, bh,
770 create ? EXT4_GET_BLOCKS_CREATE : 0);
771}
772
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700773/*
Jan Kara705965b2016-03-08 23:08:10 -0500774 * Get block function used when preparing for buffered write if we require
775 * creating an unwritten extent if blocks haven't been allocated. The extent
776 * will be converted to written after the IO is complete.
777 */
778int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
779 struct buffer_head *bh_result, int create)
780{
781 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
782 inode->i_ino, create);
783 return _ext4_get_block(inode, iblock, bh_result,
784 EXT4_GET_BLOCKS_IO_CREATE_EXT);
785}
786
Jan Karaefe70c22016-03-08 23:35:46 -0500787/* Maximum number of blocks we map for direct IO at once. */
788#define DIO_MAX_BLOCKS 4096
789
Jan Karae84dfbe2016-04-01 02:07:22 -0400790/*
791 * Get blocks function for the cases that need to start a transaction -
792 * generally difference cases of direct IO and DAX IO. It also handles retries
793 * in case of ENOSPC.
794 */
795static int ext4_get_block_trans(struct inode *inode, sector_t iblock,
796 struct buffer_head *bh_result, int flags)
Jan Karaefe70c22016-03-08 23:35:46 -0500797{
798 int dio_credits;
Jan Karae84dfbe2016-04-01 02:07:22 -0400799 handle_t *handle;
800 int retries = 0;
801 int ret;
Jan Karaefe70c22016-03-08 23:35:46 -0500802
803 /* Trim mapping request to maximum we can map at once for DIO */
804 if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS)
805 bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits;
806 dio_credits = ext4_chunk_trans_blocks(inode,
807 bh_result->b_size >> inode->i_blkbits);
Jan Karae84dfbe2016-04-01 02:07:22 -0400808retry:
809 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
810 if (IS_ERR(handle))
811 return PTR_ERR(handle);
812
813 ret = _ext4_get_block(inode, iblock, bh_result, flags);
814 ext4_journal_stop(handle);
815
816 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
817 goto retry;
818 return ret;
Jan Karaefe70c22016-03-08 23:35:46 -0500819}
820
Jan Kara705965b2016-03-08 23:08:10 -0500821/* Get block function for DIO reads and writes to inodes without extents */
822int ext4_dio_get_block(struct inode *inode, sector_t iblock,
823 struct buffer_head *bh, int create)
824{
Jan Karaefe70c22016-03-08 23:35:46 -0500825 /* We don't expect handle for direct IO */
826 WARN_ON_ONCE(ext4_journal_current_handle());
827
Jan Karae84dfbe2016-04-01 02:07:22 -0400828 if (!create)
829 return _ext4_get_block(inode, iblock, bh, 0);
830 return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE);
Jan Kara705965b2016-03-08 23:08:10 -0500831}
832
833/*
Jan Kara109811c2016-03-08 23:36:46 -0500834 * Get block function for AIO DIO writes when we create unwritten extent if
Jan Kara705965b2016-03-08 23:08:10 -0500835 * blocks are not allocated yet. The extent will be converted to written
836 * after IO is complete.
837 */
Jan Kara109811c2016-03-08 23:36:46 -0500838static int ext4_dio_get_block_unwritten_async(struct inode *inode,
839 sector_t iblock, struct buffer_head *bh_result, int create)
Jan Kara705965b2016-03-08 23:08:10 -0500840{
Jan Karaefe70c22016-03-08 23:35:46 -0500841 int ret;
842
Jan Karaefe70c22016-03-08 23:35:46 -0500843 /* We don't expect handle for direct IO */
844 WARN_ON_ONCE(ext4_journal_current_handle());
845
Jan Karae84dfbe2016-04-01 02:07:22 -0400846 ret = ext4_get_block_trans(inode, iblock, bh_result,
847 EXT4_GET_BLOCKS_IO_CREATE_EXT);
Jan Karaefe70c22016-03-08 23:35:46 -0500848
Jan Kara109811c2016-03-08 23:36:46 -0500849 /*
850 * When doing DIO using unwritten extents, we need io_end to convert
851 * unwritten extents to written on IO completion. We allocate io_end
852 * once we spot unwritten extent and store it in b_private. Generic
853 * DIO code keeps b_private set and furthermore passes the value to
854 * our completion callback in 'private' argument.
855 */
856 if (!ret && buffer_unwritten(bh_result)) {
857 if (!bh_result->b_private) {
858 ext4_io_end_t *io_end;
859
860 io_end = ext4_init_io_end(inode, GFP_KERNEL);
861 if (!io_end)
862 return -ENOMEM;
863 bh_result->b_private = io_end;
864 ext4_set_io_unwritten_flag(inode, io_end);
865 }
Jan Karaefe70c22016-03-08 23:35:46 -0500866 set_buffer_defer_completion(bh_result);
Jan Karaefe70c22016-03-08 23:35:46 -0500867 }
868
869 return ret;
Jan Kara705965b2016-03-08 23:08:10 -0500870}
871
Jan Kara109811c2016-03-08 23:36:46 -0500872/*
873 * Get block function for non-AIO DIO writes when we create unwritten extent if
874 * blocks are not allocated yet. The extent will be converted to written
875 * after IO is complete from ext4_ext_direct_IO() function.
876 */
877static int ext4_dio_get_block_unwritten_sync(struct inode *inode,
878 sector_t iblock, struct buffer_head *bh_result, int create)
879{
Jan Kara109811c2016-03-08 23:36:46 -0500880 int ret;
881
882 /* We don't expect handle for direct IO */
883 WARN_ON_ONCE(ext4_journal_current_handle());
884
Jan Karae84dfbe2016-04-01 02:07:22 -0400885 ret = ext4_get_block_trans(inode, iblock, bh_result,
886 EXT4_GET_BLOCKS_IO_CREATE_EXT);
Jan Kara109811c2016-03-08 23:36:46 -0500887
888 /*
889 * Mark inode as having pending DIO writes to unwritten extents.
890 * ext4_ext_direct_IO() checks this flag and converts extents to
891 * written.
892 */
893 if (!ret && buffer_unwritten(bh_result))
894 ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
895
896 return ret;
897}
898
Jan Kara705965b2016-03-08 23:08:10 -0500899static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock,
900 struct buffer_head *bh_result, int create)
901{
902 int ret;
903
904 ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n",
905 inode->i_ino, create);
Jan Karaefe70c22016-03-08 23:35:46 -0500906 /* We don't expect handle for direct IO */
907 WARN_ON_ONCE(ext4_journal_current_handle());
908
Jan Kara705965b2016-03-08 23:08:10 -0500909 ret = _ext4_get_block(inode, iblock, bh_result, 0);
910 /*
911 * Blocks should have been preallocated! ext4_file_write_iter() checks
912 * that.
913 */
Jan Karaefe70c22016-03-08 23:35:46 -0500914 WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result));
Jan Kara705965b2016-03-08 23:08:10 -0500915
916 return ret;
917}
918
919
920/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700921 * `handle' can be NULL if create is zero
922 */
Mingming Cao617ba132006-10-11 01:20:53 -0700923struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400924 ext4_lblk_t block, int map_flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700925{
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400926 struct ext4_map_blocks map;
927 struct buffer_head *bh;
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400928 int create = map_flags & EXT4_GET_BLOCKS_CREATE;
Theodore Ts'o10560082014-08-29 20:51:32 -0400929 int err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700930
931 J_ASSERT(handle != NULL || create == 0);
932
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400933 map.m_lblk = block;
934 map.m_len = 1;
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400935 err = ext4_map_blocks(handle, inode, &map, map_flags);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700936
Theodore Ts'o10560082014-08-29 20:51:32 -0400937 if (err == 0)
938 return create ? ERR_PTR(-ENOSPC) : NULL;
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400939 if (err < 0)
Theodore Ts'o10560082014-08-29 20:51:32 -0400940 return ERR_PTR(err);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400941
942 bh = sb_getblk(inode->i_sb, map.m_pblk);
Theodore Ts'o10560082014-08-29 20:51:32 -0400943 if (unlikely(!bh))
944 return ERR_PTR(-ENOMEM);
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400945 if (map.m_flags & EXT4_MAP_NEW) {
946 J_ASSERT(create != 0);
947 J_ASSERT(handle != NULL);
948
949 /*
950 * Now that we do not always journal data, we should
951 * keep in mind whether this should always journal the
952 * new buffer as metadata. For now, regular file
953 * writes use ext4_get_block instead, so it's not a
954 * problem.
955 */
956 lock_buffer(bh);
957 BUFFER_TRACE(bh, "call get_create_access");
Theodore Ts'o10560082014-08-29 20:51:32 -0400958 err = ext4_journal_get_create_access(handle, bh);
959 if (unlikely(err)) {
960 unlock_buffer(bh);
961 goto errout;
962 }
963 if (!buffer_uptodate(bh)) {
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400964 memset(bh->b_data, 0, inode->i_sb->s_blocksize);
965 set_buffer_uptodate(bh);
966 }
967 unlock_buffer(bh);
968 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
969 err = ext4_handle_dirty_metadata(handle, inode, bh);
Theodore Ts'o10560082014-08-29 20:51:32 -0400970 if (unlikely(err))
971 goto errout;
972 } else
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400973 BUFFER_TRACE(bh, "not a new buffer");
Theodore Ts'o2ed88682010-05-16 20:00:00 -0400974 return bh;
Theodore Ts'o10560082014-08-29 20:51:32 -0400975errout:
976 brelse(bh);
977 return ERR_PTR(err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700978}
979
Mingming Cao617ba132006-10-11 01:20:53 -0700980struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400981 ext4_lblk_t block, int map_flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700982{
Theodore Ts'oaf5bc922008-09-08 22:25:24 -0400983 struct buffer_head *bh;
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700984
Theodore Ts'oc5e298a2015-06-21 01:25:29 -0400985 bh = ext4_getblk(handle, inode, block, map_flags);
Theodore Ts'o1c215022014-08-29 20:52:15 -0400986 if (IS_ERR(bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700987 return bh;
Theodore Ts'o1c215022014-08-29 20:52:15 -0400988 if (!bh || buffer_uptodate(bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700989 return bh;
Mike Christiedfec8a12016-06-05 14:31:44 -0500990 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700991 wait_on_buffer(bh);
992 if (buffer_uptodate(bh))
993 return bh;
994 put_bh(bh);
Theodore Ts'o1c215022014-08-29 20:52:15 -0400995 return ERR_PTR(-EIO);
Dave Kleikampac27a0e2006-10-11 01:20:50 -0700996}
997
Tao Maf19d5872012-12-10 14:05:51 -0500998int ext4_walk_page_buffers(handle_t *handle,
999 struct buffer_head *head,
1000 unsigned from,
1001 unsigned to,
1002 int *partial,
1003 int (*fn)(handle_t *handle,
1004 struct buffer_head *bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001005{
1006 struct buffer_head *bh;
1007 unsigned block_start, block_end;
1008 unsigned blocksize = head->b_size;
1009 int err, ret = 0;
1010 struct buffer_head *next;
1011
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001012 for (bh = head, block_start = 0;
1013 ret == 0 && (bh != head || !block_start);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001014 block_start = block_end, bh = next) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001015 next = bh->b_this_page;
1016 block_end = block_start + blocksize;
1017 if (block_end <= from || block_start >= to) {
1018 if (partial && !buffer_uptodate(bh))
1019 *partial = 1;
1020 continue;
1021 }
1022 err = (*fn)(handle, bh);
1023 if (!ret)
1024 ret = err;
1025 }
1026 return ret;
1027}
1028
1029/*
1030 * To preserve ordering, it is essential that the hole instantiation and
1031 * the data write be encapsulated in a single transaction. We cannot
Mingming Cao617ba132006-10-11 01:20:53 -07001032 * close off a transaction and start a new one between the ext4_get_block()
Mingming Caodab291a2006-10-11 01:21:01 -07001033 * and the commit_write(). So doing the jbd2_journal_start at the start of
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001034 * prepare_write() is the right place.
1035 *
Jan Kara36ade452013-01-28 09:30:52 -05001036 * Also, this function can nest inside ext4_writepage(). In that case, we
1037 * *know* that ext4_writepage() has generated enough buffer credits to do the
1038 * whole page. So we won't block on the journal in that case, which is good,
1039 * because the caller may be PF_MEMALLOC.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001040 *
Mingming Cao617ba132006-10-11 01:20:53 -07001041 * By accident, ext4 can be reentered when a transaction is open via
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001042 * quota file writes. If we were to commit the transaction while thus
1043 * reentered, there can be a deadlock - we would be holding a quota
1044 * lock, and the commit would never complete if another thread had a
1045 * transaction open and was blocking on the quota lock - a ranking
1046 * violation.
1047 *
Mingming Caodab291a2006-10-11 01:21:01 -07001048 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001049 * will _not_ run commit under these circumstances because handle->h_ref
1050 * is elevated. We'll still have enough credits for the tiny quotafile
1051 * write.
1052 */
Tao Maf19d5872012-12-10 14:05:51 -05001053int do_journal_get_write_access(handle_t *handle,
1054 struct buffer_head *bh)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001055{
Jan Kara56d35a42010-08-05 14:41:42 -04001056 int dirty = buffer_dirty(bh);
1057 int ret;
1058
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001059 if (!buffer_mapped(bh) || buffer_freed(bh))
1060 return 0;
Jan Kara56d35a42010-08-05 14:41:42 -04001061 /*
Christoph Hellwigebdec242010-10-06 10:47:23 +02001062 * __block_write_begin() could have dirtied some buffers. Clean
Jan Kara56d35a42010-08-05 14:41:42 -04001063 * the dirty bit as jbd2_journal_get_write_access() could complain
1064 * otherwise about fs integrity issues. Setting of the dirty bit
Christoph Hellwigebdec242010-10-06 10:47:23 +02001065 * by __block_write_begin() isn't a real problem here as we clear
Jan Kara56d35a42010-08-05 14:41:42 -04001066 * the bit before releasing a page lock and thus writeback cannot
1067 * ever write the buffer.
1068 */
1069 if (dirty)
1070 clear_buffer_dirty(bh);
liang xie5d601252014-05-12 22:06:43 -04001071 BUFFER_TRACE(bh, "get write access");
Jan Kara56d35a42010-08-05 14:41:42 -04001072 ret = ext4_journal_get_write_access(handle, bh);
1073 if (!ret && dirty)
1074 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1075 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001076}
1077
Michael Halcrow2058f832015-04-12 00:55:10 -04001078#ifdef CONFIG_EXT4_FS_ENCRYPTION
1079static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1080 get_block_t *get_block)
1081{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001082 unsigned from = pos & (PAGE_SIZE - 1);
Michael Halcrow2058f832015-04-12 00:55:10 -04001083 unsigned to = from + len;
1084 struct inode *inode = page->mapping->host;
1085 unsigned block_start, block_end;
1086 sector_t block;
1087 int err = 0;
1088 unsigned blocksize = inode->i_sb->s_blocksize;
1089 unsigned bbits;
1090 struct buffer_head *bh, *head, *wait[2], **wait_bh = wait;
1091 bool decrypt = false;
1092
1093 BUG_ON(!PageLocked(page));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001094 BUG_ON(from > PAGE_SIZE);
1095 BUG_ON(to > PAGE_SIZE);
Michael Halcrow2058f832015-04-12 00:55:10 -04001096 BUG_ON(from > to);
1097
1098 if (!page_has_buffers(page))
1099 create_empty_buffers(page, blocksize, 0);
1100 head = page_buffers(page);
1101 bbits = ilog2(blocksize);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001102 block = (sector_t)page->index << (PAGE_SHIFT - bbits);
Michael Halcrow2058f832015-04-12 00:55:10 -04001103
1104 for (bh = head, block_start = 0; bh != head || !block_start;
1105 block++, block_start = block_end, bh = bh->b_this_page) {
1106 block_end = block_start + blocksize;
1107 if (block_end <= from || block_start >= to) {
1108 if (PageUptodate(page)) {
1109 if (!buffer_uptodate(bh))
1110 set_buffer_uptodate(bh);
1111 }
1112 continue;
1113 }
1114 if (buffer_new(bh))
1115 clear_buffer_new(bh);
1116 if (!buffer_mapped(bh)) {
1117 WARN_ON(bh->b_size != blocksize);
1118 err = get_block(inode, block, bh, 1);
1119 if (err)
1120 break;
1121 if (buffer_new(bh)) {
1122 unmap_underlying_metadata(bh->b_bdev,
1123 bh->b_blocknr);
1124 if (PageUptodate(page)) {
1125 clear_buffer_new(bh);
1126 set_buffer_uptodate(bh);
1127 mark_buffer_dirty(bh);
1128 continue;
1129 }
1130 if (block_end > to || block_start < from)
1131 zero_user_segments(page, to, block_end,
1132 block_start, from);
1133 continue;
1134 }
1135 }
1136 if (PageUptodate(page)) {
1137 if (!buffer_uptodate(bh))
1138 set_buffer_uptodate(bh);
1139 continue;
1140 }
1141 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1142 !buffer_unwritten(bh) &&
1143 (block_start < from || block_end > to)) {
Mike Christiedfec8a12016-06-05 14:31:44 -05001144 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
Michael Halcrow2058f832015-04-12 00:55:10 -04001145 *wait_bh++ = bh;
1146 decrypt = ext4_encrypted_inode(inode) &&
1147 S_ISREG(inode->i_mode);
1148 }
1149 }
1150 /*
1151 * If we issued read requests, let them complete.
1152 */
1153 while (wait_bh > wait) {
1154 wait_on_buffer(*--wait_bh);
1155 if (!buffer_uptodate(*wait_bh))
1156 err = -EIO;
1157 }
1158 if (unlikely(err))
1159 page_zero_new_buffers(page, from, to);
1160 else if (decrypt)
Jaegeuk Kima7550b32016-07-10 14:01:03 -04001161 err = fscrypt_decrypt_page(page);
Michael Halcrow2058f832015-04-12 00:55:10 -04001162 return err;
1163}
1164#endif
1165
Nick Pigginbfc1af62007-10-16 01:25:05 -07001166static int ext4_write_begin(struct file *file, struct address_space *mapping,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001167 loff_t pos, unsigned len, unsigned flags,
1168 struct page **pagep, void **fsdata)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001169{
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001170 struct inode *inode = mapping->host;
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001171 int ret, needed_blocks;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001172 handle_t *handle;
1173 int retries = 0;
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001174 struct page *page;
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001175 pgoff_t index;
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001176 unsigned from, to;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001177
Theodore Ts'o9bffad12009-06-17 11:48:11 -04001178 trace_ext4_write_begin(inode, pos, len, flags);
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001179 /*
1180 * Reserve one block more for addition to orphan list in case
1181 * we allocate blocks but write fails for some reason
1182 */
1183 needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001184 index = pos >> PAGE_SHIFT;
1185 from = pos & (PAGE_SIZE - 1);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001186 to = from + len;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001187
Tao Maf19d5872012-12-10 14:05:51 -05001188 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1189 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1190 flags, pagep);
1191 if (ret < 0)
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001192 return ret;
1193 if (ret == 1)
1194 return 0;
Tao Maf19d5872012-12-10 14:05:51 -05001195 }
1196
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001197 /*
1198 * grab_cache_page_write_begin() can take a long time if the
1199 * system is thrashing due to memory pressure, or if the page
1200 * is being written back. So grab it first before we start
1201 * the transaction handle. This also allows us to allocate
1202 * the page (if needed) without using GFP_NOFS.
1203 */
1204retry_grab:
Nick Piggin54566b22009-01-04 12:00:53 -08001205 page = grab_cache_page_write_begin(mapping, index, flags);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001206 if (!page)
1207 return -ENOMEM;
1208 unlock_page(page);
1209
1210retry_journal:
Theodore Ts'o9924a922013-02-08 21:59:22 -05001211 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001212 if (IS_ERR(handle)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001213 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001214 return PTR_ERR(handle);
Jan Karacf108bc2008-07-11 19:27:31 -04001215 }
Tao Maf19d5872012-12-10 14:05:51 -05001216
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001217 lock_page(page);
1218 if (page->mapping != mapping) {
1219 /* The page got truncated from under us */
1220 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001221 put_page(page);
Jan Karacf108bc2008-07-11 19:27:31 -04001222 ext4_journal_stop(handle);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001223 goto retry_grab;
Jan Karacf108bc2008-07-11 19:27:31 -04001224 }
Dmitry Monakhov7afe5aa2013-08-28 14:30:47 -04001225 /* In case writeback began while the page was unlocked */
1226 wait_for_stable_page(page);
Jan Karacf108bc2008-07-11 19:27:31 -04001227
Michael Halcrow2058f832015-04-12 00:55:10 -04001228#ifdef CONFIG_EXT4_FS_ENCRYPTION
1229 if (ext4_should_dioread_nolock(inode))
1230 ret = ext4_block_write_begin(page, pos, len,
Jan Kara705965b2016-03-08 23:08:10 -05001231 ext4_get_block_unwritten);
Michael Halcrow2058f832015-04-12 00:55:10 -04001232 else
1233 ret = ext4_block_write_begin(page, pos, len,
1234 ext4_get_block);
1235#else
Jiaying Zhang744692d2010-03-04 16:14:02 -05001236 if (ext4_should_dioread_nolock(inode))
Jan Kara705965b2016-03-08 23:08:10 -05001237 ret = __block_write_begin(page, pos, len,
1238 ext4_get_block_unwritten);
Jiaying Zhang744692d2010-03-04 16:14:02 -05001239 else
Christoph Hellwig6e1db882010-06-04 11:29:57 +02001240 ret = __block_write_begin(page, pos, len, ext4_get_block);
Michael Halcrow2058f832015-04-12 00:55:10 -04001241#endif
Nick Pigginbfc1af62007-10-16 01:25:05 -07001242 if (!ret && ext4_should_journal_data(inode)) {
Tao Maf19d5872012-12-10 14:05:51 -05001243 ret = ext4_walk_page_buffers(handle, page_buffers(page),
1244 from, to, NULL,
1245 do_journal_get_write_access);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001246 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001247
1248 if (ret) {
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04001249 unlock_page(page);
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001250 /*
Christoph Hellwig6e1db882010-06-04 11:29:57 +02001251 * __block_write_begin may have instantiated a few blocks
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001252 * outside i_size. Trim these off again. Don't need
1253 * i_size_read because we hold i_mutex.
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001254 *
1255 * Add inode to orphan list in case we crash before
1256 * truncate finishes
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04001257 */
Jan Karaffacfa72009-07-13 16:22:22 -04001258 if (pos + len > inode->i_size && ext4_can_truncate(inode))
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001259 ext4_orphan_add(handle, inode);
1260
1261 ext4_journal_stop(handle);
1262 if (pos + len > inode->i_size) {
Jan Karab9a42072009-12-08 21:24:33 -05001263 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001264 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001265 * If truncate failed early the inode might
Aneesh Kumar K.V1938a152009-06-05 01:00:26 -04001266 * still be on the orphan list; we need to
1267 * make sure the inode is removed from the
1268 * orphan list in that case.
1269 */
1270 if (inode->i_nlink)
1271 ext4_orphan_del(NULL, inode);
1272 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001273
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001274 if (ret == -ENOSPC &&
1275 ext4_should_retry_alloc(inode->i_sb, &retries))
1276 goto retry_journal;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001277 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05001278 return ret;
1279 }
1280 *pagep = page;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001281 return ret;
1282}
1283
Nick Pigginbfc1af62007-10-16 01:25:05 -07001284/* For write_end() in data=journal mode */
1285static int write_end_fn(handle_t *handle, struct buffer_head *bh)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001286{
Theodore Ts'o13fca322013-04-21 16:45:54 -04001287 int ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001288 if (!buffer_mapped(bh) || buffer_freed(bh))
1289 return 0;
1290 set_buffer_uptodate(bh);
Theodore Ts'o13fca322013-04-21 16:45:54 -04001291 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1292 clear_buffer_meta(bh);
1293 clear_buffer_prio(bh);
1294 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001295}
1296
Zheng Liueed43332013-04-03 12:41:17 -04001297/*
1298 * We need to pick up the new inode size which generic_commit_write gave us
1299 * `file' can be NULL - eg, when called from page_symlink().
1300 *
1301 * ext4 never places buffers on inode->i_mapping->private_list. metadata
1302 * buffers are managed internally.
1303 */
1304static int ext4_write_end(struct file *file,
1305 struct address_space *mapping,
1306 loff_t pos, unsigned len, unsigned copied,
1307 struct page *page, void *fsdata)
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001308{
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001309 handle_t *handle = ext4_journal_current_handle();
Zheng Liueed43332013-04-03 12:41:17 -04001310 struct inode *inode = mapping->host;
Xiaoguang Wang05726392015-02-12 23:00:17 -05001311 loff_t old_size = inode->i_size;
Zheng Liueed43332013-04-03 12:41:17 -04001312 int ret = 0, ret2;
1313 int i_size_changed = 0;
1314
1315 trace_ext4_write_end(inode, pos, len, copied);
Theodore Ts'o42c832d2013-07-01 08:12:39 -04001316 if (ext4_has_inline_data(inode)) {
1317 ret = ext4_write_inline_data_end(inode, pos, len,
1318 copied, page);
1319 if (ret < 0)
1320 goto errout;
1321 copied = ret;
1322 } else
Tao Maf19d5872012-12-10 14:05:51 -05001323 copied = block_write_end(file, mapping, pos,
1324 len, copied, page, fsdata);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001325 /*
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001326 * it's important to update i_size while still holding page lock:
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001327 * page writeout could otherwise come in and zero beyond i_size.
1328 */
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001329 i_size_changed = ext4_update_inode_size(inode, pos + copied);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001330 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001331 put_page(page);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001332
Xiaoguang Wang05726392015-02-12 23:00:17 -05001333 if (old_size < pos)
1334 pagecache_isize_extended(inode, old_size, pos);
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001335 /*
1336 * Don't mark the inode dirty under page lock. First, it unnecessarily
1337 * makes the holding time of page lock longer. Second, it forces lock
1338 * ordering of page lock and transaction start for journaling
1339 * filesystems.
1340 */
1341 if (i_size_changed)
1342 ext4_mark_inode_dirty(handle, inode);
1343
Jan Karaffacfa72009-07-13 16:22:22 -04001344 if (pos + len > inode->i_size && ext4_can_truncate(inode))
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001345 /* if we have allocated more blocks and copied
1346 * less. We will have blocks allocated outside
1347 * inode->i_size. So truncate them
1348 */
1349 ext4_orphan_add(handle, inode);
Theodore Ts'o74d553a2013-04-03 12:39:17 -04001350errout:
Mingming Cao617ba132006-10-11 01:20:53 -07001351 ret2 = ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001352 if (!ret)
1353 ret = ret2;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001354
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001355 if (pos + len > inode->i_size) {
Jan Karab9a42072009-12-08 21:24:33 -05001356 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001357 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001358 * If truncate failed early the inode might still be
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001359 * on the orphan list; we need to make sure the inode
1360 * is removed from the orphan list in that case.
1361 */
1362 if (inode->i_nlink)
1363 ext4_orphan_del(NULL, inode);
1364 }
1365
Nick Pigginbfc1af62007-10-16 01:25:05 -07001366 return ret ? ret : copied;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001367}
1368
Theodore Ts'ob90197b2015-10-15 10:29:05 -04001369/*
1370 * This is a private version of page_zero_new_buffers() which doesn't
1371 * set the buffer to be dirty, since in data=journalled mode we need
1372 * to call ext4_handle_dirty_metadata() instead.
1373 */
1374static void zero_new_buffers(struct page *page, unsigned from, unsigned to)
1375{
1376 unsigned int block_start = 0, block_end;
1377 struct buffer_head *head, *bh;
1378
1379 bh = head = page_buffers(page);
1380 do {
1381 block_end = block_start + bh->b_size;
1382 if (buffer_new(bh)) {
1383 if (block_end > from && block_start < to) {
1384 if (!PageUptodate(page)) {
1385 unsigned start, size;
1386
1387 start = max(from, block_start);
1388 size = min(to, block_end) - start;
1389
1390 zero_user(page, start, size);
1391 set_buffer_uptodate(bh);
1392 }
1393 clear_buffer_new(bh);
1394 }
1395 }
1396 block_start = block_end;
1397 bh = bh->b_this_page;
1398 } while (bh != head);
1399}
1400
Nick Pigginbfc1af62007-10-16 01:25:05 -07001401static int ext4_journalled_write_end(struct file *file,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001402 struct address_space *mapping,
1403 loff_t pos, unsigned len, unsigned copied,
1404 struct page *page, void *fsdata)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001405{
Mingming Cao617ba132006-10-11 01:20:53 -07001406 handle_t *handle = ext4_journal_current_handle();
Nick Pigginbfc1af62007-10-16 01:25:05 -07001407 struct inode *inode = mapping->host;
Xiaoguang Wang05726392015-02-12 23:00:17 -05001408 loff_t old_size = inode->i_size;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001409 int ret = 0, ret2;
1410 int partial = 0;
Nick Pigginbfc1af62007-10-16 01:25:05 -07001411 unsigned from, to;
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001412 int size_changed = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001413
Theodore Ts'o9bffad12009-06-17 11:48:11 -04001414 trace_ext4_journalled_write_end(inode, pos, len, copied);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001415 from = pos & (PAGE_SIZE - 1);
Nick Pigginbfc1af62007-10-16 01:25:05 -07001416 to = from + len;
1417
Curt Wohlgemuth441c8502011-08-13 11:25:18 -04001418 BUG_ON(!ext4_handle_valid(handle));
1419
Tao Ma3fdcfb62012-12-10 14:05:57 -05001420 if (ext4_has_inline_data(inode))
1421 copied = ext4_write_inline_data_end(inode, pos, len,
1422 copied, page);
1423 else {
1424 if (copied < len) {
1425 if (!PageUptodate(page))
1426 copied = 0;
Theodore Ts'ob90197b2015-10-15 10:29:05 -04001427 zero_new_buffers(page, from+copied, to);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001428 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001429
Tao Ma3fdcfb62012-12-10 14:05:57 -05001430 ret = ext4_walk_page_buffers(handle, page_buffers(page), from,
1431 to, &partial, write_end_fn);
1432 if (!partial)
1433 SetPageUptodate(page);
1434 }
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001435 size_changed = ext4_update_inode_size(inode, pos + copied);
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05001436 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
Jan Kara2d859db2011-07-26 09:07:11 -04001437 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001438 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001439 put_page(page);
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001440
Xiaoguang Wang05726392015-02-12 23:00:17 -05001441 if (old_size < pos)
1442 pagecache_isize_extended(inode, old_size, pos);
1443
Dmitry Monakhov4631dbf2014-08-23 17:48:28 -04001444 if (size_changed) {
Mingming Cao617ba132006-10-11 01:20:53 -07001445 ret2 = ext4_mark_inode_dirty(handle, inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001446 if (!ret)
1447 ret = ret2;
1448 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001449
Jan Karaffacfa72009-07-13 16:22:22 -04001450 if (pos + len > inode->i_size && ext4_can_truncate(inode))
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001451 /* if we have allocated more blocks and copied
1452 * less. We will have blocks allocated outside
1453 * inode->i_size. So truncate them
1454 */
1455 ext4_orphan_add(handle, inode);
1456
Mingming Cao617ba132006-10-11 01:20:53 -07001457 ret2 = ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001458 if (!ret)
1459 ret = ret2;
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001460 if (pos + len > inode->i_size) {
Jan Karab9a42072009-12-08 21:24:33 -05001461 ext4_truncate_failed_write(inode);
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04001462 /*
Jan Karaffacfa72009-07-13 16:22:22 -04001463 * If truncate failed early the inode might still be
Aneesh Kumar K.Vf8514082009-06-05 00:56:49 -04001464 * on the orphan list; we need to make sure the inode
1465 * is removed from the orphan list in that case.
1466 */
1467 if (inode->i_nlink)
1468 ext4_orphan_del(NULL, inode);
1469 }
Nick Pigginbfc1af62007-10-16 01:25:05 -07001470
1471 return ret ? ret : copied;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001472}
Mingming Caod2a17632008-07-14 17:52:37 -04001473
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001474/*
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001475 * Reserve space for a single cluster
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001476 */
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001477static int ext4_da_reserve_space(struct inode *inode)
Mingming Caod2a17632008-07-14 17:52:37 -04001478{
Mingming Cao60e58e02009-01-22 18:13:05 +01001479 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001480 struct ext4_inode_info *ei = EXT4_I(inode);
Christoph Hellwig5dd40562010-03-03 09:05:00 -05001481 int ret;
Mingming Caod2a17632008-07-14 17:52:37 -04001482
Mingming Cao60e58e02009-01-22 18:13:05 +01001483 /*
Eric Sandeen72b8ab92010-05-16 11:00:00 -04001484 * We will charge metadata quota at writeout time; this saves
1485 * us from metadata over-estimation, though we may go over by
1486 * a small amount in the end. Here we just reserve for data.
Mingming Cao60e58e02009-01-22 18:13:05 +01001487 */
Aditya Kali7b415bf2011-09-09 19:04:51 -04001488 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
Christoph Hellwig5dd40562010-03-03 09:05:00 -05001489 if (ret)
1490 return ret;
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001491
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001492 spin_lock(&ei->i_block_reservation_lock);
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -04001493 if (ext4_claim_free_clusters(sbi, 1, 0)) {
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001494 spin_unlock(&ei->i_block_reservation_lock);
Theodore Ts'o03179fe2012-07-23 00:00:20 -04001495 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
Mingming Caod2a17632008-07-14 17:52:37 -04001496 return -ENOSPC;
1497 }
Theodore Ts'o9d0be502010-01-01 02:41:30 -05001498 ei->i_reserved_data_blocks++;
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001499 trace_ext4_da_reserve_space(inode);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001500 spin_unlock(&ei->i_block_reservation_lock);
Dmitry Monakhov39bc6802009-12-10 16:36:27 +00001501
Mingming Caod2a17632008-07-14 17:52:37 -04001502 return 0; /* success */
1503}
1504
Aneesh Kumar K.V12219ae2008-07-17 16:12:08 -04001505static void ext4_da_release_space(struct inode *inode, int to_free)
Mingming Caod2a17632008-07-14 17:52:37 -04001506{
1507 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001508 struct ext4_inode_info *ei = EXT4_I(inode);
Mingming Caod2a17632008-07-14 17:52:37 -04001509
Mingming Caocd213222008-08-19 22:16:59 -04001510 if (!to_free)
1511 return; /* Nothing to release, exit */
1512
Mingming Caod2a17632008-07-14 17:52:37 -04001513 spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Caocd213222008-08-19 22:16:59 -04001514
Li Zefan5a58ec82010-05-17 02:00:00 -04001515 trace_ext4_da_release_space(inode, to_free);
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001516 if (unlikely(to_free > ei->i_reserved_data_blocks)) {
Mingming Caocd213222008-08-19 22:16:59 -04001517 /*
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001518 * if there aren't enough reserved blocks, then the
1519 * counter is messed up somewhere. Since this
1520 * function is called from invalidate page, it's
1521 * harmless to return without any action.
Mingming Caocd213222008-08-19 22:16:59 -04001522 */
Theodore Ts'o8de5c322013-02-14 15:11:41 -05001523 ext4_warning(inode->i_sb, "ext4_da_release_space: "
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001524 "ino %lu, to_free %d with only %d reserved "
Theodore Ts'o1084f252012-03-19 23:13:43 -04001525 "data blocks", inode->i_ino, to_free,
Theodore Ts'o0637c6f2009-12-30 14:20:45 -05001526 ei->i_reserved_data_blocks);
1527 WARN_ON(1);
1528 to_free = ei->i_reserved_data_blocks;
1529 }
1530 ei->i_reserved_data_blocks -= to_free;
1531
Eric Sandeen72b8ab92010-05-16 11:00:00 -04001532 /* update fs dirty data blocks counter */
Theodore Ts'o57042652011-09-09 18:56:51 -04001533 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
Mingming Caod2a17632008-07-14 17:52:37 -04001534
Mingming Caod2a17632008-07-14 17:52:37 -04001535 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
Mingming Cao60e58e02009-01-22 18:13:05 +01001536
Aditya Kali7b415bf2011-09-09 19:04:51 -04001537 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
Mingming Caod2a17632008-07-14 17:52:37 -04001538}
1539
1540static void ext4_da_page_release_reservation(struct page *page,
Lukas Czernerca99fdd2013-05-21 23:25:01 -04001541 unsigned int offset,
1542 unsigned int length)
Mingming Caod2a17632008-07-14 17:52:37 -04001543{
Lukas Czerner9705acd2015-07-03 21:13:55 -04001544 int to_release = 0, contiguous_blks = 0;
Mingming Caod2a17632008-07-14 17:52:37 -04001545 struct buffer_head *head, *bh;
1546 unsigned int curr_off = 0;
Aditya Kali7b415bf2011-09-09 19:04:51 -04001547 struct inode *inode = page->mapping->host;
1548 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Lukas Czernerca99fdd2013-05-21 23:25:01 -04001549 unsigned int stop = offset + length;
Aditya Kali7b415bf2011-09-09 19:04:51 -04001550 int num_clusters;
Zheng Liu51865fd2012-11-08 21:57:32 -05001551 ext4_fsblk_t lblk;
Mingming Caod2a17632008-07-14 17:52:37 -04001552
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001553 BUG_ON(stop > PAGE_SIZE || stop < length);
Lukas Czernerca99fdd2013-05-21 23:25:01 -04001554
Mingming Caod2a17632008-07-14 17:52:37 -04001555 head = page_buffers(page);
1556 bh = head;
1557 do {
1558 unsigned int next_off = curr_off + bh->b_size;
1559
Lukas Czernerca99fdd2013-05-21 23:25:01 -04001560 if (next_off > stop)
1561 break;
1562
Mingming Caod2a17632008-07-14 17:52:37 -04001563 if ((offset <= curr_off) && (buffer_delay(bh))) {
1564 to_release++;
Lukas Czerner9705acd2015-07-03 21:13:55 -04001565 contiguous_blks++;
Mingming Caod2a17632008-07-14 17:52:37 -04001566 clear_buffer_delay(bh);
Lukas Czerner9705acd2015-07-03 21:13:55 -04001567 } else if (contiguous_blks) {
1568 lblk = page->index <<
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001569 (PAGE_SHIFT - inode->i_blkbits);
Lukas Czerner9705acd2015-07-03 21:13:55 -04001570 lblk += (curr_off >> inode->i_blkbits) -
1571 contiguous_blks;
1572 ext4_es_remove_extent(inode, lblk, contiguous_blks);
1573 contiguous_blks = 0;
Mingming Caod2a17632008-07-14 17:52:37 -04001574 }
1575 curr_off = next_off;
1576 } while ((bh = bh->b_this_page) != head);
Aditya Kali7b415bf2011-09-09 19:04:51 -04001577
Lukas Czerner9705acd2015-07-03 21:13:55 -04001578 if (contiguous_blks) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001579 lblk = page->index << (PAGE_SHIFT - inode->i_blkbits);
Lukas Czerner9705acd2015-07-03 21:13:55 -04001580 lblk += (curr_off >> inode->i_blkbits) - contiguous_blks;
1581 ext4_es_remove_extent(inode, lblk, contiguous_blks);
Zheng Liu51865fd2012-11-08 21:57:32 -05001582 }
1583
Aditya Kali7b415bf2011-09-09 19:04:51 -04001584 /* If we have released all the blocks belonging to a cluster, then we
1585 * need to release the reserved space for that cluster. */
1586 num_clusters = EXT4_NUM_B2C(sbi, to_release);
1587 while (num_clusters > 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001588 lblk = (page->index << (PAGE_SHIFT - inode->i_blkbits)) +
Aditya Kali7b415bf2011-09-09 19:04:51 -04001589 ((num_clusters - 1) << sbi->s_cluster_bits);
1590 if (sbi->s_cluster_ratio == 1 ||
Zheng Liu7d1b1fb2012-11-08 21:57:35 -05001591 !ext4_find_delalloc_cluster(inode, lblk))
Aditya Kali7b415bf2011-09-09 19:04:51 -04001592 ext4_da_release_space(inode, 1);
1593
1594 num_clusters--;
1595 }
Mingming Caod2a17632008-07-14 17:52:37 -04001596}
Dave Kleikampac27a0e2006-10-11 01:20:50 -07001597
1598/*
Alex Tomas64769242008-07-11 19:27:31 -04001599 * Delayed allocation stuff
1600 */
1601
Jan Kara4e7ea812013-06-04 13:17:40 -04001602struct mpage_da_data {
1603 struct inode *inode;
1604 struct writeback_control *wbc;
Jan Kara6b523df2013-06-04 13:21:11 -04001605
Jan Kara4e7ea812013-06-04 13:17:40 -04001606 pgoff_t first_page; /* The first page to write */
1607 pgoff_t next_page; /* Current page to examine */
1608 pgoff_t last_page; /* Last page to examine */
Aneesh Kumar K.V791b7f02009-01-05 21:50:43 -05001609 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04001610 * Extent to map - this can be after first_page because that can be
1611 * fully mapped. We somewhat abuse m_flags to store whether the extent
1612 * is delalloc or unwritten.
Aneesh Kumar K.V791b7f02009-01-05 21:50:43 -05001613 */
Jan Kara4e7ea812013-06-04 13:17:40 -04001614 struct ext4_map_blocks map;
1615 struct ext4_io_submit io_submit; /* IO submission data */
1616};
Alex Tomas64769242008-07-11 19:27:31 -04001617
Jan Kara4e7ea812013-06-04 13:17:40 -04001618static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1619 bool invalidate)
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001620{
1621 int nr_pages, i;
1622 pgoff_t index, end;
1623 struct pagevec pvec;
1624 struct inode *inode = mpd->inode;
1625 struct address_space *mapping = inode->i_mapping;
Jan Kara4e7ea812013-06-04 13:17:40 -04001626
1627 /* This is necessary when next_page == 0. */
1628 if (mpd->first_page >= mpd->next_page)
1629 return;
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001630
Curt Wohlgemuthc7f59382011-02-26 12:27:52 -05001631 index = mpd->first_page;
1632 end = mpd->next_page - 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04001633 if (invalidate) {
1634 ext4_lblk_t start, last;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001635 start = index << (PAGE_SHIFT - inode->i_blkbits);
1636 last = end << (PAGE_SHIFT - inode->i_blkbits);
Jan Kara4e7ea812013-06-04 13:17:40 -04001637 ext4_es_remove_extent(inode, start, last - start + 1);
1638 }
Zheng Liu51865fd2012-11-08 21:57:32 -05001639
Eric Sandeen66bea922012-11-14 22:22:05 -05001640 pagevec_init(&pvec, 0);
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001641 while (index <= end) {
1642 nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE);
1643 if (nr_pages == 0)
1644 break;
1645 for (i = 0; i < nr_pages; i++) {
1646 struct page *page = pvec.pages[i];
Jan Kara9b1d09982010-03-03 16:19:32 -05001647 if (page->index > end)
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001648 break;
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001649 BUG_ON(!PageLocked(page));
1650 BUG_ON(PageWriteback(page));
Jan Kara4e7ea812013-06-04 13:17:40 -04001651 if (invalidate) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001652 block_invalidatepage(page, 0, PAGE_SIZE);
Jan Kara4e7ea812013-06-04 13:17:40 -04001653 ClearPageUptodate(page);
1654 }
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001655 unlock_page(page);
1656 }
Jan Kara9b1d09982010-03-03 16:19:32 -05001657 index = pvec.pages[nr_pages - 1]->index + 1;
1658 pagevec_release(&pvec);
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001659 }
Aneesh Kumar K.Vc4a0c462008-08-19 21:08:18 -04001660}
1661
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04001662static void ext4_print_free_blocks(struct inode *inode)
1663{
1664 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Theodore Ts'o92b97812012-03-19 23:41:49 -04001665 struct super_block *sb = inode->i_sb;
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001666 struct ext4_inode_info *ei = EXT4_I(inode);
Theodore Ts'o92b97812012-03-19 23:41:49 -04001667
1668 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
Theodore Ts'o5dee5432011-09-09 19:10:51 -04001669 EXT4_C2B(EXT4_SB(inode->i_sb),
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001670 ext4_count_free_clusters(sb)));
Theodore Ts'o92b97812012-03-19 23:41:49 -04001671 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1672 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001673 (long long) EXT4_C2B(EXT4_SB(sb),
Theodore Ts'o57042652011-09-09 18:56:51 -04001674 percpu_counter_sum(&sbi->s_freeclusters_counter)));
Theodore Ts'o92b97812012-03-19 23:41:49 -04001675 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001676 (long long) EXT4_C2B(EXT4_SB(sb),
Aditya Kali7b415bf2011-09-09 19:04:51 -04001677 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
Theodore Ts'o92b97812012-03-19 23:41:49 -04001678 ext4_msg(sb, KERN_CRIT, "Block reservation details");
1679 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
Lukas Czernerf78ee70d2013-04-03 23:33:30 -04001680 ei->i_reserved_data_blocks);
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04001681 return;
1682}
1683
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04001684static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001685{
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04001686 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001687}
1688
Alex Tomas64769242008-07-11 19:27:31 -04001689/*
Aditya Kali5356f262011-09-09 19:20:51 -04001690 * This function is grabs code from the very beginning of
1691 * ext4_map_blocks, but assumes that the caller is from delayed write
1692 * time. This function looks up the requested blocks and sets the
1693 * buffer delay bit under the protection of i_data_sem.
1694 */
1695static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1696 struct ext4_map_blocks *map,
1697 struct buffer_head *bh)
1698{
Zheng Liud100eef2013-02-18 00:29:59 -05001699 struct extent_status es;
Aditya Kali5356f262011-09-09 19:20:51 -04001700 int retval;
1701 sector_t invalid_block = ~((sector_t) 0xffff);
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001702#ifdef ES_AGGRESSIVE_TEST
1703 struct ext4_map_blocks orig_map;
1704
1705 memcpy(&orig_map, map, sizeof(*map));
1706#endif
Aditya Kali5356f262011-09-09 19:20:51 -04001707
1708 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1709 invalid_block = ~0;
1710
1711 map->m_flags = 0;
1712 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u,"
1713 "logical block %lu\n", inode->i_ino, map->m_len,
1714 (unsigned long) map->m_lblk);
Zheng Liud100eef2013-02-18 00:29:59 -05001715
1716 /* Lookup extent status tree firstly */
1717 if (ext4_es_lookup_extent(inode, iblock, &es)) {
Zheng Liud100eef2013-02-18 00:29:59 -05001718 if (ext4_es_is_hole(&es)) {
1719 retval = 0;
Lukas Czernerc8b459f2014-05-12 12:55:07 -04001720 down_read(&EXT4_I(inode)->i_data_sem);
Zheng Liud100eef2013-02-18 00:29:59 -05001721 goto add_delayed;
1722 }
1723
1724 /*
1725 * Delayed extent could be allocated by fallocate.
1726 * So we need to check it.
1727 */
1728 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1729 map_bh(bh, inode->i_sb, invalid_block);
1730 set_buffer_new(bh);
1731 set_buffer_delay(bh);
1732 return 0;
1733 }
1734
1735 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1736 retval = es.es_len - (iblock - es.es_lblk);
1737 if (retval > map->m_len)
1738 retval = map->m_len;
1739 map->m_len = retval;
1740 if (ext4_es_is_written(&es))
1741 map->m_flags |= EXT4_MAP_MAPPED;
1742 else if (ext4_es_is_unwritten(&es))
1743 map->m_flags |= EXT4_MAP_UNWRITTEN;
1744 else
1745 BUG_ON(1);
1746
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001747#ifdef ES_AGGRESSIVE_TEST
1748 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1749#endif
Zheng Liud100eef2013-02-18 00:29:59 -05001750 return retval;
1751 }
1752
Aditya Kali5356f262011-09-09 19:20:51 -04001753 /*
1754 * Try to see if we can get the block without requesting a new
1755 * file system block.
1756 */
Lukas Czernerc8b459f2014-05-12 12:55:07 -04001757 down_read(&EXT4_I(inode)->i_data_sem);
Jan Karacbd75842014-11-25 11:41:49 -05001758 if (ext4_has_inline_data(inode))
Tao Ma9c3569b2012-12-10 14:05:57 -05001759 retval = 0;
Jan Karacbd75842014-11-25 11:41:49 -05001760 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Zheng Liu2f8e0a72014-11-25 11:44:37 -05001761 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
Aditya Kali5356f262011-09-09 19:20:51 -04001762 else
Zheng Liu2f8e0a72014-11-25 11:44:37 -05001763 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
Aditya Kali5356f262011-09-09 19:20:51 -04001764
Zheng Liud100eef2013-02-18 00:29:59 -05001765add_delayed:
Aditya Kali5356f262011-09-09 19:20:51 -04001766 if (retval == 0) {
Zheng Liuf7fec032013-02-18 00:28:47 -05001767 int ret;
Aditya Kali5356f262011-09-09 19:20:51 -04001768 /*
1769 * XXX: __block_prepare_write() unmaps passed block,
1770 * is it OK?
1771 */
Lukas Czerner386ad672013-03-10 22:50:00 -04001772 /*
1773 * If the block was allocated from previously allocated cluster,
1774 * then we don't need to reserve it again. However we still need
1775 * to reserve metadata for every block we're going to write.
1776 */
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001777 if (EXT4_SB(inode->i_sb)->s_cluster_ratio == 1 ||
Jan Karacbd75842014-11-25 11:41:49 -05001778 !ext4_find_delalloc_cluster(inode, map->m_lblk)) {
Eric Whitneyc27e43a2015-06-21 21:37:05 -04001779 ret = ext4_da_reserve_space(inode);
Zheng Liuf7fec032013-02-18 00:28:47 -05001780 if (ret) {
Aditya Kali5356f262011-09-09 19:20:51 -04001781 /* not enough space to reserve */
Zheng Liuf7fec032013-02-18 00:28:47 -05001782 retval = ret;
Aditya Kali5356f262011-09-09 19:20:51 -04001783 goto out_unlock;
Zheng Liuf7fec032013-02-18 00:28:47 -05001784 }
Aditya Kali5356f262011-09-09 19:20:51 -04001785 }
1786
Zheng Liuf7fec032013-02-18 00:28:47 -05001787 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1788 ~0, EXTENT_STATUS_DELAYED);
1789 if (ret) {
1790 retval = ret;
Zheng Liu51865fd2012-11-08 21:57:32 -05001791 goto out_unlock;
Zheng Liuf7fec032013-02-18 00:28:47 -05001792 }
Zheng Liu51865fd2012-11-08 21:57:32 -05001793
Aditya Kali5356f262011-09-09 19:20:51 -04001794 map_bh(bh, inode->i_sb, invalid_block);
1795 set_buffer_new(bh);
1796 set_buffer_delay(bh);
Zheng Liuf7fec032013-02-18 00:28:47 -05001797 } else if (retval > 0) {
1798 int ret;
Theodore Ts'o3be78c72013-08-16 21:22:41 -04001799 unsigned int status;
Zheng Liuf7fec032013-02-18 00:28:47 -05001800
Zheng Liu44fb851d2013-07-29 12:51:42 -04001801 if (unlikely(retval != map->m_len)) {
1802 ext4_warning(inode->i_sb,
1803 "ES len assertion failed for inode "
1804 "%lu: retval %d != map->m_len %d",
1805 inode->i_ino, retval, map->m_len);
1806 WARN_ON(1);
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001807 }
Dmitry Monakhov921f2662013-03-10 21:01:03 -04001808
Zheng Liuf7fec032013-02-18 00:28:47 -05001809 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1810 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1811 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1812 map->m_pblk, status);
1813 if (ret != 0)
1814 retval = ret;
Aditya Kali5356f262011-09-09 19:20:51 -04001815 }
1816
1817out_unlock:
1818 up_read((&EXT4_I(inode)->i_data_sem));
1819
1820 return retval;
1821}
1822
1823/*
Seunghun Leed91bd2c2014-09-01 22:15:30 -04001824 * This is a special get_block_t callback which is used by
Theodore Ts'ob920c752009-05-14 00:54:29 -04001825 * ext4_da_write_begin(). It will either return mapped block or
1826 * reserve space for a single block.
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04001827 *
1828 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1829 * We also have b_blocknr = -1 and b_bdev initialized properly
1830 *
1831 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1832 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1833 * initialized properly.
Alex Tomas64769242008-07-11 19:27:31 -04001834 */
Tao Ma9c3569b2012-12-10 14:05:57 -05001835int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1836 struct buffer_head *bh, int create)
Alex Tomas64769242008-07-11 19:27:31 -04001837{
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001838 struct ext4_map_blocks map;
Alex Tomas64769242008-07-11 19:27:31 -04001839 int ret = 0;
1840
1841 BUG_ON(create == 0);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001842 BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1843
1844 map.m_lblk = iblock;
1845 map.m_len = 1;
Alex Tomas64769242008-07-11 19:27:31 -04001846
1847 /*
1848 * first, we need to know whether the block is allocated already
1849 * preallocated blocks are unmapped but should treated
1850 * the same as allocated blocks.
1851 */
Aditya Kali5356f262011-09-09 19:20:51 -04001852 ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1853 if (ret <= 0)
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001854 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04001855
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001856 map_bh(bh, inode->i_sb, map.m_pblk);
Jan Karaed8ad832016-02-19 00:18:25 -05001857 ext4_update_bh_state(bh, map.m_flags);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001858
1859 if (buffer_unwritten(bh)) {
1860 /* A delayed write to unwritten bh should be marked
1861 * new and mapped. Mapped ensures that we don't do
1862 * get_block multiple times when we write to the same
1863 * offset and new ensures that we do proper zero out
1864 * for partial write.
1865 */
1866 set_buffer_new(bh);
Theodore Ts'oc8205632011-04-10 22:30:07 -04001867 set_buffer_mapped(bh);
Theodore Ts'o2ed88682010-05-16 20:00:00 -04001868 }
1869 return 0;
Alex Tomas64769242008-07-11 19:27:31 -04001870}
Mingming Cao61628a32008-07-11 19:27:31 -04001871
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001872static int bget_one(handle_t *handle, struct buffer_head *bh)
1873{
1874 get_bh(bh);
1875 return 0;
1876}
1877
1878static int bput_one(handle_t *handle, struct buffer_head *bh)
1879{
1880 put_bh(bh);
1881 return 0;
1882}
1883
1884static int __ext4_journalled_writepage(struct page *page,
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001885 unsigned int len)
1886{
1887 struct address_space *mapping = page->mapping;
1888 struct inode *inode = mapping->host;
Tao Ma3fdcfb62012-12-10 14:05:57 -05001889 struct buffer_head *page_bufs = NULL;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001890 handle_t *handle = NULL;
Tao Ma3fdcfb62012-12-10 14:05:57 -05001891 int ret = 0, err = 0;
1892 int inline_data = ext4_has_inline_data(inode);
1893 struct buffer_head *inode_bh = NULL;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001894
Theodore Ts'ocb20d512010-10-27 21:30:09 -04001895 ClearPageChecked(page);
Tao Ma3fdcfb62012-12-10 14:05:57 -05001896
1897 if (inline_data) {
1898 BUG_ON(page->index != 0);
1899 BUG_ON(len > ext4_get_max_inline_size(inode));
1900 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1901 if (inode_bh == NULL)
1902 goto out;
1903 } else {
1904 page_bufs = page_buffers(page);
1905 if (!page_bufs) {
1906 BUG();
1907 goto out;
1908 }
1909 ext4_walk_page_buffers(handle, page_bufs, 0, len,
1910 NULL, bget_one);
1911 }
Theodore Ts'obdf96832015-06-12 23:45:33 -04001912 /*
1913 * We need to release the page lock before we start the
1914 * journal, so grab a reference so the page won't disappear
1915 * out from under us.
1916 */
1917 get_page(page);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001918 unlock_page(page);
1919
Theodore Ts'o9924a922013-02-08 21:59:22 -05001920 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
1921 ext4_writepage_trans_blocks(inode));
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001922 if (IS_ERR(handle)) {
1923 ret = PTR_ERR(handle);
Theodore Ts'obdf96832015-06-12 23:45:33 -04001924 put_page(page);
1925 goto out_no_pagelock;
1926 }
1927 BUG_ON(!ext4_handle_valid(handle));
1928
1929 lock_page(page);
1930 put_page(page);
1931 if (page->mapping != mapping) {
1932 /* The page got truncated from under us */
1933 ext4_journal_stop(handle);
1934 ret = 0;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001935 goto out;
1936 }
1937
Tao Ma3fdcfb62012-12-10 14:05:57 -05001938 if (inline_data) {
liang xie5d601252014-05-12 22:06:43 -04001939 BUFFER_TRACE(inode_bh, "get write access");
Tao Ma3fdcfb62012-12-10 14:05:57 -05001940 ret = ext4_journal_get_write_access(handle, inode_bh);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001941
Tao Ma3fdcfb62012-12-10 14:05:57 -05001942 err = ext4_handle_dirty_metadata(handle, inode, inode_bh);
1943
1944 } else {
1945 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
1946 do_journal_get_write_access);
1947
1948 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
1949 write_end_fn);
1950 }
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001951 if (ret == 0)
1952 ret = err;
Jan Kara2d859db2011-07-26 09:07:11 -04001953 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001954 err = ext4_journal_stop(handle);
1955 if (!ret)
1956 ret = err;
1957
Tao Ma3fdcfb62012-12-10 14:05:57 -05001958 if (!ext4_has_inline_data(inode))
Theodore Ts'o8c9367f2014-01-07 13:08:03 -05001959 ext4_walk_page_buffers(NULL, page_bufs, 0, len,
Tao Ma3fdcfb62012-12-10 14:05:57 -05001960 NULL, bput_one);
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05001961 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001962out:
Theodore Ts'obdf96832015-06-12 23:45:33 -04001963 unlock_page(page);
1964out_no_pagelock:
Tao Ma3fdcfb62012-12-10 14:05:57 -05001965 brelse(inode_bh);
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04001966 return ret;
1967}
1968
Mingming Cao61628a32008-07-11 19:27:31 -04001969/*
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001970 * Note that we don't need to start a transaction unless we're journaling data
1971 * because we should have holes filled from ext4_page_mkwrite(). We even don't
1972 * need to file the inode to the transaction's list in ordered mode because if
1973 * we are writing back data added by write(), the inode is already there and if
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001974 * we are writing back data modified via mmap(), no one guarantees in which
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001975 * transaction the data will hit the disk. In case we are journaling data, we
1976 * cannot start transaction directly because transaction start ranks above page
1977 * lock so we have to do some magic.
1978 *
Theodore Ts'ob920c752009-05-14 00:54:29 -04001979 * This function can get called via...
Theodore Ts'o20970ba2013-06-06 14:00:46 -04001980 * - ext4_writepages after taking page lock (have journal handle)
Theodore Ts'ob920c752009-05-14 00:54:29 -04001981 * - journal_submit_inode_data_buffers (no journal handle)
Artem Bityutskiyf6463b02012-07-25 18:12:04 +03001982 * - shrink_page_list via the kswapd/direct reclaim (no journal handle)
Theodore Ts'ob920c752009-05-14 00:54:29 -04001983 * - grab_page_cache when doing write_begin (have journal handle)
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001984 *
1985 * We don't do any block allocation in this function. If we have page with
1986 * multiple blocks we need to write those buffer_heads that are mapped. This
1987 * is important for mmaped based write. So if we do with blocksize 1K
1988 * truncate(f, 1024);
1989 * a = mmap(f, 0, 4096);
1990 * a[0] = 'a';
1991 * truncate(f, 4096);
1992 * we have in the page first buffer_head mapped via page_mkwrite call back
Paul Bolle90802ed2011-12-05 13:00:34 +01001993 * but other buffer_heads would be unmapped but dirty (dirty done via the
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04001994 * do_wp_page). So writepage should write the first block. If we modify
1995 * the mmap area beyond 1024 we will again get a page_fault and the
1996 * page_mkwrite callback will do the block allocation and mark the
1997 * buffer_heads mapped.
1998 *
1999 * We redirty the page if we have any buffer_heads that is either delay or
2000 * unwritten in the page.
2001 *
2002 * We can get recursively called as show below.
2003 *
2004 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
2005 * ext4_writepage()
2006 *
2007 * But since we don't do any block allocation we should not deadlock.
2008 * Page also have the dirty flag cleared so we don't get recurive page_lock.
Mingming Cao61628a32008-07-11 19:27:31 -04002009 */
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04002010static int ext4_writepage(struct page *page,
Aneesh Kumar K.V62e086b2009-06-14 17:59:34 -04002011 struct writeback_control *wbc)
Alex Tomas64769242008-07-11 19:27:31 -04002012{
Jan Karaf8bec372013-01-28 12:55:08 -05002013 int ret = 0;
Mingming Cao61628a32008-07-11 19:27:31 -04002014 loff_t size;
Theodore Ts'o498e5f22008-11-05 00:14:04 -05002015 unsigned int len;
Jiaying Zhang744692d2010-03-04 16:14:02 -05002016 struct buffer_head *page_bufs = NULL;
Mingming Cao61628a32008-07-11 19:27:31 -04002017 struct inode *inode = page->mapping->host;
Jan Kara36ade452013-01-28 09:30:52 -05002018 struct ext4_io_submit io_submit;
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002019 bool keep_towrite = false;
Alex Tomas64769242008-07-11 19:27:31 -04002020
Lukas Czernera9c667f2011-06-06 09:51:52 -04002021 trace_ext4_writepage(page);
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04002022 size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002023 if (page->index == size >> PAGE_SHIFT)
2024 len = size & ~PAGE_MASK;
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04002025 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002026 len = PAGE_SIZE;
Alex Tomas64769242008-07-11 19:27:31 -04002027
Theodore Ts'oa42afc52010-10-27 21:30:09 -04002028 page_bufs = page_buffers(page);
Aneesh Kumar K.Vc364b222009-06-14 17:57:10 -04002029 /*
Jan Karafe386132013-01-28 21:06:42 -05002030 * We cannot do block allocation or other extent handling in this
2031 * function. If there are buffers needing that, we have to redirty
2032 * the page. But we may reach here when we do a journal commit via
2033 * journal_submit_inode_data_buffers() and in that case we must write
2034 * allocated buffers to achieve data=ordered mode guarantees.
Theodore Ts'occcd1472015-10-03 10:49:23 -04002035 *
2036 * Also, if there is only one buffer per page (the fs block
2037 * size == the page size), if one buffer needs block
2038 * allocation or needs to modify the extent tree to clear the
2039 * unwritten flag, we know that the page can't be written at
2040 * all, so we might as well refuse the write immediately.
2041 * Unfortunately if the block size != page size, we can't as
2042 * easily detect this case using ext4_walk_page_buffers(), but
2043 * for the extremely common case, this is an optimization that
2044 * skips a useless round trip through ext4_bio_write_page().
Aneesh Kumar K.Vcd1aac32008-07-11 19:27:31 -04002045 */
Tao Maf19d5872012-12-10 14:05:51 -05002046 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL,
2047 ext4_bh_delay_or_unwritten)) {
Jan Karaf8bec372013-01-28 12:55:08 -05002048 redirty_page_for_writepage(wbc, page);
Theodore Ts'occcd1472015-10-03 10:49:23 -04002049 if ((current->flags & PF_MEMALLOC) ||
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002050 (inode->i_sb->s_blocksize == PAGE_SIZE)) {
Jan Karafe386132013-01-28 21:06:42 -05002051 /*
2052 * For memory cleaning there's no point in writing only
2053 * some buffers. So just bail out. Warn if we came here
2054 * from direct reclaim.
2055 */
2056 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2057 == PF_MEMALLOC);
Aneesh Kumar K.Vf0e6c982008-07-11 19:27:31 -04002058 unlock_page(page);
2059 return 0;
2060 }
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002061 keep_towrite = true;
Theodore Ts'oa42afc52010-10-27 21:30:09 -04002062 }
Alex Tomas64769242008-07-11 19:27:31 -04002063
Theodore Ts'ocb20d512010-10-27 21:30:09 -04002064 if (PageChecked(page) && ext4_should_journal_data(inode))
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04002065 /*
2066 * It's mmapped pagecache. Add buffers and journal it. There
2067 * doesn't seem much point in redirtying the page here.
2068 */
Wu Fengguang3f0ca302009-11-24 11:15:44 -05002069 return __ext4_journalled_writepage(page, len);
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04002070
Jan Kara97a851e2013-06-04 11:58:58 -04002071 ext4_io_submit_init(&io_submit, wbc);
2072 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2073 if (!io_submit.io_end) {
2074 redirty_page_for_writepage(wbc, page);
2075 unlock_page(page);
2076 return -ENOMEM;
2077 }
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002078 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite);
Jan Kara36ade452013-01-28 09:30:52 -05002079 ext4_io_submit(&io_submit);
Jan Kara97a851e2013-06-04 11:58:58 -04002080 /* Drop io_end reference we got from init */
2081 ext4_put_io_end_defer(io_submit.io_end);
Alex Tomas64769242008-07-11 19:27:31 -04002082 return ret;
2083}
2084
Jan Kara5f1132b2013-08-17 10:02:33 -04002085static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2086{
2087 int len;
2088 loff_t size = i_size_read(mpd->inode);
2089 int err;
2090
2091 BUG_ON(page->index != mpd->first_page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002092 if (page->index == size >> PAGE_SHIFT)
2093 len = size & ~PAGE_MASK;
Jan Kara5f1132b2013-08-17 10:02:33 -04002094 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002095 len = PAGE_SIZE;
Jan Kara5f1132b2013-08-17 10:02:33 -04002096 clear_page_dirty_for_io(page);
Namjae Jeon1c8349a2014-05-12 08:12:25 -04002097 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false);
Jan Kara5f1132b2013-08-17 10:02:33 -04002098 if (!err)
2099 mpd->wbc->nr_to_write--;
2100 mpd->first_page++;
2101
2102 return err;
2103}
2104
Jan Kara4e7ea812013-06-04 13:17:40 -04002105#define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay))
2106
Mingming Cao61628a32008-07-11 19:27:31 -04002107/*
Jan Karafffb2732013-06-04 13:01:11 -04002108 * mballoc gives us at most this number of blocks...
2109 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
Anatol Pomozov70261f52013-08-28 14:40:12 -04002110 * The rest of mballoc seems to handle chunks up to full group size.
Mingming Cao61628a32008-07-11 19:27:31 -04002111 */
Jan Karafffb2732013-06-04 13:01:11 -04002112#define MAX_WRITEPAGES_EXTENT_LEN 2048
Mingming Cao525f4ed2008-08-19 22:15:58 -04002113
Jan Karafffb2732013-06-04 13:01:11 -04002114/*
Jan Kara4e7ea812013-06-04 13:17:40 -04002115 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2116 *
2117 * @mpd - extent of blocks
2118 * @lblk - logical number of the block in the file
Jan Kara09930042013-08-17 09:57:56 -04002119 * @bh - buffer head we want to add to the extent
Jan Kara4e7ea812013-06-04 13:17:40 -04002120 *
Jan Kara09930042013-08-17 09:57:56 -04002121 * The function is used to collect contig. blocks in the same state. If the
2122 * buffer doesn't require mapping for writeback and we haven't started the
2123 * extent of buffers to map yet, the function returns 'true' immediately - the
2124 * caller can write the buffer right away. Otherwise the function returns true
2125 * if the block has been added to the extent, false if the block couldn't be
2126 * added.
Jan Kara4e7ea812013-06-04 13:17:40 -04002127 */
Jan Kara09930042013-08-17 09:57:56 -04002128static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2129 struct buffer_head *bh)
Jan Kara4e7ea812013-06-04 13:17:40 -04002130{
2131 struct ext4_map_blocks *map = &mpd->map;
2132
Jan Kara09930042013-08-17 09:57:56 -04002133 /* Buffer that doesn't need mapping for writeback? */
2134 if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2135 (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2136 /* So far no extent to map => we write the buffer right away */
2137 if (map->m_len == 0)
2138 return true;
2139 return false;
2140 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002141
2142 /* First block in the extent? */
2143 if (map->m_len == 0) {
2144 map->m_lblk = lblk;
2145 map->m_len = 1;
Jan Kara09930042013-08-17 09:57:56 -04002146 map->m_flags = bh->b_state & BH_FLAGS;
2147 return true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002148 }
2149
Jan Kara09930042013-08-17 09:57:56 -04002150 /* Don't go larger than mballoc is willing to allocate */
2151 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2152 return false;
2153
Jan Kara4e7ea812013-06-04 13:17:40 -04002154 /* Can we merge the block to our big extent? */
2155 if (lblk == map->m_lblk + map->m_len &&
Jan Kara09930042013-08-17 09:57:56 -04002156 (bh->b_state & BH_FLAGS) == map->m_flags) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002157 map->m_len++;
Jan Kara09930042013-08-17 09:57:56 -04002158 return true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002159 }
Jan Kara09930042013-08-17 09:57:56 -04002160 return false;
Jan Kara4e7ea812013-06-04 13:17:40 -04002161}
2162
Jan Kara5f1132b2013-08-17 10:02:33 -04002163/*
2164 * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2165 *
2166 * @mpd - extent of blocks for mapping
2167 * @head - the first buffer in the page
2168 * @bh - buffer we should start processing from
2169 * @lblk - logical number of the block in the file corresponding to @bh
2170 *
2171 * Walk through page buffers from @bh upto @head (exclusive) and either submit
2172 * the page for IO if all buffers in this page were mapped and there's no
2173 * accumulated extent of buffers to map or add buffers in the page to the
2174 * extent of buffers to map. The function returns 1 if the caller can continue
2175 * by processing the next page, 0 if it should stop adding buffers to the
2176 * extent to map because we cannot extend it anymore. It can also return value
2177 * < 0 in case of error during IO submission.
2178 */
2179static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2180 struct buffer_head *head,
2181 struct buffer_head *bh,
2182 ext4_lblk_t lblk)
Jan Kara4e7ea812013-06-04 13:17:40 -04002183{
2184 struct inode *inode = mpd->inode;
Jan Kara5f1132b2013-08-17 10:02:33 -04002185 int err;
Jan Kara4e7ea812013-06-04 13:17:40 -04002186 ext4_lblk_t blocks = (i_size_read(inode) + (1 << inode->i_blkbits) - 1)
2187 >> inode->i_blkbits;
2188
2189 do {
2190 BUG_ON(buffer_locked(bh));
2191
Jan Kara09930042013-08-17 09:57:56 -04002192 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002193 /* Found extent to map? */
2194 if (mpd->map.m_len)
Jan Kara5f1132b2013-08-17 10:02:33 -04002195 return 0;
Jan Kara09930042013-08-17 09:57:56 -04002196 /* Everything mapped so far and we hit EOF */
Jan Kara5f1132b2013-08-17 10:02:33 -04002197 break;
Jan Kara4e7ea812013-06-04 13:17:40 -04002198 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002199 } while (lblk++, (bh = bh->b_this_page) != head);
Jan Kara5f1132b2013-08-17 10:02:33 -04002200 /* So far everything mapped? Submit the page for IO. */
2201 if (mpd->map.m_len == 0) {
2202 err = mpage_submit_page(mpd, head->b_page);
2203 if (err < 0)
2204 return err;
2205 }
2206 return lblk < blocks;
Jan Kara4e7ea812013-06-04 13:17:40 -04002207}
2208
2209/*
2210 * mpage_map_buffers - update buffers corresponding to changed extent and
2211 * submit fully mapped pages for IO
2212 *
2213 * @mpd - description of extent to map, on return next extent to map
2214 *
2215 * Scan buffers corresponding to changed extent (we expect corresponding pages
2216 * to be already locked) and update buffer state according to new extent state.
2217 * We map delalloc buffers to their physical location, clear unwritten bits,
Lukas Czerner556615d2014-04-20 23:45:47 -04002218 * and mark buffers as uninit when we perform writes to unwritten extents
Jan Kara4e7ea812013-06-04 13:17:40 -04002219 * and do extent conversion after IO is finished. If the last page is not fully
2220 * mapped, we update @map to the next extent in the last page that needs
2221 * mapping. Otherwise we submit the page for IO.
2222 */
2223static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2224{
2225 struct pagevec pvec;
2226 int nr_pages, i;
2227 struct inode *inode = mpd->inode;
2228 struct buffer_head *head, *bh;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002229 int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
Jan Kara4e7ea812013-06-04 13:17:40 -04002230 pgoff_t start, end;
2231 ext4_lblk_t lblk;
2232 sector_t pblock;
2233 int err;
2234
2235 start = mpd->map.m_lblk >> bpp_bits;
2236 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2237 lblk = start << bpp_bits;
2238 pblock = mpd->map.m_pblk;
2239
2240 pagevec_init(&pvec, 0);
2241 while (start <= end) {
2242 nr_pages = pagevec_lookup(&pvec, inode->i_mapping, start,
2243 PAGEVEC_SIZE);
2244 if (nr_pages == 0)
2245 break;
2246 for (i = 0; i < nr_pages; i++) {
2247 struct page *page = pvec.pages[i];
2248
2249 if (page->index > end)
2250 break;
Anatol Pomozov70261f52013-08-28 14:40:12 -04002251 /* Up to 'end' pages must be contiguous */
Jan Kara4e7ea812013-06-04 13:17:40 -04002252 BUG_ON(page->index != start);
2253 bh = head = page_buffers(page);
2254 do {
2255 if (lblk < mpd->map.m_lblk)
2256 continue;
2257 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2258 /*
2259 * Buffer after end of mapped extent.
2260 * Find next buffer in the page to map.
2261 */
2262 mpd->map.m_len = 0;
2263 mpd->map.m_flags = 0;
Jan Kara5f1132b2013-08-17 10:02:33 -04002264 /*
2265 * FIXME: If dioread_nolock supports
2266 * blocksize < pagesize, we need to make
2267 * sure we add size mapped so far to
2268 * io_end->size as the following call
2269 * can submit the page for IO.
2270 */
2271 err = mpage_process_page_bufs(mpd, head,
2272 bh, lblk);
Jan Kara4e7ea812013-06-04 13:17:40 -04002273 pagevec_release(&pvec);
Jan Kara5f1132b2013-08-17 10:02:33 -04002274 if (err > 0)
2275 err = 0;
2276 return err;
Jan Kara4e7ea812013-06-04 13:17:40 -04002277 }
2278 if (buffer_delay(bh)) {
2279 clear_buffer_delay(bh);
2280 bh->b_blocknr = pblock++;
2281 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002282 clear_buffer_unwritten(bh);
Jan Kara5f1132b2013-08-17 10:02:33 -04002283 } while (lblk++, (bh = bh->b_this_page) != head);
Jan Kara4e7ea812013-06-04 13:17:40 -04002284
2285 /*
2286 * FIXME: This is going to break if dioread_nolock
2287 * supports blocksize < pagesize as we will try to
2288 * convert potentially unmapped parts of inode.
2289 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002290 mpd->io_submit.io_end->size += PAGE_SIZE;
Jan Kara4e7ea812013-06-04 13:17:40 -04002291 /* Page fully mapped - let IO run! */
2292 err = mpage_submit_page(mpd, page);
2293 if (err < 0) {
2294 pagevec_release(&pvec);
2295 return err;
2296 }
2297 start++;
2298 }
2299 pagevec_release(&pvec);
2300 }
2301 /* Extent fully mapped and matches with page boundary. We are done. */
2302 mpd->map.m_len = 0;
2303 mpd->map.m_flags = 0;
2304 return 0;
2305}
2306
2307static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2308{
2309 struct inode *inode = mpd->inode;
2310 struct ext4_map_blocks *map = &mpd->map;
2311 int get_blocks_flags;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002312 int err, dioread_nolock;
Jan Kara4e7ea812013-06-04 13:17:40 -04002313
2314 trace_ext4_da_write_pages_extent(inode, map);
2315 /*
2316 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
Lukas Czerner556615d2014-04-20 23:45:47 -04002317 * to convert an unwritten extent to be initialized (in the case
Jan Kara4e7ea812013-06-04 13:17:40 -04002318 * where we have written into one or more preallocated blocks). It is
2319 * possible that we're going to need more metadata blocks than
2320 * previously reserved. However we must not fail because we're in
2321 * writeback and there is nothing we can do about it so it might result
2322 * in data loss. So use reserved blocks to allocate metadata if
2323 * possible.
2324 *
Theodore Ts'o754cfed2014-09-04 18:08:22 -04002325 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2326 * the blocks in question are delalloc blocks. This indicates
2327 * that the blocks and quotas has already been checked when
2328 * the data was copied into the page cache.
Jan Kara4e7ea812013-06-04 13:17:40 -04002329 */
2330 get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
Jan Karaee0876b2016-04-24 00:56:08 -04002331 EXT4_GET_BLOCKS_METADATA_NOFAIL |
2332 EXT4_GET_BLOCKS_IO_SUBMIT;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002333 dioread_nolock = ext4_should_dioread_nolock(inode);
2334 if (dioread_nolock)
Jan Kara4e7ea812013-06-04 13:17:40 -04002335 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2336 if (map->m_flags & (1 << BH_Delay))
2337 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2338
2339 err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2340 if (err < 0)
2341 return err;
Lukas Czerner090f32e2014-04-20 23:44:47 -04002342 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
Jan Kara6b523df2013-06-04 13:21:11 -04002343 if (!mpd->io_submit.io_end->handle &&
2344 ext4_handle_valid(handle)) {
2345 mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2346 handle->h_rsv_handle = NULL;
2347 }
Jan Kara3613d222013-06-04 13:19:34 -04002348 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
Jan Kara6b523df2013-06-04 13:21:11 -04002349 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002350
2351 BUG_ON(map->m_len == 0);
2352 if (map->m_flags & EXT4_MAP_NEW) {
2353 struct block_device *bdev = inode->i_sb->s_bdev;
2354 int i;
2355
2356 for (i = 0; i < map->m_len; i++)
2357 unmap_underlying_metadata(bdev, map->m_pblk + i);
2358 }
2359 return 0;
2360}
2361
2362/*
2363 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2364 * mpd->len and submit pages underlying it for IO
2365 *
2366 * @handle - handle for journal operations
2367 * @mpd - extent to map
Jan Kara7534e852013-10-16 08:26:08 -04002368 * @give_up_on_write - we set this to true iff there is a fatal error and there
2369 * is no hope of writing the data. The caller should discard
2370 * dirty pages to avoid infinite loops.
Jan Kara4e7ea812013-06-04 13:17:40 -04002371 *
2372 * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2373 * delayed, blocks are allocated, if it is unwritten, we may need to convert
2374 * them to initialized or split the described range from larger unwritten
2375 * extent. Note that we need not map all the described range since allocation
2376 * can return less blocks or the range is covered by more unwritten extents. We
2377 * cannot map more because we are limited by reserved transaction credits. On
2378 * the other hand we always make sure that the last touched page is fully
2379 * mapped so that it can be written out (and thus forward progress is
2380 * guaranteed). After mapping we submit all mapped pages for IO.
2381 */
2382static int mpage_map_and_submit_extent(handle_t *handle,
Theodore Ts'ocb530542013-07-01 08:12:40 -04002383 struct mpage_da_data *mpd,
2384 bool *give_up_on_write)
Jan Kara4e7ea812013-06-04 13:17:40 -04002385{
2386 struct inode *inode = mpd->inode;
2387 struct ext4_map_blocks *map = &mpd->map;
2388 int err;
2389 loff_t disksize;
Dmitry Monakhov66031202014-08-27 18:40:03 -04002390 int progress = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002391
2392 mpd->io_submit.io_end->offset =
2393 ((loff_t)map->m_lblk) << inode->i_blkbits;
Jan Kara27d7c4e2013-07-05 21:57:22 -04002394 do {
Jan Kara4e7ea812013-06-04 13:17:40 -04002395 err = mpage_map_one_extent(handle, mpd);
2396 if (err < 0) {
2397 struct super_block *sb = inode->i_sb;
2398
Theodore Ts'ocb530542013-07-01 08:12:40 -04002399 if (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)
2400 goto invalidate_dirty_pages;
Jan Kara4e7ea812013-06-04 13:17:40 -04002401 /*
Theodore Ts'ocb530542013-07-01 08:12:40 -04002402 * Let the uper layers retry transient errors.
2403 * In the case of ENOSPC, if ext4_count_free_blocks()
2404 * is non-zero, a commit should free up blocks.
Jan Kara4e7ea812013-06-04 13:17:40 -04002405 */
Theodore Ts'ocb530542013-07-01 08:12:40 -04002406 if ((err == -ENOMEM) ||
Dmitry Monakhov66031202014-08-27 18:40:03 -04002407 (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2408 if (progress)
2409 goto update_disksize;
Theodore Ts'ocb530542013-07-01 08:12:40 -04002410 return err;
Dmitry Monakhov66031202014-08-27 18:40:03 -04002411 }
Theodore Ts'ocb530542013-07-01 08:12:40 -04002412 ext4_msg(sb, KERN_CRIT,
2413 "Delayed block allocation failed for "
2414 "inode %lu at logical offset %llu with"
2415 " max blocks %u with error %d",
2416 inode->i_ino,
2417 (unsigned long long)map->m_lblk,
2418 (unsigned)map->m_len, -err);
2419 ext4_msg(sb, KERN_CRIT,
2420 "This should not happen!! Data will "
2421 "be lost\n");
2422 if (err == -ENOSPC)
2423 ext4_print_free_blocks(inode);
2424 invalidate_dirty_pages:
2425 *give_up_on_write = true;
Jan Kara4e7ea812013-06-04 13:17:40 -04002426 return err;
2427 }
Dmitry Monakhov66031202014-08-27 18:40:03 -04002428 progress = 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04002429 /*
2430 * Update buffer state, submit mapped pages, and get us new
2431 * extent to map
2432 */
2433 err = mpage_map_and_submit_buffers(mpd);
2434 if (err < 0)
Dmitry Monakhov66031202014-08-27 18:40:03 -04002435 goto update_disksize;
Jan Kara27d7c4e2013-07-05 21:57:22 -04002436 } while (map->m_len);
Jan Kara4e7ea812013-06-04 13:17:40 -04002437
Dmitry Monakhov66031202014-08-27 18:40:03 -04002438update_disksize:
Theodore Ts'o622cad12014-04-11 10:35:17 -04002439 /*
2440 * Update on-disk size after IO is submitted. Races with
2441 * truncate are avoided by checking i_size under i_data_sem.
2442 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002443 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
Jan Kara4e7ea812013-06-04 13:17:40 -04002444 if (disksize > EXT4_I(inode)->i_disksize) {
2445 int err2;
Theodore Ts'o622cad12014-04-11 10:35:17 -04002446 loff_t i_size;
Jan Kara4e7ea812013-06-04 13:17:40 -04002447
Theodore Ts'o622cad12014-04-11 10:35:17 -04002448 down_write(&EXT4_I(inode)->i_data_sem);
2449 i_size = i_size_read(inode);
2450 if (disksize > i_size)
2451 disksize = i_size;
2452 if (disksize > EXT4_I(inode)->i_disksize)
2453 EXT4_I(inode)->i_disksize = disksize;
Jan Kara4e7ea812013-06-04 13:17:40 -04002454 err2 = ext4_mark_inode_dirty(handle, inode);
Theodore Ts'o622cad12014-04-11 10:35:17 -04002455 up_write(&EXT4_I(inode)->i_data_sem);
Jan Kara4e7ea812013-06-04 13:17:40 -04002456 if (err2)
2457 ext4_error(inode->i_sb,
2458 "Failed to mark inode %lu dirty",
2459 inode->i_ino);
2460 if (!err)
2461 err = err2;
2462 }
2463 return err;
2464}
2465
2466/*
Jan Karafffb2732013-06-04 13:01:11 -04002467 * Calculate the total number of credits to reserve for one writepages
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002468 * iteration. This is called from ext4_writepages(). We map an extent of
Anatol Pomozov70261f52013-08-28 14:40:12 -04002469 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
Jan Karafffb2732013-06-04 13:01:11 -04002470 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2471 * bpp - 1 blocks in bpp different extents.
2472 */
Mingming Cao525f4ed2008-08-19 22:15:58 -04002473static int ext4_da_writepages_trans_blocks(struct inode *inode)
2474{
Jan Karafffb2732013-06-04 13:01:11 -04002475 int bpp = ext4_journal_blocks_per_page(inode);
Mingming Cao525f4ed2008-08-19 22:15:58 -04002476
Jan Karafffb2732013-06-04 13:01:11 -04002477 return ext4_meta_trans_blocks(inode,
2478 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
Mingming Cao525f4ed2008-08-19 22:15:58 -04002479}
Mingming Cao61628a32008-07-11 19:27:31 -04002480
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002481/*
Jan Kara4e7ea812013-06-04 13:17:40 -04002482 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2483 * and underlying extent to map
2484 *
2485 * @mpd - where to look for pages
2486 *
2487 * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2488 * IO immediately. When we find a page which isn't mapped we start accumulating
2489 * extent of buffers underlying these pages that needs mapping (formed by
2490 * either delayed or unwritten buffers). We also lock the pages containing
2491 * these buffers. The extent found is returned in @mpd structure (starting at
2492 * mpd->lblk with length mpd->len blocks).
2493 *
2494 * Note that this function can attach bios to one io_end structure which are
2495 * neither logically nor physically contiguous. Although it may seem as an
2496 * unnecessary complication, it is actually inevitable in blocksize < pagesize
2497 * case as we need to track IO to all buffers underlying a page in one io_end.
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002498 */
Jan Kara4e7ea812013-06-04 13:17:40 -04002499static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002500{
Jan Kara4e7ea812013-06-04 13:17:40 -04002501 struct address_space *mapping = mpd->inode->i_mapping;
2502 struct pagevec pvec;
2503 unsigned int nr_pages;
Ming Leiaeac5892013-10-17 18:56:16 -04002504 long left = mpd->wbc->nr_to_write;
Jan Kara4e7ea812013-06-04 13:17:40 -04002505 pgoff_t index = mpd->first_page;
2506 pgoff_t end = mpd->last_page;
2507 int tag;
2508 int i, err = 0;
2509 int blkbits = mpd->inode->i_blkbits;
2510 ext4_lblk_t lblk;
2511 struct buffer_head *head;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002512
Jan Kara4e7ea812013-06-04 13:17:40 -04002513 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
Eric Sandeen5b41d922010-10-27 21:30:13 -04002514 tag = PAGECACHE_TAG_TOWRITE;
2515 else
2516 tag = PAGECACHE_TAG_DIRTY;
2517
Jan Kara4e7ea812013-06-04 13:17:40 -04002518 pagevec_init(&pvec, 0);
2519 mpd->map.m_len = 0;
2520 mpd->next_page = index;
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002521 while (index <= end) {
Eric Sandeen5b41d922010-10-27 21:30:13 -04002522 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002523 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
2524 if (nr_pages == 0)
Jan Kara4e7ea812013-06-04 13:17:40 -04002525 goto out;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002526
2527 for (i = 0; i < nr_pages; i++) {
2528 struct page *page = pvec.pages[i];
2529
2530 /*
2531 * At this point, the page may be truncated or
2532 * invalidated (changing page->mapping to NULL), or
2533 * even swizzled back from swapper_space to tmpfs file
2534 * mapping. However, page->index will not change
2535 * because we have a reference on the page.
2536 */
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002537 if (page->index > end)
2538 goto out;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002539
Ming Leiaeac5892013-10-17 18:56:16 -04002540 /*
2541 * Accumulated enough dirty pages? This doesn't apply
2542 * to WB_SYNC_ALL mode. For integrity sync we have to
2543 * keep going because someone may be concurrently
2544 * dirtying pages, and we might have synced a lot of
2545 * newly appeared dirty pages, but have not synced all
2546 * of the old dirty pages.
2547 */
2548 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2549 goto out;
2550
Jan Kara4e7ea812013-06-04 13:17:40 -04002551 /* If we can't merge this page, we are done. */
2552 if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2553 goto out;
Theodore Ts'o78aaced2011-02-26 14:09:14 -05002554
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002555 lock_page(page);
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002556 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002557 * If the page is no longer dirty, or its mapping no
2558 * longer corresponds to inode we are writing (which
2559 * means it has been truncated or invalidated), or the
2560 * page is already under writeback and we are not doing
2561 * a data integrity writeback, skip the page
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002562 */
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002563 if (!PageDirty(page) ||
2564 (PageWriteback(page) &&
Jan Kara4e7ea812013-06-04 13:17:40 -04002565 (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002566 unlikely(page->mapping != mapping)) {
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002567 unlock_page(page);
2568 continue;
2569 }
2570
Darrick J. Wong7cb1a532011-05-18 13:53:20 -04002571 wait_on_page_writeback(page);
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002572 BUG_ON(PageWriteback(page));
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002573
Jan Kara4e7ea812013-06-04 13:17:40 -04002574 if (mpd->map.m_len == 0)
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002575 mpd->first_page = page->index;
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002576 mpd->next_page = page->index + 1;
Jan Karaf8bec372013-01-28 12:55:08 -05002577 /* Add all dirty buffers to mpd */
Jan Kara4e7ea812013-06-04 13:17:40 -04002578 lblk = ((ext4_lblk_t)page->index) <<
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002579 (PAGE_SHIFT - blkbits);
Jan Karaf8bec372013-01-28 12:55:08 -05002580 head = page_buffers(page);
Jan Kara5f1132b2013-08-17 10:02:33 -04002581 err = mpage_process_page_bufs(mpd, head, head, lblk);
2582 if (err <= 0)
Jan Kara4e7ea812013-06-04 13:17:40 -04002583 goto out;
Jan Kara5f1132b2013-08-17 10:02:33 -04002584 err = 0;
Ming Leiaeac5892013-10-17 18:56:16 -04002585 left--;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002586 }
2587 pagevec_release(&pvec);
2588 cond_resched();
2589 }
Theodore Ts'o4f01b022011-02-26 14:07:37 -05002590 return 0;
Theodore Ts'o8eb9e5c2011-02-26 14:07:31 -05002591out:
2592 pagevec_release(&pvec);
Jan Kara4e7ea812013-06-04 13:17:40 -04002593 return err;
Theodore Ts'o8e48dcf2010-05-16 18:00:00 -04002594}
2595
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002596static int __writepage(struct page *page, struct writeback_control *wbc,
2597 void *data)
2598{
2599 struct address_space *mapping = data;
2600 int ret = ext4_writepage(page, wbc);
2601 mapping_set_error(mapping, ret);
2602 return ret;
2603}
2604
2605static int ext4_writepages(struct address_space *mapping,
2606 struct writeback_control *wbc)
Alex Tomas64769242008-07-11 19:27:31 -04002607{
Jan Kara4e7ea812013-06-04 13:17:40 -04002608 pgoff_t writeback_index = 0;
2609 long nr_to_write = wbc->nr_to_write;
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002610 int range_whole = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002611 int cycled = 1;
Mingming Cao61628a32008-07-11 19:27:31 -04002612 handle_t *handle = NULL;
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002613 struct mpage_da_data mpd;
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04002614 struct inode *inode = mapping->host;
Jan Kara6b523df2013-06-04 13:21:11 -04002615 int needed_blocks, rsv_blocks = 0, ret = 0;
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04002616 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
Jan Kara4e7ea812013-06-04 13:17:40 -04002617 bool done;
Shaohua Li1bce63d2011-10-18 10:55:51 -04002618 struct blk_plug plug;
Theodore Ts'ocb530542013-07-01 08:12:40 -04002619 bool give_up_on_write = false;
Mingming Cao61628a32008-07-11 19:27:31 -04002620
Daeho Jeongc8585c62016-04-25 23:22:35 -04002621 percpu_down_read(&sbi->s_journal_flag_rwsem);
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002622 trace_ext4_writepages(inode, wbc);
Theodore Ts'oba80b102009-01-03 20:03:21 -05002623
Daeho Jeongc8585c62016-04-25 23:22:35 -04002624 if (dax_mapping(mapping)) {
2625 ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev,
2626 wbc);
2627 goto out_writepages;
2628 }
Ross Zwisler7f6d5b52016-02-26 15:19:55 -08002629
Mingming Cao61628a32008-07-11 19:27:31 -04002630 /*
2631 * No pages to write? This is mainly a kludge to avoid starting
2632 * a transaction for special inodes like journal inode on last iput()
2633 * because that could violate lock ordering on umount
2634 */
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002635 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
Ming Leibbf023c2013-10-30 07:27:16 -04002636 goto out_writepages;
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002637
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002638 if (ext4_should_journal_data(inode)) {
2639 struct blk_plug plug;
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002640
2641 blk_start_plug(&plug);
2642 ret = write_cache_pages(mapping, wbc, __writepage, mapping);
2643 blk_finish_plug(&plug);
Ming Leibbf023c2013-10-30 07:27:16 -04002644 goto out_writepages;
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002645 }
2646
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002647 /*
2648 * If the filesystem has aborted, it is read-only, so return
2649 * right away instead of dumping stack traces later on that
2650 * will obscure the real source of the problem. We test
Theodore Ts'o4ab2f152009-06-13 10:09:36 -04002651 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002652 * the latter could be true if the filesystem is mounted
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002653 * read-only, and in that case, ext4_writepages should
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002654 * *never* be called, so if that ever happens, we would want
2655 * the stack trace.
2656 */
Ming Leibbf023c2013-10-30 07:27:16 -04002657 if (unlikely(sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) {
2658 ret = -EROFS;
2659 goto out_writepages;
2660 }
Theodore Ts'o2a21e372008-11-05 09:22:24 -05002661
Jan Kara6b523df2013-06-04 13:21:11 -04002662 if (ext4_should_dioread_nolock(inode)) {
2663 /*
Anatol Pomozov70261f52013-08-28 14:40:12 -04002664 * We may need to convert up to one extent per block in
Jan Kara6b523df2013-06-04 13:21:11 -04002665 * the page and we may dirty the inode.
2666 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002667 rsv_blocks = 1 + (PAGE_SIZE >> inode->i_blkbits);
Jan Kara6b523df2013-06-04 13:21:11 -04002668 }
2669
Jan Kara4e7ea812013-06-04 13:17:40 -04002670 /*
2671 * If we have inline data and arrive here, it means that
2672 * we will soon create the block for the 1st page, so
2673 * we'd better clear the inline data here.
2674 */
2675 if (ext4_has_inline_data(inode)) {
2676 /* Just inode will be modified... */
2677 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2678 if (IS_ERR(handle)) {
2679 ret = PTR_ERR(handle);
2680 goto out_writepages;
2681 }
2682 BUG_ON(ext4_test_inode_state(inode,
2683 EXT4_STATE_MAY_INLINE_DATA));
2684 ext4_destroy_inline_data(handle, inode);
2685 ext4_journal_stop(handle);
2686 }
2687
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002688 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2689 range_whole = 1;
Mingming Cao61628a32008-07-11 19:27:31 -04002690
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002691 if (wbc->range_cyclic) {
Jan Kara4e7ea812013-06-04 13:17:40 -04002692 writeback_index = mapping->writeback_index;
2693 if (writeback_index)
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002694 cycled = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002695 mpd.first_page = writeback_index;
2696 mpd.last_page = -1;
Eric Sandeen5b41d922010-10-27 21:30:13 -04002697 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002698 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2699 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
Eric Sandeen5b41d922010-10-27 21:30:13 -04002700 }
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002701
Jan Kara4e7ea812013-06-04 13:17:40 -04002702 mpd.inode = inode;
2703 mpd.wbc = wbc;
2704 ext4_io_submit_init(&mpd.io_submit, wbc);
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002705retry:
Wu Fengguang6e6938b2010-06-06 10:38:15 -06002706 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
Jan Kara4e7ea812013-06-04 13:17:40 -04002707 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2708 done = false;
Shaohua Li1bce63d2011-10-18 10:55:51 -04002709 blk_start_plug(&plug);
Jan Kara4e7ea812013-06-04 13:17:40 -04002710 while (!done && mpd.first_page <= mpd.last_page) {
2711 /* For each extent of pages we use new io_end */
2712 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2713 if (!mpd.io_submit.io_end) {
2714 ret = -ENOMEM;
2715 break;
2716 }
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002717
2718 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002719 * We have two constraints: We find one extent to map and we
2720 * must always write out whole page (makes a difference when
2721 * blocksize < pagesize) so that we don't block on IO when we
2722 * try to write out the rest of the page. Journalled mode is
2723 * not supported by delalloc.
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002724 */
2725 BUG_ON(ext4_should_journal_data(inode));
Mingming Cao525f4ed2008-08-19 22:15:58 -04002726 needed_blocks = ext4_da_writepages_trans_blocks(inode);
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002727
Jan Kara4e7ea812013-06-04 13:17:40 -04002728 /* start a new transaction */
Jan Kara6b523df2013-06-04 13:21:11 -04002729 handle = ext4_journal_start_with_reserve(inode,
2730 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
Mingming Cao61628a32008-07-11 19:27:31 -04002731 if (IS_ERR(handle)) {
2732 ret = PTR_ERR(handle);
Theodore Ts'o16939182009-09-26 17:43:59 -04002733 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
Curt Wohlgemuthfbe845d2010-05-16 13:00:00 -04002734 "%ld pages, ino %lu; err %d", __func__,
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002735 wbc->nr_to_write, inode->i_ino, ret);
Jan Kara4e7ea812013-06-04 13:17:40 -04002736 /* Release allocated io_end */
2737 ext4_put_io_end(mpd.io_submit.io_end);
2738 break;
Mingming Cao61628a32008-07-11 19:27:31 -04002739 }
Theodore Ts'of63e6002009-02-23 16:42:39 -05002740
Jan Kara4e7ea812013-06-04 13:17:40 -04002741 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2742 ret = mpage_prepare_extent_to_map(&mpd);
2743 if (!ret) {
2744 if (mpd.map.m_len)
Theodore Ts'ocb530542013-07-01 08:12:40 -04002745 ret = mpage_map_and_submit_extent(handle, &mpd,
2746 &give_up_on_write);
Jan Kara4e7ea812013-06-04 13:17:40 -04002747 else {
2748 /*
2749 * We scanned the whole range (or exhausted
2750 * nr_to_write), submitted what was mapped and
2751 * didn't find anything needing mapping. We are
2752 * done.
2753 */
2754 done = true;
2755 }
Theodore Ts'of63e6002009-02-23 16:42:39 -05002756 }
Jan Kara646caa92016-07-04 10:14:01 -04002757 /*
2758 * Caution: If the handle is synchronous,
2759 * ext4_journal_stop() can wait for transaction commit
2760 * to finish which may depend on writeback of pages to
2761 * complete or on page lock to be released. In that
2762 * case, we have to wait until after after we have
2763 * submitted all the IO, released page locks we hold,
2764 * and dropped io_end reference (for extent conversion
2765 * to be able to complete) before stopping the handle.
2766 */
2767 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2768 ext4_journal_stop(handle);
2769 handle = NULL;
2770 }
Jan Kara4e7ea812013-06-04 13:17:40 -04002771 /* Submit prepared bio */
2772 ext4_io_submit(&mpd.io_submit);
2773 /* Unlock pages we didn't use */
Theodore Ts'ocb530542013-07-01 08:12:40 -04002774 mpage_release_unused_pages(&mpd, give_up_on_write);
Jan Kara646caa92016-07-04 10:14:01 -04002775 /*
2776 * Drop our io_end reference we got from init. We have
2777 * to be careful and use deferred io_end finishing if
2778 * we are still holding the transaction as we can
2779 * release the last reference to io_end which may end
2780 * up doing unwritten extent conversion.
2781 */
2782 if (handle) {
2783 ext4_put_io_end_defer(mpd.io_submit.io_end);
2784 ext4_journal_stop(handle);
2785 } else
2786 ext4_put_io_end(mpd.io_submit.io_end);
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002787
Jan Kara4e7ea812013-06-04 13:17:40 -04002788 if (ret == -ENOSPC && sbi->s_journal) {
2789 /*
2790 * Commit the transaction which would
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002791 * free blocks released in the transaction
2792 * and try again
2793 */
Aneesh Kumar K.Vdf222912008-09-08 23:05:34 -04002794 jbd2_journal_force_commit_nested(sbi->s_journal);
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002795 ret = 0;
Jan Kara4e7ea812013-06-04 13:17:40 -04002796 continue;
2797 }
2798 /* Fatal error - ENOMEM, EIO... */
2799 if (ret)
Mingming Cao61628a32008-07-11 19:27:31 -04002800 break;
Mingming Cao61628a32008-07-11 19:27:31 -04002801 }
Shaohua Li1bce63d2011-10-18 10:55:51 -04002802 blk_finish_plug(&plug);
Jan Kara9c12a832013-09-16 08:24:26 -04002803 if (!ret && !cycled && wbc->nr_to_write > 0) {
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002804 cycled = 1;
Jan Kara4e7ea812013-06-04 13:17:40 -04002805 mpd.last_page = writeback_index - 1;
2806 mpd.first_page = 0;
Aneesh Kumar K.V2acf2c22009-02-14 10:42:58 -05002807 goto retry;
2808 }
Mingming Cao61628a32008-07-11 19:27:31 -04002809
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002810 /* Update index */
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002811 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2812 /*
Jan Kara4e7ea812013-06-04 13:17:40 -04002813 * Set the writeback_index so that range_cyclic
Aneesh Kumar K.V22208de2008-10-16 10:10:36 -04002814 * mode will write it back later
2815 */
Jan Kara4e7ea812013-06-04 13:17:40 -04002816 mapping->writeback_index = mpd.first_page;
Aneesh Kumar K.Va1d6cc52008-08-19 21:55:02 -04002817
Mingming Cao61628a32008-07-11 19:27:31 -04002818out_writepages:
Theodore Ts'o20970ba2013-06-06 14:00:46 -04002819 trace_ext4_writepages_result(inode, wbc, ret,
2820 nr_to_write - wbc->nr_to_write);
Daeho Jeongc8585c62016-04-25 23:22:35 -04002821 percpu_up_read(&sbi->s_journal_flag_rwsem);
Mingming Cao61628a32008-07-11 19:27:31 -04002822 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04002823}
2824
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002825static int ext4_nonda_switch(struct super_block *sb)
2826{
Eric Whitney5c1ff332013-04-09 09:27:31 -04002827 s64 free_clusters, dirty_clusters;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002828 struct ext4_sb_info *sbi = EXT4_SB(sb);
2829
2830 /*
2831 * switch to non delalloc mode if we are running low
2832 * on free block. The free block accounting via percpu
Eric Dumazet179f7eb2009-01-06 14:41:04 -08002833 * counters can get slightly wrong with percpu_counter_batch getting
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002834 * accumulated on each CPU without updating global counters
2835 * Delalloc need an accurate free block accounting. So switch
2836 * to non delalloc when we are near to error range.
2837 */
Eric Whitney5c1ff332013-04-09 09:27:31 -04002838 free_clusters =
2839 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2840 dirty_clusters =
2841 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
Theodore Ts'o00d4e732012-09-19 22:42:36 -04002842 /*
2843 * Start pushing delalloc when 1/2 of free blocks are dirty.
2844 */
Eric Whitney5c1ff332013-04-09 09:27:31 -04002845 if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
Miao Xie10ee27a2013-01-10 13:47:57 +08002846 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
Theodore Ts'o00d4e732012-09-19 22:42:36 -04002847
Eric Whitney5c1ff332013-04-09 09:27:31 -04002848 if (2 * free_clusters < 3 * dirty_clusters ||
2849 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002850 /*
Eric Sandeenc8afb442009-12-23 07:58:12 -05002851 * free block count is less than 150% of dirty blocks
2852 * or free blocks is less than watermark
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002853 */
2854 return 1;
2855 }
2856 return 0;
2857}
2858
Eric Sandeen0ff89472014-10-11 19:51:17 -04002859/* We always reserve for an inode update; the superblock could be there too */
2860static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
2861{
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04002862 if (likely(ext4_has_feature_large_file(inode->i_sb)))
Eric Sandeen0ff89472014-10-11 19:51:17 -04002863 return 1;
2864
2865 if (pos + len <= 0x7fffffffULL)
2866 return 1;
2867
2868 /* We might need to update the superblock to set LARGE_FILE */
2869 return 2;
2870}
2871
Alex Tomas64769242008-07-11 19:27:31 -04002872static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04002873 loff_t pos, unsigned len, unsigned flags,
2874 struct page **pagep, void **fsdata)
Alex Tomas64769242008-07-11 19:27:31 -04002875{
Eric Sandeen72b8ab92010-05-16 11:00:00 -04002876 int ret, retries = 0;
Alex Tomas64769242008-07-11 19:27:31 -04002877 struct page *page;
2878 pgoff_t index;
Alex Tomas64769242008-07-11 19:27:31 -04002879 struct inode *inode = mapping->host;
2880 handle_t *handle;
2881
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002882 index = pos >> PAGE_SHIFT;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04002883
2884 if (ext4_nonda_switch(inode->i_sb)) {
2885 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2886 return ext4_write_begin(file, mapping, pos,
2887 len, flags, pagep, fsdata);
2888 }
2889 *fsdata = (void *)0;
Theodore Ts'o9bffad12009-06-17 11:48:11 -04002890 trace_ext4_da_write_begin(inode, pos, len, flags);
Tao Ma9c3569b2012-12-10 14:05:57 -05002891
2892 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
2893 ret = ext4_da_write_inline_data_begin(mapping, inode,
2894 pos, len, flags,
2895 pagep, fsdata);
2896 if (ret < 0)
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002897 return ret;
2898 if (ret == 1)
2899 return 0;
Tao Ma9c3569b2012-12-10 14:05:57 -05002900 }
2901
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002902 /*
2903 * grab_cache_page_write_begin() can take a long time if the
2904 * system is thrashing due to memory pressure, or if the page
2905 * is being written back. So grab it first before we start
2906 * the transaction handle. This also allows us to allocate
2907 * the page (if needed) without using GFP_NOFS.
2908 */
2909retry_grab:
2910 page = grab_cache_page_write_begin(mapping, index, flags);
2911 if (!page)
2912 return -ENOMEM;
2913 unlock_page(page);
2914
Alex Tomas64769242008-07-11 19:27:31 -04002915 /*
2916 * With delayed allocation, we don't log the i_disksize update
2917 * if there is delayed block allocation. But we still need
2918 * to journalling the i_disksize update if writes to the end
2919 * of file which has an already mapped buffer.
2920 */
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002921retry_journal:
Eric Sandeen0ff89472014-10-11 19:51:17 -04002922 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
2923 ext4_da_write_credits(inode, pos, len));
Alex Tomas64769242008-07-11 19:27:31 -04002924 if (IS_ERR(handle)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002925 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002926 return PTR_ERR(handle);
Alex Tomas64769242008-07-11 19:27:31 -04002927 }
2928
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002929 lock_page(page);
2930 if (page->mapping != mapping) {
2931 /* The page got truncated from under us */
2932 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002933 put_page(page);
Eric Sandeend5a0d4f2008-08-02 18:51:06 -04002934 ext4_journal_stop(handle);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002935 goto retry_grab;
Eric Sandeend5a0d4f2008-08-02 18:51:06 -04002936 }
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002937 /* In case writeback began while the page was unlocked */
Dmitry Monakhov7afe5aa2013-08-28 14:30:47 -04002938 wait_for_stable_page(page);
Alex Tomas64769242008-07-11 19:27:31 -04002939
Michael Halcrow2058f832015-04-12 00:55:10 -04002940#ifdef CONFIG_EXT4_FS_ENCRYPTION
2941 ret = ext4_block_write_begin(page, pos, len,
2942 ext4_da_get_block_prep);
2943#else
Christoph Hellwig6e1db882010-06-04 11:29:57 +02002944 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
Michael Halcrow2058f832015-04-12 00:55:10 -04002945#endif
Alex Tomas64769242008-07-11 19:27:31 -04002946 if (ret < 0) {
2947 unlock_page(page);
2948 ext4_journal_stop(handle);
Aneesh Kumar K.Vae4d5372008-09-13 13:10:25 -04002949 /*
2950 * block_write_begin may have instantiated a few blocks
2951 * outside i_size. Trim these off again. Don't need
2952 * i_size_read because we hold i_mutex.
2953 */
2954 if (pos + len > inode->i_size)
Jan Karab9a42072009-12-08 21:24:33 -05002955 ext4_truncate_failed_write(inode);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002956
2957 if (ret == -ENOSPC &&
2958 ext4_should_retry_alloc(inode->i_sb, &retries))
2959 goto retry_journal;
2960
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002961 put_page(page);
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002962 return ret;
Alex Tomas64769242008-07-11 19:27:31 -04002963 }
2964
Theodore Ts'o47564bf2013-02-09 09:24:14 -05002965 *pagep = page;
Alex Tomas64769242008-07-11 19:27:31 -04002966 return ret;
2967}
2968
Mingming Cao632eaea2008-07-11 19:27:31 -04002969/*
2970 * Check if we should update i_disksize
2971 * when write to the end of file but not require block allocation
2972 */
2973static int ext4_da_should_update_i_disksize(struct page *page,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04002974 unsigned long offset)
Mingming Cao632eaea2008-07-11 19:27:31 -04002975{
2976 struct buffer_head *bh;
2977 struct inode *inode = page->mapping->host;
2978 unsigned int idx;
2979 int i;
2980
2981 bh = page_buffers(page);
2982 idx = offset >> inode->i_blkbits;
2983
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04002984 for (i = 0; i < idx; i++)
Mingming Cao632eaea2008-07-11 19:27:31 -04002985 bh = bh->b_this_page;
2986
Aneesh Kumar K.V29fa89d2009-05-12 16:30:27 -04002987 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
Mingming Cao632eaea2008-07-11 19:27:31 -04002988 return 0;
2989 return 1;
2990}
2991
Alex Tomas64769242008-07-11 19:27:31 -04002992static int ext4_da_write_end(struct file *file,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04002993 struct address_space *mapping,
2994 loff_t pos, unsigned len, unsigned copied,
2995 struct page *page, void *fsdata)
Alex Tomas64769242008-07-11 19:27:31 -04002996{
2997 struct inode *inode = mapping->host;
2998 int ret = 0, ret2;
2999 handle_t *handle = ext4_journal_current_handle();
3000 loff_t new_i_size;
Mingming Cao632eaea2008-07-11 19:27:31 -04003001 unsigned long start, end;
Aneesh Kumar K.V79f0be82008-10-08 23:13:30 -04003002 int write_mode = (int)(unsigned long)fsdata;
3003
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003004 if (write_mode == FALL_BACK_TO_NONDELALLOC)
3005 return ext4_write_end(file, mapping, pos,
3006 len, copied, page, fsdata);
Mingming Cao632eaea2008-07-11 19:27:31 -04003007
Theodore Ts'o9bffad12009-06-17 11:48:11 -04003008 trace_ext4_da_write_end(inode, pos, len, copied);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003009 start = pos & (PAGE_SIZE - 1);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04003010 end = start + copied - 1;
Alex Tomas64769242008-07-11 19:27:31 -04003011
3012 /*
3013 * generic_write_end() will run mark_inode_dirty() if i_size
3014 * changes. So let's piggyback the i_disksize mark_inode_dirty
3015 * into that.
3016 */
Alex Tomas64769242008-07-11 19:27:31 -04003017 new_i_size = pos + copied;
Andrea Arcangeliea51d132011-12-13 21:41:15 -05003018 if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
Tao Ma9c3569b2012-12-10 14:05:57 -05003019 if (ext4_has_inline_data(inode) ||
3020 ext4_da_should_update_i_disksize(page, end)) {
Dmitry Monakhovee124d22014-08-30 23:34:06 -04003021 ext4_update_i_disksize(inode, new_i_size);
Aneesh Kumar K.Vcf17fea2008-09-13 13:06:18 -04003022 /* We need to mark inode dirty even if
3023 * new_i_size is less that inode->i_size
3024 * bu greater than i_disksize.(hint delalloc)
3025 */
3026 ext4_mark_inode_dirty(handle, inode);
Alex Tomas64769242008-07-11 19:27:31 -04003027 }
Mingming Cao632eaea2008-07-11 19:27:31 -04003028 }
Tao Ma9c3569b2012-12-10 14:05:57 -05003029
3030 if (write_mode != CONVERT_INLINE_DATA &&
3031 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3032 ext4_has_inline_data(inode))
3033 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
3034 page);
3035 else
3036 ret2 = generic_write_end(file, mapping, pos, len, copied,
Alex Tomas64769242008-07-11 19:27:31 -04003037 page, fsdata);
Tao Ma9c3569b2012-12-10 14:05:57 -05003038
Alex Tomas64769242008-07-11 19:27:31 -04003039 copied = ret2;
3040 if (ret2 < 0)
3041 ret = ret2;
3042 ret2 = ext4_journal_stop(handle);
3043 if (!ret)
3044 ret = ret2;
3045
3046 return ret ? ret : copied;
3047}
3048
Lukas Czernerd47992f2013-05-21 23:17:23 -04003049static void ext4_da_invalidatepage(struct page *page, unsigned int offset,
3050 unsigned int length)
Alex Tomas64769242008-07-11 19:27:31 -04003051{
Alex Tomas64769242008-07-11 19:27:31 -04003052 /*
3053 * Drop reserved blocks
3054 */
3055 BUG_ON(!PageLocked(page));
3056 if (!page_has_buffers(page))
3057 goto out;
3058
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003059 ext4_da_page_release_reservation(page, offset, length);
Alex Tomas64769242008-07-11 19:27:31 -04003060
3061out:
Lukas Czernerd47992f2013-05-21 23:17:23 -04003062 ext4_invalidatepage(page, offset, length);
Alex Tomas64769242008-07-11 19:27:31 -04003063
3064 return;
3065}
3066
Theodore Ts'occd25062009-02-26 01:04:07 -05003067/*
3068 * Force all delayed allocation blocks to be allocated for a given inode.
3069 */
3070int ext4_alloc_da_blocks(struct inode *inode)
3071{
Theodore Ts'ofb40ba02009-09-16 19:30:40 -04003072 trace_ext4_alloc_da_blocks(inode);
3073
Theodore Ts'o71d4f7d2014-07-15 06:02:38 -04003074 if (!EXT4_I(inode)->i_reserved_data_blocks)
Theodore Ts'occd25062009-02-26 01:04:07 -05003075 return 0;
3076
3077 /*
3078 * We do something simple for now. The filemap_flush() will
3079 * also start triggering a write of the data blocks, which is
3080 * not strictly speaking necessary (and for users of
3081 * laptop_mode, not even desirable). However, to do otherwise
3082 * would require replicating code paths in:
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003083 *
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003084 * ext4_writepages() ->
Theodore Ts'occd25062009-02-26 01:04:07 -05003085 * write_cache_pages() ---> (via passed in callback function)
3086 * __mpage_da_writepage() -->
3087 * mpage_add_bh_to_extent()
3088 * mpage_da_map_blocks()
3089 *
3090 * The problem is that write_cache_pages(), located in
3091 * mm/page-writeback.c, marks pages clean in preparation for
3092 * doing I/O, which is not desirable if we're not planning on
3093 * doing I/O at all.
3094 *
3095 * We could call write_cache_pages(), and then redirty all of
Wu Fengguang380cf092010-11-11 19:23:29 +08003096 * the pages by calling redirty_page_for_writepage() but that
Theodore Ts'occd25062009-02-26 01:04:07 -05003097 * would be ugly in the extreme. So instead we would need to
3098 * replicate parts of the code in the above functions,
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003099 * simplifying them because we wouldn't actually intend to
Theodore Ts'occd25062009-02-26 01:04:07 -05003100 * write out the pages, but rather only collect contiguous
3101 * logical block extents, call the multi-block allocator, and
3102 * then update the buffer heads with the block allocations.
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04003103 *
Theodore Ts'occd25062009-02-26 01:04:07 -05003104 * For now, though, we'll cheat by calling filemap_flush(),
3105 * which will map the blocks, and start the I/O, but not
3106 * actually wait for the I/O to complete.
3107 */
3108 return filemap_flush(inode->i_mapping);
3109}
Alex Tomas64769242008-07-11 19:27:31 -04003110
3111/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003112 * bmap() is special. It gets used by applications such as lilo and by
3113 * the swapper to find the on-disk block of a specific piece of data.
3114 *
3115 * Naturally, this is dangerous if the block concerned is still in the
Mingming Cao617ba132006-10-11 01:20:53 -07003116 * journal. If somebody makes a swapfile on an ext4 data-journaling
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003117 * filesystem and enables swap, then they may get a nasty shock when the
3118 * data getting swapped to that swapfile suddenly gets overwritten by
3119 * the original zero's written out previously to the journal and
3120 * awaiting writeback in the kernel's buffer cache.
3121 *
3122 * So, if we see any bmap calls here on a modified, data-journaled file,
3123 * take extra steps to flush any blocks which might be in the cache.
3124 */
Mingming Cao617ba132006-10-11 01:20:53 -07003125static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003126{
3127 struct inode *inode = mapping->host;
3128 journal_t *journal;
3129 int err;
3130
Tao Ma46c7f252012-12-10 14:04:52 -05003131 /*
3132 * We can get here for an inline file via the FIBMAP ioctl
3133 */
3134 if (ext4_has_inline_data(inode))
3135 return 0;
3136
Alex Tomas64769242008-07-11 19:27:31 -04003137 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3138 test_opt(inode->i_sb, DELALLOC)) {
3139 /*
3140 * With delalloc we want to sync the file
3141 * so that we can make sure we allocate
3142 * blocks for file
3143 */
3144 filemap_write_and_wait(mapping);
3145 }
3146
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05003147 if (EXT4_JOURNAL(inode) &&
3148 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003149 /*
3150 * This is a REALLY heavyweight approach, but the use of
3151 * bmap on dirty files is expected to be extremely rare:
3152 * only if we run lilo or swapon on a freshly made file
3153 * do we expect this to happen.
3154 *
3155 * (bmap requires CAP_SYS_RAWIO so this does not
3156 * represent an unprivileged user DOS attack --- we'd be
3157 * in trouble if mortal users could trigger this path at
3158 * will.)
3159 *
Mingming Cao617ba132006-10-11 01:20:53 -07003160 * NB. EXT4_STATE_JDATA is not set on files other than
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003161 * regular files. If somebody wants to bmap a directory
3162 * or symlink and gets confused because the buffer
3163 * hasn't yet been flushed to disk, they deserve
3164 * everything they get.
3165 */
3166
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05003167 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
Mingming Cao617ba132006-10-11 01:20:53 -07003168 journal = EXT4_JOURNAL(inode);
Mingming Caodab291a2006-10-11 01:21:01 -07003169 jbd2_journal_lock_updates(journal);
3170 err = jbd2_journal_flush(journal);
3171 jbd2_journal_unlock_updates(journal);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003172
3173 if (err)
3174 return 0;
3175 }
3176
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04003177 return generic_block_bmap(mapping, block, ext4_get_block);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003178}
3179
Mingming Cao617ba132006-10-11 01:20:53 -07003180static int ext4_readpage(struct file *file, struct page *page)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003181{
Tao Ma46c7f252012-12-10 14:04:52 -05003182 int ret = -EAGAIN;
3183 struct inode *inode = page->mapping->host;
3184
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003185 trace_ext4_readpage(page);
Tao Ma46c7f252012-12-10 14:04:52 -05003186
3187 if (ext4_has_inline_data(inode))
3188 ret = ext4_readpage_inline(inode, page);
3189
3190 if (ret == -EAGAIN)
Theodore Ts'of64e02f2015-04-08 00:00:32 -04003191 return ext4_mpage_readpages(page->mapping, NULL, page, 1);
Tao Ma46c7f252012-12-10 14:04:52 -05003192
3193 return ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003194}
3195
3196static int
Mingming Cao617ba132006-10-11 01:20:53 -07003197ext4_readpages(struct file *file, struct address_space *mapping,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003198 struct list_head *pages, unsigned nr_pages)
3199{
Tao Ma46c7f252012-12-10 14:04:52 -05003200 struct inode *inode = mapping->host;
3201
3202 /* If the file has inline data, no need to do readpages. */
3203 if (ext4_has_inline_data(inode))
3204 return 0;
3205
Theodore Ts'of64e02f2015-04-08 00:00:32 -04003206 return ext4_mpage_readpages(mapping, pages, NULL, nr_pages);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003207}
3208
Lukas Czernerd47992f2013-05-21 23:17:23 -04003209static void ext4_invalidatepage(struct page *page, unsigned int offset,
3210 unsigned int length)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003211{
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003212 trace_ext4_invalidatepage(page, offset, length);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003213
Jan Kara4520fb32012-12-25 13:28:54 -05003214 /* No journalling happens on data buffers when this function is used */
3215 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3216
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003217 block_invalidatepage(page, offset, length);
Jan Kara4520fb32012-12-25 13:28:54 -05003218}
3219
Jan Kara53e87262012-12-25 13:29:52 -05003220static int __ext4_journalled_invalidatepage(struct page *page,
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003221 unsigned int offset,
3222 unsigned int length)
Jan Kara4520fb32012-12-25 13:28:54 -05003223{
3224 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3225
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003226 trace_ext4_journalled_invalidatepage(page, offset, length);
Jan Kara4520fb32012-12-25 13:28:54 -05003227
Jiaying Zhang744692d2010-03-04 16:14:02 -05003228 /*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003229 * If it's a full truncate we just forget about the pending dirtying
3230 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003231 if (offset == 0 && length == PAGE_SIZE)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003232 ClearPageChecked(page);
3233
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003234 return jbd2_journal_invalidatepage(journal, page, offset, length);
Jan Kara53e87262012-12-25 13:29:52 -05003235}
3236
3237/* Wrapper for aops... */
3238static void ext4_journalled_invalidatepage(struct page *page,
Lukas Czernerd47992f2013-05-21 23:17:23 -04003239 unsigned int offset,
3240 unsigned int length)
Jan Kara53e87262012-12-25 13:29:52 -05003241{
Lukas Czernerca99fdd2013-05-21 23:25:01 -04003242 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003243}
3244
Mingming Cao617ba132006-10-11 01:20:53 -07003245static int ext4_releasepage(struct page *page, gfp_t wait)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003246{
Mingming Cao617ba132006-10-11 01:20:53 -07003247 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003248
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003249 trace_ext4_releasepage(page);
3250
Jan Karae1c36592013-03-10 22:19:00 -04003251 /* Page has dirty journalled data -> cannot release */
3252 if (PageChecked(page))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003253 return 0;
Frank Mayhar03901312009-01-07 00:06:22 -05003254 if (journal)
3255 return jbd2_journal_try_to_free_buffers(journal, page, wait);
3256 else
3257 return try_to_free_buffers(page);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003258}
3259
Jan Karaba5843f2015-12-07 15:10:44 -05003260#ifdef CONFIG_FS_DAX
Jan Kara12735f82016-05-13 00:51:15 -04003261/*
3262 * Get block function for DAX IO and mmap faults. It takes care of converting
3263 * unwritten extents to written ones and initializes new / converted blocks
3264 * to zeros.
3265 */
3266int ext4_dax_get_block(struct inode *inode, sector_t iblock,
3267 struct buffer_head *bh_result, int create)
Matthew Wilcoxed923b52015-09-08 14:59:20 -07003268{
Jan Kara7cb476f2016-05-13 00:38:16 -04003269 int ret;
Jan Karac86d8db2015-12-07 15:10:26 -05003270
Jan Kara12735f82016-05-13 00:51:15 -04003271 ext4_debug("inode %lu, create flag %d\n", inode->i_ino, create);
Jan Kara7cb476f2016-05-13 00:38:16 -04003272 if (!create)
3273 return _ext4_get_block(inode, iblock, bh_result, 0);
Jan Karaba5843f2015-12-07 15:10:44 -05003274
Jan Kara7cb476f2016-05-13 00:38:16 -04003275 ret = ext4_get_block_trans(inode, iblock, bh_result,
3276 EXT4_GET_BLOCKS_PRE_IO |
3277 EXT4_GET_BLOCKS_CREATE_ZERO);
3278 if (ret < 0)
3279 return ret;
Jan Karaba5843f2015-12-07 15:10:44 -05003280
Jan Kara7cb476f2016-05-13 00:38:16 -04003281 if (buffer_unwritten(bh_result)) {
Jan Karaba5843f2015-12-07 15:10:44 -05003282 /*
Jan Kara12735f82016-05-13 00:51:15 -04003283 * We are protected by i_mmap_sem or i_mutex so we know block
3284 * cannot go away from under us even though we dropped
3285 * i_data_sem. Convert extent to written and write zeros there.
Jan Karaba5843f2015-12-07 15:10:44 -05003286 */
Jan Kara7cb476f2016-05-13 00:38:16 -04003287 ret = ext4_get_block_trans(inode, iblock, bh_result,
3288 EXT4_GET_BLOCKS_CONVERT |
3289 EXT4_GET_BLOCKS_CREATE_ZERO);
3290 if (ret < 0)
Jan Karaba5843f2015-12-07 15:10:44 -05003291 return ret;
Jan Karaba5843f2015-12-07 15:10:44 -05003292 }
Jan Kara7cb476f2016-05-13 00:38:16 -04003293 /*
3294 * At least for now we have to clear BH_New so that DAX code
3295 * doesn't attempt to zero blocks again in a racy way.
3296 */
3297 clear_buffer_new(bh_result);
3298 return 0;
Matthew Wilcoxed923b52015-09-08 14:59:20 -07003299}
Jan Kara12735f82016-05-13 00:51:15 -04003300#else
3301/* Just define empty function, it will never get called. */
3302int ext4_dax_get_block(struct inode *inode, sector_t iblock,
3303 struct buffer_head *bh_result, int create)
3304{
3305 BUG();
3306 return 0;
Matthew Wilcoxed923b52015-09-08 14:59:20 -07003307}
Jan Karaba5843f2015-12-07 15:10:44 -05003308#endif
Matthew Wilcoxed923b52015-09-08 14:59:20 -07003309
Christoph Hellwig187372a2016-02-08 14:40:51 +11003310static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
Christoph Hellwig7b7a8662013-09-04 15:04:39 +02003311 ssize_t size, void *private)
Mingming Cao4c0425f2009-09-28 15:48:41 -04003312{
Jan Kara109811c2016-03-08 23:36:46 -05003313 ext4_io_end_t *io_end = private;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003314
Jan Kara97a851e2013-06-04 11:58:58 -04003315 /* if not async direct IO just return */
Christoph Hellwig7b7a8662013-09-04 15:04:39 +02003316 if (!io_end)
Christoph Hellwig187372a2016-02-08 14:40:51 +11003317 return 0;
Mingming4b70df12009-11-03 14:44:54 -05003318
Zheng Liu88635ca2011-12-28 19:00:25 -05003319 ext_debug("ext4_end_io_dio(): io_end 0x%p "
Joe Perchesace36ad2012-03-19 23:11:43 -04003320 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
Jan Kara109811c2016-03-08 23:36:46 -05003321 io_end, io_end->inode->i_ino, iocb, offset, size);
Mingming Cao8d5d02e2009-09-28 15:48:29 -04003322
Jan Kara74c66bc2016-02-29 08:36:38 +11003323 /*
3324 * Error during AIO DIO. We cannot convert unwritten extents as the
3325 * data was not written. Just clear the unwritten flag and drop io_end.
3326 */
3327 if (size <= 0) {
3328 ext4_clear_io_unwritten_flag(io_end);
3329 size = 0;
3330 }
Mingming Cao4c0425f2009-09-28 15:48:41 -04003331 io_end->offset = offset;
3332 io_end->size = size;
Christoph Hellwig7b7a8662013-09-04 15:04:39 +02003333 ext4_put_io_end(io_end);
Christoph Hellwig187372a2016-02-08 14:40:51 +11003334
3335 return 0;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003336}
Jiaying Zhangc7064ef2010-03-02 13:28:44 -05003337
Mingming Cao4c0425f2009-09-28 15:48:41 -04003338/*
Jan Kara914f82a2016-05-13 00:44:16 -04003339 * Handling of direct IO writes.
3340 *
3341 * For ext4 extent files, ext4 will do direct-io write even to holes,
Mingming Cao4c0425f2009-09-28 15:48:41 -04003342 * preallocated extents, and those write extend the file, no need to
3343 * fall back to buffered IO.
3344 *
Lukas Czerner556615d2014-04-20 23:45:47 -04003345 * For holes, we fallocate those blocks, mark them as unwritten
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003346 * If those blocks were preallocated, we mark sure they are split, but
Lukas Czerner556615d2014-04-20 23:45:47 -04003347 * still keep the range to write as unwritten.
Mingming Cao4c0425f2009-09-28 15:48:41 -04003348 *
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003349 * The unwritten extents will be converted to written when DIO is completed.
Mingming Cao8d5d02e2009-09-28 15:48:29 -04003350 * For async direct IO, since the IO may still pending when return, we
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003351 * set up an end_io call back function, which will do the conversion
Mingming Cao8d5d02e2009-09-28 15:48:29 -04003352 * when async direct IO completed.
Mingming Cao4c0425f2009-09-28 15:48:41 -04003353 *
3354 * If the O_DIRECT write will extend the file then add this inode to the
3355 * orphan list. So recovery will truncate it back to the original size
3356 * if the machine crashes during the write.
3357 *
3358 */
Linus Torvalds0e01df12016-05-24 12:55:26 -07003359static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
Mingming Cao4c0425f2009-09-28 15:48:41 -04003360{
3361 struct file *file = iocb->ki_filp;
3362 struct inode *inode = file->f_mapping->host;
Jan Kara914f82a2016-05-13 00:44:16 -04003363 struct ext4_inode_info *ei = EXT4_I(inode);
Mingming Cao4c0425f2009-09-28 15:48:41 -04003364 ssize_t ret;
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07003365 loff_t offset = iocb->ki_pos;
Al Viroa6cbcd42014-03-04 22:38:00 -05003366 size_t count = iov_iter_count(iter);
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003367 int overwrite = 0;
3368 get_block_t *get_block_func = NULL;
3369 int dio_flags = 0;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003370 loff_t final_size = offset + count;
Jan Kara914f82a2016-05-13 00:44:16 -04003371 int orphan = 0;
3372 handle_t *handle;
Zheng Liu729f52c2012-07-09 16:29:29 -04003373
Jan Kara914f82a2016-05-13 00:44:16 -04003374 if (final_size > inode->i_size) {
3375 /* Credits for sb + inode write */
3376 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3377 if (IS_ERR(handle)) {
3378 ret = PTR_ERR(handle);
3379 goto out;
3380 }
3381 ret = ext4_orphan_add(handle, inode);
3382 if (ret) {
3383 ext4_journal_stop(handle);
3384 goto out;
3385 }
3386 orphan = 1;
3387 ei->i_disksize = inode->i_size;
3388 ext4_journal_stop(handle);
3389 }
Zheng Liu4bd809d2012-07-22 20:19:31 -04003390
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003391 BUG_ON(iocb->private == NULL);
Zheng Liu4bd809d2012-07-22 20:19:31 -04003392
Jan Karae8340392013-06-04 14:27:38 -04003393 /*
3394 * Make all waiters for direct IO properly wait also for extent
3395 * conversion. This also disallows race between truncate() and
3396 * overwrite DIO as i_dio_count needs to be incremented under i_mutex.
3397 */
Jan Kara914f82a2016-05-13 00:44:16 -04003398 inode_dio_begin(inode);
Jan Karae8340392013-06-04 14:27:38 -04003399
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003400 /* If we do a overwrite dio, i_mutex locking can be released */
3401 overwrite = *((int *)iocb->private);
Zheng Liu4bd809d2012-07-22 20:19:31 -04003402
Jan Kara2dcba472015-12-07 15:04:57 -05003403 if (overwrite)
Al Viro59551022016-01-22 15:40:57 -05003404 inode_unlock(inode);
Mingming Cao8d5d02e2009-09-28 15:48:29 -04003405
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003406 /*
Jan Kara914f82a2016-05-13 00:44:16 -04003407 * For extent mapped files we could direct write to holes and fallocate.
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003408 *
Jan Kara109811c2016-03-08 23:36:46 -05003409 * Allocated blocks to fill the hole are marked as unwritten to prevent
3410 * parallel buffered read to expose the stale data before DIO complete
3411 * the data IO.
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003412 *
Jan Kara109811c2016-03-08 23:36:46 -05003413 * As to previously fallocated extents, ext4 get_block will just simply
3414 * mark the buffer mapped but still keep the extents unwritten.
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003415 *
Jan Kara109811c2016-03-08 23:36:46 -05003416 * For non AIO case, we will convert those unwritten extents to written
3417 * after return back from blockdev_direct_IO. That way we save us from
3418 * allocating io_end structure and also the overhead of offloading
3419 * the extent convertion to a workqueue.
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003420 *
3421 * For async DIO, the conversion needs to be deferred when the
3422 * IO is completed. The ext4 end_io callback function will be
3423 * called to take care of the conversion work. Here for async
3424 * case, we allocate an io_end structure to hook to the iocb.
3425 */
3426 iocb->private = NULL;
Jan Kara109811c2016-03-08 23:36:46 -05003427 if (overwrite)
Jan Kara705965b2016-03-08 23:08:10 -05003428 get_block_func = ext4_dio_get_block_overwrite;
Jan Kara12735f82016-05-13 00:51:15 -04003429 else if (IS_DAX(inode)) {
3430 /*
3431 * We can avoid zeroing for aligned DAX writes beyond EOF. Other
3432 * writes need zeroing either because they can race with page
3433 * faults or because they use partial blocks.
3434 */
3435 if (round_down(offset, 1<<inode->i_blkbits) >= inode->i_size &&
3436 ext4_aligned_io(inode, offset, count))
3437 get_block_func = ext4_dio_get_block;
3438 else
3439 get_block_func = ext4_dax_get_block;
3440 dio_flags = DIO_LOCKING;
3441 } else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) ||
3442 round_down(offset, 1 << inode->i_blkbits) >= inode->i_size) {
Jan Kara914f82a2016-05-13 00:44:16 -04003443 get_block_func = ext4_dio_get_block;
3444 dio_flags = DIO_LOCKING | DIO_SKIP_HOLES;
3445 } else if (is_sync_kiocb(iocb)) {
Jan Kara109811c2016-03-08 23:36:46 -05003446 get_block_func = ext4_dio_get_block_unwritten_sync;
3447 dio_flags = DIO_LOCKING;
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003448 } else {
Jan Kara109811c2016-03-08 23:36:46 -05003449 get_block_func = ext4_dio_get_block_unwritten_async;
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003450 dio_flags = DIO_LOCKING;
3451 }
Michael Halcrow2058f832015-04-12 00:55:10 -04003452#ifdef CONFIG_EXT4_FS_ENCRYPTION
3453 BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode));
3454#endif
Jan Kara914f82a2016-05-13 00:44:16 -04003455 if (IS_DAX(inode)) {
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07003456 ret = dax_do_io(iocb, inode, iter, get_block_func,
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003457 ext4_end_io_dio, dio_flags);
Jan Kara914f82a2016-05-13 00:44:16 -04003458 } else
Omar Sandoval17f8c842015-03-16 04:33:50 -07003459 ret = __blockdev_direct_IO(iocb, inode,
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07003460 inode->i_sb->s_bdev, iter,
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003461 get_block_func,
3462 ext4_end_io_dio, NULL, dio_flags);
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003463
Jan Kara97a851e2013-06-04 11:58:58 -04003464 if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003465 EXT4_STATE_DIO_UNWRITTEN)) {
3466 int err;
3467 /*
3468 * for non AIO case, since the IO is already
3469 * completed, we could do the conversion right here
3470 */
Jan Kara6b523df2013-06-04 13:21:11 -04003471 err = ext4_convert_unwritten_extents(NULL, inode,
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003472 offset, ret);
3473 if (err < 0)
3474 ret = err;
3475 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
3476 }
3477
Jan Kara914f82a2016-05-13 00:44:16 -04003478 inode_dio_end(inode);
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003479 /* take i_mutex locking again if we do a ovewrite dio */
Jan Kara2dcba472015-12-07 15:04:57 -05003480 if (overwrite)
Al Viro59551022016-01-22 15:40:57 -05003481 inode_lock(inode);
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003482
Jan Kara914f82a2016-05-13 00:44:16 -04003483 if (ret < 0 && final_size > inode->i_size)
3484 ext4_truncate_failed_write(inode);
3485
3486 /* Handle extending of i_size after direct IO write */
3487 if (orphan) {
3488 int err;
3489
3490 /* Credits for sb + inode write */
3491 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3492 if (IS_ERR(handle)) {
3493 /* This is really bad luck. We've written the data
3494 * but cannot extend i_size. Bail out and pretend
3495 * the write failed... */
3496 ret = PTR_ERR(handle);
3497 if (inode->i_nlink)
3498 ext4_orphan_del(NULL, inode);
3499
3500 goto out;
3501 }
3502 if (inode->i_nlink)
3503 ext4_orphan_del(handle, inode);
3504 if (ret > 0) {
3505 loff_t end = offset + ret;
3506 if (end > inode->i_size) {
3507 ei->i_disksize = end;
3508 i_size_write(inode, end);
3509 /*
3510 * We're going to return a positive `ret'
3511 * here due to non-zero-length I/O, so there's
3512 * no way of reporting error returns from
3513 * ext4_mark_inode_dirty() to userspace. So
3514 * ignore it.
3515 */
3516 ext4_mark_inode_dirty(handle, inode);
3517 }
3518 }
3519 err = ext4_journal_stop(handle);
3520 if (ret == 0)
3521 ret = err;
3522 }
3523out:
3524 return ret;
3525}
3526
Linus Torvalds0e01df12016-05-24 12:55:26 -07003527static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter)
Jan Kara914f82a2016-05-13 00:44:16 -04003528{
3529 int unlocked = 0;
3530 struct inode *inode = iocb->ki_filp->f_mapping->host;
3531 ssize_t ret;
3532
3533 if (ext4_should_dioread_nolock(inode)) {
3534 /*
3535 * Nolock dioread optimization may be dynamically disabled
3536 * via ext4_inode_block_unlocked_dio(). Check inode's state
3537 * while holding extra i_dio_count ref.
3538 */
3539 inode_dio_begin(inode);
3540 smp_mb();
3541 if (unlikely(ext4_test_inode_state(inode,
3542 EXT4_STATE_DIOREAD_LOCK)))
3543 inode_dio_end(inode);
3544 else
3545 unlocked = 1;
3546 }
3547 if (IS_DAX(inode)) {
Linus Torvalds0e01df12016-05-24 12:55:26 -07003548 ret = dax_do_io(iocb, inode, iter, ext4_dio_get_block,
Jan Kara914f82a2016-05-13 00:44:16 -04003549 NULL, unlocked ? 0 : DIO_LOCKING);
3550 } else {
3551 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
Linus Torvalds0e01df12016-05-24 12:55:26 -07003552 iter, ext4_dio_get_block,
Jan Kara914f82a2016-05-13 00:44:16 -04003553 NULL, NULL,
3554 unlocked ? 0 : DIO_LOCKING);
3555 }
3556 if (unlocked)
3557 inode_dio_end(inode);
Theodore Ts'o69c499d2012-11-29 21:13:48 -05003558 return ret;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003559}
3560
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07003561static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
Mingming Cao4c0425f2009-09-28 15:48:41 -04003562{
3563 struct file *file = iocb->ki_filp;
3564 struct inode *inode = file->f_mapping->host;
Al Viroa6cbcd42014-03-04 22:38:00 -05003565 size_t count = iov_iter_count(iter);
Christoph Hellwigc8b8e322016-04-07 08:51:58 -07003566 loff_t offset = iocb->ki_pos;
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003567 ssize_t ret;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003568
Michael Halcrow2058f832015-04-12 00:55:10 -04003569#ifdef CONFIG_EXT4_FS_ENCRYPTION
3570 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode))
3571 return 0;
3572#endif
3573
Theodore Ts'o84ebd792011-08-31 11:56:51 -04003574 /*
3575 * If we are doing data journalling we don't support O_DIRECT
3576 */
3577 if (ext4_should_journal_data(inode))
3578 return 0;
3579
Tao Ma46c7f252012-12-10 14:04:52 -05003580 /* Let buffer I/O handle the inline data case. */
3581 if (ext4_has_inline_data(inode))
3582 return 0;
3583
Omar Sandoval6f673762015-03-16 04:33:52 -07003584 trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
Jan Kara914f82a2016-05-13 00:44:16 -04003585 if (iov_iter_rw(iter) == READ)
Linus Torvalds0e01df12016-05-24 12:55:26 -07003586 ret = ext4_direct_IO_read(iocb, iter);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003587 else
Linus Torvalds0e01df12016-05-24 12:55:26 -07003588 ret = ext4_direct_IO_write(iocb, iter);
Omar Sandoval6f673762015-03-16 04:33:52 -07003589 trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret);
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04003590 return ret;
Mingming Cao4c0425f2009-09-28 15:48:41 -04003591}
3592
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003593/*
Mingming Cao617ba132006-10-11 01:20:53 -07003594 * Pages can be marked dirty completely asynchronously from ext4's journalling
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003595 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do
3596 * much here because ->set_page_dirty is called under VFS locks. The page is
3597 * not necessarily locked.
3598 *
3599 * We cannot just dirty the page and leave attached buffers clean, because the
3600 * buffers' dirty state is "definitive". We cannot just set the buffers dirty
3601 * or jbddirty because all the journalling code will explode.
3602 *
3603 * So what we do is to mark the page "pending dirty" and next time writepage
3604 * is called, propagate that into the buffers appropriately.
3605 */
Mingming Cao617ba132006-10-11 01:20:53 -07003606static int ext4_journalled_set_page_dirty(struct page *page)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003607{
3608 SetPageChecked(page);
3609 return __set_page_dirty_nobuffers(page);
3610}
3611
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003612static const struct address_space_operations ext4_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003613 .readpage = ext4_readpage,
3614 .readpages = ext4_readpages,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003615 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003616 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003617 .write_begin = ext4_write_begin,
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003618 .write_end = ext4_write_end,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003619 .bmap = ext4_bmap,
3620 .invalidatepage = ext4_invalidatepage,
3621 .releasepage = ext4_releasepage,
3622 .direct_IO = ext4_direct_IO,
3623 .migratepage = buffer_migrate_page,
3624 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003625 .error_remove_page = generic_error_remove_page,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003626};
3627
Mingming Cao617ba132006-10-11 01:20:53 -07003628static const struct address_space_operations ext4_journalled_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003629 .readpage = ext4_readpage,
3630 .readpages = ext4_readpages,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003631 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003632 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003633 .write_begin = ext4_write_begin,
3634 .write_end = ext4_journalled_write_end,
3635 .set_page_dirty = ext4_journalled_set_page_dirty,
3636 .bmap = ext4_bmap,
Jan Kara4520fb32012-12-25 13:28:54 -05003637 .invalidatepage = ext4_journalled_invalidatepage,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003638 .releasepage = ext4_releasepage,
Theodore Ts'o84ebd792011-08-31 11:56:51 -04003639 .direct_IO = ext4_direct_IO,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003640 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003641 .error_remove_page = generic_error_remove_page,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003642};
3643
Alex Tomas64769242008-07-11 19:27:31 -04003644static const struct address_space_operations ext4_da_aops = {
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003645 .readpage = ext4_readpage,
3646 .readpages = ext4_readpages,
Aneesh Kumar K.V43ce1d22009-06-14 17:58:45 -04003647 .writepage = ext4_writepage,
Theodore Ts'o20970ba2013-06-06 14:00:46 -04003648 .writepages = ext4_writepages,
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07003649 .write_begin = ext4_da_write_begin,
3650 .write_end = ext4_da_write_end,
3651 .bmap = ext4_bmap,
3652 .invalidatepage = ext4_da_invalidatepage,
3653 .releasepage = ext4_releasepage,
3654 .direct_IO = ext4_direct_IO,
3655 .migratepage = buffer_migrate_page,
3656 .is_partially_uptodate = block_is_partially_uptodate,
Andi Kleenaa261f52009-09-16 11:50:16 +02003657 .error_remove_page = generic_error_remove_page,
Alex Tomas64769242008-07-11 19:27:31 -04003658};
3659
Mingming Cao617ba132006-10-11 01:20:53 -07003660void ext4_set_aops(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003661{
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003662 switch (ext4_inode_journal_mode(inode)) {
3663 case EXT4_INODE_ORDERED_DATA_MODE:
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003664 case EXT4_INODE_WRITEBACK_DATA_MODE:
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003665 break;
3666 case EXT4_INODE_JOURNAL_DATA_MODE:
Mingming Cao617ba132006-10-11 01:20:53 -07003667 inode->i_mapping->a_ops = &ext4_journalled_aops;
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003668 return;
Lukas Czerner3d2b1582012-02-20 17:53:00 -05003669 default:
3670 BUG();
3671 }
Theodore Ts'o74d553a2013-04-03 12:39:17 -04003672 if (test_opt(inode->i_sb, DELALLOC))
3673 inode->i_mapping->a_ops = &ext4_da_aops;
3674 else
3675 inode->i_mapping->a_ops = &ext4_aops;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003676}
3677
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003678static int __ext4_block_zero_page_range(handle_t *handle,
Lukas Czernerd863dc32013-05-27 23:32:35 -04003679 struct address_space *mapping, loff_t from, loff_t length)
3680{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003681 ext4_fsblk_t index = from >> PAGE_SHIFT;
3682 unsigned offset = from & (PAGE_SIZE-1);
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003683 unsigned blocksize, pos;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003684 ext4_lblk_t iblock;
3685 struct inode *inode = mapping->host;
3686 struct buffer_head *bh;
3687 struct page *page;
3688 int err = 0;
3689
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003690 page = find_or_create_page(mapping, from >> PAGE_SHIFT,
Michal Hockoc62d2552015-11-06 16:28:49 -08003691 mapping_gfp_constraint(mapping, ~__GFP_FS));
Lukas Czernerd863dc32013-05-27 23:32:35 -04003692 if (!page)
3693 return -ENOMEM;
3694
3695 blocksize = inode->i_sb->s_blocksize;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003696
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003697 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003698
3699 if (!page_has_buffers(page))
3700 create_empty_buffers(page, blocksize, 0);
3701
3702 /* Find the buffer that contains "offset" */
3703 bh = page_buffers(page);
3704 pos = blocksize;
3705 while (offset >= pos) {
3706 bh = bh->b_this_page;
3707 iblock++;
3708 pos += blocksize;
3709 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003710 if (buffer_freed(bh)) {
3711 BUFFER_TRACE(bh, "freed: skip");
3712 goto unlock;
3713 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003714 if (!buffer_mapped(bh)) {
3715 BUFFER_TRACE(bh, "unmapped");
3716 ext4_get_block(inode, iblock, bh, 0);
3717 /* unmapped? It's a hole - nothing to do */
3718 if (!buffer_mapped(bh)) {
3719 BUFFER_TRACE(bh, "still unmapped");
3720 goto unlock;
3721 }
3722 }
3723
3724 /* Ok, it's mapped. Make sure it's up-to-date */
3725 if (PageUptodate(page))
3726 set_buffer_uptodate(bh);
3727
3728 if (!buffer_uptodate(bh)) {
3729 err = -EIO;
Mike Christiedfec8a12016-06-05 14:31:44 -05003730 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003731 wait_on_buffer(bh);
3732 /* Uhhuh. Read error. Complain and punt. */
3733 if (!buffer_uptodate(bh))
3734 goto unlock;
Michael Halcrowc9c74292015-04-12 00:56:10 -04003735 if (S_ISREG(inode->i_mode) &&
3736 ext4_encrypted_inode(inode)) {
3737 /* We expect the key to be set. */
Jaegeuk Kima7550b32016-07-10 14:01:03 -04003738 BUG_ON(!fscrypt_has_encryption_key(inode));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003739 BUG_ON(blocksize != PAGE_SIZE);
Jaegeuk Kima7550b32016-07-10 14:01:03 -04003740 WARN_ON_ONCE(fscrypt_decrypt_page(page));
Michael Halcrowc9c74292015-04-12 00:56:10 -04003741 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003742 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003743 if (ext4_should_journal_data(inode)) {
3744 BUFFER_TRACE(bh, "get write access");
3745 err = ext4_journal_get_write_access(handle, bh);
3746 if (err)
3747 goto unlock;
3748 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003749 zero_user(page, offset, length);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003750 BUFFER_TRACE(bh, "zeroed end of block");
3751
Lukas Czernerd863dc32013-05-27 23:32:35 -04003752 if (ext4_should_journal_data(inode)) {
3753 err = ext4_handle_dirty_metadata(handle, inode, bh);
Lukas Czerner0713ed02013-05-27 23:32:35 -04003754 } else {
jon ernst353eefd2013-07-01 08:12:39 -04003755 err = 0;
Lukas Czernerd863dc32013-05-27 23:32:35 -04003756 mark_buffer_dirty(bh);
Jan Kara3957ef52016-04-24 00:56:05 -04003757 if (ext4_should_order_data(inode))
Jan Karaee0876b2016-04-24 00:56:08 -04003758 err = ext4_jbd2_inode_add_write(handle, inode);
Lukas Czerner0713ed02013-05-27 23:32:35 -04003759 }
Lukas Czernerd863dc32013-05-27 23:32:35 -04003760
3761unlock:
3762 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003763 put_page(page);
Lukas Czernerd863dc32013-05-27 23:32:35 -04003764 return err;
3765}
3766
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003767/*
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003768 * ext4_block_zero_page_range() zeros out a mapping of length 'length'
3769 * starting from file offset 'from'. The range to be zero'd must
3770 * be contained with in one block. If the specified range exceeds
3771 * the end of the block it will be shortened to end of the block
3772 * that cooresponds to 'from'
3773 */
3774static int ext4_block_zero_page_range(handle_t *handle,
3775 struct address_space *mapping, loff_t from, loff_t length)
3776{
3777 struct inode *inode = mapping->host;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003778 unsigned offset = from & (PAGE_SIZE-1);
Ross Zwisler923ae0f2015-02-16 15:59:38 -08003779 unsigned blocksize = inode->i_sb->s_blocksize;
3780 unsigned max = blocksize - (offset & (blocksize - 1));
3781
3782 /*
3783 * correct length if it does not fall between
3784 * 'from' and the end of the block
3785 */
3786 if (length > max || length < 0)
3787 length = max;
3788
3789 if (IS_DAX(inode))
3790 return dax_zero_page_range(inode, from, length, ext4_get_block);
3791 return __ext4_block_zero_page_range(handle, mapping, from, length);
3792}
3793
3794/*
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003795 * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3796 * up to the end of the block which corresponds to `from'.
3797 * This required during truncate. We need to physically zero the tail end
3798 * of that block so it doesn't yield old data if the file is later grown.
3799 */
Stephen Hemmingerc1978552014-05-12 10:50:23 -04003800static int ext4_block_truncate_page(handle_t *handle,
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003801 struct address_space *mapping, loff_t from)
3802{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003803 unsigned offset = from & (PAGE_SIZE-1);
Matthew Wilcox94350ab2014-03-24 15:09:16 -04003804 unsigned length;
3805 unsigned blocksize;
3806 struct inode *inode = mapping->host;
3807
3808 blocksize = inode->i_sb->s_blocksize;
3809 length = blocksize - (offset & (blocksize - 1));
3810
3811 return ext4_block_zero_page_range(handle, mapping, from, length);
3812}
3813
Lukas Czernera87dd182013-05-27 23:32:35 -04003814int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3815 loff_t lstart, loff_t length)
3816{
3817 struct super_block *sb = inode->i_sb;
3818 struct address_space *mapping = inode->i_mapping;
Lukas Czernere1be3a92013-07-01 08:12:39 -04003819 unsigned partial_start, partial_end;
Lukas Czernera87dd182013-05-27 23:32:35 -04003820 ext4_fsblk_t start, end;
3821 loff_t byte_end = (lstart + length - 1);
3822 int err = 0;
3823
Lukas Czernere1be3a92013-07-01 08:12:39 -04003824 partial_start = lstart & (sb->s_blocksize - 1);
3825 partial_end = byte_end & (sb->s_blocksize - 1);
3826
Lukas Czernera87dd182013-05-27 23:32:35 -04003827 start = lstart >> sb->s_blocksize_bits;
3828 end = byte_end >> sb->s_blocksize_bits;
3829
3830 /* Handle partial zero within the single block */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003831 if (start == end &&
3832 (partial_start || (partial_end != sb->s_blocksize - 1))) {
Lukas Czernera87dd182013-05-27 23:32:35 -04003833 err = ext4_block_zero_page_range(handle, mapping,
3834 lstart, length);
3835 return err;
3836 }
3837 /* Handle partial zero out on the start of the range */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003838 if (partial_start) {
Lukas Czernera87dd182013-05-27 23:32:35 -04003839 err = ext4_block_zero_page_range(handle, mapping,
3840 lstart, sb->s_blocksize);
3841 if (err)
3842 return err;
3843 }
3844 /* Handle partial zero out on the end of the range */
Lukas Czernere1be3a92013-07-01 08:12:39 -04003845 if (partial_end != sb->s_blocksize - 1)
Lukas Czernera87dd182013-05-27 23:32:35 -04003846 err = ext4_block_zero_page_range(handle, mapping,
Lukas Czernere1be3a92013-07-01 08:12:39 -04003847 byte_end - partial_end,
3848 partial_end + 1);
Lukas Czernera87dd182013-05-27 23:32:35 -04003849 return err;
3850}
3851
Duane Griffin91ef4ca2008-07-11 19:27:31 -04003852int ext4_can_truncate(struct inode *inode)
3853{
Duane Griffin91ef4ca2008-07-11 19:27:31 -04003854 if (S_ISREG(inode->i_mode))
3855 return 1;
3856 if (S_ISDIR(inode->i_mode))
3857 return 1;
3858 if (S_ISLNK(inode->i_mode))
3859 return !ext4_inode_is_fast_symlink(inode);
3860 return 0;
3861}
3862
Dave Kleikampac27a0e2006-10-11 01:20:50 -07003863/*
Jan Kara01127842015-12-07 14:34:49 -05003864 * We have to make sure i_disksize gets properly updated before we truncate
3865 * page cache due to hole punching or zero range. Otherwise i_disksize update
3866 * can get lost as it may have been postponed to submission of writeback but
3867 * that will never happen after we truncate page cache.
3868 */
3869int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3870 loff_t len)
3871{
3872 handle_t *handle;
3873 loff_t size = i_size_read(inode);
3874
Al Viro59551022016-01-22 15:40:57 -05003875 WARN_ON(!inode_is_locked(inode));
Jan Kara01127842015-12-07 14:34:49 -05003876 if (offset > size || offset + len < size)
3877 return 0;
3878
3879 if (EXT4_I(inode)->i_disksize >= size)
3880 return 0;
3881
3882 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3883 if (IS_ERR(handle))
3884 return PTR_ERR(handle);
3885 ext4_update_i_disksize(inode, size);
3886 ext4_mark_inode_dirty(handle, inode);
3887 ext4_journal_stop(handle);
3888
3889 return 0;
3890}
3891
3892/*
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003893 * ext4_punch_hole: punches a hole in a file by releaseing the blocks
3894 * associated with the given offset and length
3895 *
3896 * @inode: File inode
3897 * @offset: The offset where the hole will begin
3898 * @len: The length of the hole
3899 *
Anatol Pomozov4907cb72012-09-01 10:31:09 -07003900 * Returns: 0 on success or negative on failure
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003901 */
3902
Ashish Sangwanaeb28172013-07-01 08:12:38 -04003903int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003904{
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003905 struct super_block *sb = inode->i_sb;
3906 ext4_lblk_t first_block, stop_block;
3907 struct address_space *mapping = inode->i_mapping;
Lukas Czernera87dd182013-05-27 23:32:35 -04003908 loff_t first_block_offset, last_block_offset;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003909 handle_t *handle;
3910 unsigned int credits;
3911 int ret = 0;
3912
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003913 if (!S_ISREG(inode->i_mode))
Allison Henderson73355192012-03-21 22:23:31 -04003914 return -EOPNOTSUPP;
Allison Hendersona4bb6b62011-05-25 07:41:50 -04003915
Lukas Czernerb8a86842014-03-18 18:05:35 -04003916 trace_ext4_punch_hole(inode, offset, length, 0);
Zheng Liuaaddea82013-01-16 20:21:26 -05003917
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003918 /*
3919 * Write out all dirty pages to avoid race conditions
3920 * Then release them.
3921 */
3922 if (mapping->nrpages && mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
3923 ret = filemap_write_and_wait_range(mapping, offset,
3924 offset + length - 1);
3925 if (ret)
3926 return ret;
3927 }
3928
Al Viro59551022016-01-22 15:40:57 -05003929 inode_lock(inode);
Lukas Czerner9ef06ce2014-04-12 09:47:00 -04003930
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003931 /* No need to punch hole beyond i_size */
3932 if (offset >= inode->i_size)
3933 goto out_mutex;
3934
3935 /*
3936 * If the hole extends beyond i_size, set the hole
3937 * to end after the page that contains i_size
3938 */
3939 if (offset + length > inode->i_size) {
3940 length = inode->i_size +
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003941 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003942 offset;
3943 }
3944
Jan Karaa3612932013-08-16 21:19:41 -04003945 if (offset & (sb->s_blocksize - 1) ||
3946 (offset + length) & (sb->s_blocksize - 1)) {
3947 /*
3948 * Attach jinode to inode for jbd2 if we do any zeroing of
3949 * partial block
3950 */
3951 ret = ext4_inode_attach_jinode(inode);
3952 if (ret < 0)
3953 goto out_mutex;
3954
3955 }
3956
Jan Karaea3d7202015-12-07 14:28:03 -05003957 /* Wait all existing dio workers, newcomers will block on i_mutex */
3958 ext4_inode_block_unlocked_dio(inode);
3959 inode_dio_wait(inode);
3960
3961 /*
3962 * Prevent page faults from reinstantiating pages we have released from
3963 * page cache.
3964 */
3965 down_write(&EXT4_I(inode)->i_mmap_sem);
Lukas Czernera87dd182013-05-27 23:32:35 -04003966 first_block_offset = round_up(offset, sb->s_blocksize);
3967 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003968
Lukas Czernera87dd182013-05-27 23:32:35 -04003969 /* Now release the pages and zero block aligned part of pages*/
Jan Kara01127842015-12-07 14:34:49 -05003970 if (last_block_offset > first_block_offset) {
3971 ret = ext4_update_disksize_before_punch(inode, offset, length);
3972 if (ret)
3973 goto out_dio;
Lukas Czernera87dd182013-05-27 23:32:35 -04003974 truncate_pagecache_range(inode, first_block_offset,
3975 last_block_offset);
Jan Kara01127842015-12-07 14:34:49 -05003976 }
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003977
3978 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3979 credits = ext4_writepage_trans_blocks(inode);
3980 else
3981 credits = ext4_blocks_for_truncate(inode);
3982 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
3983 if (IS_ERR(handle)) {
3984 ret = PTR_ERR(handle);
3985 ext4_std_error(sb, ret);
3986 goto out_dio;
3987 }
3988
Lukas Czernera87dd182013-05-27 23:32:35 -04003989 ret = ext4_zero_partial_blocks(handle, inode, offset,
3990 length);
3991 if (ret)
3992 goto out_stop;
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04003993
3994 first_block = (offset + sb->s_blocksize - 1) >>
3995 EXT4_BLOCK_SIZE_BITS(sb);
3996 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
3997
3998 /* If there are no blocks to remove, return now */
3999 if (first_block >= stop_block)
4000 goto out_stop;
4001
4002 down_write(&EXT4_I(inode)->i_data_sem);
4003 ext4_discard_preallocations(inode);
4004
4005 ret = ext4_es_remove_extent(inode, first_block,
4006 stop_block - first_block);
4007 if (ret) {
4008 up_write(&EXT4_I(inode)->i_data_sem);
4009 goto out_stop;
4010 }
4011
4012 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4013 ret = ext4_ext_remove_space(inode, first_block,
4014 stop_block - 1);
4015 else
Lukas Czerner4f579ae2014-07-15 06:03:38 -04004016 ret = ext4_ind_remove_space(handle, inode, first_block,
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004017 stop_block);
4018
Theodore Ts'o819c4922013-04-03 12:47:17 -04004019 up_write(&EXT4_I(inode)->i_data_sem);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004020 if (IS_SYNC(inode))
4021 ext4_handle_sync(handle);
Maxim Patlasove251f9b2014-02-20 16:58:05 -05004022
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004023 inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
4024 ext4_mark_inode_dirty(handle, inode);
4025out_stop:
4026 ext4_journal_stop(handle);
4027out_dio:
Jan Karaea3d7202015-12-07 14:28:03 -05004028 up_write(&EXT4_I(inode)->i_mmap_sem);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004029 ext4_inode_resume_unlocked_dio(inode);
4030out_mutex:
Al Viro59551022016-01-22 15:40:57 -05004031 inode_unlock(inode);
Theodore Ts'o26a4c0c2013-04-03 12:45:17 -04004032 return ret;
Allison Hendersona4bb6b62011-05-25 07:41:50 -04004033}
4034
Jan Karaa3612932013-08-16 21:19:41 -04004035int ext4_inode_attach_jinode(struct inode *inode)
4036{
4037 struct ext4_inode_info *ei = EXT4_I(inode);
4038 struct jbd2_inode *jinode;
4039
4040 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4041 return 0;
4042
4043 jinode = jbd2_alloc_inode(GFP_KERNEL);
4044 spin_lock(&inode->i_lock);
4045 if (!ei->jinode) {
4046 if (!jinode) {
4047 spin_unlock(&inode->i_lock);
4048 return -ENOMEM;
4049 }
4050 ei->jinode = jinode;
4051 jbd2_journal_init_jbd_inode(ei->jinode, inode);
4052 jinode = NULL;
4053 }
4054 spin_unlock(&inode->i_lock);
4055 if (unlikely(jinode != NULL))
4056 jbd2_free_inode(jinode);
4057 return 0;
4058}
4059
Allison Hendersona4bb6b62011-05-25 07:41:50 -04004060/*
Mingming Cao617ba132006-10-11 01:20:53 -07004061 * ext4_truncate()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004062 *
Mingming Cao617ba132006-10-11 01:20:53 -07004063 * We block out ext4_get_block() block instantiations across the entire
4064 * transaction, and VFS/VM ensures that ext4_truncate() cannot run
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004065 * simultaneously on behalf of the same inode.
4066 *
Justin P. Mattock42b2aa82011-11-28 20:31:00 -08004067 * As we work through the truncate and commit bits of it to the journal there
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004068 * is one core, guiding principle: the file's tree must always be consistent on
4069 * disk. We must be able to restart the truncate after a crash.
4070 *
4071 * The file's tree may be transiently inconsistent in memory (although it
4072 * probably isn't), but whenever we close off and commit a journal transaction,
4073 * the contents of (the filesystem + the journal) must be consistent and
4074 * restartable. It's pretty simple, really: bottom up, right to left (although
4075 * left-to-right works OK too).
4076 *
4077 * Note that at recovery time, journal replay occurs *before* the restart of
4078 * truncate against the orphan inode list.
4079 *
4080 * The committed inode has the new, desired i_size (which is the same as
Mingming Cao617ba132006-10-11 01:20:53 -07004081 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004082 * that this inode's truncate did not complete and it will again call
Mingming Cao617ba132006-10-11 01:20:53 -07004083 * ext4_truncate() to have another go. So there will be instantiated blocks
4084 * to the right of the truncation point in a crashed ext4 filesystem. But
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004085 * that's fine - as long as they are linked from the inode, the post-crash
Mingming Cao617ba132006-10-11 01:20:53 -07004086 * ext4_truncate() run will find them and release them.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004087 */
Mingming Cao617ba132006-10-11 01:20:53 -07004088void ext4_truncate(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004089{
Theodore Ts'o819c4922013-04-03 12:47:17 -04004090 struct ext4_inode_info *ei = EXT4_I(inode);
4091 unsigned int credits;
4092 handle_t *handle;
4093 struct address_space *mapping = inode->i_mapping;
Theodore Ts'o819c4922013-04-03 12:47:17 -04004094
Theodore Ts'o19b5ef62013-04-03 21:58:52 -04004095 /*
4096 * There is a possibility that we're either freeing the inode
Matthew Wilcoxe04027e2014-03-24 15:15:07 -04004097 * or it's a completely new inode. In those cases we might not
Theodore Ts'o19b5ef62013-04-03 21:58:52 -04004098 * have i_mutex locked because it's not necessary.
4099 */
4100 if (!(inode->i_state & (I_NEW|I_FREEING)))
Al Viro59551022016-01-22 15:40:57 -05004101 WARN_ON(!inode_is_locked(inode));
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004102 trace_ext4_truncate_enter(inode);
4103
Duane Griffin91ef4ca2008-07-11 19:27:31 -04004104 if (!ext4_can_truncate(inode))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004105 return;
4106
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04004107 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
Jiaying Zhangc8d46e42010-02-24 09:52:53 -05004108
Theodore Ts'o5534fb52009-09-17 09:34:16 -04004109 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05004110 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
Theodore Ts'o7d8f9f72009-02-24 08:21:14 -05004111
Tao Maaef1c852012-12-10 14:06:02 -05004112 if (ext4_has_inline_data(inode)) {
4113 int has_inline = 1;
4114
4115 ext4_inline_data_truncate(inode, &has_inline);
4116 if (has_inline)
4117 return;
4118 }
4119
Jan Karaa3612932013-08-16 21:19:41 -04004120 /* If we zero-out tail of the page, we have to create jinode for jbd2 */
4121 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4122 if (ext4_inode_attach_jinode(inode) < 0)
4123 return;
4124 }
4125
Amir Goldsteinff9893d2011-06-27 16:36:31 -04004126 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
Theodore Ts'o819c4922013-04-03 12:47:17 -04004127 credits = ext4_writepage_trans_blocks(inode);
Amir Goldsteinff9893d2011-06-27 16:36:31 -04004128 else
Theodore Ts'o819c4922013-04-03 12:47:17 -04004129 credits = ext4_blocks_for_truncate(inode);
4130
4131 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4132 if (IS_ERR(handle)) {
4133 ext4_std_error(inode->i_sb, PTR_ERR(handle));
4134 return;
4135 }
4136
Lukas Czernereb3544c2013-05-27 23:32:35 -04004137 if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4138 ext4_block_truncate_page(handle, mapping, inode->i_size);
Theodore Ts'o819c4922013-04-03 12:47:17 -04004139
4140 /*
4141 * We add the inode to the orphan list, so that if this
4142 * truncate spans multiple transactions, and we crash, we will
4143 * resume the truncate when the filesystem recovers. It also
4144 * marks the inode dirty, to catch the new size.
4145 *
4146 * Implication: the file must always be in a sane, consistent
4147 * truncatable state while each transaction commits.
4148 */
4149 if (ext4_orphan_add(handle, inode))
4150 goto out_stop;
4151
4152 down_write(&EXT4_I(inode)->i_data_sem);
4153
4154 ext4_discard_preallocations(inode);
4155
4156 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4157 ext4_ext_truncate(handle, inode);
4158 else
4159 ext4_ind_truncate(handle, inode);
4160
4161 up_write(&ei->i_data_sem);
4162
4163 if (IS_SYNC(inode))
4164 ext4_handle_sync(handle);
4165
4166out_stop:
4167 /*
4168 * If this was a simple ftruncate() and the file will remain alive,
4169 * then we need to clear up the orphan record which we created above.
4170 * However, if this was a real unlink then we were called by
Wang Shilong58d86a52014-11-25 16:17:29 -05004171 * ext4_evict_inode(), and we allow that function to clean up the
Theodore Ts'o819c4922013-04-03 12:47:17 -04004172 * orphan info for us.
4173 */
4174 if (inode->i_nlink)
4175 ext4_orphan_del(handle, inode);
4176
4177 inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
4178 ext4_mark_inode_dirty(handle, inode);
4179 ext4_journal_stop(handle);
Alex Tomasa86c6182006-10-11 01:21:03 -07004180
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004181 trace_ext4_truncate_exit(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004182}
4183
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004184/*
Mingming Cao617ba132006-10-11 01:20:53 -07004185 * ext4_get_inode_loc returns with an extra refcount against the inode's
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004186 * underlying buffer_head on success. If 'in_mem' is true, we have all
4187 * data in memory that is needed to recreate the on-disk version of this
4188 * inode.
4189 */
Mingming Cao617ba132006-10-11 01:20:53 -07004190static int __ext4_get_inode_loc(struct inode *inode,
4191 struct ext4_iloc *iloc, int in_mem)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004192{
Theodore Ts'o240799c2008-10-09 23:53:47 -04004193 struct ext4_group_desc *gdp;
4194 struct buffer_head *bh;
4195 struct super_block *sb = inode->i_sb;
4196 ext4_fsblk_t block;
4197 int inodes_per_block, inode_offset;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004198
Aneesh Kumar K.V3a06d772008-11-22 15:04:59 -05004199 iloc->bh = NULL;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004200 if (!ext4_valid_inum(sb, inode->i_ino))
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004201 return -EFSCORRUPTED;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004202
Theodore Ts'o240799c2008-10-09 23:53:47 -04004203 iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb);
4204 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4205 if (!gdp)
4206 return -EIO;
4207
4208 /*
4209 * Figure out the offset within the block group inode table
4210 */
Tao Ma00d09882011-05-09 10:26:41 -04004211 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004212 inode_offset = ((inode->i_ino - 1) %
4213 EXT4_INODES_PER_GROUP(sb));
4214 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
4215 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4216
4217 bh = sb_getblk(sb, block);
Wang Shilongaebf0242013-01-12 16:28:47 -05004218 if (unlikely(!bh))
Theodore Ts'o860d21e2013-01-12 16:19:36 -05004219 return -ENOMEM;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004220 if (!buffer_uptodate(bh)) {
4221 lock_buffer(bh);
Hidehiro Kawai9c83a922008-07-26 16:39:26 -04004222
4223 /*
4224 * If the buffer has the write error flag, we have failed
4225 * to write out another inode in the same block. In this
4226 * case, we don't have to read the block because we may
4227 * read the old inode data successfully.
4228 */
4229 if (buffer_write_io_error(bh) && !buffer_uptodate(bh))
4230 set_buffer_uptodate(bh);
4231
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004232 if (buffer_uptodate(bh)) {
4233 /* someone brought it uptodate while we waited */
4234 unlock_buffer(bh);
4235 goto has_buffer;
4236 }
4237
4238 /*
4239 * If we have all information of the inode in memory and this
4240 * is the only valid inode in the block, we need not read the
4241 * block.
4242 */
4243 if (in_mem) {
4244 struct buffer_head *bitmap_bh;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004245 int i, start;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004246
Theodore Ts'o240799c2008-10-09 23:53:47 -04004247 start = inode_offset & ~(inodes_per_block - 1);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004248
4249 /* Is the inode bitmap in cache? */
Theodore Ts'o240799c2008-10-09 23:53:47 -04004250 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
Wang Shilongaebf0242013-01-12 16:28:47 -05004251 if (unlikely(!bitmap_bh))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004252 goto make_io;
4253
4254 /*
4255 * If the inode bitmap isn't in cache then the
4256 * optimisation may end up performing two reads instead
4257 * of one, so skip it.
4258 */
4259 if (!buffer_uptodate(bitmap_bh)) {
4260 brelse(bitmap_bh);
4261 goto make_io;
4262 }
Theodore Ts'o240799c2008-10-09 23:53:47 -04004263 for (i = start; i < start + inodes_per_block; i++) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004264 if (i == inode_offset)
4265 continue;
Mingming Cao617ba132006-10-11 01:20:53 -07004266 if (ext4_test_bit(i, bitmap_bh->b_data))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004267 break;
4268 }
4269 brelse(bitmap_bh);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004270 if (i == start + inodes_per_block) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004271 /* all other inodes are free, so skip I/O */
4272 memset(bh->b_data, 0, bh->b_size);
4273 set_buffer_uptodate(bh);
4274 unlock_buffer(bh);
4275 goto has_buffer;
4276 }
4277 }
4278
4279make_io:
4280 /*
Theodore Ts'o240799c2008-10-09 23:53:47 -04004281 * If we need to do any I/O, try to pre-readahead extra
4282 * blocks from the inode table.
4283 */
4284 if (EXT4_SB(sb)->s_inode_readahead_blks) {
4285 ext4_fsblk_t b, end, table;
4286 unsigned num;
Theodore Ts'o0d606e22013-04-23 08:59:35 -04004287 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004288
4289 table = ext4_inode_table(sb, gdp);
Theodore Ts'ob713a5e2009-03-31 09:11:14 -04004290 /* s_inode_readahead_blks is always a power of 2 */
Theodore Ts'o0d606e22013-04-23 08:59:35 -04004291 b = block & ~((ext4_fsblk_t) ra_blks - 1);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004292 if (table > b)
4293 b = table;
Theodore Ts'o0d606e22013-04-23 08:59:35 -04004294 end = b + ra_blks;
Theodore Ts'o240799c2008-10-09 23:53:47 -04004295 num = EXT4_INODES_PER_GROUP(sb);
Darrick J. Wongfeb0ab32012-04-29 18:45:10 -04004296 if (ext4_has_group_desc_csum(sb))
Aneesh Kumar K.V560671a2009-01-05 22:20:24 -05004297 num -= ext4_itable_unused_count(sb, gdp);
Theodore Ts'o240799c2008-10-09 23:53:47 -04004298 table += num / inodes_per_block;
4299 if (end > table)
4300 end = table;
4301 while (b <= end)
4302 sb_breadahead(sb, b++);
4303 }
4304
4305 /*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004306 * There are other valid inodes in the buffer, this inode
4307 * has in-inode xattrs, or we don't have this inode in memory.
4308 * Read the block from disk.
4309 */
Jiaying Zhang0562e0b2011-03-21 21:38:05 -04004310 trace_ext4_load_inode(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004311 get_bh(bh);
4312 bh->b_end_io = end_buffer_read_sync;
Mike Christie2a222ca2016-06-05 14:31:43 -05004313 submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004314 wait_on_buffer(bh);
4315 if (!buffer_uptodate(bh)) {
Theodore Ts'oc398eda2010-07-27 11:56:40 -04004316 EXT4_ERROR_INODE_BLOCK(inode, block,
4317 "unable to read itable block");
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004318 brelse(bh);
4319 return -EIO;
4320 }
4321 }
4322has_buffer:
4323 iloc->bh = bh;
4324 return 0;
4325}
4326
Mingming Cao617ba132006-10-11 01:20:53 -07004327int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004328{
4329 /* We have all inode data except xattrs in memory here. */
Mingming Cao617ba132006-10-11 01:20:53 -07004330 return __ext4_get_inode_loc(inode, iloc,
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05004331 !ext4_test_inode_state(inode, EXT4_STATE_XATTR));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004332}
4333
Mingming Cao617ba132006-10-11 01:20:53 -07004334void ext4_set_inode_flags(struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004335{
Mingming Cao617ba132006-10-11 01:20:53 -07004336 unsigned int flags = EXT4_I(inode)->i_flags;
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004337 unsigned int new_fl = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004338
Mingming Cao617ba132006-10-11 01:20:53 -07004339 if (flags & EXT4_SYNC_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004340 new_fl |= S_SYNC;
Mingming Cao617ba132006-10-11 01:20:53 -07004341 if (flags & EXT4_APPEND_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004342 new_fl |= S_APPEND;
Mingming Cao617ba132006-10-11 01:20:53 -07004343 if (flags & EXT4_IMMUTABLE_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004344 new_fl |= S_IMMUTABLE;
Mingming Cao617ba132006-10-11 01:20:53 -07004345 if (flags & EXT4_NOATIME_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004346 new_fl |= S_NOATIME;
Mingming Cao617ba132006-10-11 01:20:53 -07004347 if (flags & EXT4_DIRSYNC_FL)
Theodore Ts'o00a1a052014-03-30 10:20:01 -04004348 new_fl |= S_DIRSYNC;
Ross Zwisler0a6cf912016-02-26 15:19:46 -08004349 if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode))
Ross Zwisler923ae0f2015-02-16 15:59:38 -08004350 new_fl |= S_DAX;
Theodore Ts'o5f16f322014-03-24 14:43:12 -04004351 inode_set_flags(inode, new_fl,
Ross Zwisler923ae0f2015-02-16 15:59:38 -08004352 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004353}
4354
Jan Karaff9ddf72007-07-18 09:24:20 -04004355/* Propagate flags from i_flags to EXT4_I(inode)->i_flags */
4356void ext4_get_inode_flags(struct ext4_inode_info *ei)
4357{
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004358 unsigned int vfs_fl;
4359 unsigned long old_fl, new_fl;
Jan Karaff9ddf72007-07-18 09:24:20 -04004360
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004361 do {
4362 vfs_fl = ei->vfs_inode.i_flags;
4363 old_fl = ei->i_flags;
4364 new_fl = old_fl & ~(EXT4_SYNC_FL|EXT4_APPEND_FL|
4365 EXT4_IMMUTABLE_FL|EXT4_NOATIME_FL|
4366 EXT4_DIRSYNC_FL);
4367 if (vfs_fl & S_SYNC)
4368 new_fl |= EXT4_SYNC_FL;
4369 if (vfs_fl & S_APPEND)
4370 new_fl |= EXT4_APPEND_FL;
4371 if (vfs_fl & S_IMMUTABLE)
4372 new_fl |= EXT4_IMMUTABLE_FL;
4373 if (vfs_fl & S_NOATIME)
4374 new_fl |= EXT4_NOATIME_FL;
4375 if (vfs_fl & S_DIRSYNC)
4376 new_fl |= EXT4_DIRSYNC_FL;
4377 } while (cmpxchg(&ei->i_flags, old_fl, new_fl) != old_fl);
Jan Karaff9ddf72007-07-18 09:24:20 -04004378}
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004379
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004380static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004381 struct ext4_inode_info *ei)
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004382{
4383 blkcnt_t i_blocks ;
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004384 struct inode *inode = &(ei->vfs_inode);
4385 struct super_block *sb = inode->i_sb;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004386
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004387 if (ext4_has_feature_huge_file(sb)) {
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004388 /* we are using combined 48 bit field */
4389 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4390 le32_to_cpu(raw_inode->i_blocks_lo);
Theodore Ts'o07a03822010-06-14 09:54:48 -04004391 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004392 /* i_blocks represent file system block size */
4393 return i_blocks << (inode->i_blkbits - 9);
4394 } else {
4395 return i_blocks;
4396 }
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004397 } else {
4398 return le32_to_cpu(raw_inode->i_blocks_lo);
4399 }
4400}
Jan Karaff9ddf72007-07-18 09:24:20 -04004401
Tao Ma152a7b02012-12-02 11:13:24 -05004402static inline void ext4_iget_extra_inode(struct inode *inode,
4403 struct ext4_inode *raw_inode,
4404 struct ext4_inode_info *ei)
4405{
4406 __le32 *magic = (void *)raw_inode +
4407 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
Tao Ma67cf5b02012-12-10 14:04:46 -05004408 if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
Tao Ma152a7b02012-12-02 11:13:24 -05004409 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
Tao Ma67cf5b02012-12-10 14:04:46 -05004410 ext4_find_inline_data_nolock(inode);
Tao Maf19d5872012-12-10 14:05:51 -05004411 } else
4412 EXT4_I(inode)->i_inline_off = 0;
Tao Ma152a7b02012-12-02 11:13:24 -05004413}
4414
Li Xi040cb372016-01-08 16:01:21 -05004415int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4416{
4417 if (!EXT4_HAS_RO_COMPAT_FEATURE(inode->i_sb, EXT4_FEATURE_RO_COMPAT_PROJECT))
4418 return -EOPNOTSUPP;
4419 *projid = EXT4_I(inode)->i_projid;
4420 return 0;
4421}
4422
David Howells1d1fe1e2008-02-07 00:15:37 -08004423struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004424{
Mingming Cao617ba132006-10-11 01:20:53 -07004425 struct ext4_iloc iloc;
4426 struct ext4_inode *raw_inode;
David Howells1d1fe1e2008-02-07 00:15:37 -08004427 struct ext4_inode_info *ei;
David Howells1d1fe1e2008-02-07 00:15:37 -08004428 struct inode *inode;
Jan Karab436b9b2009-12-08 23:51:10 -05004429 journal_t *journal = EXT4_SB(sb)->s_journal;
David Howells1d1fe1e2008-02-07 00:15:37 -08004430 long ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004431 int block;
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004432 uid_t i_uid;
4433 gid_t i_gid;
Li Xi040cb372016-01-08 16:01:21 -05004434 projid_t i_projid;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004435
David Howells1d1fe1e2008-02-07 00:15:37 -08004436 inode = iget_locked(sb, ino);
4437 if (!inode)
4438 return ERR_PTR(-ENOMEM);
4439 if (!(inode->i_state & I_NEW))
4440 return inode;
4441
4442 ei = EXT4_I(inode);
Peter Huewe7dc57612011-02-21 21:01:42 -05004443 iloc.bh = NULL;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004444
David Howells1d1fe1e2008-02-07 00:15:37 -08004445 ret = __ext4_get_inode_loc(inode, &iloc, 0);
4446 if (ret < 0)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004447 goto bad_inode;
Mingming Cao617ba132006-10-11 01:20:53 -07004448 raw_inode = ext4_raw_inode(&iloc);
Darrick J. Wong814525f2012-04-29 18:31:10 -04004449
4450 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4451 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4452 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4453 EXT4_INODE_SIZE(inode->i_sb)) {
4454 EXT4_ERROR_INODE(inode, "bad extra_isize (%u != %u)",
4455 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize,
4456 EXT4_INODE_SIZE(inode->i_sb));
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004457 ret = -EFSCORRUPTED;
Darrick J. Wong814525f2012-04-29 18:31:10 -04004458 goto bad_inode;
4459 }
4460 } else
4461 ei->i_extra_isize = 0;
4462
4463 /* Precompute checksum seed for inode metadata */
Dmitry Monakhov9aa5d322014-10-13 03:36:16 -04004464 if (ext4_has_metadata_csum(sb)) {
Darrick J. Wong814525f2012-04-29 18:31:10 -04004465 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4466 __u32 csum;
4467 __le32 inum = cpu_to_le32(inode->i_ino);
4468 __le32 gen = raw_inode->i_generation;
4469 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4470 sizeof(inum));
4471 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4472 sizeof(gen));
4473 }
4474
4475 if (!ext4_inode_csum_verify(inode, raw_inode, ei)) {
4476 EXT4_ERROR_INODE(inode, "checksum invalid");
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004477 ret = -EFSBADCRC;
Darrick J. Wong814525f2012-04-29 18:31:10 -04004478 goto bad_inode;
4479 }
4480
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004481 inode->i_mode = le16_to_cpu(raw_inode->i_mode);
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004482 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4483 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
Li Xi040cb372016-01-08 16:01:21 -05004484 if (EXT4_HAS_RO_COMPAT_FEATURE(sb, EXT4_FEATURE_RO_COMPAT_PROJECT) &&
4485 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4486 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4487 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4488 else
4489 i_projid = EXT4_DEF_PROJID;
4490
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004491 if (!(test_opt(inode->i_sb, NO_UID32))) {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004492 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4493 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004494 }
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004495 i_uid_write(inode, i_uid);
4496 i_gid_write(inode, i_gid);
Li Xi040cb372016-01-08 16:01:21 -05004497 ei->i_projid = make_kprojid(&init_user_ns, i_projid);
Miklos Szeredibfe86842011-10-28 14:13:29 +02004498 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004499
Theodore Ts'o353eb832011-01-10 12:18:25 -05004500 ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */
Tao Ma67cf5b02012-12-10 14:04:46 -05004501 ei->i_inline_off = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004502 ei->i_dir_start_lookup = 0;
4503 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4504 /* We now have enough fields to check if the inode was active or not.
4505 * This is needed because nfsd might try to access dead inodes
4506 * the test is that same one that e2fsck uses
4507 * NeilBrown 1999oct15
4508 */
4509 if (inode->i_nlink == 0) {
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004510 if ((inode->i_mode == 0 ||
4511 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4512 ino != EXT4_BOOT_LOADER_INO) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004513 /* this inode is deleted */
David Howells1d1fe1e2008-02-07 00:15:37 -08004514 ret = -ESTALE;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004515 goto bad_inode;
4516 }
4517 /* The only unlinked inodes we let through here have
4518 * valid i_mode and are being read by the orphan
4519 * recovery code: that's fine, we're about to complete
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004520 * the process of deleting those.
4521 * OR it is the EXT4_BOOT_LOADER_INO which is
4522 * not initialized on a new filesystem. */
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004523 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004524 ei->i_flags = le32_to_cpu(raw_inode->i_flags);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004525 inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
Aneesh Kumar K.V7973c0c2008-01-28 23:58:27 -05004526 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004527 if (ext4_has_feature_64bit(sb))
Badari Pulavartya1ddeb72006-10-11 01:21:09 -07004528 ei->i_file_acl |=
4529 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
Aneesh Kumar K.Va48380f2008-01-28 23:58:27 -05004530 inode->i_size = ext4_isize(raw_inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004531 ei->i_disksize = inode->i_size;
Dmitry Monakhova9e7f442009-12-14 15:21:14 +03004532#ifdef CONFIG_QUOTA
4533 ei->i_reserved_quota = 0;
4534#endif
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004535 inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4536 ei->i_block_group = iloc.block_group;
Theodore Ts'oa4912122009-03-12 12:18:34 -04004537 ei->i_last_alloc_group = ~0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004538 /*
4539 * NOTE! The in-memory inode i_data array is in little-endian order
4540 * even on big-endian machines: we do NOT byteswap the block numbers!
4541 */
Mingming Cao617ba132006-10-11 01:20:53 -07004542 for (block = 0; block < EXT4_N_BLOCKS; block++)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004543 ei->i_data[block] = raw_inode->i_block[block];
4544 INIT_LIST_HEAD(&ei->i_orphan);
4545
Jan Karab436b9b2009-12-08 23:51:10 -05004546 /*
4547 * Set transaction id's of transactions that have to be committed
4548 * to finish f[data]sync. We set them to currently running transaction
4549 * as we cannot be sure that the inode or some of its metadata isn't
4550 * part of the transaction - the inode could have been reclaimed and
4551 * now it is reread from disk.
4552 */
4553 if (journal) {
4554 transaction_t *transaction;
4555 tid_t tid;
4556
Theodore Ts'oa931da62010-08-03 21:35:12 -04004557 read_lock(&journal->j_state_lock);
Jan Karab436b9b2009-12-08 23:51:10 -05004558 if (journal->j_running_transaction)
4559 transaction = journal->j_running_transaction;
4560 else
4561 transaction = journal->j_committing_transaction;
4562 if (transaction)
4563 tid = transaction->t_tid;
4564 else
4565 tid = journal->j_commit_sequence;
Theodore Ts'oa931da62010-08-03 21:35:12 -04004566 read_unlock(&journal->j_state_lock);
Jan Karab436b9b2009-12-08 23:51:10 -05004567 ei->i_sync_tid = tid;
4568 ei->i_datasync_tid = tid;
4569 }
4570
Eric Sandeen0040d982008-02-05 22:36:43 -05004571 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004572 if (ei->i_extra_isize == 0) {
4573 /* The extra space is currently unused. Use it. */
Mingming Cao617ba132006-10-11 01:20:53 -07004574 ei->i_extra_isize = sizeof(struct ext4_inode) -
4575 EXT4_GOOD_OLD_INODE_SIZE;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004576 } else {
Tao Ma152a7b02012-12-02 11:13:24 -05004577 ext4_iget_extra_inode(inode, raw_inode, ei);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004578 }
Darrick J. Wong814525f2012-04-29 18:31:10 -04004579 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004580
Kalpak Shahef7f3832007-07-18 09:15:20 -04004581 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
4582 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
4583 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
4584 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4585
Theodore Ts'oed3654e2014-03-24 14:09:06 -04004586 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
Theodore Ts'oc4f65702014-03-20 00:32:57 -04004587 inode->i_version = le32_to_cpu(raw_inode->i_disk_version);
4588 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4589 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4590 inode->i_version |=
4591 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4592 }
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05004593 }
4594
Theodore Ts'oc4b5a612009-04-24 18:45:35 -04004595 ret = 0;
Theodore Ts'o485c26e2009-04-24 13:43:20 -04004596 if (ei->i_file_acl &&
Theodore Ts'o10329882009-11-15 15:29:56 -05004597 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) {
Theodore Ts'o24676da2010-05-16 21:00:00 -04004598 EXT4_ERROR_INODE(inode, "bad extended attribute block %llu",
4599 ei->i_file_acl);
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004600 ret = -EFSCORRUPTED;
Theodore Ts'o485c26e2009-04-24 13:43:20 -04004601 goto bad_inode;
Tao Maf19d5872012-12-10 14:05:51 -05004602 } else if (!ext4_has_inline_data(inode)) {
4603 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
4604 if ((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4605 (S_ISLNK(inode->i_mode) &&
4606 !ext4_inode_is_fast_symlink(inode))))
4607 /* Validate extent which is part of inode */
4608 ret = ext4_ext_check_inode(inode);
4609 } else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4610 (S_ISLNK(inode->i_mode) &&
4611 !ext4_inode_is_fast_symlink(inode))) {
4612 /* Validate block references which are part of inode */
4613 ret = ext4_ind_check_inode(inode);
4614 }
Thiemo Nagelfe2c8192009-03-31 08:36:10 -04004615 }
Theodore Ts'o567f3e92009-11-14 08:19:05 -05004616 if (ret)
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004617 goto bad_inode;
Aneesh Kumar K.V7a262f72009-03-27 16:39:58 -04004618
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004619 if (S_ISREG(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004620 inode->i_op = &ext4_file_inode_operations;
Boaz Harroshbe64f882015-04-15 16:15:17 -07004621 inode->i_fop = &ext4_file_operations;
Mingming Cao617ba132006-10-11 01:20:53 -07004622 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004623 } else if (S_ISDIR(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004624 inode->i_op = &ext4_dir_inode_operations;
4625 inode->i_fop = &ext4_dir_operations;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004626 } else if (S_ISLNK(inode->i_mode)) {
Al Viroa7a67e82015-04-27 17:51:30 -04004627 if (ext4_encrypted_inode(inode)) {
4628 inode->i_op = &ext4_encrypted_symlink_inode_operations;
4629 ext4_set_aops(inode);
4630 } else if (ext4_inode_is_fast_symlink(inode)) {
Al Viro75e75662015-05-02 10:13:58 -04004631 inode->i_link = (char *)ei->i_data;
Mingming Cao617ba132006-10-11 01:20:53 -07004632 inode->i_op = &ext4_fast_symlink_inode_operations;
Duane Griffine83c1392008-12-19 20:47:15 +00004633 nd_terminate_link(ei->i_data, inode->i_size,
4634 sizeof(ei->i_data) - 1);
4635 } else {
Mingming Cao617ba132006-10-11 01:20:53 -07004636 inode->i_op = &ext4_symlink_inode_operations;
4637 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004638 }
Al Viro21fc61c2015-11-17 01:07:57 -05004639 inode_nohighmem(inode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004640 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4641 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
Mingming Cao617ba132006-10-11 01:20:53 -07004642 inode->i_op = &ext4_special_inode_operations;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004643 if (raw_inode->i_block[0])
4644 init_special_inode(inode, inode->i_mode,
4645 old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4646 else
4647 init_special_inode(inode, inode->i_mode,
4648 new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
Dr. Tilmann Bubeck393d1d12013-04-08 12:54:05 -04004649 } else if (ino == EXT4_BOOT_LOADER_INO) {
4650 make_bad_inode(inode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004651 } else {
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004652 ret = -EFSCORRUPTED;
Theodore Ts'o24676da2010-05-16 21:00:00 -04004653 EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode);
Theodore Ts'o563bdd62009-03-26 00:06:19 -04004654 goto bad_inode;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004655 }
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004656 brelse(iloc.bh);
Mingming Cao617ba132006-10-11 01:20:53 -07004657 ext4_set_inode_flags(inode);
David Howells1d1fe1e2008-02-07 00:15:37 -08004658 unlock_new_inode(inode);
4659 return inode;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004660
4661bad_inode:
Theodore Ts'o567f3e92009-11-14 08:19:05 -05004662 brelse(iloc.bh);
David Howells1d1fe1e2008-02-07 00:15:37 -08004663 iget_failed(inode);
4664 return ERR_PTR(ret);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004665}
4666
Theodore Ts'of4bb2982014-10-05 22:56:00 -04004667struct inode *ext4_iget_normal(struct super_block *sb, unsigned long ino)
4668{
4669 if (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO)
Darrick J. Wong6a797d22015-10-17 16:16:04 -04004670 return ERR_PTR(-EFSCORRUPTED);
Theodore Ts'of4bb2982014-10-05 22:56:00 -04004671 return ext4_iget(sb, ino);
4672}
4673
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004674static int ext4_inode_blocks_set(handle_t *handle,
4675 struct ext4_inode *raw_inode,
4676 struct ext4_inode_info *ei)
4677{
4678 struct inode *inode = &(ei->vfs_inode);
4679 u64 i_blocks = inode->i_blocks;
4680 struct super_block *sb = inode->i_sb;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004681
4682 if (i_blocks <= ~0U) {
4683 /*
Anatol Pomozov4907cb72012-09-01 10:31:09 -07004684 * i_blocks can be represented in a 32 bit variable
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004685 * as multiple of 512 bytes
4686 */
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004687 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004688 raw_inode->i_blocks_high = 0;
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004689 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004690 return 0;
4691 }
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004692 if (!ext4_has_feature_huge_file(sb))
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004693 return -EFBIG;
4694
4695 if (i_blocks <= 0xffffffffffffULL) {
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004696 /*
4697 * i_blocks can be represented in a 48 bit variable
4698 * as multiple of 512 bytes
4699 */
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004700 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004701 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004702 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004703 } else {
Dmitry Monakhov84a8dce2010-06-05 11:51:27 -04004704 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Aneesh Kumar K.V8180a562008-01-28 23:58:27 -05004705 /* i_block is stored in file system block size */
4706 i_blocks = i_blocks >> (inode->i_blkbits - 9);
4707 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4708 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004709 }
Theodore Ts'of287a1a2008-10-16 22:50:48 -04004710 return 0;
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004711}
4712
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004713struct other_inode {
4714 unsigned long orig_ino;
4715 struct ext4_inode *raw_inode;
4716};
4717
4718static int other_inode_match(struct inode * inode, unsigned long ino,
4719 void *data)
4720{
4721 struct other_inode *oi = (struct other_inode *) data;
4722
4723 if ((inode->i_ino != ino) ||
4724 (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
4725 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) ||
4726 ((inode->i_state & I_DIRTY_TIME) == 0))
4727 return 0;
4728 spin_lock(&inode->i_lock);
4729 if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
4730 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) == 0) &&
4731 (inode->i_state & I_DIRTY_TIME)) {
4732 struct ext4_inode_info *ei = EXT4_I(inode);
4733
4734 inode->i_state &= ~(I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED);
4735 spin_unlock(&inode->i_lock);
4736
4737 spin_lock(&ei->i_raw_lock);
4738 EXT4_INODE_SET_XTIME(i_ctime, inode, oi->raw_inode);
4739 EXT4_INODE_SET_XTIME(i_mtime, inode, oi->raw_inode);
4740 EXT4_INODE_SET_XTIME(i_atime, inode, oi->raw_inode);
4741 ext4_inode_csum_set(inode, oi->raw_inode, ei);
4742 spin_unlock(&ei->i_raw_lock);
4743 trace_ext4_other_inode_update_time(inode, oi->orig_ino);
4744 return -1;
4745 }
4746 spin_unlock(&inode->i_lock);
4747 return -1;
4748}
4749
4750/*
4751 * Opportunistically update the other time fields for other inodes in
4752 * the same inode table block.
4753 */
4754static void ext4_update_other_inodes_time(struct super_block *sb,
4755 unsigned long orig_ino, char *buf)
4756{
4757 struct other_inode oi;
4758 unsigned long ino;
4759 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4760 int inode_size = EXT4_INODE_SIZE(sb);
4761
4762 oi.orig_ino = orig_ino;
Theodore Ts'o0f0ff9a2015-07-01 23:37:46 -04004763 /*
4764 * Calculate the first inode in the inode table block. Inode
4765 * numbers are one-based. That is, the first inode in a block
4766 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
4767 */
4768 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004769 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
4770 if (ino == orig_ino)
4771 continue;
4772 oi.raw_inode = (struct ext4_inode *) buf;
4773 (void) find_inode_nowait(sb, ino, other_inode_match, &oi);
4774 }
4775}
4776
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004777/*
4778 * Post the struct inode info into an on-disk inode location in the
4779 * buffer-cache. This gobbles the caller's reference to the
4780 * buffer_head in the inode location struct.
4781 *
4782 * The caller must have write access to iloc->bh.
4783 */
Mingming Cao617ba132006-10-11 01:20:53 -07004784static int ext4_do_update_inode(handle_t *handle,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004785 struct inode *inode,
Frank Mayhar830156c2009-09-29 10:07:47 -04004786 struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004787{
Mingming Cao617ba132006-10-11 01:20:53 -07004788 struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
4789 struct ext4_inode_info *ei = EXT4_I(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004790 struct buffer_head *bh = iloc->bh;
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004791 struct super_block *sb = inode->i_sb;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004792 int err = 0, rc, block;
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004793 int need_datasync = 0, set_large_file = 0;
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004794 uid_t i_uid;
4795 gid_t i_gid;
Li Xi040cb372016-01-08 16:01:21 -05004796 projid_t i_projid;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004797
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004798 spin_lock(&ei->i_raw_lock);
4799
4800 /* For fields not tracked in the in-memory inode,
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004801 * initialise them to zero for new inodes. */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05004802 if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
Mingming Cao617ba132006-10-11 01:20:53 -07004803 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004804
Jan Karaff9ddf72007-07-18 09:24:20 -04004805 ext4_get_inode_flags(ei);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004806 raw_inode->i_mode = cpu_to_le16(inode->i_mode);
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004807 i_uid = i_uid_read(inode);
4808 i_gid = i_gid_read(inode);
Li Xi040cb372016-01-08 16:01:21 -05004809 i_projid = from_kprojid(&init_user_ns, ei->i_projid);
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004810 if (!(test_opt(inode->i_sb, NO_UID32))) {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004811 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
4812 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004813/*
4814 * Fix up interoperability with old kernels. Otherwise, old inodes get
4815 * re-used with the upper 16 bits of the uid/gid intact
4816 */
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004817 if (!ei->i_dtime) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004818 raw_inode->i_uid_high =
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004819 cpu_to_le16(high_16_bits(i_uid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004820 raw_inode->i_gid_high =
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004821 cpu_to_le16(high_16_bits(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004822 } else {
4823 raw_inode->i_uid_high = 0;
4824 raw_inode->i_gid_high = 0;
4825 }
4826 } else {
Eric W. Biederman08cefc72012-02-07 15:41:49 -08004827 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
4828 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004829 raw_inode->i_uid_high = 0;
4830 raw_inode->i_gid_high = 0;
4831 }
4832 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
Kalpak Shahef7f3832007-07-18 09:15:20 -04004833
4834 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
4835 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
4836 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
4837 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
4838
Li Xibce92d52014-10-01 22:11:06 -04004839 err = ext4_inode_blocks_set(handle, raw_inode, ei);
4840 if (err) {
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004841 spin_unlock(&ei->i_raw_lock);
Aneesh Kumar K.V0fc1b452008-01-28 23:58:26 -05004842 goto out_brelse;
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004843 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004844 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
Theodore Ts'o353eb832011-01-10 12:18:25 -05004845 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
Theodore Ts'oed3654e2014-03-24 14:09:06 -04004846 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
Badari Pulavartya1ddeb72006-10-11 01:21:09 -07004847 raw_inode->i_file_acl_high =
4848 cpu_to_le16(ei->i_file_acl >> 32);
Aneesh Kumar K.V7973c0c2008-01-28 23:58:27 -05004849 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
Jan Karab71fc072012-09-26 21:52:20 -04004850 if (ei->i_disksize != ext4_isize(raw_inode)) {
4851 ext4_isize_set(raw_inode, ei->i_disksize);
4852 need_datasync = 1;
4853 }
Aneesh Kumar K.Va48380f2008-01-28 23:58:27 -05004854 if (ei->i_disksize > 0x7fffffffULL) {
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004855 if (!ext4_has_feature_large_file(sb) ||
Aneesh Kumar K.Va48380f2008-01-28 23:58:27 -05004856 EXT4_SB(sb)->s_es->s_rev_level ==
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004857 cpu_to_le32(EXT4_GOOD_OLD_REV))
4858 set_large_file = 1;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004859 }
4860 raw_inode->i_generation = cpu_to_le32(inode->i_generation);
4861 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
4862 if (old_valid_dev(inode->i_rdev)) {
4863 raw_inode->i_block[0] =
4864 cpu_to_le32(old_encode_dev(inode->i_rdev));
4865 raw_inode->i_block[1] = 0;
4866 } else {
4867 raw_inode->i_block[0] = 0;
4868 raw_inode->i_block[1] =
4869 cpu_to_le32(new_encode_dev(inode->i_rdev));
4870 raw_inode->i_block[2] = 0;
4871 }
Tao Maf19d5872012-12-10 14:05:51 -05004872 } else if (!ext4_has_inline_data(inode)) {
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04004873 for (block = 0; block < EXT4_N_BLOCKS; block++)
4874 raw_inode->i_block[block] = ei->i_data[block];
Tao Maf19d5872012-12-10 14:05:51 -05004875 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004876
Theodore Ts'oed3654e2014-03-24 14:09:06 -04004877 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
Theodore Ts'oc4f65702014-03-20 00:32:57 -04004878 raw_inode->i_disk_version = cpu_to_le32(inode->i_version);
4879 if (ei->i_extra_isize) {
4880 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4881 raw_inode->i_version_hi =
4882 cpu_to_le32(inode->i_version >> 32);
4883 raw_inode->i_extra_isize =
4884 cpu_to_le16(ei->i_extra_isize);
4885 }
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05004886 }
Li Xi040cb372016-01-08 16:01:21 -05004887
4888 BUG_ON(!EXT4_HAS_RO_COMPAT_FEATURE(inode->i_sb,
4889 EXT4_FEATURE_RO_COMPAT_PROJECT) &&
4890 i_projid != EXT4_DEF_PROJID);
4891
4892 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4893 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4894 raw_inode->i_projid = cpu_to_le32(i_projid);
4895
Darrick J. Wong814525f2012-04-29 18:31:10 -04004896 ext4_inode_csum_set(inode, raw_inode, ei);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004897 spin_unlock(&ei->i_raw_lock);
Theodore Ts'oa26f4992015-02-02 00:37:02 -05004898 if (inode->i_sb->s_flags & MS_LAZYTIME)
4899 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
4900 bh->b_data);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004901
Frank Mayhar830156c2009-09-29 10:07:47 -04004902 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
Curt Wohlgemuth73b50c12010-02-16 15:06:29 -05004903 rc = ext4_handle_dirty_metadata(handle, NULL, bh);
Frank Mayhar830156c2009-09-29 10:07:47 -04004904 if (!err)
4905 err = rc;
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05004906 ext4_clear_inode_state(inode, EXT4_STATE_NEW);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004907 if (set_large_file) {
liang xie5d601252014-05-12 22:06:43 -04004908 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004909 err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh);
4910 if (err)
4911 goto out_brelse;
4912 ext4_update_dynamic_rev(sb);
Darrick J. Wonge2b911c2015-10-17 16:18:43 -04004913 ext4_set_feature_large_file(sb);
Theodore Ts'o202ee5d2014-04-21 14:37:55 -04004914 ext4_handle_sync(handle);
4915 err = ext4_handle_dirty_super(handle, sb);
4916 }
Jan Karab71fc072012-09-26 21:52:20 -04004917 ext4_update_inode_fsync_trans(handle, inode, need_datasync);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004918out_brelse:
Theodore Ts'oaf5bc922008-09-08 22:25:24 -04004919 brelse(bh);
Mingming Cao617ba132006-10-11 01:20:53 -07004920 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004921 return err;
4922}
4923
4924/*
Mingming Cao617ba132006-10-11 01:20:53 -07004925 * ext4_write_inode()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004926 *
4927 * We are called from a few places:
4928 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004929 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004930 * Here, there will be no transaction running. We wait for any running
Anatol Pomozov4907cb72012-09-01 10:31:09 -07004931 * transaction to commit.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004932 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004933 * - Within flush work (sys_sync(), kupdate and such).
4934 * We wait on commit, if told to.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004935 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004936 * - Within iput_final() -> write_inode_now()
4937 * We wait on commit, if told to.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004938 *
4939 * In all cases it is actually safe for us to return without doing anything,
4940 * because the inode has been copied into a raw inode buffer in
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004941 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
4942 * writeback.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004943 *
4944 * Note that we are absolutely dependent upon all inode dirtiers doing the
4945 * right thing: they *must* call mark_inode_dirty() after dirtying info in
4946 * which we are interested.
4947 *
4948 * It would be a bug for them to not do this. The code:
4949 *
4950 * mark_inode_dirty(inode)
4951 * stuff();
4952 * inode->i_size = expr;
4953 *
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004954 * is in error because write_inode() could occur while `stuff()' is running,
4955 * and the new i_size will be lost. Plus the inode will no longer be on the
4956 * superblock's dirty inode list.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004957 */
Christoph Hellwiga9185b42010-03-05 09:21:37 +01004958int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004959{
Frank Mayhar91ac6f42009-09-09 22:33:47 -04004960 int err;
4961
Theodore Ts'o87f7e412014-04-08 11:38:28 -04004962 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004963 return 0;
4964
Frank Mayhar91ac6f42009-09-09 22:33:47 -04004965 if (EXT4_SB(inode->i_sb)->s_journal) {
4966 if (ext4_journal_current_handle()) {
4967 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
4968 dump_stack();
4969 return -EIO;
4970 }
4971
Jan Kara10542c22014-03-04 10:50:50 -05004972 /*
4973 * No need to force transaction in WB_SYNC_NONE mode. Also
4974 * ext4_sync_fs() will force the commit after everything is
4975 * written.
4976 */
4977 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
Frank Mayhar91ac6f42009-09-09 22:33:47 -04004978 return 0;
4979
4980 err = ext4_force_commit(inode->i_sb);
4981 } else {
4982 struct ext4_iloc iloc;
4983
Curt Wohlgemuth8b472d72010-04-03 16:45:06 -04004984 err = __ext4_get_inode_loc(inode, &iloc, 0);
Frank Mayhar91ac6f42009-09-09 22:33:47 -04004985 if (err)
4986 return err;
Jan Kara10542c22014-03-04 10:50:50 -05004987 /*
4988 * sync(2) will flush the whole buffer cache. No need to do
4989 * it here separately for each inode.
4990 */
4991 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
Frank Mayhar830156c2009-09-29 10:07:47 -04004992 sync_dirty_buffer(iloc.bh);
4993 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
Theodore Ts'oc398eda2010-07-27 11:56:40 -04004994 EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr,
4995 "IO error syncing inode");
Frank Mayhar830156c2009-09-29 10:07:47 -04004996 err = -EIO;
4997 }
Curt Wohlgemuthfd2dd9f2010-04-03 17:44:16 -04004998 brelse(iloc.bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07004999 }
Frank Mayhar91ac6f42009-09-09 22:33:47 -04005000 return err;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005001}
5002
5003/*
Jan Kara53e87262012-12-25 13:29:52 -05005004 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate
5005 * buffers that are attached to a page stradding i_size and are undergoing
5006 * commit. In that case we have to wait for commit to finish and try again.
5007 */
5008static void ext4_wait_for_tail_page_commit(struct inode *inode)
5009{
5010 struct page *page;
5011 unsigned offset;
5012 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5013 tid_t commit_tid = 0;
5014 int ret;
5015
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005016 offset = inode->i_size & (PAGE_SIZE - 1);
Jan Kara53e87262012-12-25 13:29:52 -05005017 /*
5018 * All buffers in the last page remain valid? Then there's nothing to
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03005019 * do. We do the check mainly to optimize the common PAGE_SIZE ==
Jan Kara53e87262012-12-25 13:29:52 -05005020 * blocksize case
5021 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005022 if (offset > PAGE_SIZE - (1 << inode->i_blkbits))
Jan Kara53e87262012-12-25 13:29:52 -05005023 return;
5024 while (1) {
5025 page = find_lock_page(inode->i_mapping,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005026 inode->i_size >> PAGE_SHIFT);
Jan Kara53e87262012-12-25 13:29:52 -05005027 if (!page)
5028 return;
Lukas Czernerca99fdd2013-05-21 23:25:01 -04005029 ret = __ext4_journalled_invalidatepage(page, offset,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005030 PAGE_SIZE - offset);
Jan Kara53e87262012-12-25 13:29:52 -05005031 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005032 put_page(page);
Jan Kara53e87262012-12-25 13:29:52 -05005033 if (ret != -EBUSY)
5034 return;
5035 commit_tid = 0;
5036 read_lock(&journal->j_state_lock);
5037 if (journal->j_committing_transaction)
5038 commit_tid = journal->j_committing_transaction->t_tid;
5039 read_unlock(&journal->j_state_lock);
5040 if (commit_tid)
5041 jbd2_log_wait_commit(journal, commit_tid);
5042 }
5043}
5044
5045/*
Mingming Cao617ba132006-10-11 01:20:53 -07005046 * ext4_setattr()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005047 *
5048 * Called from notify_change.
5049 *
5050 * We want to trap VFS attempts to truncate the file as soon as
5051 * possible. In particular, we want to make sure that when the VFS
5052 * shrinks i_size, we put the inode on the orphan list and modify
5053 * i_disksize immediately, so that during the subsequent flushing of
5054 * dirty pages and freeing of disk blocks, we can guarantee that any
5055 * commit will leave the blocks being flushed in an unused state on
5056 * disk. (On recovery, the inode will get truncated and the blocks will
5057 * be freed, so we have a strong guarantee that no future commit will
5058 * leave these blocks visible to the user.)
5059 *
Jan Kara678aaf42008-07-11 19:27:31 -04005060 * Another thing we have to assure is that if we are in ordered mode
5061 * and inode is still attached to the committing transaction, we must
5062 * we start writeout of all the dirty pages which are being truncated.
5063 * This way we are sure that all the data written in the previous
5064 * transaction are already on disk (truncate waits for pages under
5065 * writeback).
5066 *
5067 * Called with inode->i_mutex down.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005068 */
Mingming Cao617ba132006-10-11 01:20:53 -07005069int ext4_setattr(struct dentry *dentry, struct iattr *attr)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005070{
David Howells2b0143b2015-03-17 22:25:59 +00005071 struct inode *inode = d_inode(dentry);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005072 int error, rc = 0;
Dmitry Monakhov3d287de2010-10-27 22:08:46 -04005073 int orphan = 0;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005074 const unsigned int ia_valid = attr->ia_valid;
5075
5076 error = inode_change_ok(inode, attr);
5077 if (error)
5078 return error;
5079
Jan Karaa7cdade2015-06-29 16:22:54 +02005080 if (is_quota_modification(inode, attr)) {
5081 error = dquot_initialize(inode);
5082 if (error)
5083 return error;
5084 }
Eric W. Biederman08cefc72012-02-07 15:41:49 -08005085 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
5086 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005087 handle_t *handle;
5088
5089 /* (user+group)*(old+new) structure, inode write (sb,
5090 * inode block, ? - but truncate inode update has it) */
Theodore Ts'o9924a922013-02-08 21:59:22 -05005091 handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5092 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5093 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005094 if (IS_ERR(handle)) {
5095 error = PTR_ERR(handle);
5096 goto err_out;
5097 }
Christoph Hellwigb43fa822010-03-03 09:05:03 -05005098 error = dquot_transfer(inode, attr);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005099 if (error) {
Mingming Cao617ba132006-10-11 01:20:53 -07005100 ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005101 return error;
5102 }
5103 /* Update corresponding info in inode so that everything is in
5104 * one transaction */
5105 if (attr->ia_valid & ATTR_UID)
5106 inode->i_uid = attr->ia_uid;
5107 if (attr->ia_valid & ATTR_GID)
5108 inode->i_gid = attr->ia_gid;
Mingming Cao617ba132006-10-11 01:20:53 -07005109 error = ext4_mark_inode_dirty(handle, inode);
5110 ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005111 }
5112
Josef Bacik3da40c72015-06-22 00:31:26 -04005113 if (attr->ia_valid & ATTR_SIZE) {
Jan Kara52083862013-08-17 10:07:17 -04005114 handle_t *handle;
Josef Bacik3da40c72015-06-22 00:31:26 -04005115 loff_t oldsize = inode->i_size;
5116 int shrink = (attr->ia_size <= inode->i_size);
Christoph Hellwig562c72a2011-06-24 14:29:45 -04005117
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005118 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
Eric Sandeene2b46572008-01-28 23:58:27 -05005119 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5120
Theodore Ts'o0c095c72010-07-27 11:56:06 -04005121 if (attr->ia_size > sbi->s_bitmap_maxbytes)
5122 return -EFBIG;
Eric Sandeene2b46572008-01-28 23:58:27 -05005123 }
Josef Bacik3da40c72015-06-22 00:31:26 -04005124 if (!S_ISREG(inode->i_mode))
5125 return -EINVAL;
Christoph Hellwigdff6efc2013-11-19 07:17:07 -08005126
5127 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
5128 inode_inc_iversion(inode);
5129
Josef Bacik3da40c72015-06-22 00:31:26 -04005130 if (ext4_should_order_data(inode) &&
Jan Kara52083862013-08-17 10:07:17 -04005131 (attr->ia_size < inode->i_size)) {
Josef Bacik3da40c72015-06-22 00:31:26 -04005132 error = ext4_begin_ordered_truncate(inode,
Jan Kara678aaf42008-07-11 19:27:31 -04005133 attr->ia_size);
Josef Bacik3da40c72015-06-22 00:31:26 -04005134 if (error)
5135 goto err_out;
5136 }
5137 if (attr->ia_size != inode->i_size) {
Jan Kara52083862013-08-17 10:07:17 -04005138 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5139 if (IS_ERR(handle)) {
5140 error = PTR_ERR(handle);
5141 goto err_out;
5142 }
Josef Bacik3da40c72015-06-22 00:31:26 -04005143 if (ext4_handle_valid(handle) && shrink) {
Jan Kara52083862013-08-17 10:07:17 -04005144 error = ext4_orphan_add(handle, inode);
5145 orphan = 1;
5146 }
Eryu Guan911af572015-07-28 15:08:41 -04005147 /*
5148 * Update c/mtime on truncate up, ext4_truncate() will
5149 * update c/mtime in shrink case below
5150 */
5151 if (!shrink) {
5152 inode->i_mtime = ext4_current_time(inode);
5153 inode->i_ctime = inode->i_mtime;
5154 }
Jan Kara90e775b2013-08-17 10:09:31 -04005155 down_write(&EXT4_I(inode)->i_data_sem);
Jan Kara52083862013-08-17 10:07:17 -04005156 EXT4_I(inode)->i_disksize = attr->ia_size;
5157 rc = ext4_mark_inode_dirty(handle, inode);
5158 if (!error)
5159 error = rc;
Jan Kara90e775b2013-08-17 10:09:31 -04005160 /*
5161 * We have to update i_size under i_data_sem together
5162 * with i_disksize to avoid races with writeback code
5163 * running ext4_wb_update_i_disksize().
5164 */
5165 if (!error)
5166 i_size_write(inode, attr->ia_size);
5167 up_write(&EXT4_I(inode)->i_data_sem);
Jan Kara52083862013-08-17 10:07:17 -04005168 ext4_journal_stop(handle);
5169 if (error) {
Josef Bacik3da40c72015-06-22 00:31:26 -04005170 if (orphan)
5171 ext4_orphan_del(NULL, inode);
Jan Kara678aaf42008-07-11 19:27:31 -04005172 goto err_out;
5173 }
Jan Karad6320cb2014-10-01 21:49:46 -04005174 }
Josef Bacik3da40c72015-06-22 00:31:26 -04005175 if (!shrink)
5176 pagecache_isize_extended(inode, oldsize, inode->i_size);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005177
Jan Kara52083862013-08-17 10:07:17 -04005178 /*
5179 * Blocks are going to be removed from the inode. Wait
5180 * for dio in flight. Temporarily disable
5181 * dioread_nolock to prevent livelock.
5182 */
5183 if (orphan) {
5184 if (!ext4_should_journal_data(inode)) {
5185 ext4_inode_block_unlocked_dio(inode);
5186 inode_dio_wait(inode);
5187 ext4_inode_resume_unlocked_dio(inode);
5188 } else
5189 ext4_wait_for_tail_page_commit(inode);
Dmitry Monakhov1c9114f2012-09-29 00:55:23 -04005190 }
Jan Karaea3d7202015-12-07 14:28:03 -05005191 down_write(&EXT4_I(inode)->i_mmap_sem);
Jan Kara52083862013-08-17 10:07:17 -04005192 /*
5193 * Truncate pagecache after we've waited for commit
5194 * in data=journal mode to make pages freeable.
5195 */
Ross Zwisler923ae0f2015-02-16 15:59:38 -08005196 truncate_pagecache(inode, inode->i_size);
Josef Bacik3da40c72015-06-22 00:31:26 -04005197 if (shrink)
5198 ext4_truncate(inode);
Jan Karaea3d7202015-12-07 14:28:03 -05005199 up_write(&EXT4_I(inode)->i_mmap_sem);
Theodore Ts'o072bd7e2011-05-23 15:13:02 -04005200 }
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005201
Christoph Hellwig10257742010-06-04 11:30:02 +02005202 if (!rc) {
5203 setattr_copy(inode, attr);
5204 mark_inode_dirty(inode);
5205 }
5206
5207 /*
5208 * If the call to ext4_truncate failed to get a transaction handle at
5209 * all, we need to clean up the in-core orphan list manually.
5210 */
Dmitry Monakhov3d287de2010-10-27 22:08:46 -04005211 if (orphan && inode->i_nlink)
Mingming Cao617ba132006-10-11 01:20:53 -07005212 ext4_orphan_del(NULL, inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005213
5214 if (!rc && (ia_valid & ATTR_MODE))
Christoph Hellwig64e178a2013-12-20 05:16:44 -08005215 rc = posix_acl_chmod(inode, inode->i_mode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005216
5217err_out:
Mingming Cao617ba132006-10-11 01:20:53 -07005218 ext4_std_error(inode->i_sb, error);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005219 if (!error)
5220 error = rc;
5221 return error;
5222}
5223
Mingming Cao3e3398a2008-07-11 19:27:31 -04005224int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry,
5225 struct kstat *stat)
5226{
5227 struct inode *inode;
Jan Kara8af8eec2013-05-31 19:39:56 -04005228 unsigned long long delalloc_blocks;
Mingming Cao3e3398a2008-07-11 19:27:31 -04005229
David Howells2b0143b2015-03-17 22:25:59 +00005230 inode = d_inode(dentry);
Mingming Cao3e3398a2008-07-11 19:27:31 -04005231 generic_fillattr(inode, stat);
5232
5233 /*
Andreas Dilger9206c562013-11-11 22:38:12 -05005234 * If there is inline data in the inode, the inode will normally not
5235 * have data blocks allocated (it may have an external xattr block).
5236 * Report at least one sector for such files, so tools like tar, rsync,
5237 * others doen't incorrectly think the file is completely sparse.
5238 */
5239 if (unlikely(ext4_has_inline_data(inode)))
5240 stat->blocks += (stat->size + 511) >> 9;
5241
5242 /*
Mingming Cao3e3398a2008-07-11 19:27:31 -04005243 * We can't update i_blocks if the block allocation is delayed
5244 * otherwise in the case of system crash before the real block
5245 * allocation is done, we will have i_blocks inconsistent with
5246 * on-disk file blocks.
5247 * We always keep i_blocks updated together with real
5248 * allocation. But to not confuse with user, stat
5249 * will return the blocks that include the delayed allocation
5250 * blocks for this file.
5251 */
Tao Ma96607552012-05-31 22:54:16 -04005252 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
Andreas Dilger9206c562013-11-11 22:38:12 -05005253 EXT4_I(inode)->i_reserved_data_blocks);
5254 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
Mingming Cao3e3398a2008-07-11 19:27:31 -04005255 return 0;
5256}
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005257
Jan Karafffb2732013-06-04 13:01:11 -04005258static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5259 int pextents)
Mingming Caoa02908f2008-08-19 22:16:07 -04005260{
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005261 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
Jan Karafffb2732013-06-04 13:01:11 -04005262 return ext4_ind_trans_blocks(inode, lblocks);
5263 return ext4_ext_index_trans_blocks(inode, pextents);
Mingming Caoa02908f2008-08-19 22:16:07 -04005264}
Theodore Ts'oac51d832008-11-06 16:49:36 -05005265
Mingming Caoa02908f2008-08-19 22:16:07 -04005266/*
5267 * Account for index blocks, block groups bitmaps and block group
5268 * descriptor blocks if modify datablocks and index blocks
5269 * worse case, the indexs blocks spread over different block groups
5270 *
5271 * If datablocks are discontiguous, they are possible to spread over
Anatol Pomozov4907cb72012-09-01 10:31:09 -07005272 * different block groups too. If they are contiguous, with flexbg,
Mingming Caoa02908f2008-08-19 22:16:07 -04005273 * they could still across block group boundary.
5274 *
5275 * Also account for superblock, inode, quota and xattr blocks
5276 */
Jan Karafffb2732013-06-04 13:01:11 -04005277static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5278 int pextents)
Mingming Caoa02908f2008-08-19 22:16:07 -04005279{
Theodore Ts'o8df96752009-05-01 08:50:38 -04005280 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5281 int gdpblocks;
Mingming Caoa02908f2008-08-19 22:16:07 -04005282 int idxblocks;
5283 int ret = 0;
5284
5285 /*
Jan Karafffb2732013-06-04 13:01:11 -04005286 * How many index blocks need to touch to map @lblocks logical blocks
5287 * to @pextents physical extents?
Mingming Caoa02908f2008-08-19 22:16:07 -04005288 */
Jan Karafffb2732013-06-04 13:01:11 -04005289 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
Mingming Caoa02908f2008-08-19 22:16:07 -04005290
5291 ret = idxblocks;
5292
5293 /*
5294 * Now let's see how many group bitmaps and group descriptors need
5295 * to account
5296 */
Jan Karafffb2732013-06-04 13:01:11 -04005297 groups = idxblocks + pextents;
Mingming Caoa02908f2008-08-19 22:16:07 -04005298 gdpblocks = groups;
Theodore Ts'o8df96752009-05-01 08:50:38 -04005299 if (groups > ngroups)
5300 groups = ngroups;
Mingming Caoa02908f2008-08-19 22:16:07 -04005301 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5302 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5303
5304 /* bitmaps and block group descriptor blocks */
5305 ret += groups + gdpblocks;
5306
5307 /* Blocks for super block, inode, quota and xattr blocks */
5308 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005309
5310 return ret;
5311}
5312
5313/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03005314 * Calculate the total number of credits to reserve to fit
Mingming Caof3bd1f32008-08-19 22:16:03 -04005315 * the modification of a single pages into a single transaction,
5316 * which may include multiple chunks of block allocations.
Mingming Caoa02908f2008-08-19 22:16:07 -04005317 *
Mingming Cao525f4ed2008-08-19 22:15:58 -04005318 * This could be called via ext4_write_begin()
Mingming Caoa02908f2008-08-19 22:16:07 -04005319 *
Mingming Cao525f4ed2008-08-19 22:15:58 -04005320 * We need to consider the worse case, when
Mingming Caoa02908f2008-08-19 22:16:07 -04005321 * one new block per extent.
Mingming Caoa02908f2008-08-19 22:16:07 -04005322 */
5323int ext4_writepage_trans_blocks(struct inode *inode)
5324{
5325 int bpp = ext4_journal_blocks_per_page(inode);
5326 int ret;
5327
Jan Karafffb2732013-06-04 13:01:11 -04005328 ret = ext4_meta_trans_blocks(inode, bpp, bpp);
Mingming Caoa02908f2008-08-19 22:16:07 -04005329
5330 /* Account for data blocks for journalled mode */
5331 if (ext4_should_journal_data(inode))
5332 ret += bpp;
5333 return ret;
5334}
Mingming Caof3bd1f32008-08-19 22:16:03 -04005335
5336/*
5337 * Calculate the journal credits for a chunk of data modification.
5338 *
5339 * This is called from DIO, fallocate or whoever calling
Eric Sandeen79e83032010-07-27 11:56:07 -04005340 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
Mingming Caof3bd1f32008-08-19 22:16:03 -04005341 *
5342 * journal buffers for data blocks are not included here, as DIO
5343 * and fallocate do no need to journal data buffers.
5344 */
5345int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5346{
5347 return ext4_meta_trans_blocks(inode, nrblocks, 1);
5348}
5349
Mingming Caoa02908f2008-08-19 22:16:07 -04005350/*
Mingming Cao617ba132006-10-11 01:20:53 -07005351 * The caller must have previously called ext4_reserve_inode_write().
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005352 * Give this, we know that the caller already has write access to iloc->bh.
5353 */
Mingming Cao617ba132006-10-11 01:20:53 -07005354int ext4_mark_iloc_dirty(handle_t *handle,
Theodore Ts'ode9a55b2009-06-14 17:45:34 -04005355 struct inode *inode, struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005356{
5357 int err = 0;
5358
Theodore Ts'oc64db502012-03-02 12:23:11 -05005359 if (IS_I_VERSION(inode))
Jean Noel Cordenner25ec56b2008-01-28 23:58:27 -05005360 inode_inc_iversion(inode);
5361
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005362 /* the do_update_inode consumes one bh->b_count */
5363 get_bh(iloc->bh);
5364
Mingming Caodab291a2006-10-11 01:21:01 -07005365 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
Frank Mayhar830156c2009-09-29 10:07:47 -04005366 err = ext4_do_update_inode(handle, inode, iloc);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005367 put_bh(iloc->bh);
5368 return err;
5369}
5370
5371/*
5372 * On success, We end up with an outstanding reference count against
5373 * iloc->bh. This _must_ be cleaned up later.
5374 */
5375
5376int
Mingming Cao617ba132006-10-11 01:20:53 -07005377ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5378 struct ext4_iloc *iloc)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005379{
Frank Mayhar03901312009-01-07 00:06:22 -05005380 int err;
5381
5382 err = ext4_get_inode_loc(inode, iloc);
5383 if (!err) {
5384 BUFFER_TRACE(iloc->bh, "get_write_access");
5385 err = ext4_journal_get_write_access(handle, iloc->bh);
5386 if (err) {
5387 brelse(iloc->bh);
5388 iloc->bh = NULL;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005389 }
5390 }
Mingming Cao617ba132006-10-11 01:20:53 -07005391 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005392 return err;
5393}
5394
5395/*
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005396 * Expand an inode by new_extra_isize bytes.
5397 * Returns 0 on success or negative error number on failure.
5398 */
Aneesh Kumar K.V1d03ec92008-01-28 23:58:27 -05005399static int ext4_expand_extra_isize(struct inode *inode,
5400 unsigned int new_extra_isize,
5401 struct ext4_iloc iloc,
5402 handle_t *handle)
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005403{
5404 struct ext4_inode *raw_inode;
5405 struct ext4_xattr_ibody_header *header;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005406
5407 if (EXT4_I(inode)->i_extra_isize >= new_extra_isize)
5408 return 0;
5409
5410 raw_inode = ext4_raw_inode(&iloc);
5411
5412 header = IHDR(inode, raw_inode);
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005413
5414 /* No extended attributes present */
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005415 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5416 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005417 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0,
5418 new_extra_isize);
5419 EXT4_I(inode)->i_extra_isize = new_extra_isize;
5420 return 0;
5421 }
5422
5423 /* try to expand with EAs present */
5424 return ext4_expand_extra_isize_ea(inode, new_extra_isize,
5425 raw_inode, handle);
5426}
5427
5428/*
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005429 * What we do here is to mark the in-core inode as clean with respect to inode
5430 * dirtiness (it may still be data-dirty).
5431 * This means that the in-core inode may be reaped by prune_icache
5432 * without having to perform any I/O. This is a very good thing,
5433 * because *any* task may call prune_icache - even ones which
5434 * have a transaction open against a different journal.
5435 *
5436 * Is this cheating? Not really. Sure, we haven't written the
5437 * inode out, but prune_icache isn't a user-visible syncing function.
5438 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
5439 * we start and wait on commits.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005440 */
Mingming Cao617ba132006-10-11 01:20:53 -07005441int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005442{
Mingming Cao617ba132006-10-11 01:20:53 -07005443 struct ext4_iloc iloc;
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005444 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5445 static unsigned int mnt_count;
5446 int err, ret;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005447
5448 might_sleep();
Theodore Ts'o7ff9c072010-11-08 13:51:33 -05005449 trace_ext4_mark_inode_dirty(inode, _RET_IP_);
Mingming Cao617ba132006-10-11 01:20:53 -07005450 err = ext4_reserve_inode_write(handle, inode, &iloc);
Eryu Guan5e1021f2016-03-12 21:40:32 -05005451 if (err)
5452 return err;
Frank Mayhar03901312009-01-07 00:06:22 -05005453 if (ext4_handle_valid(handle) &&
5454 EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize &&
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005455 !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005456 /*
5457 * We need extra buffer credits since we may write into EA block
5458 * with this same handle. If journal_extend fails, then it will
5459 * only result in a minor loss of functionality for that inode.
5460 * If this is felt to be critical, then e2fsck should be run to
5461 * force a large enough s_min_extra_isize.
5462 */
5463 if ((jbd2_journal_extend(handle,
5464 EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) {
5465 ret = ext4_expand_extra_isize(inode,
5466 sbi->s_want_extra_isize,
5467 iloc, handle);
5468 if (ret) {
Theodore Ts'o19f5fb72010-01-24 14:34:07 -05005469 ext4_set_inode_state(inode,
5470 EXT4_STATE_NO_EXPAND);
Aneesh Kumar K.Vc1bddad2007-10-16 18:38:25 -04005471 if (mnt_count !=
5472 le16_to_cpu(sbi->s_es->s_mnt_count)) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05005473 ext4_warning(inode->i_sb,
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005474 "Unable to expand inode %lu. Delete"
5475 " some EAs or run e2fsck.",
5476 inode->i_ino);
Aneesh Kumar K.Vc1bddad2007-10-16 18:38:25 -04005477 mnt_count =
5478 le16_to_cpu(sbi->s_es->s_mnt_count);
Kalpak Shah6dd4ee72007-07-18 09:19:57 -04005479 }
5480 }
5481 }
5482 }
Eryu Guan5e1021f2016-03-12 21:40:32 -05005483 return ext4_mark_iloc_dirty(handle, inode, &iloc);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005484}
5485
5486/*
Mingming Cao617ba132006-10-11 01:20:53 -07005487 * ext4_dirty_inode() is called from __mark_inode_dirty()
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005488 *
5489 * We're really interested in the case where a file is being extended.
5490 * i_size has been changed by generic_commit_write() and we thus need
5491 * to include the updated inode in the current transaction.
5492 *
Christoph Hellwig5dd40562010-03-03 09:05:00 -05005493 * Also, dquot_alloc_block() will always dirty the inode when blocks
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005494 * are allocated to the file.
5495 *
5496 * If the inode is marked synchronous, we don't honour that here - doing
5497 * so would cause a commit on atime updates, which we don't bother doing.
5498 * We handle synchronous inodes at the highest possible level.
Theodore Ts'o0ae45f62015-02-02 00:37:00 -05005499 *
5500 * If only the I_DIRTY_TIME flag is set, we can skip everything. If
5501 * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need
5502 * to copy into the on-disk inode structure are the timestamp files.
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005503 */
Christoph Hellwigaa385722011-05-27 06:53:02 -04005504void ext4_dirty_inode(struct inode *inode, int flags)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005505{
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005506 handle_t *handle;
5507
Theodore Ts'o0ae45f62015-02-02 00:37:00 -05005508 if (flags == I_DIRTY_TIME)
5509 return;
Theodore Ts'o9924a922013-02-08 21:59:22 -05005510 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005511 if (IS_ERR(handle))
5512 goto out;
Curt Wohlgemuthf3dc2722009-09-29 16:06:01 -04005513
Curt Wohlgemuthf3dc2722009-09-29 16:06:01 -04005514 ext4_mark_inode_dirty(handle, inode);
5515
Mingming Cao617ba132006-10-11 01:20:53 -07005516 ext4_journal_stop(handle);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005517out:
5518 return;
5519}
5520
5521#if 0
5522/*
5523 * Bind an inode's backing buffer_head into this transaction, to prevent
5524 * it from being flushed to disk early. Unlike
Mingming Cao617ba132006-10-11 01:20:53 -07005525 * ext4_reserve_inode_write, this leaves behind no bh reference and
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005526 * returns no iloc structure, so the caller needs to repeat the iloc
5527 * lookup to mark the inode dirty later.
5528 */
Mingming Cao617ba132006-10-11 01:20:53 -07005529static int ext4_pin_inode(handle_t *handle, struct inode *inode)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005530{
Mingming Cao617ba132006-10-11 01:20:53 -07005531 struct ext4_iloc iloc;
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005532
5533 int err = 0;
5534 if (handle) {
Mingming Cao617ba132006-10-11 01:20:53 -07005535 err = ext4_get_inode_loc(inode, &iloc);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005536 if (!err) {
5537 BUFFER_TRACE(iloc.bh, "get_write_access");
Mingming Caodab291a2006-10-11 01:21:01 -07005538 err = jbd2_journal_get_write_access(handle, iloc.bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005539 if (!err)
Frank Mayhar03901312009-01-07 00:06:22 -05005540 err = ext4_handle_dirty_metadata(handle,
Curt Wohlgemuth73b50c12010-02-16 15:06:29 -05005541 NULL,
Frank Mayhar03901312009-01-07 00:06:22 -05005542 iloc.bh);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005543 brelse(iloc.bh);
5544 }
5545 }
Mingming Cao617ba132006-10-11 01:20:53 -07005546 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005547 return err;
5548}
5549#endif
5550
Mingming Cao617ba132006-10-11 01:20:53 -07005551int ext4_change_inode_journal_flag(struct inode *inode, int val)
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005552{
5553 journal_t *journal;
5554 handle_t *handle;
5555 int err;
Daeho Jeongc8585c62016-04-25 23:22:35 -04005556 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005557
5558 /*
5559 * We have to be very careful here: changing a data block's
5560 * journaling status dynamically is dangerous. If we write a
5561 * data block to the journal, change the status and then delete
5562 * that block, we risk forgetting to revoke the old log record
5563 * from the journal and so a subsequent replay can corrupt data.
5564 * So, first we make sure that the journal is empty and that
5565 * nobody is changing anything.
5566 */
5567
Mingming Cao617ba132006-10-11 01:20:53 -07005568 journal = EXT4_JOURNAL(inode);
Frank Mayhar03901312009-01-07 00:06:22 -05005569 if (!journal)
5570 return 0;
Dave Hansend6995942007-07-18 08:33:51 -04005571 if (is_journal_aborted(journal))
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005572 return -EROFS;
5573
Dmitry Monakhov17335dc2012-09-29 00:41:21 -04005574 /* Wait for all existing dio workers */
5575 ext4_inode_block_unlocked_dio(inode);
5576 inode_dio_wait(inode);
5577
Daeho Jeong4c546592016-04-25 23:21:00 -04005578 /*
5579 * Before flushing the journal and switching inode's aops, we have
5580 * to flush all dirty data the inode has. There can be outstanding
5581 * delayed allocations, there can be unwritten extents created by
5582 * fallocate or buffered writes in dioread_nolock mode covered by
5583 * dirty data which can be converted only after flushing the dirty
5584 * data (and journalled aops don't know how to handle these cases).
5585 */
5586 if (val) {
5587 down_write(&EXT4_I(inode)->i_mmap_sem);
5588 err = filemap_write_and_wait(inode->i_mapping);
5589 if (err < 0) {
5590 up_write(&EXT4_I(inode)->i_mmap_sem);
5591 ext4_inode_resume_unlocked_dio(inode);
5592 return err;
5593 }
5594 }
5595
Daeho Jeongc8585c62016-04-25 23:22:35 -04005596 percpu_down_write(&sbi->s_journal_flag_rwsem);
Mingming Caodab291a2006-10-11 01:21:01 -07005597 jbd2_journal_lock_updates(journal);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005598
5599 /*
5600 * OK, there are no updates running now, and all cached data is
5601 * synced to disk. We are now in a completely consistent state
5602 * which doesn't have anything in the journal, and we know that
5603 * no filesystem updates are running, so it is safe to modify
5604 * the inode's in-core data-journaling state flag now.
5605 */
5606
5607 if (val)
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005608 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
Yongqiang Yang5872dda2011-12-28 13:55:51 -05005609 else {
Jan Kara4f879ca2014-10-30 10:53:17 -04005610 err = jbd2_journal_flush(journal);
5611 if (err < 0) {
5612 jbd2_journal_unlock_updates(journal);
Daeho Jeongc8585c62016-04-25 23:22:35 -04005613 percpu_up_write(&sbi->s_journal_flag_rwsem);
Jan Kara4f879ca2014-10-30 10:53:17 -04005614 ext4_inode_resume_unlocked_dio(inode);
5615 return err;
5616 }
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04005617 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
Yongqiang Yang5872dda2011-12-28 13:55:51 -05005618 }
Mingming Cao617ba132006-10-11 01:20:53 -07005619 ext4_set_aops(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005620
Mingming Caodab291a2006-10-11 01:21:01 -07005621 jbd2_journal_unlock_updates(journal);
Daeho Jeongc8585c62016-04-25 23:22:35 -04005622 percpu_up_write(&sbi->s_journal_flag_rwsem);
5623
Daeho Jeong4c546592016-04-25 23:21:00 -04005624 if (val)
5625 up_write(&EXT4_I(inode)->i_mmap_sem);
Dmitry Monakhov17335dc2012-09-29 00:41:21 -04005626 ext4_inode_resume_unlocked_dio(inode);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005627
5628 /* Finally we can mark the inode as dirty. */
5629
Theodore Ts'o9924a922013-02-08 21:59:22 -05005630 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005631 if (IS_ERR(handle))
5632 return PTR_ERR(handle);
5633
Mingming Cao617ba132006-10-11 01:20:53 -07005634 err = ext4_mark_inode_dirty(handle, inode);
Frank Mayhar03901312009-01-07 00:06:22 -05005635 ext4_handle_sync(handle);
Mingming Cao617ba132006-10-11 01:20:53 -07005636 ext4_journal_stop(handle);
5637 ext4_std_error(inode->i_sb, err);
Dave Kleikampac27a0e2006-10-11 01:20:50 -07005638
5639 return err;
5640}
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005641
5642static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh)
5643{
5644 return !buffer_mapped(bh);
5645}
5646
Nick Pigginc2ec1752009-03-31 15:23:21 -07005647int ext4_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005648{
Nick Pigginc2ec1752009-03-31 15:23:21 -07005649 struct page *page = vmf->page;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005650 loff_t size;
5651 unsigned long len;
Jan Kara9ea7df52011-06-24 14:29:41 -04005652 int ret;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005653 struct file *file = vma->vm_file;
Al Viro496ad9a2013-01-23 17:07:38 -05005654 struct inode *inode = file_inode(file);
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005655 struct address_space *mapping = inode->i_mapping;
Jan Kara9ea7df52011-06-24 14:29:41 -04005656 handle_t *handle;
5657 get_block_t *get_block;
5658 int retries = 0;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005659
Jan Kara8e8ad8a2012-06-12 16:20:38 +02005660 sb_start_pagefault(inode->i_sb);
Theodore Ts'o041bbb6d2012-09-30 23:04:56 -04005661 file_update_time(vma->vm_file);
Jan Karaea3d7202015-12-07 14:28:03 -05005662
5663 down_read(&EXT4_I(inode)->i_mmap_sem);
Jan Kara9ea7df52011-06-24 14:29:41 -04005664 /* Delalloc case is easy... */
5665 if (test_opt(inode->i_sb, DELALLOC) &&
5666 !ext4_should_journal_data(inode) &&
5667 !ext4_nonda_switch(inode->i_sb)) {
5668 do {
Ross Zwisler5c500022015-10-13 16:51:02 -06005669 ret = block_page_mkwrite(vma, vmf,
Jan Kara9ea7df52011-06-24 14:29:41 -04005670 ext4_da_get_block_prep);
5671 } while (ret == -ENOSPC &&
5672 ext4_should_retry_alloc(inode->i_sb, &retries));
5673 goto out_ret;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005674 }
Darrick J. Wong0e499892011-05-18 13:55:20 -04005675
5676 lock_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04005677 size = i_size_read(inode);
5678 /* Page got truncated from under us? */
5679 if (page->mapping != mapping || page_offset(page) > size) {
5680 unlock_page(page);
5681 ret = VM_FAULT_NOPAGE;
5682 goto out;
Darrick J. Wong0e499892011-05-18 13:55:20 -04005683 }
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005684
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005685 if (page->index == size >> PAGE_SHIFT)
5686 len = size & ~PAGE_MASK;
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005687 else
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005688 len = PAGE_SIZE;
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04005689 /*
Jan Kara9ea7df52011-06-24 14:29:41 -04005690 * Return if we have all the buffers mapped. This avoids the need to do
5691 * journal_start/journal_stop which can block and take a long time
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04005692 */
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005693 if (page_has_buffers(page)) {
Tao Maf19d5872012-12-10 14:05:51 -05005694 if (!ext4_walk_page_buffers(NULL, page_buffers(page),
5695 0, len, NULL,
5696 ext4_bh_unmapped)) {
Jan Kara9ea7df52011-06-24 14:29:41 -04005697 /* Wait so that we don't change page under IO */
Darrick J. Wong1d1d1a72013-02-21 16:42:51 -08005698 wait_for_stable_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04005699 ret = VM_FAULT_LOCKED;
5700 goto out;
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04005701 }
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005702 }
Aneesh Kumar K.Va827eaf2009-09-09 22:36:03 -04005703 unlock_page(page);
Jan Kara9ea7df52011-06-24 14:29:41 -04005704 /* OK, we need to fill the hole... */
5705 if (ext4_should_dioread_nolock(inode))
Jan Kara705965b2016-03-08 23:08:10 -05005706 get_block = ext4_get_block_unwritten;
Jan Kara9ea7df52011-06-24 14:29:41 -04005707 else
5708 get_block = ext4_get_block;
5709retry_alloc:
Theodore Ts'o9924a922013-02-08 21:59:22 -05005710 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
5711 ext4_writepage_trans_blocks(inode));
Jan Kara9ea7df52011-06-24 14:29:41 -04005712 if (IS_ERR(handle)) {
Nick Pigginc2ec1752009-03-31 15:23:21 -07005713 ret = VM_FAULT_SIGBUS;
Jan Kara9ea7df52011-06-24 14:29:41 -04005714 goto out;
5715 }
Ross Zwisler5c500022015-10-13 16:51:02 -06005716 ret = block_page_mkwrite(vma, vmf, get_block);
Jan Kara9ea7df52011-06-24 14:29:41 -04005717 if (!ret && ext4_should_journal_data(inode)) {
Tao Maf19d5872012-12-10 14:05:51 -05005718 if (ext4_walk_page_buffers(handle, page_buffers(page), 0,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005719 PAGE_SIZE, NULL, do_journal_get_write_access)) {
Jan Kara9ea7df52011-06-24 14:29:41 -04005720 unlock_page(page);
5721 ret = VM_FAULT_SIGBUS;
Yongqiang Yangfcbb5512011-10-26 05:00:19 -04005722 ext4_journal_stop(handle);
Jan Kara9ea7df52011-06-24 14:29:41 -04005723 goto out;
5724 }
5725 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
5726 }
5727 ext4_journal_stop(handle);
5728 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
5729 goto retry_alloc;
5730out_ret:
5731 ret = block_page_mkwrite_return(ret);
5732out:
Jan Karaea3d7202015-12-07 14:28:03 -05005733 up_read(&EXT4_I(inode)->i_mmap_sem);
Jan Kara8e8ad8a2012-06-12 16:20:38 +02005734 sb_end_pagefault(inode->i_sb);
Aneesh Kumar K.V2e9ee852008-07-11 19:27:31 -04005735 return ret;
5736}
Jan Karaea3d7202015-12-07 14:28:03 -05005737
5738int ext4_filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
5739{
5740 struct inode *inode = file_inode(vma->vm_file);
5741 int err;
5742
5743 down_read(&EXT4_I(inode)->i_mmap_sem);
5744 err = filemap_fault(vma, vmf);
5745 up_read(&EXT4_I(inode)->i_mmap_sem);
5746
5747 return err;
5748}
Jan Kara2d90c162016-03-09 23:11:13 -05005749
5750/*
5751 * Find the first extent at or after @lblk in an inode that is not a hole.
5752 * Search for @map_len blocks at most. The extent is returned in @result.
5753 *
5754 * The function returns 1 if we found an extent. The function returns 0 in
5755 * case there is no extent at or after @lblk and in that case also sets
5756 * @result->es_len to 0. In case of error, the error code is returned.
5757 */
5758int ext4_get_next_extent(struct inode *inode, ext4_lblk_t lblk,
5759 unsigned int map_len, struct extent_status *result)
5760{
5761 struct ext4_map_blocks map;
5762 struct extent_status es = {};
5763 int ret;
5764
5765 map.m_lblk = lblk;
5766 map.m_len = map_len;
5767
5768 /*
5769 * For non-extent based files this loop may iterate several times since
5770 * we do not determine full hole size.
5771 */
5772 while (map.m_len > 0) {
5773 ret = ext4_map_blocks(NULL, inode, &map, 0);
5774 if (ret < 0)
5775 return ret;
5776 /* There's extent covering m_lblk? Just return it. */
5777 if (ret > 0) {
5778 int status;
5779
5780 ext4_es_store_pblock(result, map.m_pblk);
5781 result->es_lblk = map.m_lblk;
5782 result->es_len = map.m_len;
5783 if (map.m_flags & EXT4_MAP_UNWRITTEN)
5784 status = EXTENT_STATUS_UNWRITTEN;
5785 else
5786 status = EXTENT_STATUS_WRITTEN;
5787 ext4_es_store_status(result, status);
5788 return 1;
5789 }
5790 ext4_es_find_delayed_extent_range(inode, map.m_lblk,
5791 map.m_lblk + map.m_len - 1,
5792 &es);
5793 /* Is delalloc data before next block in extent tree? */
5794 if (es.es_len && es.es_lblk < map.m_lblk + map.m_len) {
5795 ext4_lblk_t offset = 0;
5796
5797 if (es.es_lblk < lblk)
5798 offset = lblk - es.es_lblk;
5799 result->es_lblk = es.es_lblk + offset;
5800 ext4_es_store_pblock(result,
5801 ext4_es_pblock(&es) + offset);
5802 result->es_len = es.es_len - offset;
5803 ext4_es_store_status(result, ext4_es_status(&es));
5804
5805 return 1;
5806 }
5807 /* There's a hole at m_lblk, advance us after it */
5808 map.m_lblk += map.m_len;
5809 map_len -= map.m_len;
5810 map.m_len = map_len;
5811 cond_resched();
5812 }
5813 result->es_len = 0;
5814 return 0;
5815}