blob: c4eabbfc90df9a266994631f9ac29ba0489de20b [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/fs/read_write.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
Ingo Molnarb12fb7f2017-02-08 18:51:33 +01008#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/stat.h>
Ingo Molnarb12fb7f2017-02-08 18:51:33 +010010#include <linux/sched/xacct.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/fcntl.h>
12#include <linux/file.h>
13#include <linux/uio.h>
Robert Love0eeca282005-07-12 17:06:03 -040014#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/security.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050016#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080018#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020019#include <linux/splice.h>
Al Viro561c6732013-02-24 10:52:26 -050020#include <linux/compat.h>
Zach Brown29732932015-11-10 16:53:30 -050021#include <linux/mount.h>
Wouter van Kesteren2feb55f2016-02-16 22:20:59 +010022#include <linux/fs.h>
Al Viro06ae43f2013-03-20 13:19:30 -040023#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080025#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <asm/unistd.h>
27
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080028const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070029 .llseek = generic_file_llseek,
Al Viroaad4f8b2014-04-02 14:33:16 -040030 .read_iter = generic_file_read_iter,
Linus Torvalds1da177e2005-04-16 15:20:36 -070031 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020032 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070033};
34
35EXPORT_SYMBOL(generic_ro_fops);
36
Christoph Hellwigddef7ed2017-07-06 18:58:37 +020037static inline bool unsigned_offsets(struct file *file)
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070038{
Al Virocccb5a12010-12-17 07:44:05 -050039 return file->f_mode & FMODE_UNSIGNED_OFFSET;
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070040}
41
Jie Liu46a1c2c2013-06-25 12:02:13 +080042/**
43 * vfs_setpos - update the file offset for lseek
44 * @file: file structure in question
45 * @offset: file offset to seek to
46 * @maxsize: maximum file size
47 *
48 * This is a low-level filesystem helper for updating the file offset to
49 * the value specified by @offset if the given offset is valid and it is
50 * not equal to the current file offset.
51 *
52 * Return the specified offset on success and -EINVAL on invalid offset.
53 */
54loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
Andi Kleenef3d0fd2011-09-15 16:06:48 -070055{
56 if (offset < 0 && !unsigned_offsets(file))
57 return -EINVAL;
58 if (offset > maxsize)
59 return -EINVAL;
60
61 if (offset != file->f_pos) {
62 file->f_pos = offset;
63 file->f_version = 0;
64 }
65 return offset;
66}
Jie Liu46a1c2c2013-06-25 12:02:13 +080067EXPORT_SYMBOL(vfs_setpos);
Andi Kleenef3d0fd2011-09-15 16:06:48 -070068
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020069/**
Andi Kleen57604952011-09-15 16:06:50 -070070 * generic_file_llseek_size - generic llseek implementation for regular files
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020071 * @file: file structure to seek on
72 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -080073 * @whence: type of seek
Eric Sandeene8b96eb2012-04-30 13:11:29 -050074 * @size: max size of this file in file system
75 * @eof: offset used for SEEK_END position
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020076 *
Andi Kleen57604952011-09-15 16:06:50 -070077 * This is a variant of generic_file_llseek that allows passing in a custom
Eric Sandeene8b96eb2012-04-30 13:11:29 -050078 * maximum file size and a custom EOF position, for e.g. hashed directories
Andi Kleenef3d0fd2011-09-15 16:06:48 -070079 *
80 * Synchronization:
Andi Kleen57604952011-09-15 16:06:50 -070081 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
Andi Kleenef3d0fd2011-09-15 16:06:48 -070082 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
83 * read/writes behave like SEEK_SET against seeks.
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020084 */
Andi Kleen9465efc2008-06-27 11:05:24 +020085loff_t
Andrew Morton965c8e52012-12-17 15:59:39 -080086generic_file_llseek_size(struct file *file, loff_t offset, int whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -050087 loff_t maxsize, loff_t eof)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088{
Andrew Morton965c8e52012-12-17 15:59:39 -080089 switch (whence) {
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020090 case SEEK_END:
Eric Sandeene8b96eb2012-04-30 13:11:29 -050091 offset += eof;
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020092 break;
93 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -080094 /*
95 * Here we special-case the lseek(fd, 0, SEEK_CUR)
96 * position-querying operation. Avoid rewriting the "same"
97 * f_pos value back to the file because a concurrent read(),
98 * write() or lseek() might have altered it
99 */
100 if (offset == 0)
101 return file->f_pos;
Andi Kleenef3d0fd2011-09-15 16:06:48 -0700102 /*
103 * f_lock protects against read/modify/write race with other
104 * SEEK_CURs. Note that parallel writes and reads behave
105 * like SEEK_SET.
106 */
107 spin_lock(&file->f_lock);
Jie Liu46a1c2c2013-06-25 12:02:13 +0800108 offset = vfs_setpos(file, file->f_pos + offset, maxsize);
Andi Kleenef3d0fd2011-09-15 16:06:48 -0700109 spin_unlock(&file->f_lock);
110 return offset;
Josef Bacik982d8162011-07-18 13:21:35 -0400111 case SEEK_DATA:
112 /*
113 * In the generic case the entire file is data, so as long as
114 * offset isn't at the end of the file then the offset is data.
115 */
Andreas Gruenbacherfc468202017-09-25 12:23:03 +0200116 if ((unsigned long long)offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400117 return -ENXIO;
118 break;
119 case SEEK_HOLE:
120 /*
121 * There is a virtual hole at the end of the file, so as long as
122 * offset isn't i_size or larger, return i_size.
123 */
Andreas Gruenbacherfc468202017-09-25 12:23:03 +0200124 if ((unsigned long long)offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400125 return -ENXIO;
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500126 offset = eof;
Josef Bacik982d8162011-07-18 13:21:35 -0400127 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +0200129
Jie Liu46a1c2c2013-06-25 12:02:13 +0800130 return vfs_setpos(file, offset, maxsize);
Andi Kleen57604952011-09-15 16:06:50 -0700131}
132EXPORT_SYMBOL(generic_file_llseek_size);
133
134/**
135 * generic_file_llseek - generic llseek implementation for regular files
136 * @file: file structure to seek on
137 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800138 * @whence: type of seek
Andi Kleen57604952011-09-15 16:06:50 -0700139 *
140 * This is a generic implemenation of ->llseek useable for all normal local
141 * filesystems. It just updates the file offset to the value specified by
Ming Lei546ae2d2013-04-29 15:06:07 -0700142 * @offset and @whence.
Andi Kleen57604952011-09-15 16:06:50 -0700143 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800144loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
Andi Kleen57604952011-09-15 16:06:50 -0700145{
146 struct inode *inode = file->f_mapping->host;
147
Andrew Morton965c8e52012-12-17 15:59:39 -0800148 return generic_file_llseek_size(file, offset, whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500149 inode->i_sb->s_maxbytes,
150 i_size_read(inode));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151}
Andi Kleen9465efc2008-06-27 11:05:24 +0200152EXPORT_SYMBOL(generic_file_llseek);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
jan Blunckae6afc32010-05-26 14:44:48 -0700154/**
Al Viro1bf9d142013-06-16 20:27:42 +0400155 * fixed_size_llseek - llseek implementation for fixed-sized devices
156 * @file: file structure to seek on
157 * @offset: file offset to seek to
158 * @whence: type of seek
159 * @size: size of the file
160 *
161 */
162loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
163{
164 switch (whence) {
165 case SEEK_SET: case SEEK_CUR: case SEEK_END:
166 return generic_file_llseek_size(file, offset, whence,
167 size, size);
168 default:
169 return -EINVAL;
170 }
171}
172EXPORT_SYMBOL(fixed_size_llseek);
173
174/**
Al Virob25472f2015-12-05 22:04:48 -0500175 * no_seek_end_llseek - llseek implementation for fixed-sized devices
176 * @file: file structure to seek on
177 * @offset: file offset to seek to
178 * @whence: type of seek
179 *
180 */
181loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
182{
183 switch (whence) {
184 case SEEK_SET: case SEEK_CUR:
185 return generic_file_llseek_size(file, offset, whence,
Wouter van Kesteren2feb55f2016-02-16 22:20:59 +0100186 OFFSET_MAX, 0);
Al Virob25472f2015-12-05 22:04:48 -0500187 default:
188 return -EINVAL;
189 }
190}
191EXPORT_SYMBOL(no_seek_end_llseek);
192
193/**
194 * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
195 * @file: file structure to seek on
196 * @offset: file offset to seek to
197 * @whence: type of seek
198 * @size: maximal offset allowed
199 *
200 */
201loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
202{
203 switch (whence) {
204 case SEEK_SET: case SEEK_CUR:
205 return generic_file_llseek_size(file, offset, whence,
206 size, 0);
207 default:
208 return -EINVAL;
209 }
210}
211EXPORT_SYMBOL(no_seek_end_llseek_size);
212
213/**
jan Blunckae6afc32010-05-26 14:44:48 -0700214 * noop_llseek - No Operation Performed llseek implementation
215 * @file: file structure to seek on
216 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800217 * @whence: type of seek
jan Blunckae6afc32010-05-26 14:44:48 -0700218 *
219 * This is an implementation of ->llseek useable for the rare special case when
220 * userspace expects the seek to succeed but the (device) file is actually not
221 * able to perform the seek. In this case you use noop_llseek() instead of
222 * falling back to the default implementation of ->llseek.
223 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800224loff_t noop_llseek(struct file *file, loff_t offset, int whence)
jan Blunckae6afc32010-05-26 14:44:48 -0700225{
226 return file->f_pos;
227}
228EXPORT_SYMBOL(noop_llseek);
229
Andrew Morton965c8e52012-12-17 15:59:39 -0800230loff_t no_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231{
232 return -ESPIPE;
233}
234EXPORT_SYMBOL(no_llseek);
235
Andrew Morton965c8e52012-12-17 15:59:39 -0800236loff_t default_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237{
Al Viro496ad9a2013-01-23 17:07:38 -0500238 struct inode *inode = file_inode(file);
David Sterba16abef02008-04-22 15:09:22 +0200239 loff_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
Al Viro59551022016-01-22 15:40:57 -0500241 inode_lock(inode);
Andrew Morton965c8e52012-12-17 15:59:39 -0800242 switch (whence) {
Chris Snook7b8e8922007-05-08 00:24:13 -0700243 case SEEK_END:
Josef Bacik982d8162011-07-18 13:21:35 -0400244 offset += i_size_read(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700246 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800247 if (offset == 0) {
248 retval = file->f_pos;
249 goto out;
250 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 offset += file->f_pos;
Josef Bacik982d8162011-07-18 13:21:35 -0400252 break;
253 case SEEK_DATA:
254 /*
255 * In the generic case the entire file is data, so as
256 * long as offset isn't at the end of the file then the
257 * offset is data.
258 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300259 if (offset >= inode->i_size) {
260 retval = -ENXIO;
261 goto out;
262 }
Josef Bacik982d8162011-07-18 13:21:35 -0400263 break;
264 case SEEK_HOLE:
265 /*
266 * There is a virtual hole at the end of the file, so
267 * as long as offset isn't i_size or larger, return
268 * i_size.
269 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300270 if (offset >= inode->i_size) {
271 retval = -ENXIO;
272 goto out;
273 }
Josef Bacik982d8162011-07-18 13:21:35 -0400274 offset = inode->i_size;
275 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 }
277 retval = -EINVAL;
Al Virocccb5a12010-12-17 07:44:05 -0500278 if (offset >= 0 || unsigned_offsets(file)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 if (offset != file->f_pos) {
280 file->f_pos = offset;
281 file->f_version = 0;
282 }
283 retval = offset;
284 }
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800285out:
Al Viro59551022016-01-22 15:40:57 -0500286 inode_unlock(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 return retval;
288}
289EXPORT_SYMBOL(default_llseek);
290
Andrew Morton965c8e52012-12-17 15:59:39 -0800291loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292{
293 loff_t (*fn)(struct file *, loff_t, int);
294
295 fn = no_llseek;
296 if (file->f_mode & FMODE_LSEEK) {
Al Viro72c2d532013-09-22 16:27:52 -0400297 if (file->f_op->llseek)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 fn = file->f_op->llseek;
299 }
Andrew Morton965c8e52012-12-17 15:59:39 -0800300 return fn(file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301}
302EXPORT_SYMBOL(vfs_llseek);
303
Dominik Brodowski76847e42018-03-13 21:51:17 +0100304off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305{
306 off_t retval;
Linus Torvalds9c225f22014-03-03 09:36:58 -0800307 struct fd f = fdget_pos(fd);
Al Viro2903ff02012-08-28 12:52:22 -0400308 if (!f.file)
309 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310
311 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800312 if (whence <= SEEK_MAX) {
313 loff_t res = vfs_llseek(f.file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 retval = res;
315 if (res != (loff_t)retval)
316 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
317 }
Linus Torvalds9c225f22014-03-03 09:36:58 -0800318 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 return retval;
320}
321
Dominik Brodowski76847e42018-03-13 21:51:17 +0100322SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
323{
324 return ksys_lseek(fd, offset, whence);
325}
326
Al Viro561c6732013-02-24 10:52:26 -0500327#ifdef CONFIG_COMPAT
328COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
329{
Dominik Brodowski76847e42018-03-13 21:51:17 +0100330 return ksys_lseek(fd, offset, whence);
Al Viro561c6732013-02-24 10:52:26 -0500331}
332#endif
333
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334#ifdef __ARCH_WANT_SYS_LLSEEK
Heiko Carstens003d7ab2009-01-14 14:14:21 +0100335SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
336 unsigned long, offset_low, loff_t __user *, result,
Andrew Morton965c8e52012-12-17 15:59:39 -0800337 unsigned int, whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338{
339 int retval;
Eric Biggersd7a15f82014-03-16 14:24:08 -0500340 struct fd f = fdget_pos(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 loff_t offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342
Al Viro2903ff02012-08-28 12:52:22 -0400343 if (!f.file)
344 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345
346 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800347 if (whence > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 goto out_putf;
349
Al Viro2903ff02012-08-28 12:52:22 -0400350 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
Andrew Morton965c8e52012-12-17 15:59:39 -0800351 whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352
353 retval = (int)offset;
354 if (offset >= 0) {
355 retval = -EFAULT;
356 if (!copy_to_user(result, &offset, sizeof(offset)))
357 retval = 0;
358 }
359out_putf:
Eric Biggersd7a15f82014-03-16 14:24:08 -0500360 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 return retval;
362}
363#endif
364
Al Viro68d70d02013-06-19 15:26:04 +0400365int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366{
367 struct inode *inode;
368 loff_t pos;
James Morrisc43e2592008-01-12 22:05:48 +1100369 int retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
Al Viro496ad9a2013-01-23 17:07:38 -0500371 inode = file_inode(file);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800372 if (unlikely((ssize_t) count < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100373 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 pos = *ppos;
Al Virocccb5a12010-12-17 07:44:05 -0500375 if (unlikely(pos < 0)) {
376 if (!unsigned_offsets(file))
377 return retval;
378 if (count >= -pos) /* both values are in 0..LLONG_MAX */
379 return -EOVERFLOW;
380 } else if (unlikely((loff_t) (pos + count) < 0)) {
381 if (!unsigned_offsets(file))
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -0700382 return retval;
383 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
Jeff Laytonbd61e0a2015-01-16 15:05:55 -0500385 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
Christoph Hellwigacc15572015-12-03 12:59:49 +0100386 retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
387 read_write == READ ? F_RDLCK : F_WRLCK);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800388 if (retval < 0)
389 return retval;
390 }
Al Virobc613842016-03-31 21:48:20 -0400391 return security_file_permission(file,
James Morrisc43e2592008-01-12 22:05:48 +1100392 read_write == READ ? MAY_READ : MAY_WRITE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393}
394
Al Viro5d5d5682015-04-03 15:41:18 -0400395static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
Al Viro293bc982014-02-11 18:37:41 -0500396{
397 struct iovec iov = { .iov_base = buf, .iov_len = len };
398 struct kiocb kiocb;
399 struct iov_iter iter;
400 ssize_t ret;
401
402 init_sync_kiocb(&kiocb, filp);
403 kiocb.ki_pos = *ppos;
Al Viro293bc982014-02-11 18:37:41 -0500404 iov_iter_init(&iter, READ, &iov, 1, len);
405
Miklos Szeredibb7462b2017-02-20 16:51:23 +0100406 ret = call_read_iter(filp, &kiocb, &iter);
Christoph Hellwig599bd192015-02-11 19:59:44 +0100407 BUG_ON(ret == -EIOCBQUEUED);
Al Viro293bc982014-02-11 18:37:41 -0500408 *ppos = kiocb.ki_pos;
409 return ret;
410}
411
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200412ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
413 loff_t *pos)
414{
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200415 if (file->f_op->read)
Al Viro3d04c8a2015-04-03 15:09:18 -0400416 return file->f_op->read(file, buf, count, pos);
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200417 else if (file->f_op->read_iter)
Al Viro3d04c8a2015-04-03 15:09:18 -0400418 return new_sync_read(file, buf, count, pos);
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200419 else
Al Viro3d04c8a2015-04-03 15:09:18 -0400420 return -EINVAL;
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200421}
422
Christoph Hellwigbdd1d2d2017-09-01 17:39:13 +0200423ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
Christoph Hellwigc41fbad2017-09-01 17:39:12 +0200424{
425 mm_segment_t old_fs;
Christoph Hellwigbdd1d2d2017-09-01 17:39:13 +0200426 ssize_t result;
Christoph Hellwigc41fbad2017-09-01 17:39:12 +0200427
428 old_fs = get_fs();
429 set_fs(get_ds());
430 /* The cast to a user pointer is valid due to the set_fs() */
Christoph Hellwigbdd1d2d2017-09-01 17:39:13 +0200431 result = vfs_read(file, (void __user *)buf, count, pos);
Christoph Hellwigc41fbad2017-09-01 17:39:12 +0200432 set_fs(old_fs);
433 return result;
434}
435EXPORT_SYMBOL(kernel_read);
Al Viro293bc982014-02-11 18:37:41 -0500436
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
438{
439 ssize_t ret;
440
441 if (!(file->f_mode & FMODE_READ))
442 return -EBADF;
Al Viro7f7f25e2014-02-11 17:49:24 -0500443 if (!(file->f_mode & FMODE_CAN_READ))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 return -EINVAL;
445 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
446 return -EFAULT;
447
448 ret = rw_verify_area(READ, file, pos, count);
Al Virobc613842016-03-31 21:48:20 -0400449 if (!ret) {
450 if (count > MAX_RW_COUNT)
451 count = MAX_RW_COUNT;
Dmitry Kasatkin6fb50322014-11-05 17:01:17 +0200452 ret = __vfs_read(file, buf, count, pos);
James Morrisc43e2592008-01-12 22:05:48 +1100453 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500454 fsnotify_access(file);
James Morrisc43e2592008-01-12 22:05:48 +1100455 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 }
James Morrisc43e2592008-01-12 22:05:48 +1100457 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 }
459
460 return ret;
461}
462
Al Viro5d5d5682015-04-03 15:41:18 -0400463static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
Al Viro293bc982014-02-11 18:37:41 -0500464{
465 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
466 struct kiocb kiocb;
467 struct iov_iter iter;
468 ssize_t ret;
469
470 init_sync_kiocb(&kiocb, filp);
471 kiocb.ki_pos = *ppos;
Al Viro293bc982014-02-11 18:37:41 -0500472 iov_iter_init(&iter, WRITE, &iov, 1, len);
473
Miklos Szeredibb7462b2017-02-20 16:51:23 +0100474 ret = call_write_iter(filp, &kiocb, &iter);
Christoph Hellwig599bd192015-02-11 19:59:44 +0100475 BUG_ON(ret == -EIOCBQUEUED);
Al Virof765b132015-04-06 20:50:38 -0400476 if (ret > 0)
477 *ppos = kiocb.ki_pos;
Al Viro293bc982014-02-11 18:37:41 -0500478 return ret;
479}
480
Al Viro493c84c2015-04-03 15:06:43 -0400481ssize_t __vfs_write(struct file *file, const char __user *p, size_t count,
482 loff_t *pos)
483{
484 if (file->f_op->write)
485 return file->f_op->write(file, p, count, pos);
Al Viro493c84c2015-04-03 15:06:43 -0400486 else if (file->f_op->write_iter)
487 return new_sync_write(file, p, count, pos);
488 else
489 return -EINVAL;
490}
Al Viro493c84c2015-04-03 15:06:43 -0400491
Christoph Hellwig73e18f72017-09-01 17:39:15 +0200492ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
Al Viro06ae43f2013-03-20 13:19:30 -0400493{
494 mm_segment_t old_fs;
495 const char __user *p;
496 ssize_t ret;
497
Al Viro7f7f25e2014-02-11 17:49:24 -0500498 if (!(file->f_mode & FMODE_CAN_WRITE))
Al Viro3e84f482013-03-27 15:20:30 +0000499 return -EINVAL;
500
Al Viro06ae43f2013-03-20 13:19:30 -0400501 old_fs = get_fs();
502 set_fs(get_ds());
503 p = (__force const char __user *)buf;
504 if (count > MAX_RW_COUNT)
505 count = MAX_RW_COUNT;
Al Viro493c84c2015-04-03 15:06:43 -0400506 ret = __vfs_write(file, p, count, pos);
Al Viro06ae43f2013-03-20 13:19:30 -0400507 set_fs(old_fs);
508 if (ret > 0) {
509 fsnotify_modify(file);
510 add_wchar(current, ret);
511 }
512 inc_syscw(current);
513 return ret;
514}
Al Viro2ec3a122014-08-19 11:48:09 -0400515EXPORT_SYMBOL(__kernel_write);
516
Christoph Hellwige13ec932017-09-01 17:39:14 +0200517ssize_t kernel_write(struct file *file, const void *buf, size_t count,
518 loff_t *pos)
Christoph Hellwigac452ac2017-09-01 17:39:11 +0200519{
520 mm_segment_t old_fs;
521 ssize_t res;
522
523 old_fs = get_fs();
524 set_fs(get_ds());
525 /* The cast to a user pointer is valid due to the set_fs() */
Christoph Hellwige13ec932017-09-01 17:39:14 +0200526 res = vfs_write(file, (__force const char __user *)buf, count, pos);
Christoph Hellwigac452ac2017-09-01 17:39:11 +0200527 set_fs(old_fs);
528
529 return res;
530}
531EXPORT_SYMBOL(kernel_write);
532
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
534{
535 ssize_t ret;
536
537 if (!(file->f_mode & FMODE_WRITE))
538 return -EBADF;
Al Viro7f7f25e2014-02-11 17:49:24 -0500539 if (!(file->f_mode & FMODE_CAN_WRITE))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 return -EINVAL;
541 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
542 return -EFAULT;
543
544 ret = rw_verify_area(WRITE, file, pos, count);
Al Virobc613842016-03-31 21:48:20 -0400545 if (!ret) {
546 if (count > MAX_RW_COUNT)
547 count = MAX_RW_COUNT;
Al Viro03d95eb2013-03-20 13:04:20 -0400548 file_start_write(file);
Al Viro493c84c2015-04-03 15:06:43 -0400549 ret = __vfs_write(file, buf, count, pos);
James Morrisc43e2592008-01-12 22:05:48 +1100550 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500551 fsnotify_modify(file);
James Morrisc43e2592008-01-12 22:05:48 +1100552 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 }
James Morrisc43e2592008-01-12 22:05:48 +1100554 inc_syscw(current);
Al Viro03d95eb2013-03-20 13:04:20 -0400555 file_end_write(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 }
557
558 return ret;
559}
560
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561static inline loff_t file_pos_read(struct file *file)
562{
563 return file->f_pos;
564}
565
566static inline void file_pos_write(struct file *file, loff_t pos)
567{
568 file->f_pos = pos;
569}
570
Dominik Brodowski3ce4a7b2018-03-13 21:56:26 +0100571ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572{
Linus Torvalds9c225f22014-03-03 09:36:58 -0800573 struct fd f = fdget_pos(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575
Al Viro2903ff02012-08-28 12:52:22 -0400576 if (f.file) {
577 loff_t pos = file_pos_read(f.file);
578 ret = vfs_read(f.file, buf, count, &pos);
Al Viro5faf1532013-06-15 05:49:36 +0400579 if (ret >= 0)
580 file_pos_write(f.file, pos);
Linus Torvalds9c225f22014-03-03 09:36:58 -0800581 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 return ret;
584}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
Dominik Brodowski3ce4a7b2018-03-13 21:56:26 +0100586SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
587{
588 return ksys_read(fd, buf, count);
589}
590
Dominik Brodowskie7a3e8b2018-03-11 11:34:41 +0100591ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592{
Linus Torvalds9c225f22014-03-03 09:36:58 -0800593 struct fd f = fdget_pos(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Al Viro2903ff02012-08-28 12:52:22 -0400596 if (f.file) {
597 loff_t pos = file_pos_read(f.file);
598 ret = vfs_write(f.file, buf, count, &pos);
Al Viro5faf1532013-06-15 05:49:36 +0400599 if (ret >= 0)
600 file_pos_write(f.file, pos);
Linus Torvalds9c225f22014-03-03 09:36:58 -0800601 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 }
603
604 return ret;
605}
606
Dominik Brodowskie7a3e8b2018-03-11 11:34:41 +0100607SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
608 size_t, count)
609{
610 return ksys_write(fd, buf, count);
611}
612
Dominik Brodowski36028d52018-03-19 17:38:31 +0100613ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
614 loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615{
Al Viro2903ff02012-08-28 12:52:22 -0400616 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
619 if (pos < 0)
620 return -EINVAL;
621
Al Viro2903ff02012-08-28 12:52:22 -0400622 f = fdget(fd);
623 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400625 if (f.file->f_mode & FMODE_PREAD)
626 ret = vfs_read(f.file, buf, count, &pos);
627 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 }
629
630 return ret;
631}
632
Dominik Brodowski36028d52018-03-19 17:38:31 +0100633SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
634 size_t, count, loff_t, pos)
635{
636 return ksys_pread64(fd, buf, count, pos);
637}
638
639ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
640 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641{
Al Viro2903ff02012-08-28 12:52:22 -0400642 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
645 if (pos < 0)
646 return -EINVAL;
647
Al Viro2903ff02012-08-28 12:52:22 -0400648 f = fdget(fd);
649 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400651 if (f.file->f_mode & FMODE_PWRITE)
652 ret = vfs_write(f.file, buf, count, &pos);
653 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 }
655
656 return ret;
657}
658
Dominik Brodowski36028d52018-03-19 17:38:31 +0100659SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
660 size_t, count, loff_t, pos)
661{
662 return ksys_pwrite64(fd, buf, count, pos);
663}
664
Al Viroac15ac02015-03-20 20:10:21 -0400665static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200666 loff_t *ppos, int type, rwf_t flags)
Al Viro293bc982014-02-11 18:37:41 -0500667{
668 struct kiocb kiocb;
Al Viro293bc982014-02-11 18:37:41 -0500669 ssize_t ret;
670
671 init_sync_kiocb(&kiocb, filp);
Goldwyn Rodriguesfdd2f5b2017-06-20 07:05:40 -0500672 ret = kiocb_set_rw_flags(&kiocb, flags);
673 if (ret)
674 return ret;
Al Viro293bc982014-02-11 18:37:41 -0500675 kiocb.ki_pos = *ppos;
Al Viro293bc982014-02-11 18:37:41 -0500676
Miklos Szeredi0f78d062017-02-20 16:51:23 +0100677 if (type == READ)
Miklos Szeredibb7462b2017-02-20 16:51:23 +0100678 ret = call_read_iter(filp, &kiocb, iter);
Miklos Szeredi0f78d062017-02-20 16:51:23 +0100679 else
Miklos Szeredibb7462b2017-02-20 16:51:23 +0100680 ret = call_write_iter(filp, &kiocb, iter);
Christoph Hellwig599bd192015-02-11 19:59:44 +0100681 BUG_ON(ret == -EIOCBQUEUED);
Al Viro293bc982014-02-11 18:37:41 -0500682 *ppos = kiocb.ki_pos;
683 return ret;
684}
685
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700686/* Do it by hand, with file-ops */
Al Viroac15ac02015-03-20 20:10:21 -0400687static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200688 loff_t *ppos, int type, rwf_t flags)
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700689{
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700690 ssize_t ret = 0;
691
Christoph Hellwig97be7eb2016-03-03 16:04:01 +0100692 if (flags & ~RWF_HIPRI)
Christoph Hellwig793b80e2016-03-03 16:03:58 +0100693 return -EOPNOTSUPP;
694
Al Viroac15ac02015-03-20 20:10:21 -0400695 while (iov_iter_count(iter)) {
696 struct iovec iovec = iov_iter_iovec(iter);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700697 ssize_t nr;
698
Miklos Szeredi0f78d062017-02-20 16:51:23 +0100699 if (type == READ) {
700 nr = filp->f_op->read(filp, iovec.iov_base,
701 iovec.iov_len, ppos);
702 } else {
703 nr = filp->f_op->write(filp, iovec.iov_base,
704 iovec.iov_len, ppos);
705 }
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700706
707 if (nr < 0) {
708 if (!ret)
709 ret = nr;
710 break;
711 }
712 ret += nr;
Al Viroac15ac02015-03-20 20:10:21 -0400713 if (nr != iovec.iov_len)
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700714 break;
Al Viroac15ac02015-03-20 20:10:21 -0400715 iov_iter_advance(iter, nr);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700716 }
717
718 return ret;
719}
720
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721/* A write operation does a read from user space and vice versa */
722#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
723
Vegard Nossumffecee42016-10-08 11:18:07 +0200724/**
725 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace
726 * into the kernel and check that it is valid.
727 *
728 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.
729 * @uvector: Pointer to the userspace array.
730 * @nr_segs: Number of elements in userspace array.
731 * @fast_segs: Number of elements in @fast_pointer.
732 * @fast_pointer: Pointer to (usually small on-stack) kernel array.
733 * @ret_pointer: (output parameter) Pointer to a variable that will point to
734 * either @fast_pointer, a newly allocated kernel array, or NULL,
735 * depending on which array was used.
736 *
737 * This function copies an array of &struct iovec of @nr_segs from
738 * userspace into the kernel and checks that each element is valid (e.g.
739 * it does not point to a kernel address or cause overflow by being too
740 * large, etc.).
741 *
742 * As an optimization, the caller may provide a pointer to a small
743 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long
744 * (the size of this array, or 0 if unused, should be given in @fast_segs).
745 *
746 * @ret_pointer will always point to the array that was used, so the
747 * caller must take care not to call kfree() on it e.g. in case the
748 * @fast_pointer array was used and it was allocated on the stack.
749 *
750 * Return: The total number of bytes covered by the iovec array on success
751 * or a negative error code on error.
752 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700753ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
754 unsigned long nr_segs, unsigned long fast_segs,
755 struct iovec *fast_pointer,
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700756 struct iovec **ret_pointer)
Linus Torvalds435f49a2010-10-29 10:36:49 -0700757{
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700758 unsigned long seg;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700759 ssize_t ret;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700760 struct iovec *iov = fast_pointer;
761
Linus Torvalds435f49a2010-10-29 10:36:49 -0700762 /*
763 * SuS says "The readv() function *may* fail if the iovcnt argument
764 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
765 * traditionally returned zero for zero segments, so...
766 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700767 if (nr_segs == 0) {
768 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700769 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700770 }
771
Linus Torvalds435f49a2010-10-29 10:36:49 -0700772 /*
773 * First get the "struct iovec" from user memory and
774 * verify all the pointers
775 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700776 if (nr_segs > UIO_MAXIOV) {
777 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700778 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700779 }
780 if (nr_segs > fast_segs) {
Linus Torvalds435f49a2010-10-29 10:36:49 -0700781 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700782 if (iov == NULL) {
783 ret = -ENOMEM;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700784 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700785 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700786 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700787 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
788 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700789 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700790 }
791
Linus Torvalds435f49a2010-10-29 10:36:49 -0700792 /*
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700793 * According to the Single Unix Specification we should return EINVAL
794 * if an element length is < 0 when cast to ssize_t or if the
795 * total length would overflow the ssize_t return value of the
796 * system call.
Linus Torvalds435f49a2010-10-29 10:36:49 -0700797 *
798 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
799 * overflow case.
800 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700801 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700802 for (seg = 0; seg < nr_segs; seg++) {
803 void __user *buf = iov[seg].iov_base;
804 ssize_t len = (ssize_t)iov[seg].iov_len;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700805
806 /* see if we we're about to use an invalid len or if
807 * it's about to overflow ssize_t */
Linus Torvalds435f49a2010-10-29 10:36:49 -0700808 if (len < 0) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700809 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700810 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700811 }
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700812 if (type >= 0
Christopher Yeohfcf63402011-10-31 17:06:39 -0700813 && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700814 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700815 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700816 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700817 if (len > MAX_RW_COUNT - ret) {
818 len = MAX_RW_COUNT - ret;
819 iov[seg].iov_len = len;
820 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700821 ret += len;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700822 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700823out:
824 *ret_pointer = iov;
825 return ret;
826}
827
Al Virof5029852017-04-08 18:18:48 -0400828#ifdef CONFIG_COMPAT
829ssize_t compat_rw_copy_check_uvector(int type,
830 const struct compat_iovec __user *uvector, unsigned long nr_segs,
831 unsigned long fast_segs, struct iovec *fast_pointer,
832 struct iovec **ret_pointer)
833{
834 compat_ssize_t tot_len;
835 struct iovec *iov = *ret_pointer = fast_pointer;
836 ssize_t ret = 0;
837 int seg;
838
839 /*
840 * SuS says "The readv() function *may* fail if the iovcnt argument
841 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
842 * traditionally returned zero for zero segments, so...
843 */
844 if (nr_segs == 0)
845 goto out;
846
847 ret = -EINVAL;
848 if (nr_segs > UIO_MAXIOV)
849 goto out;
850 if (nr_segs > fast_segs) {
851 ret = -ENOMEM;
852 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
853 if (iov == NULL)
854 goto out;
855 }
856 *ret_pointer = iov;
857
858 ret = -EFAULT;
859 if (!access_ok(VERIFY_READ, uvector, nr_segs*sizeof(*uvector)))
860 goto out;
861
862 /*
863 * Single unix specification:
864 * We should -EINVAL if an element length is not >= 0 and fitting an
865 * ssize_t.
866 *
867 * In Linux, the total length is limited to MAX_RW_COUNT, there is
868 * no overflow possibility.
869 */
870 tot_len = 0;
871 ret = -EINVAL;
872 for (seg = 0; seg < nr_segs; seg++) {
873 compat_uptr_t buf;
874 compat_ssize_t len;
875
876 if (__get_user(len, &uvector->iov_len) ||
877 __get_user(buf, &uvector->iov_base)) {
878 ret = -EFAULT;
879 goto out;
880 }
881 if (len < 0) /* size_t not fitting in compat_ssize_t .. */
882 goto out;
883 if (type >= 0 &&
884 !access_ok(vrfy_dir(type), compat_ptr(buf), len)) {
885 ret = -EFAULT;
886 goto out;
887 }
888 if (len > MAX_RW_COUNT - tot_len)
889 len = MAX_RW_COUNT - tot_len;
890 tot_len += len;
891 iov->iov_base = compat_ptr(buf);
892 iov->iov_len = (compat_size_t) len;
893 uvector++;
894 iov++;
895 }
896 ret = tot_len;
897
898out:
899 return ret;
900}
901#endif
902
Christoph Hellwig19c73582017-05-27 11:16:48 +0300903static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200904 loff_t *pos, rwf_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 size_t tot_len;
Miklos Szeredi7687a7a2017-02-20 16:51:23 +0100907 ssize_t ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
Christoph Hellwigedab5fe2017-05-27 11:16:49 +0300909 if (!(file->f_mode & FMODE_READ))
910 return -EBADF;
911 if (!(file->f_mode & FMODE_CAN_READ))
912 return -EINVAL;
913
Miklos Szeredi7687a7a2017-02-20 16:51:23 +0100914 tot_len = iov_iter_count(iter);
Al Viro0504c072015-03-21 19:40:11 -0400915 if (!tot_len)
916 goto out;
Christoph Hellwig19c73582017-05-27 11:16:48 +0300917 ret = rw_verify_area(READ, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800918 if (ret < 0)
Christoph Hellwig19c73582017-05-27 11:16:48 +0300919 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Christoph Hellwig19c73582017-05-27 11:16:48 +0300921 if (file->f_op->read_iter)
922 ret = do_iter_readv_writev(file, iter, pos, READ, flags);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700923 else
Christoph Hellwig19c73582017-05-27 11:16:48 +0300924 ret = do_loop_readv_writev(file, iter, pos, READ, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925out:
Christoph Hellwig19c73582017-05-27 11:16:48 +0300926 if (ret >= 0)
927 fsnotify_access(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929}
930
Christoph Hellwig18e97102017-05-27 11:16:51 +0300931ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200932 rwf_t flags)
Christoph Hellwig18e97102017-05-27 11:16:51 +0300933{
934 if (!file->f_op->read_iter)
935 return -EINVAL;
936 return do_iter_read(file, iter, ppos, flags);
937}
938EXPORT_SYMBOL(vfs_iter_read);
939
Christoph Hellwig19c73582017-05-27 11:16:48 +0300940static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200941 loff_t *pos, rwf_t flags)
Christoph Hellwig19c73582017-05-27 11:16:48 +0300942{
943 size_t tot_len;
944 ssize_t ret = 0;
945
Christoph Hellwigedab5fe2017-05-27 11:16:49 +0300946 if (!(file->f_mode & FMODE_WRITE))
947 return -EBADF;
948 if (!(file->f_mode & FMODE_CAN_WRITE))
949 return -EINVAL;
950
Christoph Hellwig19c73582017-05-27 11:16:48 +0300951 tot_len = iov_iter_count(iter);
952 if (!tot_len)
953 return 0;
954 ret = rw_verify_area(WRITE, file, pos, tot_len);
955 if (ret < 0)
956 return ret;
957
Christoph Hellwig19c73582017-05-27 11:16:48 +0300958 if (file->f_op->write_iter)
959 ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
960 else
961 ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
Christoph Hellwig19c73582017-05-27 11:16:48 +0300962 if (ret > 0)
963 fsnotify_modify(file);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700964 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965}
966
Christoph Hellwigabbb65892017-05-27 11:16:52 +0300967ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200968 rwf_t flags)
Christoph Hellwigabbb65892017-05-27 11:16:52 +0300969{
970 if (!file->f_op->write_iter)
971 return -EINVAL;
972 return do_iter_write(file, iter, ppos, flags);
973}
974EXPORT_SYMBOL(vfs_iter_write);
975
Christoph Hellwig251b42a2017-05-27 11:16:46 +0300976ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200977 unsigned long vlen, loff_t *pos, rwf_t flags)
Miklos Szeredi7687a7a2017-02-20 16:51:23 +0100978{
979 struct iovec iovstack[UIO_FASTIOV];
980 struct iovec *iov = iovstack;
981 struct iov_iter iter;
982 ssize_t ret;
983
Christoph Hellwig251b42a2017-05-27 11:16:46 +0300984 ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +0300985 if (ret >= 0) {
986 ret = do_iter_read(file, &iter, pos, flags);
987 kfree(iov);
988 }
Miklos Szeredi7687a7a2017-02-20 16:51:23 +0100989
990 return ret;
991}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992
Christoph Hellwig9725d4c2017-09-01 17:39:25 +0200993static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +0200994 unsigned long vlen, loff_t *pos, rwf_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
Christoph Hellwig251b42a2017-05-27 11:16:46 +0300996 struct iovec iovstack[UIO_FASTIOV];
997 struct iovec *iov = iovstack;
998 struct iov_iter iter;
999 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000
Christoph Hellwig251b42a2017-05-27 11:16:46 +03001001 ret = import_iovec(WRITE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001002 if (ret >= 0) {
Al Viro62473a22017-07-06 09:15:47 -04001003 file_start_write(file);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001004 ret = do_iter_write(file, &iter, pos, flags);
Al Viro62473a22017-07-06 09:15:47 -04001005 file_end_write(file);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001006 kfree(iov);
1007 }
Christoph Hellwig251b42a2017-05-27 11:16:46 +03001008 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001011static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001012 unsigned long vlen, rwf_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013{
Linus Torvalds9c225f22014-03-03 09:36:58 -08001014 struct fd f = fdget_pos(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016
Al Viro2903ff02012-08-28 12:52:22 -04001017 if (f.file) {
1018 loff_t pos = file_pos_read(f.file);
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001019 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
Al Viro5faf1532013-06-15 05:49:36 +04001020 if (ret >= 0)
1021 file_pos_write(f.file, pos);
Linus Torvalds9c225f22014-03-03 09:36:58 -08001022 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 }
1024
1025 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001026 add_rchar(current, ret);
1027 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028 return ret;
1029}
1030
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001031static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001032 unsigned long vlen, rwf_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033{
Linus Torvalds9c225f22014-03-03 09:36:58 -08001034 struct fd f = fdget_pos(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036
Al Viro2903ff02012-08-28 12:52:22 -04001037 if (f.file) {
1038 loff_t pos = file_pos_read(f.file);
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001039 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
Al Viro5faf1532013-06-15 05:49:36 +04001040 if (ret >= 0)
1041 file_pos_write(f.file, pos);
Linus Torvalds9c225f22014-03-03 09:36:58 -08001042 fdput_pos(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 }
1044
1045 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001046 add_wchar(current, ret);
1047 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 return ret;
1049}
1050
Linus Torvalds601cc112009-04-03 08:03:22 -07001051static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001052{
Linus Torvalds601cc112009-04-03 08:03:22 -07001053#define HALF_LONG_BITS (BITS_PER_LONG / 2)
1054 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
1055}
1056
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001057static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001058 unsigned long vlen, loff_t pos, rwf_t flags)
Linus Torvalds601cc112009-04-03 08:03:22 -07001059{
Al Viro2903ff02012-08-28 12:52:22 -04001060 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001061 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001062
1063 if (pos < 0)
1064 return -EINVAL;
1065
Al Viro2903ff02012-08-28 12:52:22 -04001066 f = fdget(fd);
1067 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001068 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -04001069 if (f.file->f_mode & FMODE_PREAD)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001070 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
Al Viro2903ff02012-08-28 12:52:22 -04001071 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001072 }
1073
1074 if (ret > 0)
1075 add_rchar(current, ret);
1076 inc_syscr(current);
1077 return ret;
1078}
1079
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001080static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001081 unsigned long vlen, loff_t pos, rwf_t flags)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001082{
Al Viro2903ff02012-08-28 12:52:22 -04001083 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001084 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001085
1086 if (pos < 0)
1087 return -EINVAL;
1088
Al Viro2903ff02012-08-28 12:52:22 -04001089 f = fdget(fd);
1090 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001091 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -04001092 if (f.file->f_mode & FMODE_PWRITE)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001093 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
Al Viro2903ff02012-08-28 12:52:22 -04001094 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -07001095 }
1096
1097 if (ret > 0)
1098 add_wchar(current, ret);
1099 inc_syscw(current);
1100 return ret;
1101}
1102
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001103SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1104 unsigned long, vlen)
1105{
1106 return do_readv(fd, vec, vlen, 0);
1107}
1108
1109SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1110 unsigned long, vlen)
1111{
1112 return do_writev(fd, vec, vlen, 0);
1113}
1114
1115SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1116 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1117{
1118 loff_t pos = pos_from_hilo(pos_h, pos_l);
1119
1120 return do_preadv(fd, vec, vlen, pos, 0);
1121}
1122
1123SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1124 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001125 rwf_t, flags)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001126{
1127 loff_t pos = pos_from_hilo(pos_h, pos_l);
1128
1129 if (pos == -1)
1130 return do_readv(fd, vec, vlen, flags);
1131
1132 return do_preadv(fd, vec, vlen, pos, flags);
1133}
1134
1135SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1136 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1137{
1138 loff_t pos = pos_from_hilo(pos_h, pos_l);
1139
1140 return do_pwritev(fd, vec, vlen, pos, 0);
1141}
1142
1143SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1144 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001145 rwf_t, flags)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001146{
1147 loff_t pos = pos_from_hilo(pos_h, pos_l);
1148
1149 if (pos == -1)
1150 return do_writev(fd, vec, vlen, flags);
1151
1152 return do_pwritev(fd, vec, vlen, pos, flags);
1153}
1154
Al Viro72ec3512013-03-20 10:42:10 -04001155#ifdef CONFIG_COMPAT
Al Viro72ec3512013-03-20 10:42:10 -04001156static size_t compat_readv(struct file *file,
1157 const struct compat_iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001158 unsigned long vlen, loff_t *pos, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001159{
1160 struct iovec iovstack[UIO_FASTIOV];
1161 struct iovec *iov = iovstack;
1162 struct iov_iter iter;
1163 ssize_t ret;
1164
Christoph Hellwig26c87fb2017-05-27 11:16:47 +03001165 ret = compat_import_iovec(READ, vec, vlen, UIO_FASTIOV, &iov, &iter);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001166 if (ret >= 0) {
1167 ret = do_iter_read(file, &iter, pos, flags);
1168 kfree(iov);
1169 }
Al Viro72ec3512013-03-20 10:42:10 -04001170 if (ret > 0)
1171 add_rchar(current, ret);
1172 inc_syscr(current);
1173 return ret;
1174}
1175
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001176static size_t do_compat_readv(compat_ulong_t fd,
1177 const struct compat_iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001178 compat_ulong_t vlen, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001179{
Linus Torvalds9c225f22014-03-03 09:36:58 -08001180 struct fd f = fdget_pos(fd);
Al Viro72ec3512013-03-20 10:42:10 -04001181 ssize_t ret;
1182 loff_t pos;
1183
1184 if (!f.file)
1185 return -EBADF;
1186 pos = f.file->f_pos;
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001187 ret = compat_readv(f.file, vec, vlen, &pos, flags);
Al Viro5faf1532013-06-15 05:49:36 +04001188 if (ret >= 0)
1189 f.file->f_pos = pos;
Linus Torvalds9c225f22014-03-03 09:36:58 -08001190 fdput_pos(f);
Al Viro72ec3512013-03-20 10:42:10 -04001191 return ret;
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001192
Al Viro72ec3512013-03-20 10:42:10 -04001193}
1194
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001195COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
1196 const struct compat_iovec __user *,vec,
1197 compat_ulong_t, vlen)
1198{
1199 return do_compat_readv(fd, vec, vlen, 0);
1200}
1201
1202static long do_compat_preadv64(unsigned long fd,
Heiko Carstens378a10f2014-03-05 10:43:51 +01001203 const struct compat_iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001204 unsigned long vlen, loff_t pos, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001205{
1206 struct fd f;
1207 ssize_t ret;
1208
1209 if (pos < 0)
1210 return -EINVAL;
1211 f = fdget(fd);
1212 if (!f.file)
1213 return -EBADF;
1214 ret = -ESPIPE;
1215 if (f.file->f_mode & FMODE_PREAD)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001216 ret = compat_readv(f.file, vec, vlen, &pos, flags);
Al Viro72ec3512013-03-20 10:42:10 -04001217 fdput(f);
1218 return ret;
1219}
1220
Heiko Carstens378a10f2014-03-05 10:43:51 +01001221#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1222COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1223 const struct compat_iovec __user *,vec,
1224 unsigned long, vlen, loff_t, pos)
1225{
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001226 return do_compat_preadv64(fd, vec, vlen, pos, 0);
Heiko Carstens378a10f2014-03-05 10:43:51 +01001227}
1228#endif
1229
Heiko Carstensdfd948e2014-01-29 14:05:44 -08001230COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
Al Viro72ec3512013-03-20 10:42:10 -04001231 const struct compat_iovec __user *,vec,
Heiko Carstensdfd948e2014-01-29 14:05:44 -08001232 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
Al Viro72ec3512013-03-20 10:42:10 -04001233{
1234 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
Heiko Carstens378a10f2014-03-05 10:43:51 +01001235
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001236 return do_compat_preadv64(fd, vec, vlen, pos, 0);
1237}
1238
H.J. Lu3ebfd812016-07-14 12:31:53 -07001239#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1240COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1241 const struct compat_iovec __user *,vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001242 unsigned long, vlen, loff_t, pos, rwf_t, flags)
H.J. Lu3ebfd812016-07-14 12:31:53 -07001243{
1244 return do_compat_preadv64(fd, vec, vlen, pos, flags);
1245}
1246#endif
1247
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001248COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1249 const struct compat_iovec __user *,vec,
1250 compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001251 rwf_t, flags)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001252{
1253 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1254
1255 if (pos == -1)
1256 return do_compat_readv(fd, vec, vlen, flags);
1257
1258 return do_compat_preadv64(fd, vec, vlen, pos, flags);
Al Viro72ec3512013-03-20 10:42:10 -04001259}
1260
1261static size_t compat_writev(struct file *file,
1262 const struct compat_iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001263 unsigned long vlen, loff_t *pos, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001264{
Christoph Hellwig26c87fb2017-05-27 11:16:47 +03001265 struct iovec iovstack[UIO_FASTIOV];
1266 struct iovec *iov = iovstack;
1267 struct iov_iter iter;
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001268 ssize_t ret;
Al Viro72ec3512013-03-20 10:42:10 -04001269
Christoph Hellwig26c87fb2017-05-27 11:16:47 +03001270 ret = compat_import_iovec(WRITE, vec, vlen, UIO_FASTIOV, &iov, &iter);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001271 if (ret >= 0) {
Al Viro62473a22017-07-06 09:15:47 -04001272 file_start_write(file);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001273 ret = do_iter_write(file, &iter, pos, flags);
Al Viro62473a22017-07-06 09:15:47 -04001274 file_end_write(file);
Christoph Hellwigedab5fe2017-05-27 11:16:49 +03001275 kfree(iov);
1276 }
Al Viro72ec3512013-03-20 10:42:10 -04001277 if (ret > 0)
1278 add_wchar(current, ret);
1279 inc_syscw(current);
1280 return ret;
1281}
1282
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001283static size_t do_compat_writev(compat_ulong_t fd,
1284 const struct compat_iovec __user* vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001285 compat_ulong_t vlen, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001286{
Linus Torvalds9c225f22014-03-03 09:36:58 -08001287 struct fd f = fdget_pos(fd);
Al Viro72ec3512013-03-20 10:42:10 -04001288 ssize_t ret;
1289 loff_t pos;
1290
1291 if (!f.file)
1292 return -EBADF;
1293 pos = f.file->f_pos;
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001294 ret = compat_writev(f.file, vec, vlen, &pos, flags);
Al Viro5faf1532013-06-15 05:49:36 +04001295 if (ret >= 0)
1296 f.file->f_pos = pos;
Linus Torvalds9c225f22014-03-03 09:36:58 -08001297 fdput_pos(f);
Al Viro72ec3512013-03-20 10:42:10 -04001298 return ret;
1299}
1300
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001301COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
1302 const struct compat_iovec __user *, vec,
1303 compat_ulong_t, vlen)
1304{
1305 return do_compat_writev(fd, vec, vlen, 0);
1306}
1307
1308static long do_compat_pwritev64(unsigned long fd,
Heiko Carstens378a10f2014-03-05 10:43:51 +01001309 const struct compat_iovec __user *vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001310 unsigned long vlen, loff_t pos, rwf_t flags)
Al Viro72ec3512013-03-20 10:42:10 -04001311{
1312 struct fd f;
1313 ssize_t ret;
1314
1315 if (pos < 0)
1316 return -EINVAL;
1317 f = fdget(fd);
1318 if (!f.file)
1319 return -EBADF;
1320 ret = -ESPIPE;
1321 if (f.file->f_mode & FMODE_PWRITE)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001322 ret = compat_writev(f.file, vec, vlen, &pos, flags);
Al Viro72ec3512013-03-20 10:42:10 -04001323 fdput(f);
1324 return ret;
1325}
1326
Heiko Carstens378a10f2014-03-05 10:43:51 +01001327#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1328COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1329 const struct compat_iovec __user *,vec,
1330 unsigned long, vlen, loff_t, pos)
1331{
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001332 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
Heiko Carstens378a10f2014-03-05 10:43:51 +01001333}
1334#endif
1335
Heiko Carstensdfd948e2014-01-29 14:05:44 -08001336COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
Al Viro72ec3512013-03-20 10:42:10 -04001337 const struct compat_iovec __user *,vec,
Heiko Carstensdfd948e2014-01-29 14:05:44 -08001338 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
Al Viro72ec3512013-03-20 10:42:10 -04001339{
1340 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
Heiko Carstens378a10f2014-03-05 10:43:51 +01001341
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001342 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
Al Viro72ec3512013-03-20 10:42:10 -04001343}
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001344
H.J. Lu3ebfd812016-07-14 12:31:53 -07001345#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1346COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1347 const struct compat_iovec __user *,vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001348 unsigned long, vlen, loff_t, pos, rwf_t, flags)
H.J. Lu3ebfd812016-07-14 12:31:53 -07001349{
1350 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1351}
1352#endif
1353
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001354COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1355 const struct compat_iovec __user *,vec,
Christoph Hellwigddef7ed2017-07-06 18:58:37 +02001356 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
Milosz Tanskif17d8b32016-03-03 16:03:59 +01001357{
1358 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1359
1360 if (pos == -1)
1361 return do_compat_writev(fd, vec, vlen, flags);
1362
1363 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1364}
1365
Al Viro72ec3512013-03-20 10:42:10 -04001366#endif
1367
Al Viro19f4fc32013-02-24 02:17:03 -05001368static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1369 size_t count, loff_t max)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370{
Al Viro2903ff02012-08-28 12:52:22 -04001371 struct fd in, out;
1372 struct inode *in_inode, *out_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 loff_t pos;
Al Viro7995bd22013-06-20 18:58:36 +04001374 loff_t out_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 ssize_t retval;
Al Viro2903ff02012-08-28 12:52:22 -04001376 int fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377
1378 /*
1379 * Get input file, and verify that it is ok..
1380 */
1381 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -04001382 in = fdget(in_fd);
1383 if (!in.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 goto out;
Al Viro2903ff02012-08-28 12:52:22 -04001385 if (!(in.file->f_mode & FMODE_READ))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 goto fput_in;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387 retval = -ESPIPE;
Al Viro7995bd22013-06-20 18:58:36 +04001388 if (!ppos) {
1389 pos = in.file->f_pos;
1390 } else {
1391 pos = *ppos;
Al Viro2903ff02012-08-28 12:52:22 -04001392 if (!(in.file->f_mode & FMODE_PREAD))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 goto fput_in;
Al Viro7995bd22013-06-20 18:58:36 +04001394 }
1395 retval = rw_verify_area(READ, in.file, &pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -08001396 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 goto fput_in;
Al Virobc613842016-03-31 21:48:20 -04001398 if (count > MAX_RW_COUNT)
1399 count = MAX_RW_COUNT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 /*
1402 * Get output file, and verify that it is ok..
1403 */
1404 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -04001405 out = fdget(out_fd);
1406 if (!out.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 goto fput_in;
Al Viro2903ff02012-08-28 12:52:22 -04001408 if (!(out.file->f_mode & FMODE_WRITE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 goto fput_out;
1410 retval = -EINVAL;
Al Viro496ad9a2013-01-23 17:07:38 -05001411 in_inode = file_inode(in.file);
1412 out_inode = file_inode(out.file);
Al Viro7995bd22013-06-20 18:58:36 +04001413 out_pos = out.file->f_pos;
1414 retval = rw_verify_area(WRITE, out.file, &out_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -08001415 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416 goto fput_out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 if (!max)
1419 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1420
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 if (unlikely(pos + count > max)) {
1422 retval = -EOVERFLOW;
1423 if (pos >= max)
1424 goto fput_out;
1425 count = max - pos;
1426 }
1427
Jens Axboed96e6e72007-06-11 12:18:52 +02001428 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +02001429#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +02001430 /*
1431 * We need to debate whether we can enable this or not. The
1432 * man page documents EAGAIN return for the output at least,
1433 * and the application is arguably buggy if it doesn't expect
1434 * EAGAIN on a non-blocking file descriptor.
1435 */
Al Viro2903ff02012-08-28 12:52:22 -04001436 if (in.file->f_flags & O_NONBLOCK)
Jens Axboed96e6e72007-06-11 12:18:52 +02001437 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +02001438#endif
Al Viro50cd2c52013-05-23 20:10:34 -04001439 file_start_write(out.file);
Al Viro7995bd22013-06-20 18:58:36 +04001440 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
Al Viro50cd2c52013-05-23 20:10:34 -04001441 file_end_write(out.file);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442
1443 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001444 add_rchar(current, retval);
1445 add_wchar(current, retval);
Scott Wolchoka68c2f12012-12-20 15:05:52 -08001446 fsnotify_access(in.file);
1447 fsnotify_modify(out.file);
Al Viro7995bd22013-06-20 18:58:36 +04001448 out.file->f_pos = out_pos;
1449 if (ppos)
1450 *ppos = pos;
1451 else
1452 in.file->f_pos = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001455 inc_syscr(current);
1456 inc_syscw(current);
Al Viro7995bd22013-06-20 18:58:36 +04001457 if (pos > max)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 retval = -EOVERFLOW;
1459
1460fput_out:
Al Viro2903ff02012-08-28 12:52:22 -04001461 fdput(out);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462fput_in:
Al Viro2903ff02012-08-28 12:52:22 -04001463 fdput(in);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464out:
1465 return retval;
1466}
1467
Heiko Carstens002c8972009-01-14 14:14:18 +01001468SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469{
1470 loff_t pos;
1471 off_t off;
1472 ssize_t ret;
1473
1474 if (offset) {
1475 if (unlikely(get_user(off, offset)))
1476 return -EFAULT;
1477 pos = off;
1478 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1479 if (unlikely(put_user(pos, offset)))
1480 return -EFAULT;
1481 return ret;
1482 }
1483
1484 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1485}
1486
Heiko Carstens002c8972009-01-14 14:14:18 +01001487SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488{
1489 loff_t pos;
1490 ssize_t ret;
1491
1492 if (offset) {
1493 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1494 return -EFAULT;
1495 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1496 if (unlikely(put_user(pos, offset)))
1497 return -EFAULT;
1498 return ret;
1499 }
1500
1501 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1502}
Al Viro19f4fc32013-02-24 02:17:03 -05001503
1504#ifdef CONFIG_COMPAT
1505COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1506 compat_off_t __user *, offset, compat_size_t, count)
1507{
1508 loff_t pos;
1509 off_t off;
1510 ssize_t ret;
1511
1512 if (offset) {
1513 if (unlikely(get_user(off, offset)))
1514 return -EFAULT;
1515 pos = off;
1516 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1517 if (unlikely(put_user(pos, offset)))
1518 return -EFAULT;
1519 return ret;
1520 }
1521
1522 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1523}
1524
1525COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1526 compat_loff_t __user *, offset, compat_size_t, count)
1527{
1528 loff_t pos;
1529 ssize_t ret;
1530
1531 if (offset) {
1532 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1533 return -EFAULT;
1534 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1535 if (unlikely(put_user(pos, offset)))
1536 return -EFAULT;
1537 return ret;
1538 }
1539
1540 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1541}
1542#endif
Zach Brown29732932015-11-10 16:53:30 -05001543
1544/*
1545 * copy_file_range() differs from regular file read and write in that it
1546 * specifically allows return partial success. When it does so is up to
1547 * the copy_file_range method.
1548 */
1549ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1550 struct file *file_out, loff_t pos_out,
1551 size_t len, unsigned int flags)
1552{
1553 struct inode *inode_in = file_inode(file_in);
1554 struct inode *inode_out = file_inode(file_out);
1555 ssize_t ret;
1556
1557 if (flags != 0)
1558 return -EINVAL;
1559
Amir Goldstein11cbfb12017-01-31 10:34:56 +02001560 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1561 return -EISDIR;
1562 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1563 return -EINVAL;
1564
Zach Brown29732932015-11-10 16:53:30 -05001565 ret = rw_verify_area(READ, file_in, &pos_in, len);
Al Virobc613842016-03-31 21:48:20 -04001566 if (unlikely(ret))
1567 return ret;
1568
1569 ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1570 if (unlikely(ret))
Zach Brown29732932015-11-10 16:53:30 -05001571 return ret;
1572
1573 if (!(file_in->f_mode & FMODE_READ) ||
1574 !(file_out->f_mode & FMODE_WRITE) ||
Anna Schumakereac70052015-11-10 16:53:33 -05001575 (file_out->f_flags & O_APPEND))
Zach Brown29732932015-11-10 16:53:30 -05001576 return -EBADF;
1577
1578 /* this could be relaxed once a method supports cross-fs copies */
1579 if (inode_in->i_sb != inode_out->i_sb)
1580 return -EXDEV;
1581
1582 if (len == 0)
1583 return 0;
1584
Amir Goldsteinbfe219d2017-01-31 10:34:57 +02001585 file_start_write(file_out);
Zach Brown29732932015-11-10 16:53:30 -05001586
Christoph Hellwiga76b5b02016-12-09 16:17:19 -08001587 /*
1588 * Try cloning first, this is supported by more file systems, and
1589 * more efficient if both clone and copy are supported (e.g. NFS).
1590 */
1591 if (file_in->f_op->clone_file_range) {
1592 ret = file_in->f_op->clone_file_range(file_in, pos_in,
1593 file_out, pos_out, len);
1594 if (ret == 0) {
1595 ret = len;
1596 goto done;
1597 }
1598 }
1599
1600 if (file_out->f_op->copy_file_range) {
Anna Schumakereac70052015-11-10 16:53:33 -05001601 ret = file_out->f_op->copy_file_range(file_in, pos_in, file_out,
1602 pos_out, len, flags);
Christoph Hellwiga76b5b02016-12-09 16:17:19 -08001603 if (ret != -EOPNOTSUPP)
1604 goto done;
1605 }
Anna Schumakereac70052015-11-10 16:53:33 -05001606
Christoph Hellwiga76b5b02016-12-09 16:17:19 -08001607 ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1608 len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1609
1610done:
Zach Brown29732932015-11-10 16:53:30 -05001611 if (ret > 0) {
1612 fsnotify_access(file_in);
1613 add_rchar(current, ret);
1614 fsnotify_modify(file_out);
1615 add_wchar(current, ret);
1616 }
Christoph Hellwiga76b5b02016-12-09 16:17:19 -08001617
Zach Brown29732932015-11-10 16:53:30 -05001618 inc_syscr(current);
1619 inc_syscw(current);
1620
Amir Goldsteinbfe219d2017-01-31 10:34:57 +02001621 file_end_write(file_out);
Zach Brown29732932015-11-10 16:53:30 -05001622
1623 return ret;
1624}
1625EXPORT_SYMBOL(vfs_copy_file_range);
1626
1627SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1628 int, fd_out, loff_t __user *, off_out,
1629 size_t, len, unsigned int, flags)
1630{
1631 loff_t pos_in;
1632 loff_t pos_out;
1633 struct fd f_in;
1634 struct fd f_out;
1635 ssize_t ret = -EBADF;
1636
1637 f_in = fdget(fd_in);
1638 if (!f_in.file)
1639 goto out2;
1640
1641 f_out = fdget(fd_out);
1642 if (!f_out.file)
1643 goto out1;
1644
1645 ret = -EFAULT;
1646 if (off_in) {
1647 if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1648 goto out;
1649 } else {
1650 pos_in = f_in.file->f_pos;
1651 }
1652
1653 if (off_out) {
1654 if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1655 goto out;
1656 } else {
1657 pos_out = f_out.file->f_pos;
1658 }
1659
1660 ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1661 flags);
1662 if (ret > 0) {
1663 pos_in += ret;
1664 pos_out += ret;
1665
1666 if (off_in) {
1667 if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1668 ret = -EFAULT;
1669 } else {
1670 f_in.file->f_pos = pos_in;
1671 }
1672
1673 if (off_out) {
1674 if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1675 ret = -EFAULT;
1676 } else {
1677 f_out.file->f_pos = pos_out;
1678 }
1679 }
1680
1681out:
1682 fdput(f_out);
1683out1:
1684 fdput(f_in);
1685out2:
1686 return ret;
1687}
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001688
1689static int clone_verify_area(struct file *file, loff_t pos, u64 len, bool write)
1690{
1691 struct inode *inode = file_inode(file);
1692
1693 if (unlikely(pos < 0))
1694 return -EINVAL;
1695
1696 if (unlikely((loff_t) (pos + len) < 0))
1697 return -EINVAL;
1698
1699 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
1700 loff_t end = len ? pos + len - 1 : OFFSET_MAX;
1701 int retval;
1702
1703 retval = locks_mandatory_area(inode, file, pos, end,
1704 write ? F_WRLCK : F_RDLCK);
1705 if (retval < 0)
1706 return retval;
1707 }
1708
1709 return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
1710}
1711
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001712/*
1713 * Check that the two inodes are eligible for cloning, the ranges make
1714 * sense, and then flush all dirty data. Caller must ensure that the
1715 * inodes have been locked against any other modifications.
Darrick J. Wong22725ce2016-12-19 15:13:26 -08001716 *
1717 * Returns: 0 for "nothing to clone", 1 for "something to clone", or
1718 * the usual negative error code.
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001719 */
1720int vfs_clone_file_prep_inodes(struct inode *inode_in, loff_t pos_in,
1721 struct inode *inode_out, loff_t pos_out,
1722 u64 *len, bool is_dedupe)
1723{
1724 loff_t bs = inode_out->i_sb->s_blocksize;
1725 loff_t blen;
1726 loff_t isize;
1727 bool same_inode = (inode_in == inode_out);
1728 int ret;
1729
1730 /* Don't touch certain kinds of inodes */
1731 if (IS_IMMUTABLE(inode_out))
1732 return -EPERM;
1733
1734 if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1735 return -ETXTBSY;
1736
1737 /* Don't reflink dirs, pipes, sockets... */
1738 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1739 return -EISDIR;
1740 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1741 return -EINVAL;
1742
1743 /* Are we going all the way to the end? */
1744 isize = i_size_read(inode_in);
Darrick J. Wong22725ce2016-12-19 15:13:26 -08001745 if (isize == 0)
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001746 return 0;
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001747
1748 /* Zero length dedupe exits immediately; reflink goes to EOF. */
1749 if (*len == 0) {
Darrick J. Wong22725ce2016-12-19 15:13:26 -08001750 if (is_dedupe || pos_in == isize)
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001751 return 0;
Darrick J. Wong22725ce2016-12-19 15:13:26 -08001752 if (pos_in > isize)
1753 return -EINVAL;
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001754 *len = isize - pos_in;
1755 }
1756
1757 /* Ensure offsets don't wrap and the input is inside i_size */
1758 if (pos_in + *len < pos_in || pos_out + *len < pos_out ||
1759 pos_in + *len > isize)
1760 return -EINVAL;
1761
1762 /* Don't allow dedupe past EOF in the dest file */
1763 if (is_dedupe) {
1764 loff_t disize;
1765
1766 disize = i_size_read(inode_out);
1767 if (pos_out >= disize || pos_out + *len > disize)
1768 return -EINVAL;
1769 }
1770
1771 /* If we're linking to EOF, continue to the block boundary. */
1772 if (pos_in + *len == isize)
1773 blen = ALIGN(isize, bs) - pos_in;
1774 else
1775 blen = *len;
1776
1777 /* Only reflink if we're aligned to block boundaries */
1778 if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_in + blen, bs) ||
1779 !IS_ALIGNED(pos_out, bs) || !IS_ALIGNED(pos_out + blen, bs))
1780 return -EINVAL;
1781
1782 /* Don't allow overlapped reflink within the same file */
1783 if (same_inode) {
1784 if (pos_out + blen > pos_in && pos_out < pos_in + blen)
1785 return -EINVAL;
1786 }
1787
1788 /* Wait for the completion of any pending IOs on both files */
1789 inode_dio_wait(inode_in);
1790 if (!same_inode)
1791 inode_dio_wait(inode_out);
1792
1793 ret = filemap_write_and_wait_range(inode_in->i_mapping,
1794 pos_in, pos_in + *len - 1);
1795 if (ret)
1796 return ret;
1797
1798 ret = filemap_write_and_wait_range(inode_out->i_mapping,
1799 pos_out, pos_out + *len - 1);
1800 if (ret)
1801 return ret;
1802
1803 /*
1804 * Check that the extents are the same.
1805 */
1806 if (is_dedupe) {
1807 bool is_same = false;
1808
1809 ret = vfs_dedupe_file_range_compare(inode_in, pos_in,
1810 inode_out, pos_out, *len, &is_same);
1811 if (ret)
1812 return ret;
1813 if (!is_same)
1814 return -EBADE;
1815 }
1816
Darrick J. Wong22725ce2016-12-19 15:13:26 -08001817 return 1;
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001818}
1819EXPORT_SYMBOL(vfs_clone_file_prep_inodes);
1820
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001821int vfs_clone_file_range(struct file *file_in, loff_t pos_in,
1822 struct file *file_out, loff_t pos_out, u64 len)
1823{
1824 struct inode *inode_in = file_inode(file_in);
1825 struct inode *inode_out = file_inode(file_out);
1826 int ret;
1827
Amir Goldsteinb335e9d2016-10-26 22:34:01 +03001828 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1829 return -EISDIR;
1830 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1831 return -EINVAL;
1832
Amir Goldstein913b86e2016-09-23 11:38:10 +03001833 /*
1834 * FICLONE/FICLONERANGE ioctls enforce that src and dest files are on
1835 * the same mount. Practically, they only need to be on the same file
1836 * system.
1837 */
1838 if (inode_in->i_sb != inode_out->i_sb)
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001839 return -EXDEV;
1840
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001841 if (!(file_in->f_mode & FMODE_READ) ||
1842 !(file_out->f_mode & FMODE_WRITE) ||
Christoph Hellwig0fcbf992016-02-26 18:53:12 +01001843 (file_out->f_flags & O_APPEND))
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001844 return -EBADF;
1845
Christoph Hellwig0fcbf992016-02-26 18:53:12 +01001846 if (!file_in->f_op->clone_file_range)
1847 return -EOPNOTSUPP;
1848
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001849 ret = clone_verify_area(file_in, pos_in, len, false);
1850 if (ret)
1851 return ret;
1852
1853 ret = clone_verify_area(file_out, pos_out, len, true);
1854 if (ret)
1855 return ret;
1856
1857 if (pos_in + len > i_size_read(inode_in))
1858 return -EINVAL;
1859
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001860 ret = file_in->f_op->clone_file_range(file_in, pos_in,
1861 file_out, pos_out, len);
1862 if (!ret) {
1863 fsnotify_access(file_in);
1864 fsnotify_modify(file_out);
1865 }
1866
Christoph Hellwig04b38d62015-12-03 12:59:50 +01001867 return ret;
1868}
1869EXPORT_SYMBOL(vfs_clone_file_range);
Darrick J. Wong54dbc152015-12-19 00:55:59 -08001870
Darrick J. Wong876bec6f2016-12-09 16:18:30 -08001871/*
1872 * Read a page's worth of file data into the page cache. Return the page
1873 * locked.
1874 */
1875static struct page *vfs_dedupe_get_page(struct inode *inode, loff_t offset)
1876{
1877 struct address_space *mapping;
1878 struct page *page;
1879 pgoff_t n;
1880
1881 n = offset >> PAGE_SHIFT;
1882 mapping = inode->i_mapping;
1883 page = read_mapping_page(mapping, n, NULL);
1884 if (IS_ERR(page))
1885 return page;
1886 if (!PageUptodate(page)) {
1887 put_page(page);
1888 return ERR_PTR(-EIO);
1889 }
1890 lock_page(page);
1891 return page;
1892}
1893
1894/*
1895 * Compare extents of two files to see if they are the same.
1896 * Caller must have locked both inodes to prevent write races.
1897 */
1898int vfs_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
1899 struct inode *dest, loff_t destoff,
1900 loff_t len, bool *is_same)
1901{
1902 loff_t src_poff;
1903 loff_t dest_poff;
1904 void *src_addr;
1905 void *dest_addr;
1906 struct page *src_page;
1907 struct page *dest_page;
1908 loff_t cmp_len;
1909 bool same;
1910 int error;
1911
1912 error = -EINVAL;
1913 same = true;
1914 while (len) {
1915 src_poff = srcoff & (PAGE_SIZE - 1);
1916 dest_poff = destoff & (PAGE_SIZE - 1);
1917 cmp_len = min(PAGE_SIZE - src_poff,
1918 PAGE_SIZE - dest_poff);
1919 cmp_len = min(cmp_len, len);
1920 if (cmp_len <= 0)
1921 goto out_error;
1922
1923 src_page = vfs_dedupe_get_page(src, srcoff);
1924 if (IS_ERR(src_page)) {
1925 error = PTR_ERR(src_page);
1926 goto out_error;
1927 }
1928 dest_page = vfs_dedupe_get_page(dest, destoff);
1929 if (IS_ERR(dest_page)) {
1930 error = PTR_ERR(dest_page);
1931 unlock_page(src_page);
1932 put_page(src_page);
1933 goto out_error;
1934 }
1935 src_addr = kmap_atomic(src_page);
1936 dest_addr = kmap_atomic(dest_page);
1937
1938 flush_dcache_page(src_page);
1939 flush_dcache_page(dest_page);
1940
1941 if (memcmp(src_addr + src_poff, dest_addr + dest_poff, cmp_len))
1942 same = false;
1943
1944 kunmap_atomic(dest_addr);
1945 kunmap_atomic(src_addr);
1946 unlock_page(dest_page);
1947 unlock_page(src_page);
1948 put_page(dest_page);
1949 put_page(src_page);
1950
1951 if (!same)
1952 break;
1953
1954 srcoff += cmp_len;
1955 destoff += cmp_len;
1956 len -= cmp_len;
1957 }
1958
1959 *is_same = same;
1960 return 0;
1961
1962out_error:
1963 return error;
1964}
1965EXPORT_SYMBOL(vfs_dedupe_file_range_compare);
1966
Darrick J. Wong54dbc152015-12-19 00:55:59 -08001967int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
1968{
1969 struct file_dedupe_range_info *info;
1970 struct inode *src = file_inode(file);
1971 u64 off;
1972 u64 len;
1973 int i;
1974 int ret;
1975 bool is_admin = capable(CAP_SYS_ADMIN);
1976 u16 count = same->dest_count;
1977 struct file *dst_file;
1978 loff_t dst_off;
1979 ssize_t deduped;
1980
1981 if (!(file->f_mode & FMODE_READ))
1982 return -EINVAL;
1983
1984 if (same->reserved1 || same->reserved2)
1985 return -EINVAL;
1986
1987 off = same->src_offset;
1988 len = same->src_length;
1989
1990 ret = -EISDIR;
1991 if (S_ISDIR(src->i_mode))
1992 goto out;
1993
1994 ret = -EINVAL;
1995 if (!S_ISREG(src->i_mode))
1996 goto out;
1997
1998 ret = clone_verify_area(file, off, len, false);
1999 if (ret < 0)
2000 goto out;
2001 ret = 0;
2002
Darrick J. Wong22725ce2016-12-19 15:13:26 -08002003 if (off + len > i_size_read(src))
2004 return -EINVAL;
2005
Darrick J. Wong54dbc152015-12-19 00:55:59 -08002006 /* pre-format output fields to sane values */
2007 for (i = 0; i < count; i++) {
2008 same->info[i].bytes_deduped = 0ULL;
2009 same->info[i].status = FILE_DEDUPE_RANGE_SAME;
2010 }
2011
2012 for (i = 0, info = same->info; i < count; i++, info++) {
2013 struct inode *dst;
2014 struct fd dst_fd = fdget(info->dest_fd);
2015
2016 dst_file = dst_fd.file;
2017 if (!dst_file) {
2018 info->status = -EBADF;
2019 goto next_loop;
2020 }
2021 dst = file_inode(dst_file);
2022
2023 ret = mnt_want_write_file(dst_file);
2024 if (ret) {
2025 info->status = ret;
2026 goto next_loop;
2027 }
2028
2029 dst_off = info->dest_offset;
2030 ret = clone_verify_area(dst_file, dst_off, len, true);
2031 if (ret < 0) {
2032 info->status = ret;
2033 goto next_file;
2034 }
2035 ret = 0;
2036
2037 if (info->reserved) {
2038 info->status = -EINVAL;
2039 } else if (!(is_admin || (dst_file->f_mode & FMODE_WRITE))) {
2040 info->status = -EINVAL;
2041 } else if (file->f_path.mnt != dst_file->f_path.mnt) {
2042 info->status = -EXDEV;
2043 } else if (S_ISDIR(dst->i_mode)) {
2044 info->status = -EISDIR;
2045 } else if (dst_file->f_op->dedupe_file_range == NULL) {
2046 info->status = -EINVAL;
2047 } else {
2048 deduped = dst_file->f_op->dedupe_file_range(file, off,
2049 len, dst_file,
2050 info->dest_offset);
2051 if (deduped == -EBADE)
2052 info->status = FILE_DEDUPE_RANGE_DIFFERS;
2053 else if (deduped < 0)
2054 info->status = deduped;
2055 else
2056 info->bytes_deduped += deduped;
2057 }
2058
2059next_file:
2060 mnt_drop_write_file(dst_file);
2061next_loop:
2062 fdput(dst_fd);
Darrick J. Wonge62e5602016-01-22 16:58:28 -08002063
2064 if (fatal_signal_pending(current))
2065 goto out;
Darrick J. Wong54dbc152015-12-19 00:55:59 -08002066 }
2067
2068out:
2069 return ret;
2070}
2071EXPORT_SYMBOL(vfs_dedupe_file_range);