blob: 2cefa417be349b0016ffa8606635db9e85751015 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
Kent Overstreeta27bb332013-05-07 16:19:08 -070012#include <linux/aio.h>
Robert Love0eeca282005-07-12 17:06:03 -040013#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/security.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050015#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080017#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020018#include <linux/splice.h>
Al Viro561c6732013-02-24 10:52:26 -050019#include <linux/compat.h>
Al Viro06ae43f2013-03-20 13:19:30 -040020#include "internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#include <asm/uaccess.h>
23#include <asm/unistd.h>
24
Al Viroc0bd14af2013-05-04 15:00:54 -040025typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
26typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *,
27 unsigned long, loff_t);
28
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080029const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 .llseek = generic_file_llseek,
Badari Pulavarty543ade12006-09-30 23:28:48 -070031 .read = do_sync_read,
32 .aio_read = generic_file_aio_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020034 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070035};
36
37EXPORT_SYMBOL(generic_ro_fops);
38
Al Virocccb5a12010-12-17 07:44:05 -050039static inline int unsigned_offsets(struct file *file)
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070040{
Al Virocccb5a12010-12-17 07:44:05 -050041 return file->f_mode & FMODE_UNSIGNED_OFFSET;
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070042}
43
Andi Kleenef3d0fd2011-09-15 16:06:48 -070044static loff_t lseek_execute(struct file *file, struct inode *inode,
45 loff_t offset, loff_t maxsize)
46{
47 if (offset < 0 && !unsigned_offsets(file))
48 return -EINVAL;
49 if (offset > maxsize)
50 return -EINVAL;
51
52 if (offset != file->f_pos) {
53 file->f_pos = offset;
54 file->f_version = 0;
55 }
56 return offset;
57}
58
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020059/**
Andi Kleen57604952011-09-15 16:06:50 -070060 * generic_file_llseek_size - generic llseek implementation for regular files
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020061 * @file: file structure to seek on
62 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -080063 * @whence: type of seek
Eric Sandeene8b96eb2012-04-30 13:11:29 -050064 * @size: max size of this file in file system
65 * @eof: offset used for SEEK_END position
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020066 *
Andi Kleen57604952011-09-15 16:06:50 -070067 * This is a variant of generic_file_llseek that allows passing in a custom
Eric Sandeene8b96eb2012-04-30 13:11:29 -050068 * maximum file size and a custom EOF position, for e.g. hashed directories
Andi Kleenef3d0fd2011-09-15 16:06:48 -070069 *
70 * Synchronization:
Andi Kleen57604952011-09-15 16:06:50 -070071 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
Andi Kleenef3d0fd2011-09-15 16:06:48 -070072 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
73 * read/writes behave like SEEK_SET against seeks.
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020074 */
Andi Kleen9465efc2008-06-27 11:05:24 +020075loff_t
Andrew Morton965c8e52012-12-17 15:59:39 -080076generic_file_llseek_size(struct file *file, loff_t offset, int whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -050077 loff_t maxsize, loff_t eof)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078{
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 struct inode *inode = file->f_mapping->host;
80
Andrew Morton965c8e52012-12-17 15:59:39 -080081 switch (whence) {
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020082 case SEEK_END:
Eric Sandeene8b96eb2012-04-30 13:11:29 -050083 offset += eof;
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020084 break;
85 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -080086 /*
87 * Here we special-case the lseek(fd, 0, SEEK_CUR)
88 * position-querying operation. Avoid rewriting the "same"
89 * f_pos value back to the file because a concurrent read(),
90 * write() or lseek() might have altered it
91 */
92 if (offset == 0)
93 return file->f_pos;
Andi Kleenef3d0fd2011-09-15 16:06:48 -070094 /*
95 * f_lock protects against read/modify/write race with other
96 * SEEK_CURs. Note that parallel writes and reads behave
97 * like SEEK_SET.
98 */
99 spin_lock(&file->f_lock);
100 offset = lseek_execute(file, inode, file->f_pos + offset,
Andi Kleen57604952011-09-15 16:06:50 -0700101 maxsize);
Andi Kleenef3d0fd2011-09-15 16:06:48 -0700102 spin_unlock(&file->f_lock);
103 return offset;
Josef Bacik982d8162011-07-18 13:21:35 -0400104 case SEEK_DATA:
105 /*
106 * In the generic case the entire file is data, so as long as
107 * offset isn't at the end of the file then the offset is data.
108 */
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500109 if (offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400110 return -ENXIO;
111 break;
112 case SEEK_HOLE:
113 /*
114 * There is a virtual hole at the end of the file, so as long as
115 * offset isn't i_size or larger, return i_size.
116 */
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500117 if (offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400118 return -ENXIO;
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500119 offset = eof;
Josef Bacik982d8162011-07-18 13:21:35 -0400120 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +0200122
Andi Kleen57604952011-09-15 16:06:50 -0700123 return lseek_execute(file, inode, offset, maxsize);
124}
125EXPORT_SYMBOL(generic_file_llseek_size);
126
127/**
128 * generic_file_llseek - generic llseek implementation for regular files
129 * @file: file structure to seek on
130 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800131 * @whence: type of seek
Andi Kleen57604952011-09-15 16:06:50 -0700132 *
133 * This is a generic implemenation of ->llseek useable for all normal local
134 * filesystems. It just updates the file offset to the value specified by
Ming Lei546ae2d2013-04-29 15:06:07 -0700135 * @offset and @whence.
Andi Kleen57604952011-09-15 16:06:50 -0700136 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800137loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
Andi Kleen57604952011-09-15 16:06:50 -0700138{
139 struct inode *inode = file->f_mapping->host;
140
Andrew Morton965c8e52012-12-17 15:59:39 -0800141 return generic_file_llseek_size(file, offset, whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500142 inode->i_sb->s_maxbytes,
143 i_size_read(inode));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144}
Andi Kleen9465efc2008-06-27 11:05:24 +0200145EXPORT_SYMBOL(generic_file_llseek);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
jan Blunckae6afc32010-05-26 14:44:48 -0700147/**
148 * noop_llseek - No Operation Performed llseek implementation
149 * @file: file structure to seek on
150 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800151 * @whence: type of seek
jan Blunckae6afc32010-05-26 14:44:48 -0700152 *
153 * This is an implementation of ->llseek useable for the rare special case when
154 * userspace expects the seek to succeed but the (device) file is actually not
155 * able to perform the seek. In this case you use noop_llseek() instead of
156 * falling back to the default implementation of ->llseek.
157 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800158loff_t noop_llseek(struct file *file, loff_t offset, int whence)
jan Blunckae6afc32010-05-26 14:44:48 -0700159{
160 return file->f_pos;
161}
162EXPORT_SYMBOL(noop_llseek);
163
Andrew Morton965c8e52012-12-17 15:59:39 -0800164loff_t no_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165{
166 return -ESPIPE;
167}
168EXPORT_SYMBOL(no_llseek);
169
Andrew Morton965c8e52012-12-17 15:59:39 -0800170loff_t default_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171{
Al Viro496ad9a2013-01-23 17:07:38 -0500172 struct inode *inode = file_inode(file);
David Sterba16abef02008-04-22 15:09:22 +0200173 loff_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Josef Bacik982d8162011-07-18 13:21:35 -0400175 mutex_lock(&inode->i_mutex);
Andrew Morton965c8e52012-12-17 15:59:39 -0800176 switch (whence) {
Chris Snook7b8e8922007-05-08 00:24:13 -0700177 case SEEK_END:
Josef Bacik982d8162011-07-18 13:21:35 -0400178 offset += i_size_read(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700180 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800181 if (offset == 0) {
182 retval = file->f_pos;
183 goto out;
184 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 offset += file->f_pos;
Josef Bacik982d8162011-07-18 13:21:35 -0400186 break;
187 case SEEK_DATA:
188 /*
189 * In the generic case the entire file is data, so as
190 * long as offset isn't at the end of the file then the
191 * offset is data.
192 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300193 if (offset >= inode->i_size) {
194 retval = -ENXIO;
195 goto out;
196 }
Josef Bacik982d8162011-07-18 13:21:35 -0400197 break;
198 case SEEK_HOLE:
199 /*
200 * There is a virtual hole at the end of the file, so
201 * as long as offset isn't i_size or larger, return
202 * i_size.
203 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300204 if (offset >= inode->i_size) {
205 retval = -ENXIO;
206 goto out;
207 }
Josef Bacik982d8162011-07-18 13:21:35 -0400208 offset = inode->i_size;
209 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 }
211 retval = -EINVAL;
Al Virocccb5a12010-12-17 07:44:05 -0500212 if (offset >= 0 || unsigned_offsets(file)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 if (offset != file->f_pos) {
214 file->f_pos = offset;
215 file->f_version = 0;
216 }
217 retval = offset;
218 }
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800219out:
Josef Bacik982d8162011-07-18 13:21:35 -0400220 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 return retval;
222}
223EXPORT_SYMBOL(default_llseek);
224
Andrew Morton965c8e52012-12-17 15:59:39 -0800225loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226{
227 loff_t (*fn)(struct file *, loff_t, int);
228
229 fn = no_llseek;
230 if (file->f_mode & FMODE_LSEEK) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 if (file->f_op && file->f_op->llseek)
232 fn = file->f_op->llseek;
233 }
Andrew Morton965c8e52012-12-17 15:59:39 -0800234 return fn(file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235}
236EXPORT_SYMBOL(vfs_llseek);
237
Andrew Morton965c8e52012-12-17 15:59:39 -0800238SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239{
240 off_t retval;
Al Viro2903ff02012-08-28 12:52:22 -0400241 struct fd f = fdget(fd);
242 if (!f.file)
243 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
245 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800246 if (whence <= SEEK_MAX) {
247 loff_t res = vfs_llseek(f.file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 retval = res;
249 if (res != (loff_t)retval)
250 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
251 }
Al Viro2903ff02012-08-28 12:52:22 -0400252 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 return retval;
254}
255
Al Viro561c6732013-02-24 10:52:26 -0500256#ifdef CONFIG_COMPAT
257COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
258{
259 return sys_lseek(fd, offset, whence);
260}
261#endif
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263#ifdef __ARCH_WANT_SYS_LLSEEK
Heiko Carstens003d7ab2009-01-14 14:14:21 +0100264SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
265 unsigned long, offset_low, loff_t __user *, result,
Andrew Morton965c8e52012-12-17 15:59:39 -0800266 unsigned int, whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267{
268 int retval;
Al Viro2903ff02012-08-28 12:52:22 -0400269 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 loff_t offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
Al Viro2903ff02012-08-28 12:52:22 -0400272 if (!f.file)
273 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274
275 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800276 if (whence > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 goto out_putf;
278
Al Viro2903ff02012-08-28 12:52:22 -0400279 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
Andrew Morton965c8e52012-12-17 15:59:39 -0800280 whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
282 retval = (int)offset;
283 if (offset >= 0) {
284 retval = -EFAULT;
285 if (!copy_to_user(result, &offset, sizeof(offset)))
286 retval = 0;
287 }
288out_putf:
Al Viro2903ff02012-08-28 12:52:22 -0400289 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 return retval;
291}
292#endif
293
Linus Torvaldse28cc712006-01-04 16:20:40 -0800294/*
295 * rw_verify_area doesn't like huge counts. We limit
296 * them to something that fits in "int" so that others
297 * won't have to do range checks all the time.
298 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
300{
301 struct inode *inode;
302 loff_t pos;
James Morrisc43e2592008-01-12 22:05:48 +1100303 int retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304
Al Viro496ad9a2013-01-23 17:07:38 -0500305 inode = file_inode(file);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800306 if (unlikely((ssize_t) count < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100307 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 pos = *ppos;
Al Virocccb5a12010-12-17 07:44:05 -0500309 if (unlikely(pos < 0)) {
310 if (!unsigned_offsets(file))
311 return retval;
312 if (count >= -pos) /* both values are in 0..LLONG_MAX */
313 return -EOVERFLOW;
314 } else if (unlikely((loff_t) (pos + count) < 0)) {
315 if (!unsigned_offsets(file))
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -0700316 return retval;
317 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
Pavel Emelyanova16877c2007-10-01 14:41:11 -0700319 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
James Morrisc43e2592008-01-12 22:05:48 +1100320 retval = locks_mandatory_area(
Linus Torvaldse28cc712006-01-04 16:20:40 -0800321 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
322 inode, file, pos, count);
323 if (retval < 0)
324 return retval;
325 }
James Morrisc43e2592008-01-12 22:05:48 +1100326 retval = security_file_permission(file,
327 read_write == READ ? MAY_READ : MAY_WRITE);
328 if (retval)
329 return retval;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800330 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
333ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
334{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700335 struct iovec iov = { .iov_base = buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 struct kiocb kiocb;
337 ssize_t ret;
338
339 init_sync_kiocb(&kiocb, filp);
340 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700341 kiocb.ki_left = len;
David Howells61964eb2010-03-24 17:09:19 +0000342 kiocb.ki_nbytes = len;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700343
Zach Brown41003a72013-05-07 16:18:25 -0700344 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 if (-EIOCBQUEUED == ret)
346 ret = wait_on_sync_kiocb(&kiocb);
347 *ppos = kiocb.ki_pos;
348 return ret;
349}
350
351EXPORT_SYMBOL(do_sync_read);
352
353ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
354{
355 ssize_t ret;
356
357 if (!(file->f_mode & FMODE_READ))
358 return -EBADF;
359 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
360 return -EINVAL;
361 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
362 return -EFAULT;
363
364 ret = rw_verify_area(READ, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800365 if (ret >= 0) {
366 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100367 if (file->f_op->read)
368 ret = file->f_op->read(file, buf, count, pos);
369 else
370 ret = do_sync_read(file, buf, count, pos);
371 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500372 fsnotify_access(file);
James Morrisc43e2592008-01-12 22:05:48 +1100373 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 }
James Morrisc43e2592008-01-12 22:05:48 +1100375 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 }
377
378 return ret;
379}
380
381EXPORT_SYMBOL(vfs_read);
382
383ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
384{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700385 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 struct kiocb kiocb;
387 ssize_t ret;
388
389 init_sync_kiocb(&kiocb, filp);
390 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700391 kiocb.ki_left = len;
David Howells61964eb2010-03-24 17:09:19 +0000392 kiocb.ki_nbytes = len;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700393
Zach Brown41003a72013-05-07 16:18:25 -0700394 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 if (-EIOCBQUEUED == ret)
396 ret = wait_on_sync_kiocb(&kiocb);
397 *ppos = kiocb.ki_pos;
398 return ret;
399}
400
401EXPORT_SYMBOL(do_sync_write);
402
Al Viro06ae43f2013-03-20 13:19:30 -0400403ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
404{
405 mm_segment_t old_fs;
406 const char __user *p;
407 ssize_t ret;
408
Al Viro3e84f482013-03-27 15:20:30 +0000409 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
410 return -EINVAL;
411
Al Viro06ae43f2013-03-20 13:19:30 -0400412 old_fs = get_fs();
413 set_fs(get_ds());
414 p = (__force const char __user *)buf;
415 if (count > MAX_RW_COUNT)
416 count = MAX_RW_COUNT;
417 if (file->f_op->write)
418 ret = file->f_op->write(file, p, count, pos);
419 else
420 ret = do_sync_write(file, p, count, pos);
421 set_fs(old_fs);
422 if (ret > 0) {
423 fsnotify_modify(file);
424 add_wchar(current, ret);
425 }
426 inc_syscw(current);
427 return ret;
428}
429
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
431{
432 ssize_t ret;
433
434 if (!(file->f_mode & FMODE_WRITE))
435 return -EBADF;
436 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
437 return -EINVAL;
438 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
439 return -EFAULT;
440
441 ret = rw_verify_area(WRITE, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800442 if (ret >= 0) {
443 count = ret;
Al Viro03d95eb2013-03-20 13:04:20 -0400444 file_start_write(file);
James Morrisc43e2592008-01-12 22:05:48 +1100445 if (file->f_op->write)
446 ret = file->f_op->write(file, buf, count, pos);
447 else
448 ret = do_sync_write(file, buf, count, pos);
449 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500450 fsnotify_modify(file);
James Morrisc43e2592008-01-12 22:05:48 +1100451 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 }
James Morrisc43e2592008-01-12 22:05:48 +1100453 inc_syscw(current);
Al Viro03d95eb2013-03-20 13:04:20 -0400454 file_end_write(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 }
456
457 return ret;
458}
459
460EXPORT_SYMBOL(vfs_write);
461
462static inline loff_t file_pos_read(struct file *file)
463{
464 return file->f_pos;
465}
466
467static inline void file_pos_write(struct file *file, loff_t pos)
468{
469 file->f_pos = pos;
470}
471
Heiko Carstens3cdad422009-01-14 14:14:22 +0100472SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473{
Al Viro2903ff02012-08-28 12:52:22 -0400474 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Al Viro2903ff02012-08-28 12:52:22 -0400477 if (f.file) {
478 loff_t pos = file_pos_read(f.file);
479 ret = vfs_read(f.file, buf, count, &pos);
480 file_pos_write(f.file, pos);
481 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 return ret;
484}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
Heiko Carstens3cdad422009-01-14 14:14:22 +0100486SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
487 size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488{
Al Viro2903ff02012-08-28 12:52:22 -0400489 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
Al Viro2903ff02012-08-28 12:52:22 -0400492 if (f.file) {
493 loff_t pos = file_pos_read(f.file);
494 ret = vfs_write(f.file, buf, count, &pos);
495 file_pos_write(f.file, pos);
496 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 }
498
499 return ret;
500}
501
Al Viro4a0fd5b2013-01-21 15:16:58 -0500502SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
503 size_t, count, loff_t, pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504{
Al Viro2903ff02012-08-28 12:52:22 -0400505 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
508 if (pos < 0)
509 return -EINVAL;
510
Al Viro2903ff02012-08-28 12:52:22 -0400511 f = fdget(fd);
512 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400514 if (f.file->f_mode & FMODE_PREAD)
515 ret = vfs_read(f.file, buf, count, &pos);
516 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 }
518
519 return ret;
520}
521
Al Viro4a0fd5b2013-01-21 15:16:58 -0500522SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
523 size_t, count, loff_t, pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524{
Al Viro2903ff02012-08-28 12:52:22 -0400525 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527
528 if (pos < 0)
529 return -EINVAL;
530
Al Viro2903ff02012-08-28 12:52:22 -0400531 f = fdget(fd);
532 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400534 if (f.file->f_mode & FMODE_PWRITE)
535 ret = vfs_write(f.file, buf, count, &pos);
536 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 }
538
539 return ret;
540}
541
542/*
543 * Reduce an iovec's length in-place. Return the resulting number of segments
544 */
545unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
546{
547 unsigned long seg = 0;
548 size_t len = 0;
549
550 while (seg < nr_segs) {
551 seg++;
552 if (len + iov->iov_len >= to) {
553 iov->iov_len = to - len;
554 break;
555 }
556 len += iov->iov_len;
557 iov++;
558 }
559 return seg;
560}
Eric Sandeen19295522008-01-28 23:58:27 -0500561EXPORT_SYMBOL(iov_shorten);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562
Al Viro72ec3512013-03-20 10:42:10 -0400563static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700564 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
565{
566 struct kiocb kiocb;
567 ssize_t ret;
568
569 init_sync_kiocb(&kiocb, filp);
570 kiocb.ki_pos = *ppos;
571 kiocb.ki_left = len;
572 kiocb.ki_nbytes = len;
573
Zach Brown41003a72013-05-07 16:18:25 -0700574 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700575 if (ret == -EIOCBQUEUED)
576 ret = wait_on_sync_kiocb(&kiocb);
577 *ppos = kiocb.ki_pos;
578 return ret;
579}
580
581/* Do it by hand, with file-ops */
Al Viro72ec3512013-03-20 10:42:10 -0400582static ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700583 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
584{
585 struct iovec *vector = iov;
586 ssize_t ret = 0;
587
588 while (nr_segs > 0) {
589 void __user *base;
590 size_t len;
591 ssize_t nr;
592
593 base = vector->iov_base;
594 len = vector->iov_len;
595 vector++;
596 nr_segs--;
597
598 nr = fn(filp, base, len, ppos);
599
600 if (nr < 0) {
601 if (!ret)
602 ret = nr;
603 break;
604 }
605 ret += nr;
606 if (nr != len)
607 break;
608 }
609
610 return ret;
611}
612
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613/* A write operation does a read from user space and vice versa */
614#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
615
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700616ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
617 unsigned long nr_segs, unsigned long fast_segs,
618 struct iovec *fast_pointer,
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700619 struct iovec **ret_pointer)
Linus Torvalds435f49a2010-10-29 10:36:49 -0700620{
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700621 unsigned long seg;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700622 ssize_t ret;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700623 struct iovec *iov = fast_pointer;
624
Linus Torvalds435f49a2010-10-29 10:36:49 -0700625 /*
626 * SuS says "The readv() function *may* fail if the iovcnt argument
627 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
628 * traditionally returned zero for zero segments, so...
629 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700630 if (nr_segs == 0) {
631 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700632 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700633 }
634
Linus Torvalds435f49a2010-10-29 10:36:49 -0700635 /*
636 * First get the "struct iovec" from user memory and
637 * verify all the pointers
638 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700639 if (nr_segs > UIO_MAXIOV) {
640 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700641 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700642 }
643 if (nr_segs > fast_segs) {
Linus Torvalds435f49a2010-10-29 10:36:49 -0700644 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700645 if (iov == NULL) {
646 ret = -ENOMEM;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700647 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700648 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700649 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700650 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
651 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700652 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700653 }
654
Linus Torvalds435f49a2010-10-29 10:36:49 -0700655 /*
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700656 * According to the Single Unix Specification we should return EINVAL
657 * if an element length is < 0 when cast to ssize_t or if the
658 * total length would overflow the ssize_t return value of the
659 * system call.
Linus Torvalds435f49a2010-10-29 10:36:49 -0700660 *
661 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
662 * overflow case.
663 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700664 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700665 for (seg = 0; seg < nr_segs; seg++) {
666 void __user *buf = iov[seg].iov_base;
667 ssize_t len = (ssize_t)iov[seg].iov_len;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700668
669 /* see if we we're about to use an invalid len or if
670 * it's about to overflow ssize_t */
Linus Torvalds435f49a2010-10-29 10:36:49 -0700671 if (len < 0) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700672 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700673 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700674 }
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700675 if (type >= 0
Christopher Yeohfcf63402011-10-31 17:06:39 -0700676 && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700677 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700678 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700679 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700680 if (len > MAX_RW_COUNT - ret) {
681 len = MAX_RW_COUNT - ret;
682 iov[seg].iov_len = len;
683 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700684 ret += len;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700685 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700686out:
687 *ret_pointer = iov;
688 return ret;
689}
690
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691static ssize_t do_readv_writev(int type, struct file *file,
692 const struct iovec __user * uvector,
693 unsigned long nr_segs, loff_t *pos)
694{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 size_t tot_len;
696 struct iovec iovstack[UIO_FASTIOV];
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700697 struct iovec *iov = iovstack;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 io_fn_t fn;
700 iov_fn_t fnv;
701
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700702 if (!file->f_op) {
703 ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 goto out;
705 }
706
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700707 ret = rw_copy_check_uvector(type, uvector, nr_segs,
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700708 ARRAY_SIZE(iovstack), iovstack, &iov);
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700709 if (ret <= 0)
710 goto out;
711
712 tot_len = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 ret = rw_verify_area(type, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800714 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 goto out;
716
717 fnv = NULL;
718 if (type == READ) {
719 fn = file->f_op->read;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700720 fnv = file->f_op->aio_read;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 } else {
722 fn = (io_fn_t)file->f_op->write;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700723 fnv = file->f_op->aio_write;
Al Viro03d95eb2013-03-20 13:04:20 -0400724 file_start_write(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 }
726
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700727 if (fnv)
728 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
729 pos, fnv);
730 else
731 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Al Viro03d95eb2013-03-20 13:04:20 -0400733 if (type != READ)
734 file_end_write(file);
735
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736out:
737 if (iov != iovstack)
738 kfree(iov);
Robert Love0eeca282005-07-12 17:06:03 -0400739 if ((ret + (type == READ)) > 0) {
740 if (type == READ)
Eric Paris2a12a9d2009-12-17 21:24:21 -0500741 fsnotify_access(file);
Robert Love0eeca282005-07-12 17:06:03 -0400742 else
Eric Paris2a12a9d2009-12-17 21:24:21 -0500743 fsnotify_modify(file);
Robert Love0eeca282005-07-12 17:06:03 -0400744 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746}
747
748ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
749 unsigned long vlen, loff_t *pos)
750{
751 if (!(file->f_mode & FMODE_READ))
752 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700753 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 return -EINVAL;
755
756 return do_readv_writev(READ, file, vec, vlen, pos);
757}
758
759EXPORT_SYMBOL(vfs_readv);
760
761ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
762 unsigned long vlen, loff_t *pos)
763{
764 if (!(file->f_mode & FMODE_WRITE))
765 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700766 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 return -EINVAL;
768
769 return do_readv_writev(WRITE, file, vec, vlen, pos);
770}
771
772EXPORT_SYMBOL(vfs_writev);
773
Heiko Carstens3cdad422009-01-14 14:14:22 +0100774SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
775 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776{
Al Viro2903ff02012-08-28 12:52:22 -0400777 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779
Al Viro2903ff02012-08-28 12:52:22 -0400780 if (f.file) {
781 loff_t pos = file_pos_read(f.file);
782 ret = vfs_readv(f.file, vec, vlen, &pos);
783 file_pos_write(f.file, pos);
784 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 }
786
787 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800788 add_rchar(current, ret);
789 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 return ret;
791}
792
Heiko Carstens3cdad422009-01-14 14:14:22 +0100793SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
794 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795{
Al Viro2903ff02012-08-28 12:52:22 -0400796 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
Al Viro2903ff02012-08-28 12:52:22 -0400799 if (f.file) {
800 loff_t pos = file_pos_read(f.file);
801 ret = vfs_writev(f.file, vec, vlen, &pos);
802 file_pos_write(f.file, pos);
803 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 }
805
806 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800807 add_wchar(current, ret);
808 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 return ret;
810}
811
Linus Torvalds601cc112009-04-03 08:03:22 -0700812static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700813{
Linus Torvalds601cc112009-04-03 08:03:22 -0700814#define HALF_LONG_BITS (BITS_PER_LONG / 2)
815 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
816}
817
818SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
819 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
820{
821 loff_t pos = pos_from_hilo(pos_h, pos_l);
Al Viro2903ff02012-08-28 12:52:22 -0400822 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700823 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700824
825 if (pos < 0)
826 return -EINVAL;
827
Al Viro2903ff02012-08-28 12:52:22 -0400828 f = fdget(fd);
829 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700830 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400831 if (f.file->f_mode & FMODE_PREAD)
832 ret = vfs_readv(f.file, vec, vlen, &pos);
833 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700834 }
835
836 if (ret > 0)
837 add_rchar(current, ret);
838 inc_syscr(current);
839 return ret;
840}
841
842SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
Linus Torvalds601cc112009-04-03 08:03:22 -0700843 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700844{
Linus Torvalds601cc112009-04-03 08:03:22 -0700845 loff_t pos = pos_from_hilo(pos_h, pos_l);
Al Viro2903ff02012-08-28 12:52:22 -0400846 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700847 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700848
849 if (pos < 0)
850 return -EINVAL;
851
Al Viro2903ff02012-08-28 12:52:22 -0400852 f = fdget(fd);
853 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700854 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400855 if (f.file->f_mode & FMODE_PWRITE)
856 ret = vfs_writev(f.file, vec, vlen, &pos);
857 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700858 }
859
860 if (ret > 0)
861 add_wchar(current, ret);
862 inc_syscw(current);
863 return ret;
864}
865
Al Viro72ec3512013-03-20 10:42:10 -0400866#ifdef CONFIG_COMPAT
867
868static ssize_t compat_do_readv_writev(int type, struct file *file,
869 const struct compat_iovec __user *uvector,
870 unsigned long nr_segs, loff_t *pos)
871{
872 compat_ssize_t tot_len;
873 struct iovec iovstack[UIO_FASTIOV];
874 struct iovec *iov = iovstack;
875 ssize_t ret;
876 io_fn_t fn;
877 iov_fn_t fnv;
878
879 ret = -EINVAL;
880 if (!file->f_op)
881 goto out;
882
883 ret = -EFAULT;
884 if (!access_ok(VERIFY_READ, uvector, nr_segs*sizeof(*uvector)))
885 goto out;
886
887 ret = compat_rw_copy_check_uvector(type, uvector, nr_segs,
888 UIO_FASTIOV, iovstack, &iov);
889 if (ret <= 0)
890 goto out;
891
892 tot_len = ret;
893 ret = rw_verify_area(type, file, pos, tot_len);
894 if (ret < 0)
895 goto out;
896
897 fnv = NULL;
898 if (type == READ) {
899 fn = file->f_op->read;
900 fnv = file->f_op->aio_read;
901 } else {
902 fn = (io_fn_t)file->f_op->write;
903 fnv = file->f_op->aio_write;
Al Viro03d95eb2013-03-20 13:04:20 -0400904 file_start_write(file);
Al Viro72ec3512013-03-20 10:42:10 -0400905 }
906
Al Viro03d95eb2013-03-20 13:04:20 -0400907 if (fnv)
Al Viro72ec3512013-03-20 10:42:10 -0400908 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
909 pos, fnv);
Al Viro03d95eb2013-03-20 13:04:20 -0400910 else
Al Viro72ec3512013-03-20 10:42:10 -0400911 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
912
Al Viro03d95eb2013-03-20 13:04:20 -0400913 if (type != READ)
914 file_end_write(file);
915
Al Viro72ec3512013-03-20 10:42:10 -0400916out:
917 if (iov != iovstack)
918 kfree(iov);
919 if ((ret + (type == READ)) > 0) {
920 if (type == READ)
921 fsnotify_access(file);
922 else
923 fsnotify_modify(file);
924 }
925 return ret;
926}
927
928static size_t compat_readv(struct file *file,
929 const struct compat_iovec __user *vec,
930 unsigned long vlen, loff_t *pos)
931{
932 ssize_t ret = -EBADF;
933
934 if (!(file->f_mode & FMODE_READ))
935 goto out;
936
937 ret = -EINVAL;
938 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
939 goto out;
940
941 ret = compat_do_readv_writev(READ, file, vec, vlen, pos);
942
943out:
944 if (ret > 0)
945 add_rchar(current, ret);
946 inc_syscr(current);
947 return ret;
948}
949
950COMPAT_SYSCALL_DEFINE3(readv, unsigned long, fd,
951 const struct compat_iovec __user *,vec,
952 unsigned long, vlen)
953{
954 struct fd f = fdget(fd);
955 ssize_t ret;
956 loff_t pos;
957
958 if (!f.file)
959 return -EBADF;
960 pos = f.file->f_pos;
961 ret = compat_readv(f.file, vec, vlen, &pos);
962 f.file->f_pos = pos;
963 fdput(f);
964 return ret;
965}
966
967COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
968 const struct compat_iovec __user *,vec,
969 unsigned long, vlen, loff_t, pos)
970{
971 struct fd f;
972 ssize_t ret;
973
974 if (pos < 0)
975 return -EINVAL;
976 f = fdget(fd);
977 if (!f.file)
978 return -EBADF;
979 ret = -ESPIPE;
980 if (f.file->f_mode & FMODE_PREAD)
981 ret = compat_readv(f.file, vec, vlen, &pos);
982 fdput(f);
983 return ret;
984}
985
986COMPAT_SYSCALL_DEFINE5(preadv, unsigned long, fd,
987 const struct compat_iovec __user *,vec,
988 unsigned long, vlen, u32, pos_low, u32, pos_high)
989{
990 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
991 return compat_sys_preadv64(fd, vec, vlen, pos);
992}
993
994static size_t compat_writev(struct file *file,
995 const struct compat_iovec __user *vec,
996 unsigned long vlen, loff_t *pos)
997{
998 ssize_t ret = -EBADF;
999
1000 if (!(file->f_mode & FMODE_WRITE))
1001 goto out;
1002
1003 ret = -EINVAL;
1004 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
1005 goto out;
1006
1007 ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos);
1008
1009out:
1010 if (ret > 0)
1011 add_wchar(current, ret);
1012 inc_syscw(current);
1013 return ret;
1014}
1015
1016COMPAT_SYSCALL_DEFINE3(writev, unsigned long, fd,
1017 const struct compat_iovec __user *, vec,
1018 unsigned long, vlen)
1019{
1020 struct fd f = fdget(fd);
1021 ssize_t ret;
1022 loff_t pos;
1023
1024 if (!f.file)
1025 return -EBADF;
1026 pos = f.file->f_pos;
1027 ret = compat_writev(f.file, vec, vlen, &pos);
1028 f.file->f_pos = pos;
1029 fdput(f);
1030 return ret;
1031}
1032
1033COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1034 const struct compat_iovec __user *,vec,
1035 unsigned long, vlen, loff_t, pos)
1036{
1037 struct fd f;
1038 ssize_t ret;
1039
1040 if (pos < 0)
1041 return -EINVAL;
1042 f = fdget(fd);
1043 if (!f.file)
1044 return -EBADF;
1045 ret = -ESPIPE;
1046 if (f.file->f_mode & FMODE_PWRITE)
1047 ret = compat_writev(f.file, vec, vlen, &pos);
1048 fdput(f);
1049 return ret;
1050}
1051
1052COMPAT_SYSCALL_DEFINE5(pwritev, unsigned long, fd,
1053 const struct compat_iovec __user *,vec,
1054 unsigned long, vlen, u32, pos_low, u32, pos_high)
1055{
1056 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1057 return compat_sys_pwritev64(fd, vec, vlen, pos);
1058}
1059#endif
1060
Al Viro19f4fc32013-02-24 02:17:03 -05001061static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1062 size_t count, loff_t max)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063{
Al Viro2903ff02012-08-28 12:52:22 -04001064 struct fd in, out;
1065 struct inode *in_inode, *out_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 loff_t pos;
Al Viro7995bd22013-06-20 18:58:36 +04001067 loff_t out_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068 ssize_t retval;
Al Viro2903ff02012-08-28 12:52:22 -04001069 int fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
1071 /*
1072 * Get input file, and verify that it is ok..
1073 */
1074 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -04001075 in = fdget(in_fd);
1076 if (!in.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 goto out;
Al Viro2903ff02012-08-28 12:52:22 -04001078 if (!(in.file->f_mode & FMODE_READ))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 goto fput_in;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 retval = -ESPIPE;
Al Viro7995bd22013-06-20 18:58:36 +04001081 if (!ppos) {
1082 pos = in.file->f_pos;
1083 } else {
1084 pos = *ppos;
Al Viro2903ff02012-08-28 12:52:22 -04001085 if (!(in.file->f_mode & FMODE_PREAD))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086 goto fput_in;
Al Viro7995bd22013-06-20 18:58:36 +04001087 }
1088 retval = rw_verify_area(READ, in.file, &pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -08001089 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 goto fput_in;
Linus Torvaldse28cc712006-01-04 16:20:40 -08001091 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 /*
1094 * Get output file, and verify that it is ok..
1095 */
1096 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -04001097 out = fdget(out_fd);
1098 if (!out.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 goto fput_in;
Al Viro2903ff02012-08-28 12:52:22 -04001100 if (!(out.file->f_mode & FMODE_WRITE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101 goto fput_out;
1102 retval = -EINVAL;
Al Viro496ad9a2013-01-23 17:07:38 -05001103 in_inode = file_inode(in.file);
1104 out_inode = file_inode(out.file);
Al Viro7995bd22013-06-20 18:58:36 +04001105 out_pos = out.file->f_pos;
1106 retval = rw_verify_area(WRITE, out.file, &out_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -08001107 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 goto fput_out;
Linus Torvaldse28cc712006-01-04 16:20:40 -08001109 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 if (!max)
1112 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1113
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 if (unlikely(pos + count > max)) {
1115 retval = -EOVERFLOW;
1116 if (pos >= max)
1117 goto fput_out;
1118 count = max - pos;
1119 }
1120
Jens Axboed96e6e72007-06-11 12:18:52 +02001121 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +02001122#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +02001123 /*
1124 * We need to debate whether we can enable this or not. The
1125 * man page documents EAGAIN return for the output at least,
1126 * and the application is arguably buggy if it doesn't expect
1127 * EAGAIN on a non-blocking file descriptor.
1128 */
Al Viro2903ff02012-08-28 12:52:22 -04001129 if (in.file->f_flags & O_NONBLOCK)
Jens Axboed96e6e72007-06-11 12:18:52 +02001130 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +02001131#endif
Al Viro7995bd22013-06-20 18:58:36 +04001132 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133
1134 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001135 add_rchar(current, retval);
1136 add_wchar(current, retval);
Scott Wolchoka68c2f12012-12-20 15:05:52 -08001137 fsnotify_access(in.file);
1138 fsnotify_modify(out.file);
Al Viro7995bd22013-06-20 18:58:36 +04001139 out.file->f_pos = out_pos;
1140 if (ppos)
1141 *ppos = pos;
1142 else
1143 in.file->f_pos = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145
Alexey Dobriyan4b98d112007-02-10 01:46:45 -08001146 inc_syscr(current);
1147 inc_syscw(current);
Al Viro7995bd22013-06-20 18:58:36 +04001148 if (pos > max)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 retval = -EOVERFLOW;
1150
1151fput_out:
Al Viro2903ff02012-08-28 12:52:22 -04001152 fdput(out);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153fput_in:
Al Viro2903ff02012-08-28 12:52:22 -04001154 fdput(in);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155out:
1156 return retval;
1157}
1158
Heiko Carstens002c8972009-01-14 14:14:18 +01001159SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160{
1161 loff_t pos;
1162 off_t off;
1163 ssize_t ret;
1164
1165 if (offset) {
1166 if (unlikely(get_user(off, offset)))
1167 return -EFAULT;
1168 pos = off;
1169 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1170 if (unlikely(put_user(pos, offset)))
1171 return -EFAULT;
1172 return ret;
1173 }
1174
1175 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1176}
1177
Heiko Carstens002c8972009-01-14 14:14:18 +01001178SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179{
1180 loff_t pos;
1181 ssize_t ret;
1182
1183 if (offset) {
1184 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1185 return -EFAULT;
1186 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1187 if (unlikely(put_user(pos, offset)))
1188 return -EFAULT;
1189 return ret;
1190 }
1191
1192 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1193}
Al Viro19f4fc32013-02-24 02:17:03 -05001194
1195#ifdef CONFIG_COMPAT
1196COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1197 compat_off_t __user *, offset, compat_size_t, count)
1198{
1199 loff_t pos;
1200 off_t off;
1201 ssize_t ret;
1202
1203 if (offset) {
1204 if (unlikely(get_user(off, offset)))
1205 return -EFAULT;
1206 pos = off;
1207 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1208 if (unlikely(put_user(pos, offset)))
1209 return -EFAULT;
1210 return ret;
1211 }
1212
1213 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1214}
1215
1216COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1217 compat_loff_t __user *, offset, compat_size_t, count)
1218{
1219 loff_t pos;
1220 ssize_t ret;
1221
1222 if (offset) {
1223 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1224 return -EFAULT;
1225 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1226 if (unlikely(put_user(pos, offset)))
1227 return -EFAULT;
1228 return ret;
1229 }
1230
1231 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1232}
1233#endif