blob: bb34af315280328acedfd4122829b3c8125d8a6f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
Robert Love0eeca282005-07-12 17:06:03 -040012#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/security.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050014#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080016#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020017#include <linux/splice.h>
Badari Pulavartyee0b3e62006-09-30 23:28:47 -070018#include "read_write.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019
20#include <asm/uaccess.h>
21#include <asm/unistd.h>
22
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080023const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 .llseek = generic_file_llseek,
Badari Pulavarty543ade12006-09-30 23:28:48 -070025 .read = do_sync_read,
26 .aio_read = generic_file_aio_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020028 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070029};
30
31EXPORT_SYMBOL(generic_ro_fops);
32
Al Virocccb5a12010-12-17 07:44:05 -050033static inline int unsigned_offsets(struct file *file)
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070034{
Al Virocccb5a12010-12-17 07:44:05 -050035 return file->f_mode & FMODE_UNSIGNED_OFFSET;
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -070036}
37
Andi Kleenef3d0fd2011-09-15 16:06:48 -070038static loff_t lseek_execute(struct file *file, struct inode *inode,
39 loff_t offset, loff_t maxsize)
40{
41 if (offset < 0 && !unsigned_offsets(file))
42 return -EINVAL;
43 if (offset > maxsize)
44 return -EINVAL;
45
46 if (offset != file->f_pos) {
47 file->f_pos = offset;
48 file->f_version = 0;
49 }
50 return offset;
51}
52
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020053/**
Andi Kleen57604952011-09-15 16:06:50 -070054 * generic_file_llseek_size - generic llseek implementation for regular files
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020055 * @file: file structure to seek on
56 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -080057 * @whence: type of seek
Eric Sandeene8b96eb2012-04-30 13:11:29 -050058 * @size: max size of this file in file system
59 * @eof: offset used for SEEK_END position
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020060 *
Andi Kleen57604952011-09-15 16:06:50 -070061 * This is a variant of generic_file_llseek that allows passing in a custom
Eric Sandeene8b96eb2012-04-30 13:11:29 -050062 * maximum file size and a custom EOF position, for e.g. hashed directories
Andi Kleenef3d0fd2011-09-15 16:06:48 -070063 *
64 * Synchronization:
Andi Kleen57604952011-09-15 16:06:50 -070065 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
Andi Kleenef3d0fd2011-09-15 16:06:48 -070066 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
67 * read/writes behave like SEEK_SET against seeks.
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020068 */
Andi Kleen9465efc2008-06-27 11:05:24 +020069loff_t
Andrew Morton965c8e52012-12-17 15:59:39 -080070generic_file_llseek_size(struct file *file, loff_t offset, int whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -050071 loff_t maxsize, loff_t eof)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 struct inode *inode = file->f_mapping->host;
74
Andrew Morton965c8e52012-12-17 15:59:39 -080075 switch (whence) {
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020076 case SEEK_END:
Eric Sandeene8b96eb2012-04-30 13:11:29 -050077 offset += eof;
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020078 break;
79 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -080080 /*
81 * Here we special-case the lseek(fd, 0, SEEK_CUR)
82 * position-querying operation. Avoid rewriting the "same"
83 * f_pos value back to the file because a concurrent read(),
84 * write() or lseek() might have altered it
85 */
86 if (offset == 0)
87 return file->f_pos;
Andi Kleenef3d0fd2011-09-15 16:06:48 -070088 /*
89 * f_lock protects against read/modify/write race with other
90 * SEEK_CURs. Note that parallel writes and reads behave
91 * like SEEK_SET.
92 */
93 spin_lock(&file->f_lock);
94 offset = lseek_execute(file, inode, file->f_pos + offset,
Andi Kleen57604952011-09-15 16:06:50 -070095 maxsize);
Andi Kleenef3d0fd2011-09-15 16:06:48 -070096 spin_unlock(&file->f_lock);
97 return offset;
Josef Bacik982d8162011-07-18 13:21:35 -040098 case SEEK_DATA:
99 /*
100 * In the generic case the entire file is data, so as long as
101 * offset isn't at the end of the file then the offset is data.
102 */
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500103 if (offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400104 return -ENXIO;
105 break;
106 case SEEK_HOLE:
107 /*
108 * There is a virtual hole at the end of the file, so as long as
109 * offset isn't i_size or larger, return i_size.
110 */
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500111 if (offset >= eof)
Josef Bacik982d8162011-07-18 13:21:35 -0400112 return -ENXIO;
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500113 offset = eof;
Josef Bacik982d8162011-07-18 13:21:35 -0400114 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +0200116
Andi Kleen57604952011-09-15 16:06:50 -0700117 return lseek_execute(file, inode, offset, maxsize);
118}
119EXPORT_SYMBOL(generic_file_llseek_size);
120
121/**
122 * generic_file_llseek - generic llseek implementation for regular files
123 * @file: file structure to seek on
124 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800125 * @whence: type of seek
Andi Kleen57604952011-09-15 16:06:50 -0700126 *
127 * This is a generic implemenation of ->llseek useable for all normal local
128 * filesystems. It just updates the file offset to the value specified by
Andrew Morton965c8e52012-12-17 15:59:39 -0800129 * @offset and @whence under i_mutex.
Andi Kleen57604952011-09-15 16:06:50 -0700130 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800131loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
Andi Kleen57604952011-09-15 16:06:50 -0700132{
133 struct inode *inode = file->f_mapping->host;
134
Andrew Morton965c8e52012-12-17 15:59:39 -0800135 return generic_file_llseek_size(file, offset, whence,
Eric Sandeene8b96eb2012-04-30 13:11:29 -0500136 inode->i_sb->s_maxbytes,
137 i_size_read(inode));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138}
Andi Kleen9465efc2008-06-27 11:05:24 +0200139EXPORT_SYMBOL(generic_file_llseek);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
jan Blunckae6afc32010-05-26 14:44:48 -0700141/**
142 * noop_llseek - No Operation Performed llseek implementation
143 * @file: file structure to seek on
144 * @offset: file offset to seek to
Andrew Morton965c8e52012-12-17 15:59:39 -0800145 * @whence: type of seek
jan Blunckae6afc32010-05-26 14:44:48 -0700146 *
147 * This is an implementation of ->llseek useable for the rare special case when
148 * userspace expects the seek to succeed but the (device) file is actually not
149 * able to perform the seek. In this case you use noop_llseek() instead of
150 * falling back to the default implementation of ->llseek.
151 */
Andrew Morton965c8e52012-12-17 15:59:39 -0800152loff_t noop_llseek(struct file *file, loff_t offset, int whence)
jan Blunckae6afc32010-05-26 14:44:48 -0700153{
154 return file->f_pos;
155}
156EXPORT_SYMBOL(noop_llseek);
157
Andrew Morton965c8e52012-12-17 15:59:39 -0800158loff_t no_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159{
160 return -ESPIPE;
161}
162EXPORT_SYMBOL(no_llseek);
163
Andrew Morton965c8e52012-12-17 15:59:39 -0800164loff_t default_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165{
Josef Bacik982d8162011-07-18 13:21:35 -0400166 struct inode *inode = file->f_path.dentry->d_inode;
David Sterba16abef02008-04-22 15:09:22 +0200167 loff_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Josef Bacik982d8162011-07-18 13:21:35 -0400169 mutex_lock(&inode->i_mutex);
Andrew Morton965c8e52012-12-17 15:59:39 -0800170 switch (whence) {
Chris Snook7b8e8922007-05-08 00:24:13 -0700171 case SEEK_END:
Josef Bacik982d8162011-07-18 13:21:35 -0400172 offset += i_size_read(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700174 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800175 if (offset == 0) {
176 retval = file->f_pos;
177 goto out;
178 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 offset += file->f_pos;
Josef Bacik982d8162011-07-18 13:21:35 -0400180 break;
181 case SEEK_DATA:
182 /*
183 * In the generic case the entire file is data, so as
184 * long as offset isn't at the end of the file then the
185 * offset is data.
186 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300187 if (offset >= inode->i_size) {
188 retval = -ENXIO;
189 goto out;
190 }
Josef Bacik982d8162011-07-18 13:21:35 -0400191 break;
192 case SEEK_HOLE:
193 /*
194 * There is a virtual hole at the end of the file, so
195 * as long as offset isn't i_size or larger, return
196 * i_size.
197 */
Dan Carpenterbacb2d82011-07-26 17:25:20 +0300198 if (offset >= inode->i_size) {
199 retval = -ENXIO;
200 goto out;
201 }
Josef Bacik982d8162011-07-18 13:21:35 -0400202 offset = inode->i_size;
203 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 }
205 retval = -EINVAL;
Al Virocccb5a12010-12-17 07:44:05 -0500206 if (offset >= 0 || unsigned_offsets(file)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 if (offset != file->f_pos) {
208 file->f_pos = offset;
209 file->f_version = 0;
210 }
211 retval = offset;
212 }
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800213out:
Josef Bacik982d8162011-07-18 13:21:35 -0400214 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 return retval;
216}
217EXPORT_SYMBOL(default_llseek);
218
Andrew Morton965c8e52012-12-17 15:59:39 -0800219loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220{
221 loff_t (*fn)(struct file *, loff_t, int);
222
223 fn = no_llseek;
224 if (file->f_mode & FMODE_LSEEK) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 if (file->f_op && file->f_op->llseek)
226 fn = file->f_op->llseek;
227 }
Andrew Morton965c8e52012-12-17 15:59:39 -0800228 return fn(file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229}
230EXPORT_SYMBOL(vfs_llseek);
231
Andrew Morton965c8e52012-12-17 15:59:39 -0800232SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 off_t retval;
Al Viro2903ff02012-08-28 12:52:22 -0400235 struct fd f = fdget(fd);
236 if (!f.file)
237 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800240 if (whence <= SEEK_MAX) {
241 loff_t res = vfs_llseek(f.file, offset, whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 retval = res;
243 if (res != (loff_t)retval)
244 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
245 }
Al Viro2903ff02012-08-28 12:52:22 -0400246 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 return retval;
248}
249
250#ifdef __ARCH_WANT_SYS_LLSEEK
Heiko Carstens003d7ab2009-01-14 14:14:21 +0100251SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
252 unsigned long, offset_low, loff_t __user *, result,
Andrew Morton965c8e52012-12-17 15:59:39 -0800253 unsigned int, whence)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254{
255 int retval;
Al Viro2903ff02012-08-28 12:52:22 -0400256 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 loff_t offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
Al Viro2903ff02012-08-28 12:52:22 -0400259 if (!f.file)
260 return -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261
262 retval = -EINVAL;
Andrew Morton965c8e52012-12-17 15:59:39 -0800263 if (whence > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 goto out_putf;
265
Al Viro2903ff02012-08-28 12:52:22 -0400266 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
Andrew Morton965c8e52012-12-17 15:59:39 -0800267 whence);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268
269 retval = (int)offset;
270 if (offset >= 0) {
271 retval = -EFAULT;
272 if (!copy_to_user(result, &offset, sizeof(offset)))
273 retval = 0;
274 }
275out_putf:
Al Viro2903ff02012-08-28 12:52:22 -0400276 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 return retval;
278}
279#endif
280
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -0700281
Linus Torvaldse28cc712006-01-04 16:20:40 -0800282/*
283 * rw_verify_area doesn't like huge counts. We limit
284 * them to something that fits in "int" so that others
285 * won't have to do range checks all the time.
286 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
288{
289 struct inode *inode;
290 loff_t pos;
James Morrisc43e2592008-01-12 22:05:48 +1100291 int retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292
Eric Dumazet163da952007-02-12 00:52:24 -0800293 inode = file->f_path.dentry->d_inode;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800294 if (unlikely((ssize_t) count < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100295 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 pos = *ppos;
Al Virocccb5a12010-12-17 07:44:05 -0500297 if (unlikely(pos < 0)) {
298 if (!unsigned_offsets(file))
299 return retval;
300 if (count >= -pos) /* both values are in 0..LLONG_MAX */
301 return -EOVERFLOW;
302 } else if (unlikely((loff_t) (pos + count) < 0)) {
303 if (!unsigned_offsets(file))
KAMEZAWA Hiroyuki4a3956c2010-10-01 14:20:22 -0700304 return retval;
305 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Pavel Emelyanova16877c2007-10-01 14:41:11 -0700307 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
James Morrisc43e2592008-01-12 22:05:48 +1100308 retval = locks_mandatory_area(
Linus Torvaldse28cc712006-01-04 16:20:40 -0800309 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
310 inode, file, pos, count);
311 if (retval < 0)
312 return retval;
313 }
James Morrisc43e2592008-01-12 22:05:48 +1100314 retval = security_file_permission(file,
315 read_write == READ ? MAY_READ : MAY_WRITE);
316 if (retval)
317 return retval;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800318 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319}
320
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700321static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
322{
323 set_current_state(TASK_UNINTERRUPTIBLE);
324 if (!kiocbIsKicked(iocb))
325 schedule();
326 else
327 kiocbClearKicked(iocb);
328 __set_current_state(TASK_RUNNING);
329}
330
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
332{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700333 struct iovec iov = { .iov_base = buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 struct kiocb kiocb;
335 ssize_t ret;
336
337 init_sync_kiocb(&kiocb, filp);
338 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700339 kiocb.ki_left = len;
David Howells61964eb2010-03-24 17:09:19 +0000340 kiocb.ki_nbytes = len;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700341
342 for (;;) {
343 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
344 if (ret != -EIOCBRETRY)
345 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700346 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700347 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 if (-EIOCBQUEUED == ret)
350 ret = wait_on_sync_kiocb(&kiocb);
351 *ppos = kiocb.ki_pos;
352 return ret;
353}
354
355EXPORT_SYMBOL(do_sync_read);
356
357ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
358{
359 ssize_t ret;
360
361 if (!(file->f_mode & FMODE_READ))
362 return -EBADF;
363 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
364 return -EINVAL;
365 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
366 return -EFAULT;
367
368 ret = rw_verify_area(READ, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800369 if (ret >= 0) {
370 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100371 if (file->f_op->read)
372 ret = file->f_op->read(file, buf, count, pos);
373 else
374 ret = do_sync_read(file, buf, count, pos);
375 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500376 fsnotify_access(file);
James Morrisc43e2592008-01-12 22:05:48 +1100377 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 }
James Morrisc43e2592008-01-12 22:05:48 +1100379 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 }
381
382 return ret;
383}
384
385EXPORT_SYMBOL(vfs_read);
386
387ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
388{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700389 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 struct kiocb kiocb;
391 ssize_t ret;
392
393 init_sync_kiocb(&kiocb, filp);
394 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700395 kiocb.ki_left = len;
David Howells61964eb2010-03-24 17:09:19 +0000396 kiocb.ki_nbytes = len;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700397
398 for (;;) {
399 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
400 if (ret != -EIOCBRETRY)
401 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700402 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700403 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700404
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 if (-EIOCBQUEUED == ret)
406 ret = wait_on_sync_kiocb(&kiocb);
407 *ppos = kiocb.ki_pos;
408 return ret;
409}
410
411EXPORT_SYMBOL(do_sync_write);
412
413ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
414{
415 ssize_t ret;
416
417 if (!(file->f_mode & FMODE_WRITE))
418 return -EBADF;
419 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
420 return -EINVAL;
421 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
422 return -EFAULT;
423
424 ret = rw_verify_area(WRITE, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800425 if (ret >= 0) {
426 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100427 if (file->f_op->write)
428 ret = file->f_op->write(file, buf, count, pos);
429 else
430 ret = do_sync_write(file, buf, count, pos);
431 if (ret > 0) {
Eric Paris2a12a9d2009-12-17 21:24:21 -0500432 fsnotify_modify(file);
James Morrisc43e2592008-01-12 22:05:48 +1100433 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 }
James Morrisc43e2592008-01-12 22:05:48 +1100435 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 }
437
438 return ret;
439}
440
441EXPORT_SYMBOL(vfs_write);
442
443static inline loff_t file_pos_read(struct file *file)
444{
445 return file->f_pos;
446}
447
448static inline void file_pos_write(struct file *file, loff_t pos)
449{
450 file->f_pos = pos;
451}
452
Heiko Carstens3cdad422009-01-14 14:14:22 +0100453SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454{
Al Viro2903ff02012-08-28 12:52:22 -0400455 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457
Al Viro2903ff02012-08-28 12:52:22 -0400458 if (f.file) {
459 loff_t pos = file_pos_read(f.file);
460 ret = vfs_read(f.file, buf, count, &pos);
461 file_pos_write(f.file, pos);
462 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 return ret;
465}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466
Heiko Carstens3cdad422009-01-14 14:14:22 +0100467SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
468 size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469{
Al Viro2903ff02012-08-28 12:52:22 -0400470 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472
Al Viro2903ff02012-08-28 12:52:22 -0400473 if (f.file) {
474 loff_t pos = file_pos_read(f.file);
475 ret = vfs_write(f.file, buf, count, &pos);
476 file_pos_write(f.file, pos);
477 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 }
479
480 return ret;
481}
482
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100483SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
484 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485{
Al Viro2903ff02012-08-28 12:52:22 -0400486 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488
489 if (pos < 0)
490 return -EINVAL;
491
Al Viro2903ff02012-08-28 12:52:22 -0400492 f = fdget(fd);
493 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400495 if (f.file->f_mode & FMODE_PREAD)
496 ret = vfs_read(f.file, buf, count, &pos);
497 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 }
499
500 return ret;
501}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100502#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
503asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
504{
505 return SYSC_pread64((unsigned int) fd, (char __user *) buf,
506 (size_t) count, pos);
507}
508SYSCALL_ALIAS(sys_pread64, SyS_pread64);
509#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100511SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
512 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513{
Al Viro2903ff02012-08-28 12:52:22 -0400514 struct fd f;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516
517 if (pos < 0)
518 return -EINVAL;
519
Al Viro2903ff02012-08-28 12:52:22 -0400520 f = fdget(fd);
521 if (f.file) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400523 if (f.file->f_mode & FMODE_PWRITE)
524 ret = vfs_write(f.file, buf, count, &pos);
525 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 }
527
528 return ret;
529}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100530#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
531asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
532{
533 return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
534 (size_t) count, pos);
535}
536SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
537#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538
539/*
540 * Reduce an iovec's length in-place. Return the resulting number of segments
541 */
542unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
543{
544 unsigned long seg = 0;
545 size_t len = 0;
546
547 while (seg < nr_segs) {
548 seg++;
549 if (len + iov->iov_len >= to) {
550 iov->iov_len = to - len;
551 break;
552 }
553 len += iov->iov_len;
554 iov++;
555 }
556 return seg;
557}
Eric Sandeen19295522008-01-28 23:58:27 -0500558EXPORT_SYMBOL(iov_shorten);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700560ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
561 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
562{
563 struct kiocb kiocb;
564 ssize_t ret;
565
566 init_sync_kiocb(&kiocb, filp);
567 kiocb.ki_pos = *ppos;
568 kiocb.ki_left = len;
569 kiocb.ki_nbytes = len;
570
571 for (;;) {
572 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
573 if (ret != -EIOCBRETRY)
574 break;
575 wait_on_retry_sync_kiocb(&kiocb);
576 }
577
578 if (ret == -EIOCBQUEUED)
579 ret = wait_on_sync_kiocb(&kiocb);
580 *ppos = kiocb.ki_pos;
581 return ret;
582}
583
584/* Do it by hand, with file-ops */
585ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
586 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
587{
588 struct iovec *vector = iov;
589 ssize_t ret = 0;
590
591 while (nr_segs > 0) {
592 void __user *base;
593 size_t len;
594 ssize_t nr;
595
596 base = vector->iov_base;
597 len = vector->iov_len;
598 vector++;
599 nr_segs--;
600
601 nr = fn(filp, base, len, ppos);
602
603 if (nr < 0) {
604 if (!ret)
605 ret = nr;
606 break;
607 }
608 ret += nr;
609 if (nr != len)
610 break;
611 }
612
613 return ret;
614}
615
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616/* A write operation does a read from user space and vice versa */
617#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
618
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700619ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
620 unsigned long nr_segs, unsigned long fast_segs,
621 struct iovec *fast_pointer,
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700622 struct iovec **ret_pointer)
Linus Torvalds435f49a2010-10-29 10:36:49 -0700623{
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700624 unsigned long seg;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700625 ssize_t ret;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700626 struct iovec *iov = fast_pointer;
627
Linus Torvalds435f49a2010-10-29 10:36:49 -0700628 /*
629 * SuS says "The readv() function *may* fail if the iovcnt argument
630 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
631 * traditionally returned zero for zero segments, so...
632 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700633 if (nr_segs == 0) {
634 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700635 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700636 }
637
Linus Torvalds435f49a2010-10-29 10:36:49 -0700638 /*
639 * First get the "struct iovec" from user memory and
640 * verify all the pointers
641 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700642 if (nr_segs > UIO_MAXIOV) {
643 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700644 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700645 }
646 if (nr_segs > fast_segs) {
Linus Torvalds435f49a2010-10-29 10:36:49 -0700647 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700648 if (iov == NULL) {
649 ret = -ENOMEM;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700650 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700651 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700652 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700653 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
654 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700655 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700656 }
657
Linus Torvalds435f49a2010-10-29 10:36:49 -0700658 /*
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700659 * According to the Single Unix Specification we should return EINVAL
660 * if an element length is < 0 when cast to ssize_t or if the
661 * total length would overflow the ssize_t return value of the
662 * system call.
Linus Torvalds435f49a2010-10-29 10:36:49 -0700663 *
664 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
665 * overflow case.
666 */
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700667 ret = 0;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700668 for (seg = 0; seg < nr_segs; seg++) {
669 void __user *buf = iov[seg].iov_base;
670 ssize_t len = (ssize_t)iov[seg].iov_len;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700671
672 /* see if we we're about to use an invalid len or if
673 * it's about to overflow ssize_t */
Linus Torvalds435f49a2010-10-29 10:36:49 -0700674 if (len < 0) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700675 ret = -EINVAL;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700676 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700677 }
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700678 if (type >= 0
Christopher Yeohfcf63402011-10-31 17:06:39 -0700679 && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700680 ret = -EFAULT;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700681 goto out;
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700682 }
Linus Torvalds435f49a2010-10-29 10:36:49 -0700683 if (len > MAX_RW_COUNT - ret) {
684 len = MAX_RW_COUNT - ret;
685 iov[seg].iov_len = len;
686 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700687 ret += len;
Linus Torvalds435f49a2010-10-29 10:36:49 -0700688 }
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700689out:
690 *ret_pointer = iov;
691 return ret;
692}
693
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694static ssize_t do_readv_writev(int type, struct file *file,
695 const struct iovec __user * uvector,
696 unsigned long nr_segs, loff_t *pos)
697{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 size_t tot_len;
699 struct iovec iovstack[UIO_FASTIOV];
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700700 struct iovec *iov = iovstack;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 io_fn_t fn;
703 iov_fn_t fnv;
704
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700705 if (!file->f_op) {
706 ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 goto out;
708 }
709
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700710 ret = rw_copy_check_uvector(type, uvector, nr_segs,
Christopher Yeohac34ebb2012-05-31 16:26:42 -0700711 ARRAY_SIZE(iovstack), iovstack, &iov);
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700712 if (ret <= 0)
713 goto out;
714
715 tot_len = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 ret = rw_verify_area(type, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800717 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 goto out;
719
720 fnv = NULL;
721 if (type == READ) {
722 fn = file->f_op->read;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700723 fnv = file->f_op->aio_read;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 } else {
725 fn = (io_fn_t)file->f_op->write;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700726 fnv = file->f_op->aio_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 }
728
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700729 if (fnv)
730 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
731 pos, fnv);
732 else
733 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735out:
736 if (iov != iovstack)
737 kfree(iov);
Robert Love0eeca282005-07-12 17:06:03 -0400738 if ((ret + (type == READ)) > 0) {
739 if (type == READ)
Eric Paris2a12a9d2009-12-17 21:24:21 -0500740 fsnotify_access(file);
Robert Love0eeca282005-07-12 17:06:03 -0400741 else
Eric Paris2a12a9d2009-12-17 21:24:21 -0500742 fsnotify_modify(file);
Robert Love0eeca282005-07-12 17:06:03 -0400743 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
747ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
748 unsigned long vlen, loff_t *pos)
749{
750 if (!(file->f_mode & FMODE_READ))
751 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700752 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 return -EINVAL;
754
755 return do_readv_writev(READ, file, vec, vlen, pos);
756}
757
758EXPORT_SYMBOL(vfs_readv);
759
760ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
761 unsigned long vlen, loff_t *pos)
762{
763 if (!(file->f_mode & FMODE_WRITE))
764 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700765 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 return -EINVAL;
767
768 return do_readv_writev(WRITE, file, vec, vlen, pos);
769}
770
771EXPORT_SYMBOL(vfs_writev);
772
Heiko Carstens3cdad422009-01-14 14:14:22 +0100773SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
774 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775{
Al Viro2903ff02012-08-28 12:52:22 -0400776 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Al Viro2903ff02012-08-28 12:52:22 -0400779 if (f.file) {
780 loff_t pos = file_pos_read(f.file);
781 ret = vfs_readv(f.file, vec, vlen, &pos);
782 file_pos_write(f.file, pos);
783 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 }
785
786 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800787 add_rchar(current, ret);
788 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 return ret;
790}
791
Heiko Carstens3cdad422009-01-14 14:14:22 +0100792SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
793 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794{
Al Viro2903ff02012-08-28 12:52:22 -0400795 struct fd f = fdget(fd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 ssize_t ret = -EBADF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
Al Viro2903ff02012-08-28 12:52:22 -0400798 if (f.file) {
799 loff_t pos = file_pos_read(f.file);
800 ret = vfs_writev(f.file, vec, vlen, &pos);
801 file_pos_write(f.file, pos);
802 fdput(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 }
804
805 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800806 add_wchar(current, ret);
807 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 return ret;
809}
810
Linus Torvalds601cc112009-04-03 08:03:22 -0700811static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700812{
Linus Torvalds601cc112009-04-03 08:03:22 -0700813#define HALF_LONG_BITS (BITS_PER_LONG / 2)
814 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
815}
816
817SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
818 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
819{
820 loff_t pos = pos_from_hilo(pos_h, pos_l);
Al Viro2903ff02012-08-28 12:52:22 -0400821 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700822 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700823
824 if (pos < 0)
825 return -EINVAL;
826
Al Viro2903ff02012-08-28 12:52:22 -0400827 f = fdget(fd);
828 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700829 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400830 if (f.file->f_mode & FMODE_PREAD)
831 ret = vfs_readv(f.file, vec, vlen, &pos);
832 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700833 }
834
835 if (ret > 0)
836 add_rchar(current, ret);
837 inc_syscr(current);
838 return ret;
839}
840
841SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
Linus Torvalds601cc112009-04-03 08:03:22 -0700842 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700843{
Linus Torvalds601cc112009-04-03 08:03:22 -0700844 loff_t pos = pos_from_hilo(pos_h, pos_l);
Al Viro2903ff02012-08-28 12:52:22 -0400845 struct fd f;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700846 ssize_t ret = -EBADF;
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700847
848 if (pos < 0)
849 return -EINVAL;
850
Al Viro2903ff02012-08-28 12:52:22 -0400851 f = fdget(fd);
852 if (f.file) {
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700853 ret = -ESPIPE;
Al Viro2903ff02012-08-28 12:52:22 -0400854 if (f.file->f_mode & FMODE_PWRITE)
855 ret = vfs_writev(f.file, vec, vlen, &pos);
856 fdput(f);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700857 }
858
859 if (ret > 0)
860 add_wchar(current, ret);
861 inc_syscw(current);
862 return ret;
863}
864
Catalin Marinas8f9c0112012-09-19 12:01:52 +0100865ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, size_t count,
866 loff_t max)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867{
Al Viro2903ff02012-08-28 12:52:22 -0400868 struct fd in, out;
869 struct inode *in_inode, *out_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 loff_t pos;
871 ssize_t retval;
Al Viro2903ff02012-08-28 12:52:22 -0400872 int fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873
874 /*
875 * Get input file, and verify that it is ok..
876 */
877 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -0400878 in = fdget(in_fd);
879 if (!in.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 goto out;
Al Viro2903ff02012-08-28 12:52:22 -0400881 if (!(in.file->f_mode & FMODE_READ))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 goto fput_in;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 retval = -ESPIPE;
884 if (!ppos)
Al Viro2903ff02012-08-28 12:52:22 -0400885 ppos = &in.file->f_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 else
Al Viro2903ff02012-08-28 12:52:22 -0400887 if (!(in.file->f_mode & FMODE_PREAD))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 goto fput_in;
Al Viro2903ff02012-08-28 12:52:22 -0400889 retval = rw_verify_area(READ, in.file, ppos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800890 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 goto fput_in;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800892 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 /*
895 * Get output file, and verify that it is ok..
896 */
897 retval = -EBADF;
Al Viro2903ff02012-08-28 12:52:22 -0400898 out = fdget(out_fd);
899 if (!out.file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 goto fput_in;
Al Viro2903ff02012-08-28 12:52:22 -0400901 if (!(out.file->f_mode & FMODE_WRITE))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 goto fput_out;
903 retval = -EINVAL;
Al Viro2903ff02012-08-28 12:52:22 -0400904 in_inode = in.file->f_path.dentry->d_inode;
905 out_inode = out.file->f_path.dentry->d_inode;
906 retval = rw_verify_area(WRITE, out.file, &out.file->f_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800907 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 goto fput_out;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800909 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 if (!max)
912 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
913
914 pos = *ppos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 if (unlikely(pos + count > max)) {
916 retval = -EOVERFLOW;
917 if (pos >= max)
918 goto fput_out;
919 count = max - pos;
920 }
921
Jens Axboed96e6e72007-06-11 12:18:52 +0200922 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200923#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +0200924 /*
925 * We need to debate whether we can enable this or not. The
926 * man page documents EAGAIN return for the output at least,
927 * and the application is arguably buggy if it doesn't expect
928 * EAGAIN on a non-blocking file descriptor.
929 */
Al Viro2903ff02012-08-28 12:52:22 -0400930 if (in.file->f_flags & O_NONBLOCK)
Jens Axboed96e6e72007-06-11 12:18:52 +0200931 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200932#endif
Al Viro2903ff02012-08-28 12:52:22 -0400933 retval = do_splice_direct(in.file, ppos, out.file, count, fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934
935 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800936 add_rchar(current, retval);
937 add_wchar(current, retval);
Scott Wolchoka68c2f12012-12-20 15:05:52 -0800938 fsnotify_access(in.file);
939 fsnotify_modify(out.file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800942 inc_syscr(current);
943 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 if (*ppos > max)
945 retval = -EOVERFLOW;
946
947fput_out:
Al Viro2903ff02012-08-28 12:52:22 -0400948 fdput(out);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949fput_in:
Al Viro2903ff02012-08-28 12:52:22 -0400950 fdput(in);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951out:
952 return retval;
953}
954
Heiko Carstens002c8972009-01-14 14:14:18 +0100955SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956{
957 loff_t pos;
958 off_t off;
959 ssize_t ret;
960
961 if (offset) {
962 if (unlikely(get_user(off, offset)))
963 return -EFAULT;
964 pos = off;
965 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
966 if (unlikely(put_user(pos, offset)))
967 return -EFAULT;
968 return ret;
969 }
970
971 return do_sendfile(out_fd, in_fd, NULL, count, 0);
972}
973
Heiko Carstens002c8972009-01-14 14:14:18 +0100974SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975{
976 loff_t pos;
977 ssize_t ret;
978
979 if (offset) {
980 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
981 return -EFAULT;
982 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
983 if (unlikely(put_user(pos, offset)))
984 return -EFAULT;
985 return ret;
986 }
987
988 return do_sendfile(out_fd, in_fd, NULL, count, 0);
989}