blob: 9d1e76bb9ee147b8236630ba953a53873b1847d4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
12#include <linux/smp_lock.h>
Robert Love0eeca282005-07-12 17:06:03 -040013#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/security.h>
15#include <linux/module.h>
16#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080017#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020018#include <linux/splice.h>
Badari Pulavartyee0b3e62006-09-30 23:28:47 -070019#include "read_write.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21#include <asm/uaccess.h>
22#include <asm/unistd.h>
23
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080024const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070025 .llseek = generic_file_llseek,
Badari Pulavarty543ade12006-09-30 23:28:48 -070026 .read = do_sync_read,
27 .aio_read = generic_file_aio_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020029 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070030};
31
32EXPORT_SYMBOL(generic_ro_fops);
33
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020034/**
35 * generic_file_llseek_unlocked - lockless generic llseek implementation
36 * @file: file structure to seek on
37 * @offset: file offset to seek to
38 * @origin: type of seek
39 *
40 * Updates the file offset to the value specified by @offset and @origin.
41 * Locking must be provided by the caller.
42 */
Andi Kleen9465efc2008-06-27 11:05:24 +020043loff_t
44generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -070045{
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 struct inode *inode = file->f_mapping->host;
47
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 switch (origin) {
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020049 case SEEK_END:
50 offset += inode->i_size;
51 break;
52 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -080053 /*
54 * Here we special-case the lseek(fd, 0, SEEK_CUR)
55 * position-querying operation. Avoid rewriting the "same"
56 * f_pos value back to the file because a concurrent read(),
57 * write() or lseek() might have altered it
58 */
59 if (offset == 0)
60 return file->f_pos;
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020061 offset += file->f_pos;
62 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020064
65 if (offset < 0 || offset > inode->i_sb->s_maxbytes)
66 return -EINVAL;
67
68 /* Special lock needed here? */
69 if (offset != file->f_pos) {
70 file->f_pos = offset;
71 file->f_version = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020073
74 return offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075}
Andi Kleen9465efc2008-06-27 11:05:24 +020076EXPORT_SYMBOL(generic_file_llseek_unlocked);
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020078/**
79 * generic_file_llseek - generic llseek implementation for regular files
80 * @file: file structure to seek on
81 * @offset: file offset to seek to
82 * @origin: type of seek
83 *
84 * This is a generic implemenation of ->llseek useable for all normal local
85 * filesystems. It just updates the file offset to the value specified by
86 * @offset and @origin under i_mutex.
87 */
Andi Kleen9465efc2008-06-27 11:05:24 +020088loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089{
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020090 loff_t rval;
91
Andi Kleen9465efc2008-06-27 11:05:24 +020092 mutex_lock(&file->f_dentry->d_inode->i_mutex);
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020093 rval = generic_file_llseek_unlocked(file, offset, origin);
Andi Kleen9465efc2008-06-27 11:05:24 +020094 mutex_unlock(&file->f_dentry->d_inode->i_mutex);
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020095
96 return rval;
Linus Torvalds1da177e2005-04-16 15:20:36 -070097}
Andi Kleen9465efc2008-06-27 11:05:24 +020098EXPORT_SYMBOL(generic_file_llseek);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
100loff_t no_llseek(struct file *file, loff_t offset, int origin)
101{
102 return -ESPIPE;
103}
104EXPORT_SYMBOL(no_llseek);
105
106loff_t default_llseek(struct file *file, loff_t offset, int origin)
107{
David Sterba16abef02008-04-22 15:09:22 +0200108 loff_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110 lock_kernel();
111 switch (origin) {
Chris Snook7b8e8922007-05-08 00:24:13 -0700112 case SEEK_END:
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800113 offset += i_size_read(file->f_path.dentry->d_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700115 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800116 if (offset == 0) {
117 retval = file->f_pos;
118 goto out;
119 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 offset += file->f_pos;
121 }
122 retval = -EINVAL;
123 if (offset >= 0) {
124 if (offset != file->f_pos) {
125 file->f_pos = offset;
126 file->f_version = 0;
127 }
128 retval = offset;
129 }
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800130out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 unlock_kernel();
132 return retval;
133}
134EXPORT_SYMBOL(default_llseek);
135
136loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
137{
138 loff_t (*fn)(struct file *, loff_t, int);
139
140 fn = no_llseek;
141 if (file->f_mode & FMODE_LSEEK) {
142 fn = default_llseek;
143 if (file->f_op && file->f_op->llseek)
144 fn = file->f_op->llseek;
145 }
146 return fn(file, offset, origin);
147}
148EXPORT_SYMBOL(vfs_llseek);
149
Heiko Carstens003d7ab2009-01-14 14:14:21 +0100150SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151{
152 off_t retval;
153 struct file * file;
154 int fput_needed;
155
156 retval = -EBADF;
157 file = fget_light(fd, &fput_needed);
158 if (!file)
159 goto bad;
160
161 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700162 if (origin <= SEEK_MAX) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 loff_t res = vfs_llseek(file, offset, origin);
164 retval = res;
165 if (res != (loff_t)retval)
166 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
167 }
168 fput_light(file, fput_needed);
169bad:
170 return retval;
171}
172
173#ifdef __ARCH_WANT_SYS_LLSEEK
Heiko Carstens003d7ab2009-01-14 14:14:21 +0100174SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
175 unsigned long, offset_low, loff_t __user *, result,
176 unsigned int, origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
178 int retval;
179 struct file * file;
180 loff_t offset;
181 int fput_needed;
182
183 retval = -EBADF;
184 file = fget_light(fd, &fput_needed);
185 if (!file)
186 goto bad;
187
188 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700189 if (origin > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 goto out_putf;
191
192 offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
193 origin);
194
195 retval = (int)offset;
196 if (offset >= 0) {
197 retval = -EFAULT;
198 if (!copy_to_user(result, &offset, sizeof(offset)))
199 retval = 0;
200 }
201out_putf:
202 fput_light(file, fput_needed);
203bad:
204 return retval;
205}
206#endif
207
Linus Torvaldse28cc712006-01-04 16:20:40 -0800208/*
209 * rw_verify_area doesn't like huge counts. We limit
210 * them to something that fits in "int" so that others
211 * won't have to do range checks all the time.
212 */
213#define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
215int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
216{
217 struct inode *inode;
218 loff_t pos;
James Morrisc43e2592008-01-12 22:05:48 +1100219 int retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Eric Dumazet163da952007-02-12 00:52:24 -0800221 inode = file->f_path.dentry->d_inode;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800222 if (unlikely((ssize_t) count < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100223 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 pos = *ppos;
225 if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100226 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
Pavel Emelyanova16877c2007-10-01 14:41:11 -0700228 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
James Morrisc43e2592008-01-12 22:05:48 +1100229 retval = locks_mandatory_area(
Linus Torvaldse28cc712006-01-04 16:20:40 -0800230 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
231 inode, file, pos, count);
232 if (retval < 0)
233 return retval;
234 }
James Morrisc43e2592008-01-12 22:05:48 +1100235 retval = security_file_permission(file,
236 read_write == READ ? MAY_READ : MAY_WRITE);
237 if (retval)
238 return retval;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800239 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240}
241
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700242static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
243{
244 set_current_state(TASK_UNINTERRUPTIBLE);
245 if (!kiocbIsKicked(iocb))
246 schedule();
247 else
248 kiocbClearKicked(iocb);
249 __set_current_state(TASK_RUNNING);
250}
251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
253{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700254 struct iovec iov = { .iov_base = buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 struct kiocb kiocb;
256 ssize_t ret;
257
258 init_sync_kiocb(&kiocb, filp);
259 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700260 kiocb.ki_left = len;
261
262 for (;;) {
263 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
264 if (ret != -EIOCBRETRY)
265 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700266 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700267 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700268
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 if (-EIOCBQUEUED == ret)
270 ret = wait_on_sync_kiocb(&kiocb);
271 *ppos = kiocb.ki_pos;
272 return ret;
273}
274
275EXPORT_SYMBOL(do_sync_read);
276
277ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
278{
279 ssize_t ret;
280
281 if (!(file->f_mode & FMODE_READ))
282 return -EBADF;
283 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
284 return -EINVAL;
285 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
286 return -EFAULT;
287
288 ret = rw_verify_area(READ, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800289 if (ret >= 0) {
290 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100291 if (file->f_op->read)
292 ret = file->f_op->read(file, buf, count, pos);
293 else
294 ret = do_sync_read(file, buf, count, pos);
295 if (ret > 0) {
296 fsnotify_access(file->f_path.dentry);
297 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 }
James Morrisc43e2592008-01-12 22:05:48 +1100299 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 }
301
302 return ret;
303}
304
305EXPORT_SYMBOL(vfs_read);
306
307ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
308{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700309 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 struct kiocb kiocb;
311 ssize_t ret;
312
313 init_sync_kiocb(&kiocb, filp);
314 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700315 kiocb.ki_left = len;
316
317 for (;;) {
318 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
319 if (ret != -EIOCBRETRY)
320 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700321 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700322 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700323
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 if (-EIOCBQUEUED == ret)
325 ret = wait_on_sync_kiocb(&kiocb);
326 *ppos = kiocb.ki_pos;
327 return ret;
328}
329
330EXPORT_SYMBOL(do_sync_write);
331
332ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
333{
334 ssize_t ret;
335
336 if (!(file->f_mode & FMODE_WRITE))
337 return -EBADF;
338 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
339 return -EINVAL;
340 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
341 return -EFAULT;
342
343 ret = rw_verify_area(WRITE, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800344 if (ret >= 0) {
345 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100346 if (file->f_op->write)
347 ret = file->f_op->write(file, buf, count, pos);
348 else
349 ret = do_sync_write(file, buf, count, pos);
350 if (ret > 0) {
351 fsnotify_modify(file->f_path.dentry);
352 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 }
James Morrisc43e2592008-01-12 22:05:48 +1100354 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 }
356
357 return ret;
358}
359
360EXPORT_SYMBOL(vfs_write);
361
362static inline loff_t file_pos_read(struct file *file)
363{
364 return file->f_pos;
365}
366
367static inline void file_pos_write(struct file *file, loff_t pos)
368{
369 file->f_pos = pos;
370}
371
Heiko Carstens3cdad422009-01-14 14:14:22 +0100372SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 struct file *file;
375 ssize_t ret = -EBADF;
376 int fput_needed;
377
378 file = fget_light(fd, &fput_needed);
379 if (file) {
380 loff_t pos = file_pos_read(file);
381 ret = vfs_read(file, buf, count, &pos);
382 file_pos_write(file, pos);
383 fput_light(file, fput_needed);
384 }
385
386 return ret;
387}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
Heiko Carstens3cdad422009-01-14 14:14:22 +0100389SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
390 size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391{
392 struct file *file;
393 ssize_t ret = -EBADF;
394 int fput_needed;
395
396 file = fget_light(fd, &fput_needed);
397 if (file) {
398 loff_t pos = file_pos_read(file);
399 ret = vfs_write(file, buf, count, &pos);
400 file_pos_write(file, pos);
401 fput_light(file, fput_needed);
402 }
403
404 return ret;
405}
406
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100407SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
408 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409{
410 struct file *file;
411 ssize_t ret = -EBADF;
412 int fput_needed;
413
414 if (pos < 0)
415 return -EINVAL;
416
417 file = fget_light(fd, &fput_needed);
418 if (file) {
419 ret = -ESPIPE;
420 if (file->f_mode & FMODE_PREAD)
421 ret = vfs_read(file, buf, count, &pos);
422 fput_light(file, fput_needed);
423 }
424
425 return ret;
426}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100427#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
428asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
429{
430 return SYSC_pread64((unsigned int) fd, (char __user *) buf,
431 (size_t) count, pos);
432}
433SYSCALL_ALIAS(sys_pread64, SyS_pread64);
434#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100436SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
437 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438{
439 struct file *file;
440 ssize_t ret = -EBADF;
441 int fput_needed;
442
443 if (pos < 0)
444 return -EINVAL;
445
446 file = fget_light(fd, &fput_needed);
447 if (file) {
448 ret = -ESPIPE;
449 if (file->f_mode & FMODE_PWRITE)
450 ret = vfs_write(file, buf, count, &pos);
451 fput_light(file, fput_needed);
452 }
453
454 return ret;
455}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100456#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
457asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
458{
459 return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
460 (size_t) count, pos);
461}
462SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
463#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
465/*
466 * Reduce an iovec's length in-place. Return the resulting number of segments
467 */
468unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
469{
470 unsigned long seg = 0;
471 size_t len = 0;
472
473 while (seg < nr_segs) {
474 seg++;
475 if (len + iov->iov_len >= to) {
476 iov->iov_len = to - len;
477 break;
478 }
479 len += iov->iov_len;
480 iov++;
481 }
482 return seg;
483}
Eric Sandeen19295522008-01-28 23:58:27 -0500484EXPORT_SYMBOL(iov_shorten);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700486ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
487 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
488{
489 struct kiocb kiocb;
490 ssize_t ret;
491
492 init_sync_kiocb(&kiocb, filp);
493 kiocb.ki_pos = *ppos;
494 kiocb.ki_left = len;
495 kiocb.ki_nbytes = len;
496
497 for (;;) {
498 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
499 if (ret != -EIOCBRETRY)
500 break;
501 wait_on_retry_sync_kiocb(&kiocb);
502 }
503
504 if (ret == -EIOCBQUEUED)
505 ret = wait_on_sync_kiocb(&kiocb);
506 *ppos = kiocb.ki_pos;
507 return ret;
508}
509
510/* Do it by hand, with file-ops */
511ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
512 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
513{
514 struct iovec *vector = iov;
515 ssize_t ret = 0;
516
517 while (nr_segs > 0) {
518 void __user *base;
519 size_t len;
520 ssize_t nr;
521
522 base = vector->iov_base;
523 len = vector->iov_len;
524 vector++;
525 nr_segs--;
526
527 nr = fn(filp, base, len, ppos);
528
529 if (nr < 0) {
530 if (!ret)
531 ret = nr;
532 break;
533 }
534 ret += nr;
535 if (nr != len)
536 break;
537 }
538
539 return ret;
540}
541
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542/* A write operation does a read from user space and vice versa */
543#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
544
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700545ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
546 unsigned long nr_segs, unsigned long fast_segs,
547 struct iovec *fast_pointer,
548 struct iovec **ret_pointer)
549 {
550 unsigned long seg;
551 ssize_t ret;
552 struct iovec *iov = fast_pointer;
553
554 /*
555 * SuS says "The readv() function *may* fail if the iovcnt argument
556 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
557 * traditionally returned zero for zero segments, so...
558 */
559 if (nr_segs == 0) {
560 ret = 0;
561 goto out;
562 }
563
564 /*
565 * First get the "struct iovec" from user memory and
566 * verify all the pointers
567 */
568 if (nr_segs > UIO_MAXIOV) {
569 ret = -EINVAL;
570 goto out;
571 }
572 if (nr_segs > fast_segs) {
573 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
574 if (iov == NULL) {
575 ret = -ENOMEM;
576 goto out;
577 }
578 }
579 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
580 ret = -EFAULT;
581 goto out;
582 }
583
584 /*
585 * According to the Single Unix Specification we should return EINVAL
586 * if an element length is < 0 when cast to ssize_t or if the
587 * total length would overflow the ssize_t return value of the
588 * system call.
589 */
590 ret = 0;
591 for (seg = 0; seg < nr_segs; seg++) {
592 void __user *buf = iov[seg].iov_base;
593 ssize_t len = (ssize_t)iov[seg].iov_len;
594
595 /* see if we we're about to use an invalid len or if
596 * it's about to overflow ssize_t */
597 if (len < 0 || (ret + len < ret)) {
598 ret = -EINVAL;
599 goto out;
600 }
601 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
602 ret = -EFAULT;
603 goto out;
604 }
605
606 ret += len;
607 }
608out:
609 *ret_pointer = iov;
610 return ret;
611}
612
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613static ssize_t do_readv_writev(int type, struct file *file,
614 const struct iovec __user * uvector,
615 unsigned long nr_segs, loff_t *pos)
616{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 size_t tot_len;
618 struct iovec iovstack[UIO_FASTIOV];
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700619 struct iovec *iov = iovstack;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 io_fn_t fn;
622 iov_fn_t fnv;
623
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700624 if (!file->f_op) {
625 ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 goto out;
627 }
628
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700629 ret = rw_copy_check_uvector(type, uvector, nr_segs,
630 ARRAY_SIZE(iovstack), iovstack, &iov);
631 if (ret <= 0)
632 goto out;
633
634 tot_len = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 ret = rw_verify_area(type, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800636 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 goto out;
638
639 fnv = NULL;
640 if (type == READ) {
641 fn = file->f_op->read;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700642 fnv = file->f_op->aio_read;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 } else {
644 fn = (io_fn_t)file->f_op->write;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700645 fnv = file->f_op->aio_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 }
647
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700648 if (fnv)
649 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
650 pos, fnv);
651 else
652 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654out:
655 if (iov != iovstack)
656 kfree(iov);
Robert Love0eeca282005-07-12 17:06:03 -0400657 if ((ret + (type == READ)) > 0) {
658 if (type == READ)
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800659 fsnotify_access(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400660 else
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800661 fsnotify_modify(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400662 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664}
665
666ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
667 unsigned long vlen, loff_t *pos)
668{
669 if (!(file->f_mode & FMODE_READ))
670 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700671 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 return -EINVAL;
673
674 return do_readv_writev(READ, file, vec, vlen, pos);
675}
676
677EXPORT_SYMBOL(vfs_readv);
678
679ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
680 unsigned long vlen, loff_t *pos)
681{
682 if (!(file->f_mode & FMODE_WRITE))
683 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700684 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 return -EINVAL;
686
687 return do_readv_writev(WRITE, file, vec, vlen, pos);
688}
689
690EXPORT_SYMBOL(vfs_writev);
691
Heiko Carstens3cdad422009-01-14 14:14:22 +0100692SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
693 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694{
695 struct file *file;
696 ssize_t ret = -EBADF;
697 int fput_needed;
698
699 file = fget_light(fd, &fput_needed);
700 if (file) {
701 loff_t pos = file_pos_read(file);
702 ret = vfs_readv(file, vec, vlen, &pos);
703 file_pos_write(file, pos);
704 fput_light(file, fput_needed);
705 }
706
707 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800708 add_rchar(current, ret);
709 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 return ret;
711}
712
Heiko Carstens3cdad422009-01-14 14:14:22 +0100713SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
714 unsigned long, vlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715{
716 struct file *file;
717 ssize_t ret = -EBADF;
718 int fput_needed;
719
720 file = fget_light(fd, &fput_needed);
721 if (file) {
722 loff_t pos = file_pos_read(file);
723 ret = vfs_writev(file, vec, vlen, &pos);
724 file_pos_write(file, pos);
725 fput_light(file, fput_needed);
726 }
727
728 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800729 add_wchar(current, ret);
730 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 return ret;
732}
733
Linus Torvalds601cc112009-04-03 08:03:22 -0700734static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700735{
Linus Torvalds601cc112009-04-03 08:03:22 -0700736#define HALF_LONG_BITS (BITS_PER_LONG / 2)
737 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
738}
739
740SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
741 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
742{
743 loff_t pos = pos_from_hilo(pos_h, pos_l);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700744 struct file *file;
745 ssize_t ret = -EBADF;
746 int fput_needed;
747
748 if (pos < 0)
749 return -EINVAL;
750
751 file = fget_light(fd, &fput_needed);
752 if (file) {
753 ret = -ESPIPE;
754 if (file->f_mode & FMODE_PREAD)
755 ret = vfs_readv(file, vec, vlen, &pos);
756 fput_light(file, fput_needed);
757 }
758
759 if (ret > 0)
760 add_rchar(current, ret);
761 inc_syscr(current);
762 return ret;
763}
764
765SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
Linus Torvalds601cc112009-04-03 08:03:22 -0700766 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700767{
Linus Torvalds601cc112009-04-03 08:03:22 -0700768 loff_t pos = pos_from_hilo(pos_h, pos_l);
Gerd Hoffmannf3554f42009-04-02 16:59:23 -0700769 struct file *file;
770 ssize_t ret = -EBADF;
771 int fput_needed;
772
773 if (pos < 0)
774 return -EINVAL;
775
776 file = fget_light(fd, &fput_needed);
777 if (file) {
778 ret = -ESPIPE;
779 if (file->f_mode & FMODE_PWRITE)
780 ret = vfs_writev(file, vec, vlen, &pos);
781 fput_light(file, fput_needed);
782 }
783
784 if (ret > 0)
785 add_wchar(current, ret);
786 inc_syscw(current);
787 return ret;
788}
789
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
791 size_t count, loff_t max)
792{
793 struct file * in_file, * out_file;
794 struct inode * in_inode, * out_inode;
795 loff_t pos;
796 ssize_t retval;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200797 int fput_needed_in, fput_needed_out, fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
799 /*
800 * Get input file, and verify that it is ok..
801 */
802 retval = -EBADF;
803 in_file = fget_light(in_fd, &fput_needed_in);
804 if (!in_file)
805 goto out;
806 if (!(in_file->f_mode & FMODE_READ))
807 goto fput_in;
808 retval = -EINVAL;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800809 in_inode = in_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 if (!in_inode)
811 goto fput_in;
Jens Axboed96e6e72007-06-11 12:18:52 +0200812 if (!in_file->f_op || !in_file->f_op->splice_read)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 goto fput_in;
814 retval = -ESPIPE;
815 if (!ppos)
816 ppos = &in_file->f_pos;
817 else
818 if (!(in_file->f_mode & FMODE_PREAD))
819 goto fput_in;
820 retval = rw_verify_area(READ, in_file, ppos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800821 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 goto fput_in;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800823 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 /*
826 * Get output file, and verify that it is ok..
827 */
828 retval = -EBADF;
829 out_file = fget_light(out_fd, &fput_needed_out);
830 if (!out_file)
831 goto fput_in;
832 if (!(out_file->f_mode & FMODE_WRITE))
833 goto fput_out;
834 retval = -EINVAL;
835 if (!out_file->f_op || !out_file->f_op->sendpage)
836 goto fput_out;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800837 out_inode = out_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800839 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 goto fput_out;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800841 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 if (!max)
844 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
845
846 pos = *ppos;
847 retval = -EINVAL;
848 if (unlikely(pos < 0))
849 goto fput_out;
850 if (unlikely(pos + count > max)) {
851 retval = -EOVERFLOW;
852 if (pos >= max)
853 goto fput_out;
854 count = max - pos;
855 }
856
Jens Axboed96e6e72007-06-11 12:18:52 +0200857 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200858#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +0200859 /*
860 * We need to debate whether we can enable this or not. The
861 * man page documents EAGAIN return for the output at least,
862 * and the application is arguably buggy if it doesn't expect
863 * EAGAIN on a non-blocking file descriptor.
864 */
865 if (in_file->f_flags & O_NONBLOCK)
866 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200867#endif
Jens Axboed96e6e72007-06-11 12:18:52 +0200868 retval = do_splice_direct(in_file, ppos, out_file, count, fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869
870 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800871 add_rchar(current, retval);
872 add_wchar(current, retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800875 inc_syscr(current);
876 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 if (*ppos > max)
878 retval = -EOVERFLOW;
879
880fput_out:
881 fput_light(out_file, fput_needed_out);
882fput_in:
883 fput_light(in_file, fput_needed_in);
884out:
885 return retval;
886}
887
Heiko Carstens002c8972009-01-14 14:14:18 +0100888SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889{
890 loff_t pos;
891 off_t off;
892 ssize_t ret;
893
894 if (offset) {
895 if (unlikely(get_user(off, offset)))
896 return -EFAULT;
897 pos = off;
898 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
899 if (unlikely(put_user(pos, offset)))
900 return -EFAULT;
901 return ret;
902 }
903
904 return do_sendfile(out_fd, in_fd, NULL, count, 0);
905}
906
Heiko Carstens002c8972009-01-14 14:14:18 +0100907SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908{
909 loff_t pos;
910 ssize_t ret;
911
912 if (offset) {
913 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
914 return -EFAULT;
915 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
916 if (unlikely(put_user(pos, offset)))
917 return -EFAULT;
918 return ret;
919 }
920
921 return do_sendfile(out_fd, in_fd, NULL, count, 0);
922}