blob: ea1f94cc722e0c942111b9849cedf8b1aef1d86a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
12#include <linux/smp_lock.h>
Robert Love0eeca282005-07-12 17:06:03 -040013#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/security.h>
15#include <linux/module.h>
16#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080017#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020018#include <linux/splice.h>
Badari Pulavartyee0b3e62006-09-30 23:28:47 -070019#include "read_write.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21#include <asm/uaccess.h>
22#include <asm/unistd.h>
23
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080024const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070025 .llseek = generic_file_llseek,
Badari Pulavarty543ade12006-09-30 23:28:48 -070026 .read = do_sync_read,
27 .aio_read = generic_file_aio_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020029 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070030};
31
32EXPORT_SYMBOL(generic_ro_fops);
33
34loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
35{
36 long long retval;
37 struct inode *inode = file->f_mapping->host;
38
Jes Sorensen1b1dcc12006-01-09 15:59:24 -080039 mutex_lock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 switch (origin) {
Chris Snook7b8e8922007-05-08 00:24:13 -070041 case SEEK_END:
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 offset += inode->i_size;
43 break;
Chris Snook7b8e8922007-05-08 00:24:13 -070044 case SEEK_CUR:
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 offset += file->f_pos;
46 }
47 retval = -EINVAL;
48 if (offset>=0 && offset<=inode->i_sb->s_maxbytes) {
49 if (offset != file->f_pos) {
50 file->f_pos = offset;
51 file->f_version = 0;
52 }
53 retval = offset;
54 }
Jes Sorensen1b1dcc12006-01-09 15:59:24 -080055 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 return retval;
57}
58
59EXPORT_SYMBOL(generic_file_llseek);
60
61loff_t remote_llseek(struct file *file, loff_t offset, int origin)
62{
63 long long retval;
64
65 lock_kernel();
66 switch (origin) {
Chris Snook7b8e8922007-05-08 00:24:13 -070067 case SEEK_END:
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -080068 offset += i_size_read(file->f_path.dentry->d_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 break;
Chris Snook7b8e8922007-05-08 00:24:13 -070070 case SEEK_CUR:
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 offset += file->f_pos;
72 }
73 retval = -EINVAL;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -080074 if (offset>=0 && offset<=file->f_path.dentry->d_inode->i_sb->s_maxbytes) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070075 if (offset != file->f_pos) {
76 file->f_pos = offset;
77 file->f_version = 0;
78 }
79 retval = offset;
80 }
81 unlock_kernel();
82 return retval;
83}
84EXPORT_SYMBOL(remote_llseek);
85
86loff_t no_llseek(struct file *file, loff_t offset, int origin)
87{
88 return -ESPIPE;
89}
90EXPORT_SYMBOL(no_llseek);
91
92loff_t default_llseek(struct file *file, loff_t offset, int origin)
93{
94 long long retval;
95
96 lock_kernel();
97 switch (origin) {
Chris Snook7b8e8922007-05-08 00:24:13 -070098 case SEEK_END:
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -080099 offset += i_size_read(file->f_path.dentry->d_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700101 case SEEK_CUR:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 offset += file->f_pos;
103 }
104 retval = -EINVAL;
105 if (offset >= 0) {
106 if (offset != file->f_pos) {
107 file->f_pos = offset;
108 file->f_version = 0;
109 }
110 retval = offset;
111 }
112 unlock_kernel();
113 return retval;
114}
115EXPORT_SYMBOL(default_llseek);
116
117loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
118{
119 loff_t (*fn)(struct file *, loff_t, int);
120
121 fn = no_llseek;
122 if (file->f_mode & FMODE_LSEEK) {
123 fn = default_llseek;
124 if (file->f_op && file->f_op->llseek)
125 fn = file->f_op->llseek;
126 }
127 return fn(file, offset, origin);
128}
129EXPORT_SYMBOL(vfs_llseek);
130
131asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
132{
133 off_t retval;
134 struct file * file;
135 int fput_needed;
136
137 retval = -EBADF;
138 file = fget_light(fd, &fput_needed);
139 if (!file)
140 goto bad;
141
142 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700143 if (origin <= SEEK_MAX) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 loff_t res = vfs_llseek(file, offset, origin);
145 retval = res;
146 if (res != (loff_t)retval)
147 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
148 }
149 fput_light(file, fput_needed);
150bad:
151 return retval;
152}
153
154#ifdef __ARCH_WANT_SYS_LLSEEK
155asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
156 unsigned long offset_low, loff_t __user * result,
157 unsigned int origin)
158{
159 int retval;
160 struct file * file;
161 loff_t offset;
162 int fput_needed;
163
164 retval = -EBADF;
165 file = fget_light(fd, &fput_needed);
166 if (!file)
167 goto bad;
168
169 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700170 if (origin > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 goto out_putf;
172
173 offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
174 origin);
175
176 retval = (int)offset;
177 if (offset >= 0) {
178 retval = -EFAULT;
179 if (!copy_to_user(result, &offset, sizeof(offset)))
180 retval = 0;
181 }
182out_putf:
183 fput_light(file, fput_needed);
184bad:
185 return retval;
186}
187#endif
188
Linus Torvaldse28cc712006-01-04 16:20:40 -0800189/*
190 * rw_verify_area doesn't like huge counts. We limit
191 * them to something that fits in "int" so that others
192 * won't have to do range checks all the time.
193 */
194#define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
196int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
197{
198 struct inode *inode;
199 loff_t pos;
200
Eric Dumazet163da952007-02-12 00:52:24 -0800201 inode = file->f_path.dentry->d_inode;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800202 if (unlikely((ssize_t) count < 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 goto Einval;
204 pos = *ppos;
205 if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
206 goto Einval;
207
Pavel Emelyanova16877c2007-10-01 14:41:11 -0700208 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
Linus Torvaldse28cc712006-01-04 16:20:40 -0800209 int retval = locks_mandatory_area(
210 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
211 inode, file, pos, count);
212 if (retval < 0)
213 return retval;
214 }
215 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217Einval:
218 return -EINVAL;
219}
220
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700221static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
222{
223 set_current_state(TASK_UNINTERRUPTIBLE);
224 if (!kiocbIsKicked(iocb))
225 schedule();
226 else
227 kiocbClearKicked(iocb);
228 __set_current_state(TASK_RUNNING);
229}
230
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
232{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700233 struct iovec iov = { .iov_base = buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 struct kiocb kiocb;
235 ssize_t ret;
236
237 init_sync_kiocb(&kiocb, filp);
238 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700239 kiocb.ki_left = len;
240
241 for (;;) {
242 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
243 if (ret != -EIOCBRETRY)
244 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700245 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700246 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700247
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 if (-EIOCBQUEUED == ret)
249 ret = wait_on_sync_kiocb(&kiocb);
250 *ppos = kiocb.ki_pos;
251 return ret;
252}
253
254EXPORT_SYMBOL(do_sync_read);
255
256ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
257{
258 ssize_t ret;
259
260 if (!(file->f_mode & FMODE_READ))
261 return -EBADF;
262 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
263 return -EINVAL;
264 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
265 return -EFAULT;
266
267 ret = rw_verify_area(READ, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800268 if (ret >= 0) {
269 count = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 ret = security_file_permission (file, MAY_READ);
271 if (!ret) {
272 if (file->f_op->read)
273 ret = file->f_op->read(file, buf, count, pos);
274 else
275 ret = do_sync_read(file, buf, count, pos);
276 if (ret > 0) {
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800277 fsnotify_access(file->f_path.dentry);
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800278 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 }
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800280 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 }
282 }
283
284 return ret;
285}
286
287EXPORT_SYMBOL(vfs_read);
288
289ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
290{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700291 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 struct kiocb kiocb;
293 ssize_t ret;
294
295 init_sync_kiocb(&kiocb, filp);
296 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700297 kiocb.ki_left = len;
298
299 for (;;) {
300 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
301 if (ret != -EIOCBRETRY)
302 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700303 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700304 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700305
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 if (-EIOCBQUEUED == ret)
307 ret = wait_on_sync_kiocb(&kiocb);
308 *ppos = kiocb.ki_pos;
309 return ret;
310}
311
312EXPORT_SYMBOL(do_sync_write);
313
314ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
315{
316 ssize_t ret;
317
318 if (!(file->f_mode & FMODE_WRITE))
319 return -EBADF;
320 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
321 return -EINVAL;
322 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
323 return -EFAULT;
324
325 ret = rw_verify_area(WRITE, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800326 if (ret >= 0) {
327 count = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 ret = security_file_permission (file, MAY_WRITE);
329 if (!ret) {
330 if (file->f_op->write)
331 ret = file->f_op->write(file, buf, count, pos);
332 else
333 ret = do_sync_write(file, buf, count, pos);
334 if (ret > 0) {
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800335 fsnotify_modify(file->f_path.dentry);
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800336 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 }
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800338 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 }
340 }
341
342 return ret;
343}
344
345EXPORT_SYMBOL(vfs_write);
346
347static inline loff_t file_pos_read(struct file *file)
348{
349 return file->f_pos;
350}
351
352static inline void file_pos_write(struct file *file, loff_t pos)
353{
354 file->f_pos = pos;
355}
356
357asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count)
358{
359 struct file *file;
360 ssize_t ret = -EBADF;
361 int fput_needed;
362
363 file = fget_light(fd, &fput_needed);
364 if (file) {
365 loff_t pos = file_pos_read(file);
366 ret = vfs_read(file, buf, count, &pos);
367 file_pos_write(file, pos);
368 fput_light(file, fput_needed);
369 }
370
371 return ret;
372}
Arjan van de Vencb51f972007-11-14 17:00:10 -0800373EXPORT_UNUSED_SYMBOL_GPL(sys_read); /* to be deleted for 2.6.25 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374
375asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t count)
376{
377 struct file *file;
378 ssize_t ret = -EBADF;
379 int fput_needed;
380
381 file = fget_light(fd, &fput_needed);
382 if (file) {
383 loff_t pos = file_pos_read(file);
384 ret = vfs_write(file, buf, count, &pos);
385 file_pos_write(file, pos);
386 fput_light(file, fput_needed);
387 }
388
389 return ret;
390}
391
392asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf,
393 size_t count, loff_t pos)
394{
395 struct file *file;
396 ssize_t ret = -EBADF;
397 int fput_needed;
398
399 if (pos < 0)
400 return -EINVAL;
401
402 file = fget_light(fd, &fput_needed);
403 if (file) {
404 ret = -ESPIPE;
405 if (file->f_mode & FMODE_PREAD)
406 ret = vfs_read(file, buf, count, &pos);
407 fput_light(file, fput_needed);
408 }
409
410 return ret;
411}
412
413asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf,
414 size_t count, loff_t pos)
415{
416 struct file *file;
417 ssize_t ret = -EBADF;
418 int fput_needed;
419
420 if (pos < 0)
421 return -EINVAL;
422
423 file = fget_light(fd, &fput_needed);
424 if (file) {
425 ret = -ESPIPE;
426 if (file->f_mode & FMODE_PWRITE)
427 ret = vfs_write(file, buf, count, &pos);
428 fput_light(file, fput_needed);
429 }
430
431 return ret;
432}
433
434/*
435 * Reduce an iovec's length in-place. Return the resulting number of segments
436 */
437unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
438{
439 unsigned long seg = 0;
440 size_t len = 0;
441
442 while (seg < nr_segs) {
443 seg++;
444 if (len + iov->iov_len >= to) {
445 iov->iov_len = to - len;
446 break;
447 }
448 len += iov->iov_len;
449 iov++;
450 }
451 return seg;
452}
453
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700454ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
455 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
456{
457 struct kiocb kiocb;
458 ssize_t ret;
459
460 init_sync_kiocb(&kiocb, filp);
461 kiocb.ki_pos = *ppos;
462 kiocb.ki_left = len;
463 kiocb.ki_nbytes = len;
464
465 for (;;) {
466 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
467 if (ret != -EIOCBRETRY)
468 break;
469 wait_on_retry_sync_kiocb(&kiocb);
470 }
471
472 if (ret == -EIOCBQUEUED)
473 ret = wait_on_sync_kiocb(&kiocb);
474 *ppos = kiocb.ki_pos;
475 return ret;
476}
477
478/* Do it by hand, with file-ops */
479ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
480 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
481{
482 struct iovec *vector = iov;
483 ssize_t ret = 0;
484
485 while (nr_segs > 0) {
486 void __user *base;
487 size_t len;
488 ssize_t nr;
489
490 base = vector->iov_base;
491 len = vector->iov_len;
492 vector++;
493 nr_segs--;
494
495 nr = fn(filp, base, len, ppos);
496
497 if (nr < 0) {
498 if (!ret)
499 ret = nr;
500 break;
501 }
502 ret += nr;
503 if (nr != len)
504 break;
505 }
506
507 return ret;
508}
509
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510/* A write operation does a read from user space and vice versa */
511#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
512
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700513ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
514 unsigned long nr_segs, unsigned long fast_segs,
515 struct iovec *fast_pointer,
516 struct iovec **ret_pointer)
517 {
518 unsigned long seg;
519 ssize_t ret;
520 struct iovec *iov = fast_pointer;
521
522 /*
523 * SuS says "The readv() function *may* fail if the iovcnt argument
524 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
525 * traditionally returned zero for zero segments, so...
526 */
527 if (nr_segs == 0) {
528 ret = 0;
529 goto out;
530 }
531
532 /*
533 * First get the "struct iovec" from user memory and
534 * verify all the pointers
535 */
536 if (nr_segs > UIO_MAXIOV) {
537 ret = -EINVAL;
538 goto out;
539 }
540 if (nr_segs > fast_segs) {
541 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
542 if (iov == NULL) {
543 ret = -ENOMEM;
544 goto out;
545 }
546 }
547 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
548 ret = -EFAULT;
549 goto out;
550 }
551
552 /*
553 * According to the Single Unix Specification we should return EINVAL
554 * if an element length is < 0 when cast to ssize_t or if the
555 * total length would overflow the ssize_t return value of the
556 * system call.
557 */
558 ret = 0;
559 for (seg = 0; seg < nr_segs; seg++) {
560 void __user *buf = iov[seg].iov_base;
561 ssize_t len = (ssize_t)iov[seg].iov_len;
562
563 /* see if we we're about to use an invalid len or if
564 * it's about to overflow ssize_t */
565 if (len < 0 || (ret + len < ret)) {
566 ret = -EINVAL;
567 goto out;
568 }
569 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
570 ret = -EFAULT;
571 goto out;
572 }
573
574 ret += len;
575 }
576out:
577 *ret_pointer = iov;
578 return ret;
579}
580
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581static ssize_t do_readv_writev(int type, struct file *file,
582 const struct iovec __user * uvector,
583 unsigned long nr_segs, loff_t *pos)
584{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 size_t tot_len;
586 struct iovec iovstack[UIO_FASTIOV];
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700587 struct iovec *iov = iovstack;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 io_fn_t fn;
590 iov_fn_t fnv;
591
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700592 if (!file->f_op) {
593 ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 goto out;
595 }
596
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700597 ret = rw_copy_check_uvector(type, uvector, nr_segs,
598 ARRAY_SIZE(iovstack), iovstack, &iov);
599 if (ret <= 0)
600 goto out;
601
602 tot_len = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 ret = rw_verify_area(type, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800604 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 goto out;
Kostik Belousov411b67b2005-09-28 18:21:28 +0300606 ret = security_file_permission(file, type == READ ? MAY_READ : MAY_WRITE);
607 if (ret)
608 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
610 fnv = NULL;
611 if (type == READ) {
612 fn = file->f_op->read;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700613 fnv = file->f_op->aio_read;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 } else {
615 fn = (io_fn_t)file->f_op->write;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700616 fnv = file->f_op->aio_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 }
618
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700619 if (fnv)
620 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
621 pos, fnv);
622 else
623 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625out:
626 if (iov != iovstack)
627 kfree(iov);
Robert Love0eeca282005-07-12 17:06:03 -0400628 if ((ret + (type == READ)) > 0) {
629 if (type == READ)
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800630 fsnotify_access(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400631 else
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800632 fsnotify_modify(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400633 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635}
636
637ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
638 unsigned long vlen, loff_t *pos)
639{
640 if (!(file->f_mode & FMODE_READ))
641 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700642 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 return -EINVAL;
644
645 return do_readv_writev(READ, file, vec, vlen, pos);
646}
647
648EXPORT_SYMBOL(vfs_readv);
649
650ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
651 unsigned long vlen, loff_t *pos)
652{
653 if (!(file->f_mode & FMODE_WRITE))
654 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700655 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 return -EINVAL;
657
658 return do_readv_writev(WRITE, file, vec, vlen, pos);
659}
660
661EXPORT_SYMBOL(vfs_writev);
662
663asmlinkage ssize_t
664sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
665{
666 struct file *file;
667 ssize_t ret = -EBADF;
668 int fput_needed;
669
670 file = fget_light(fd, &fput_needed);
671 if (file) {
672 loff_t pos = file_pos_read(file);
673 ret = vfs_readv(file, vec, vlen, &pos);
674 file_pos_write(file, pos);
675 fput_light(file, fput_needed);
676 }
677
678 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800679 add_rchar(current, ret);
680 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 return ret;
682}
683
684asmlinkage ssize_t
685sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
686{
687 struct file *file;
688 ssize_t ret = -EBADF;
689 int fput_needed;
690
691 file = fget_light(fd, &fput_needed);
692 if (file) {
693 loff_t pos = file_pos_read(file);
694 ret = vfs_writev(file, vec, vlen, &pos);
695 file_pos_write(file, pos);
696 fput_light(file, fput_needed);
697 }
698
699 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800700 add_wchar(current, ret);
701 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 return ret;
703}
704
705static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
706 size_t count, loff_t max)
707{
708 struct file * in_file, * out_file;
709 struct inode * in_inode, * out_inode;
710 loff_t pos;
711 ssize_t retval;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200712 int fput_needed_in, fput_needed_out, fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713
714 /*
715 * Get input file, and verify that it is ok..
716 */
717 retval = -EBADF;
718 in_file = fget_light(in_fd, &fput_needed_in);
719 if (!in_file)
720 goto out;
721 if (!(in_file->f_mode & FMODE_READ))
722 goto fput_in;
723 retval = -EINVAL;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800724 in_inode = in_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 if (!in_inode)
726 goto fput_in;
Jens Axboed96e6e72007-06-11 12:18:52 +0200727 if (!in_file->f_op || !in_file->f_op->splice_read)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 goto fput_in;
729 retval = -ESPIPE;
730 if (!ppos)
731 ppos = &in_file->f_pos;
732 else
733 if (!(in_file->f_mode & FMODE_PREAD))
734 goto fput_in;
735 retval = rw_verify_area(READ, in_file, ppos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800736 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 goto fput_in;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800738 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
740 retval = security_file_permission (in_file, MAY_READ);
741 if (retval)
742 goto fput_in;
743
744 /*
745 * Get output file, and verify that it is ok..
746 */
747 retval = -EBADF;
748 out_file = fget_light(out_fd, &fput_needed_out);
749 if (!out_file)
750 goto fput_in;
751 if (!(out_file->f_mode & FMODE_WRITE))
752 goto fput_out;
753 retval = -EINVAL;
754 if (!out_file->f_op || !out_file->f_op->sendpage)
755 goto fput_out;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800756 out_inode = out_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800758 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 goto fput_out;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800760 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
762 retval = security_file_permission (out_file, MAY_WRITE);
763 if (retval)
764 goto fput_out;
765
766 if (!max)
767 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
768
769 pos = *ppos;
770 retval = -EINVAL;
771 if (unlikely(pos < 0))
772 goto fput_out;
773 if (unlikely(pos + count > max)) {
774 retval = -EOVERFLOW;
775 if (pos >= max)
776 goto fput_out;
777 count = max - pos;
778 }
779
Jens Axboed96e6e72007-06-11 12:18:52 +0200780 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200781#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +0200782 /*
783 * We need to debate whether we can enable this or not. The
784 * man page documents EAGAIN return for the output at least,
785 * and the application is arguably buggy if it doesn't expect
786 * EAGAIN on a non-blocking file descriptor.
787 */
788 if (in_file->f_flags & O_NONBLOCK)
789 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200790#endif
Jens Axboed96e6e72007-06-11 12:18:52 +0200791 retval = do_splice_direct(in_file, ppos, out_file, count, fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
793 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800794 add_rchar(current, retval);
795 add_wchar(current, retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800798 inc_syscr(current);
799 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 if (*ppos > max)
801 retval = -EOVERFLOW;
802
803fput_out:
804 fput_light(out_file, fput_needed_out);
805fput_in:
806 fput_light(in_file, fput_needed_in);
807out:
808 return retval;
809}
810
811asmlinkage ssize_t sys_sendfile(int out_fd, int in_fd, off_t __user *offset, size_t count)
812{
813 loff_t pos;
814 off_t off;
815 ssize_t ret;
816
817 if (offset) {
818 if (unlikely(get_user(off, offset)))
819 return -EFAULT;
820 pos = off;
821 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
822 if (unlikely(put_user(pos, offset)))
823 return -EFAULT;
824 return ret;
825 }
826
827 return do_sendfile(out_fd, in_fd, NULL, count, 0);
828}
829
830asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t __user *offset, size_t count)
831{
832 loff_t pos;
833 ssize_t ret;
834
835 if (offset) {
836 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
837 return -EFAULT;
838 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
839 if (unlikely(put_user(pos, offset)))
840 return -EFAULT;
841 return ret;
842 }
843
844 return do_sendfile(out_fd, in_fd, NULL, count, 0);
845}