blob: 48b4382cc114a0786e5016953cf70b4315a723f2 [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboec2058e02006-04-11 13:56:34 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/pipe_fs_i.h>
24#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe5274f052006-03-30 15:15:30 +020030
31/*
32 * Passed to the actors
33 */
34struct splice_desc {
35 unsigned int len, total_len; /* current and remaining length */
36 unsigned int flags; /* splice flags */
37 struct file *file; /* file to read/write */
38 loff_t pos; /* file position */
39};
40
Jens Axboe83f91352006-04-02 23:05:09 +020041/*
42 * Attempt to steal a page from a pipe buffer. This should perhaps go into
43 * a vm helper function, it's already simplified quite a bit by the
44 * addition of remove_mapping(). If success is returned, the caller may
45 * attempt to reuse this page for another destination.
46 */
Jens Axboe5abc97a2006-03-30 15:16:46 +020047static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
48 struct pipe_buffer *buf)
49{
50 struct page *page = buf->page;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020051 struct address_space *mapping = page_mapping(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020052
53 WARN_ON(!PageLocked(page));
54 WARN_ON(!PageUptodate(page));
55
Jens Axboead8d6f02006-04-02 23:10:32 +020056 /*
57 * At least for ext2 with nobh option, we need to wait on writeback
58 * completing on this page, since we'll remove it from the pagecache.
59 * Otherwise truncate wont wait on the page, allowing the disk
60 * blocks to be reused by someone else before we actually wrote our
61 * data to them. fs corruption ensues.
62 */
63 wait_on_page_writeback(page);
64
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020065 if (PagePrivate(page))
66 try_to_release_page(page, mapping_gfp_mask(mapping));
67
68 if (!remove_mapping(mapping, page))
Jens Axboe5abc97a2006-03-30 15:16:46 +020069 return 1;
70
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020071 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
Jens Axboe5abc97a2006-03-30 15:16:46 +020072 return 0;
73}
74
Jens Axboe5274f052006-03-30 15:15:30 +020075static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
76 struct pipe_buffer *buf)
77{
78 page_cache_release(buf->page);
79 buf->page = NULL;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020080 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
Jens Axboe5274f052006-03-30 15:15:30 +020081}
82
83static void *page_cache_pipe_buf_map(struct file *file,
84 struct pipe_inode_info *info,
85 struct pipe_buffer *buf)
86{
87 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020088 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020089
90 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +020091 lock_page(page);
92
93 /*
94 * Page got truncated/unhashed. This will cause a 0-byte
95 * splice, if this is the first page
96 */
97 if (!page->mapping) {
98 err = -ENODATA;
99 goto error;
100 }
101
102 /*
103 * uh oh, read-error from disk
104 */
105 if (!PageUptodate(page)) {
106 err = -EIO;
107 goto error;
108 }
109
110 /*
111 * page is ok afterall, fall through to mapping
112 */
Jens Axboe5274f052006-03-30 15:15:30 +0200113 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200114 }
115
Jens Axboe49d0b212006-04-10 09:04:41 +0200116 return kmap(page);
117error:
118 unlock_page(page);
119 return ERR_PTR(err);
Jens Axboe5274f052006-03-30 15:15:30 +0200120}
121
122static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
123 struct pipe_buffer *buf)
124{
Jens Axboe5274f052006-03-30 15:15:30 +0200125 kunmap(buf->page);
126}
127
128static struct pipe_buf_operations page_cache_pipe_buf_ops = {
129 .can_merge = 0,
130 .map = page_cache_pipe_buf_map,
131 .unmap = page_cache_pipe_buf_unmap,
132 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200133 .steal = page_cache_pipe_buf_steal,
Jens Axboe5274f052006-03-30 15:15:30 +0200134};
135
Jens Axboe83f91352006-04-02 23:05:09 +0200136/*
137 * Pipe output worker. This sets up our pipe format with the page cache
138 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
139 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200140static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
Jens Axboe5274f052006-03-30 15:15:30 +0200141 int nr_pages, unsigned long offset,
Linus Torvalds29e35092006-04-02 12:46:35 -0700142 unsigned long len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200143{
Jens Axboe5274f052006-03-30 15:15:30 +0200144 int ret, do_wakeup, i;
145
146 ret = 0;
147 do_wakeup = 0;
148 i = 0;
149
Ingo Molnar3a326a22006-04-10 15:18:35 +0200150 if (pipe->inode)
151 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200152
Jens Axboe5274f052006-03-30 15:15:30 +0200153 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200154 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200155 send_sig(SIGPIPE, current, 0);
156 if (!ret)
157 ret = -EPIPE;
158 break;
159 }
160
Jens Axboe6f767b02006-04-11 13:53:56 +0200161 if (pipe->nrbufs < PIPE_BUFFERS) {
162 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200163 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200164 struct page *page = pages[i++];
165 unsigned long this_len;
166
167 this_len = PAGE_CACHE_SIZE - offset;
168 if (this_len > len)
169 this_len = len;
170
171 buf->page = page;
172 buf->offset = offset;
173 buf->len = this_len;
174 buf->ops = &page_cache_pipe_buf_ops;
Jens Axboe6f767b02006-04-11 13:53:56 +0200175 pipe->nrbufs++;
176 if (pipe->inode)
177 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200178
179 ret += this_len;
180 len -= this_len;
181 offset = 0;
182 if (!--nr_pages)
183 break;
184 if (!len)
185 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200186 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200187 continue;
188
189 break;
190 }
191
Linus Torvalds29e35092006-04-02 12:46:35 -0700192 if (flags & SPLICE_F_NONBLOCK) {
193 if (!ret)
194 ret = -EAGAIN;
195 break;
196 }
197
Jens Axboe5274f052006-03-30 15:15:30 +0200198 if (signal_pending(current)) {
199 if (!ret)
200 ret = -ERESTARTSYS;
201 break;
202 }
203
204 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200205 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200206 if (waitqueue_active(&pipe->wait))
207 wake_up_interruptible_sync(&pipe->wait);
208 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200209 do_wakeup = 0;
210 }
211
Ingo Molnar3a326a22006-04-10 15:18:35 +0200212 pipe->waiting_writers++;
213 pipe_wait(pipe);
214 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200215 }
216
Ingo Molnar3a326a22006-04-10 15:18:35 +0200217 if (pipe->inode)
218 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200219
220 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200221 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200222 if (waitqueue_active(&pipe->wait))
223 wake_up_interruptible(&pipe->wait);
224 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200225 }
226
227 while (i < nr_pages)
228 page_cache_release(pages[i++]);
229
230 return ret;
231}
232
Ingo Molnar3a326a22006-04-10 15:18:35 +0200233static int
234__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
235 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200236{
237 struct address_space *mapping = in->f_mapping;
238 unsigned int offset, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200239 struct page *pages[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200240 struct page *page;
Jens Axboe16c523d2006-04-10 09:03:58 +0200241 pgoff_t index;
Jens Axboe7480a902006-04-11 13:52:47 +0200242 int i, error;
Jens Axboe5274f052006-03-30 15:15:30 +0200243
244 index = in->f_pos >> PAGE_CACHE_SHIFT;
245 offset = in->f_pos & ~PAGE_CACHE_MASK;
246 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
247
248 if (nr_pages > PIPE_BUFFERS)
249 nr_pages = PIPE_BUFFERS;
250
251 /*
Jens Axboe0b749ce2006-04-10 09:05:04 +0200252 * initiate read-ahead on this page range. however, don't call into
253 * read-ahead if this is a non-zero offset (we are likely doing small
254 * chunk splice and the page is already there) for a single page.
Jens Axboe5274f052006-03-30 15:15:30 +0200255 */
Jens Axboe0b749ce2006-04-10 09:05:04 +0200256 if (!offset || nr_pages > 1)
257 do_page_cache_readahead(mapping, in, index, nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200258
259 /*
Jens Axboe5274f052006-03-30 15:15:30 +0200260 * now fill in the holes
261 */
Jens Axboe7480a902006-04-11 13:52:47 +0200262 error = 0;
Jens Axboe16c523d2006-04-10 09:03:58 +0200263 for (i = 0; i < nr_pages; i++, index++) {
Jens Axboe7480a902006-04-11 13:52:47 +0200264find_page:
Jens Axboe5274f052006-03-30 15:15:30 +0200265 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200266 * lookup the page for this index
Jens Axboe5274f052006-03-30 15:15:30 +0200267 */
Jens Axboe7480a902006-04-11 13:52:47 +0200268 page = find_get_page(mapping, index);
269 if (!page) {
270 /*
271 * If in nonblock mode then dont block on
272 * readpage (we've kicked readahead so there
273 * will be asynchronous progress):
274 */
275 if (flags & SPLICE_F_NONBLOCK)
276 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200277
Jens Axboe7480a902006-04-11 13:52:47 +0200278 /*
279 * page didn't exist, allocate one
280 */
281 page = page_cache_alloc_cold(mapping);
282 if (!page)
283 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200284
Jens Axboe7480a902006-04-11 13:52:47 +0200285 error = add_to_page_cache_lru(page, mapping, index,
286 mapping_gfp_mask(mapping));
Jens Axboe5274f052006-03-30 15:15:30 +0200287 if (unlikely(error)) {
288 page_cache_release(page);
289 break;
290 }
Jens Axboe7480a902006-04-11 13:52:47 +0200291
292 goto readpage;
Jens Axboe5274f052006-03-30 15:15:30 +0200293 }
Jens Axboe7480a902006-04-11 13:52:47 +0200294
295 /*
296 * If the page isn't uptodate, we may need to start io on it
297 */
298 if (!PageUptodate(page)) {
299 lock_page(page);
300
301 /*
302 * page was truncated, stop here. if this isn't the
303 * first page, we'll just complete what we already
304 * added
305 */
306 if (!page->mapping) {
307 unlock_page(page);
308 page_cache_release(page);
309 break;
310 }
311 /*
312 * page was already under io and is now done, great
313 */
314 if (PageUptodate(page)) {
315 unlock_page(page);
316 goto fill_it;
317 }
318
319readpage:
320 /*
321 * need to read in the page
322 */
323 error = mapping->a_ops->readpage(in, page);
324
325 if (unlikely(error)) {
326 page_cache_release(page);
327 if (error == AOP_TRUNCATED_PAGE)
328 goto find_page;
329 break;
330 }
331 }
332fill_it:
Jens Axboe16c523d2006-04-10 09:03:58 +0200333 pages[i] = page;
Jens Axboe5274f052006-03-30 15:15:30 +0200334 }
335
Jens Axboe16c523d2006-04-10 09:03:58 +0200336 if (i)
337 return move_to_pipe(pipe, pages, i, offset, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200338
Jens Axboe7480a902006-04-11 13:52:47 +0200339 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200340}
341
Jens Axboe83f91352006-04-02 23:05:09 +0200342/**
343 * generic_file_splice_read - splice data from file to a pipe
344 * @in: file to splice from
345 * @pipe: pipe to splice to
346 * @len: number of bytes to splice
347 * @flags: splice modifier flags
348 *
349 * Will read pages from given file and fill them into a pipe.
Jens Axboe83f91352006-04-02 23:05:09 +0200350 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200351ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
Jens Axboe5274f052006-03-30 15:15:30 +0200352 size_t len, unsigned int flags)
353{
354 ssize_t spliced;
355 int ret;
356
357 ret = 0;
358 spliced = 0;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200359
Jens Axboe5274f052006-03-30 15:15:30 +0200360 while (len) {
Linus Torvalds29e35092006-04-02 12:46:35 -0700361 ret = __generic_file_splice_read(in, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200362
363 if (ret <= 0)
364 break;
365
366 in->f_pos += ret;
367 len -= ret;
368 spliced += ret;
Linus Torvalds29e35092006-04-02 12:46:35 -0700369
370 if (!(flags & SPLICE_F_NONBLOCK))
371 continue;
372 ret = -EAGAIN;
373 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200374 }
375
376 if (spliced)
377 return spliced;
378
379 return ret;
380}
381
Jens Axboe059a8f32006-04-02 23:06:05 +0200382EXPORT_SYMBOL(generic_file_splice_read);
383
Jens Axboe5274f052006-03-30 15:15:30 +0200384/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200385 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
386 * using sendpage().
Jens Axboe5274f052006-03-30 15:15:30 +0200387 */
388static int pipe_to_sendpage(struct pipe_inode_info *info,
389 struct pipe_buffer *buf, struct splice_desc *sd)
390{
391 struct file *file = sd->file;
392 loff_t pos = sd->pos;
393 unsigned int offset;
394 ssize_t ret;
395 void *ptr;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200396 int more;
Jens Axboe5274f052006-03-30 15:15:30 +0200397
398 /*
399 * sub-optimal, but we are limited by the pipe ->map. we don't
400 * need a kmap'ed buffer here, we just want to make sure we
401 * have the page pinned if the pipe page originates from the
402 * page cache
403 */
404 ptr = buf->ops->map(file, info, buf);
405 if (IS_ERR(ptr))
406 return PTR_ERR(ptr);
407
408 offset = pos & ~PAGE_CACHE_MASK;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200409 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200410
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200411 ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
Jens Axboe5274f052006-03-30 15:15:30 +0200412
413 buf->ops->unmap(info, buf);
414 if (ret == sd->len)
415 return 0;
416
417 return -EIO;
418}
419
420/*
421 * This is a little more tricky than the file -> pipe splicing. There are
422 * basically three cases:
423 *
424 * - Destination page already exists in the address space and there
425 * are users of it. For that case we have no other option that
426 * copying the data. Tough luck.
427 * - Destination page already exists in the address space, but there
428 * are no users of it. Make sure it's uptodate, then drop it. Fall
429 * through to last case.
430 * - Destination page does not exist, we can add the pipe page to
431 * the page cache and avoid the copy.
432 *
Jens Axboe83f91352006-04-02 23:05:09 +0200433 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
434 * sd->flags), we attempt to migrate pages from the pipe to the output
435 * file address space page cache. This is possible if no one else has
436 * the pipe page referenced outside of the pipe and page cache. If
437 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
438 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200439 */
440static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
441 struct splice_desc *sd)
442{
443 struct file *file = sd->file;
444 struct address_space *mapping = file->f_mapping;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200445 gfp_t gfp_mask = mapping_gfp_mask(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200446 unsigned int offset;
447 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200448 pgoff_t index;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200449 char *src;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200450 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200451
452 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200453 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200454 */
455 src = buf->ops->map(file, info, buf);
456 if (IS_ERR(src))
457 return PTR_ERR(src);
458
459 index = sd->pos >> PAGE_CACHE_SHIFT;
460 offset = sd->pos & ~PAGE_CACHE_MASK;
461
Jens Axboe5274f052006-03-30 15:15:30 +0200462 /*
Jens Axboe5abc97a2006-03-30 15:16:46 +0200463 * reuse buf page, if SPLICE_F_MOVE is set
Jens Axboe5274f052006-03-30 15:15:30 +0200464 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200465 if (sd->flags & SPLICE_F_MOVE) {
Jens Axboe83f91352006-04-02 23:05:09 +0200466 /*
467 * If steal succeeds, buf->page is now pruned from the vm
468 * side (LRU and page cache) and we can reuse it.
469 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200470 if (buf->ops->steal(info, buf))
471 goto find_page;
Jens Axboe5274f052006-03-30 15:15:30 +0200472
Jens Axboe49d0b212006-04-10 09:04:41 +0200473 /*
474 * this will also set the page locked
475 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200476 page = buf->page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200477 if (add_to_page_cache(page, mapping, index, gfp_mask))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200478 goto find_page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200479
480 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
481 lru_cache_add(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200482 } else {
483find_page:
484 ret = -ENOMEM;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200485 page = find_or_create_page(mapping, index, gfp_mask);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200486 if (!page)
Dave Jones9aefe432006-04-10 09:02:40 +0200487 goto out_nomem;
Jens Axboe5274f052006-03-30 15:15:30 +0200488
Jens Axboe5abc97a2006-03-30 15:16:46 +0200489 /*
490 * If the page is uptodate, it is also locked. If it isn't
491 * uptodate, we can mark it uptodate if we are filling the
492 * full page. Otherwise we need to read it in first...
493 */
494 if (!PageUptodate(page)) {
495 if (sd->len < PAGE_CACHE_SIZE) {
496 ret = mapping->a_ops->readpage(file, page);
497 if (unlikely(ret))
498 goto out;
499
500 lock_page(page);
501
502 if (!PageUptodate(page)) {
503 /*
504 * page got invalidated, repeat
505 */
506 if (!page->mapping) {
507 unlock_page(page);
508 page_cache_release(page);
509 goto find_page;
510 }
511 ret = -EIO;
512 goto out;
Jens Axboe5274f052006-03-30 15:15:30 +0200513 }
Jens Axboe5abc97a2006-03-30 15:16:46 +0200514 } else {
515 WARN_ON(!PageLocked(page));
516 SetPageUptodate(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200517 }
Jens Axboe5274f052006-03-30 15:15:30 +0200518 }
519 }
520
521 ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200522 if (ret == AOP_TRUNCATED_PAGE) {
523 page_cache_release(page);
524 goto find_page;
525 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200526 goto out;
527
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200528 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200529 char *dst = kmap_atomic(page, KM_USER0);
530
531 memcpy(dst + offset, src + buf->offset, sd->len);
532 flush_dcache_page(page);
533 kunmap_atomic(dst, KM_USER0);
534 }
Jens Axboe5274f052006-03-30 15:15:30 +0200535
536 ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200537 if (ret == AOP_TRUNCATED_PAGE) {
538 page_cache_release(page);
539 goto find_page;
540 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200541 goto out;
542
Jens Axboec7f21e42006-04-10 09:01:01 +0200543 mark_page_accessed(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200544 balance_dirty_pages_ratelimited(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200545out:
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200546 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200547 page_cache_release(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200548 unlock_page(page);
549 }
Dave Jones9aefe432006-04-10 09:02:40 +0200550out_nomem:
Jens Axboe5274f052006-03-30 15:15:30 +0200551 buf->ops->unmap(info, buf);
552 return ret;
553}
554
555typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
556 struct splice_desc *);
557
Jens Axboe83f91352006-04-02 23:05:09 +0200558/*
559 * Pipe input worker. Most of this logic works like a regular pipe, the
560 * key here is the 'actor' worker passed in that actually moves the data
561 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
562 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200563static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
Jens Axboe5274f052006-03-30 15:15:30 +0200564 size_t len, unsigned int flags,
565 splice_actor *actor)
566{
Jens Axboe5274f052006-03-30 15:15:30 +0200567 int ret, do_wakeup, err;
568 struct splice_desc sd;
569
570 ret = 0;
571 do_wakeup = 0;
572
573 sd.total_len = len;
574 sd.flags = flags;
575 sd.file = out;
576 sd.pos = out->f_pos;
577
Ingo Molnar3a326a22006-04-10 15:18:35 +0200578 if (pipe->inode)
579 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200580
Jens Axboe5274f052006-03-30 15:15:30 +0200581 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200582 if (pipe->nrbufs) {
583 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200584 struct pipe_buf_operations *ops = buf->ops;
585
586 sd.len = buf->len;
587 if (sd.len > sd.total_len)
588 sd.len = sd.total_len;
589
Ingo Molnar3a326a22006-04-10 15:18:35 +0200590 err = actor(pipe, buf, &sd);
Jens Axboe5274f052006-03-30 15:15:30 +0200591 if (err) {
592 if (!ret && err != -ENODATA)
593 ret = err;
594
595 break;
596 }
597
598 ret += sd.len;
599 buf->offset += sd.len;
600 buf->len -= sd.len;
601 if (!buf->len) {
602 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200603 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200604 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
605 pipe->nrbufs--;
606 if (pipe->inode)
607 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200608 }
609
610 sd.pos += sd.len;
611 sd.total_len -= sd.len;
612 if (!sd.total_len)
613 break;
614 }
615
Jens Axboe6f767b02006-04-11 13:53:56 +0200616 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200617 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200618 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200619 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200620 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200621 if (ret)
622 break;
623 }
624
Linus Torvalds29e35092006-04-02 12:46:35 -0700625 if (flags & SPLICE_F_NONBLOCK) {
626 if (!ret)
627 ret = -EAGAIN;
628 break;
629 }
630
Jens Axboe5274f052006-03-30 15:15:30 +0200631 if (signal_pending(current)) {
632 if (!ret)
633 ret = -ERESTARTSYS;
634 break;
635 }
636
637 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200638 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200639 if (waitqueue_active(&pipe->wait))
640 wake_up_interruptible_sync(&pipe->wait);
641 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200642 do_wakeup = 0;
643 }
644
Ingo Molnar3a326a22006-04-10 15:18:35 +0200645 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200646 }
647
Ingo Molnar3a326a22006-04-10 15:18:35 +0200648 if (pipe->inode)
649 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200650
651 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200652 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200653 if (waitqueue_active(&pipe->wait))
654 wake_up_interruptible(&pipe->wait);
655 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200656 }
657
Jens Axboe5274f052006-03-30 15:15:30 +0200658 out->f_pos = sd.pos;
Jens Axboe5274f052006-03-30 15:15:30 +0200659 return ret;
660
661}
662
Jens Axboe83f91352006-04-02 23:05:09 +0200663/**
664 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200665 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200666 * @out: file to write to
667 * @len: number of bytes to splice
668 * @flags: splice modifier flags
669 *
670 * Will either move or copy pages (determined by @flags options) from
671 * the given pipe inode to the given file.
672 *
673 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200674ssize_t
675generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
676 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200677{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200678 struct address_space *mapping = out->f_mapping;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200679 ssize_t ret;
680
681 ret = move_from_pipe(pipe, out, len, flags, pipe_to_file);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200682
683 /*
684 * if file or inode is SYNC and we actually wrote some data, sync it
685 */
686 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
687 && ret > 0) {
688 struct inode *inode = mapping->host;
689 int err;
690
691 mutex_lock(&inode->i_mutex);
692 err = generic_osync_inode(mapping->host, mapping,
Jens Axboe49570e92006-04-11 13:56:09 +0200693 OSYNC_METADATA|OSYNC_DATA);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200694 mutex_unlock(&inode->i_mutex);
695
696 if (err)
697 ret = err;
698 }
699
700 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200701}
702
Jens Axboe059a8f32006-04-02 23:06:05 +0200703EXPORT_SYMBOL(generic_file_splice_write);
704
Jens Axboe83f91352006-04-02 23:05:09 +0200705/**
706 * generic_splice_sendpage - splice data from a pipe to a socket
707 * @inode: pipe inode
708 * @out: socket to write to
709 * @len: number of bytes to splice
710 * @flags: splice modifier flags
711 *
712 * Will send @len bytes from the pipe to a network socket. No data copying
713 * is involved.
714 *
715 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200716ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboe5274f052006-03-30 15:15:30 +0200717 size_t len, unsigned int flags)
718{
Ingo Molnar3a326a22006-04-10 15:18:35 +0200719 return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200720}
721
Jens Axboe059a8f32006-04-02 23:06:05 +0200722EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500723
Jens Axboe83f91352006-04-02 23:05:09 +0200724/*
725 * Attempt to initiate a splice from pipe to file.
726 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200727static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboeb92ce552006-04-11 13:52:07 +0200728 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200729{
730 loff_t pos;
731 int ret;
732
Jens Axboe49570e92006-04-11 13:56:09 +0200733 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200734 return -EINVAL;
735
Jens Axboe49570e92006-04-11 13:56:09 +0200736 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200737 return -EBADF;
738
739 pos = out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200740
Jens Axboe5274f052006-03-30 15:15:30 +0200741 ret = rw_verify_area(WRITE, out, &pos, len);
742 if (unlikely(ret < 0))
743 return ret;
744
745 return out->f_op->splice_write(pipe, out, len, flags);
746}
747
Jens Axboe83f91352006-04-02 23:05:09 +0200748/*
749 * Attempt to initiate a splice from a file to a pipe.
750 */
Jens Axboeb92ce552006-04-11 13:52:07 +0200751static long do_splice_to(struct file *in, struct pipe_inode_info *pipe,
752 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200753{
754 loff_t pos, isize, left;
755 int ret;
756
Jens Axboe49570e92006-04-11 13:56:09 +0200757 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200758 return -EINVAL;
759
Jens Axboe49570e92006-04-11 13:56:09 +0200760 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200761 return -EBADF;
762
763 pos = in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200764
Jens Axboe5274f052006-03-30 15:15:30 +0200765 ret = rw_verify_area(READ, in, &pos, len);
766 if (unlikely(ret < 0))
767 return ret;
768
769 isize = i_size_read(in->f_mapping->host);
770 if (unlikely(in->f_pos >= isize))
771 return 0;
772
773 left = isize - in->f_pos;
Jens Axboe49570e92006-04-11 13:56:09 +0200774 if (unlikely(left < len))
Jens Axboe5274f052006-03-30 15:15:30 +0200775 len = left;
776
777 return in->f_op->splice_read(in, pipe, len, flags);
778}
779
Jens Axboeb92ce552006-04-11 13:52:07 +0200780long do_splice_direct(struct file *in, struct file *out, size_t len,
781 unsigned int flags)
782{
783 struct pipe_inode_info *pipe;
784 long ret, bytes;
785 umode_t i_mode;
786 int i;
787
788 /*
789 * We require the input being a regular file, as we don't want to
790 * randomly drop data for eg socket -> socket splicing. Use the
791 * piped splicing for that!
792 */
793 i_mode = in->f_dentry->d_inode->i_mode;
794 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
795 return -EINVAL;
796
797 /*
798 * neither in nor out is a pipe, setup an internal pipe attached to
799 * 'out' and transfer the wanted data from 'in' to 'out' through that
800 */
801 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +0200802 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +0200803 pipe = alloc_pipe_info(NULL);
804 if (!pipe)
805 return -ENOMEM;
806
807 /*
808 * We don't have an immediate reader, but we'll read the stuff
809 * out of the pipe right after the move_to_pipe(). So set
810 * PIPE_READERS appropriately.
811 */
812 pipe->readers = 1;
813
814 current->splice_pipe = pipe;
815 }
816
817 /*
818 * do the splice
819 */
820 ret = 0;
821 bytes = 0;
822
823 while (len) {
824 size_t read_len, max_read_len;
825
826 /*
827 * Do at most PIPE_BUFFERS pages worth of transfer:
828 */
829 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
830
831 ret = do_splice_to(in, pipe, max_read_len, flags);
832 if (unlikely(ret < 0))
833 goto out_release;
834
835 read_len = ret;
836
837 /*
838 * NOTE: nonblocking mode only applies to the input. We
839 * must not do the output in nonblocking mode as then we
840 * could get stuck data in the internal pipe:
841 */
842 ret = do_splice_from(pipe, out, read_len,
843 flags & ~SPLICE_F_NONBLOCK);
844 if (unlikely(ret < 0))
845 goto out_release;
846
847 bytes += ret;
848 len -= ret;
849
850 /*
851 * In nonblocking mode, if we got back a short read then
852 * that was due to either an IO error or due to the
853 * pagecache entry not being there. In the IO error case
854 * the _next_ splice attempt will produce a clean IO error
855 * return value (not a short read), so in both cases it's
856 * correct to break out of the loop here:
857 */
858 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
859 break;
860 }
861
862 pipe->nrbufs = pipe->curbuf = 0;
863
864 return bytes;
865
866out_release:
867 /*
868 * If we did an incomplete transfer we must release
869 * the pipe buffers in question:
870 */
871 for (i = 0; i < PIPE_BUFFERS; i++) {
872 struct pipe_buffer *buf = pipe->bufs + i;
873
874 if (buf->ops) {
875 buf->ops->release(pipe, buf);
876 buf->ops = NULL;
877 }
878 }
879 pipe->nrbufs = pipe->curbuf = 0;
880
881 /*
882 * If we transferred some data, return the number of bytes:
883 */
884 if (bytes > 0)
885 return bytes;
886
887 return ret;
888}
889
890EXPORT_SYMBOL(do_splice_direct);
891
Jens Axboe83f91352006-04-02 23:05:09 +0200892/*
893 * Determine where to splice to/from.
894 */
Ingo Molnar529565d2006-04-10 15:18:58 +0200895static long do_splice(struct file *in, loff_t __user *off_in,
896 struct file *out, loff_t __user *off_out,
897 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200898{
Ingo Molnar3a326a22006-04-10 15:18:35 +0200899 struct pipe_inode_info *pipe;
Jens Axboe5274f052006-03-30 15:15:30 +0200900
Ingo Molnar3a326a22006-04-10 15:18:35 +0200901 pipe = in->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200902 if (pipe) {
903 if (off_in)
904 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200905 if (off_out) {
906 if (out->f_op->llseek == no_llseek)
907 return -EINVAL;
908 if (copy_from_user(&out->f_pos, off_out,
909 sizeof(loff_t)))
910 return -EFAULT;
911 }
Ingo Molnar529565d2006-04-10 15:18:58 +0200912
Jens Axboeb92ce552006-04-11 13:52:07 +0200913 return do_splice_from(pipe, out, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200914 }
Jens Axboe5274f052006-03-30 15:15:30 +0200915
Ingo Molnar3a326a22006-04-10 15:18:35 +0200916 pipe = out->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200917 if (pipe) {
918 if (off_out)
919 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200920 if (off_in) {
921 if (in->f_op->llseek == no_llseek)
922 return -EINVAL;
923 if (copy_from_user(&in->f_pos, off_in, sizeof(loff_t)))
924 return -EFAULT;
925 }
Ingo Molnar529565d2006-04-10 15:18:58 +0200926
Jens Axboeb92ce552006-04-11 13:52:07 +0200927 return do_splice_to(in, pipe, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200928 }
Jens Axboe5274f052006-03-30 15:15:30 +0200929
930 return -EINVAL;
931}
932
Ingo Molnar529565d2006-04-10 15:18:58 +0200933asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
934 int fd_out, loff_t __user *off_out,
935 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200936{
937 long error;
938 struct file *in, *out;
939 int fput_in, fput_out;
940
941 if (unlikely(!len))
942 return 0;
943
944 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +0200945 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +0200946 if (in) {
947 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +0200948 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +0200949 if (out) {
950 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +0200951 error = do_splice(in, off_in,
952 out, off_out,
953 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200954 fput_light(out, fput_out);
955 }
956 }
957
958 fput_light(in, fput_in);
959 }
960
961 return error;
962}