blob: 4f5e6b09fb2688b9f9e0f4f5bcb93d8f3103db1f [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboec2058e02006-04-11 13:56:34 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/pipe_fs_i.h>
24#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe5274f052006-03-30 15:15:30 +020030
31/*
32 * Passed to the actors
33 */
34struct splice_desc {
35 unsigned int len, total_len; /* current and remaining length */
36 unsigned int flags; /* splice flags */
37 struct file *file; /* file to read/write */
38 loff_t pos; /* file position */
39};
40
Jens Axboe83f91352006-04-02 23:05:09 +020041/*
42 * Attempt to steal a page from a pipe buffer. This should perhaps go into
43 * a vm helper function, it's already simplified quite a bit by the
44 * addition of remove_mapping(). If success is returned, the caller may
45 * attempt to reuse this page for another destination.
46 */
Jens Axboe5abc97a2006-03-30 15:16:46 +020047static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
48 struct pipe_buffer *buf)
49{
50 struct page *page = buf->page;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020051 struct address_space *mapping = page_mapping(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020052
53 WARN_ON(!PageLocked(page));
54 WARN_ON(!PageUptodate(page));
55
Jens Axboead8d6f02006-04-02 23:10:32 +020056 /*
57 * At least for ext2 with nobh option, we need to wait on writeback
58 * completing on this page, since we'll remove it from the pagecache.
59 * Otherwise truncate wont wait on the page, allowing the disk
60 * blocks to be reused by someone else before we actually wrote our
61 * data to them. fs corruption ensues.
62 */
63 wait_on_page_writeback(page);
64
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020065 if (PagePrivate(page))
66 try_to_release_page(page, mapping_gfp_mask(mapping));
67
68 if (!remove_mapping(mapping, page))
Jens Axboe5abc97a2006-03-30 15:16:46 +020069 return 1;
70
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020071 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
Jens Axboe5abc97a2006-03-30 15:16:46 +020072 return 0;
73}
74
Jens Axboe5274f052006-03-30 15:15:30 +020075static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
76 struct pipe_buffer *buf)
77{
78 page_cache_release(buf->page);
79 buf->page = NULL;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020080 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
Jens Axboe5274f052006-03-30 15:15:30 +020081}
82
83static void *page_cache_pipe_buf_map(struct file *file,
84 struct pipe_inode_info *info,
85 struct pipe_buffer *buf)
86{
87 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020088 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020089
90 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +020091 lock_page(page);
92
93 /*
94 * Page got truncated/unhashed. This will cause a 0-byte
Ingo Molnar73d62d82006-04-11 13:57:21 +020095 * splice, if this is the first page.
Jens Axboe49d0b212006-04-10 09:04:41 +020096 */
97 if (!page->mapping) {
98 err = -ENODATA;
99 goto error;
100 }
101
102 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200103 * Uh oh, read-error from disk.
Jens Axboe49d0b212006-04-10 09:04:41 +0200104 */
105 if (!PageUptodate(page)) {
106 err = -EIO;
107 goto error;
108 }
109
110 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200111 * Page is ok afterall, fall through to mapping.
Jens Axboe49d0b212006-04-10 09:04:41 +0200112 */
Jens Axboe5274f052006-03-30 15:15:30 +0200113 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200114 }
115
Jens Axboe49d0b212006-04-10 09:04:41 +0200116 return kmap(page);
117error:
118 unlock_page(page);
119 return ERR_PTR(err);
Jens Axboe5274f052006-03-30 15:15:30 +0200120}
121
122static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
123 struct pipe_buffer *buf)
124{
Jens Axboe5274f052006-03-30 15:15:30 +0200125 kunmap(buf->page);
126}
127
Jens Axboe70524492006-04-11 15:51:17 +0200128static void page_cache_pipe_buf_get(struct pipe_inode_info *info,
129 struct pipe_buffer *buf)
130{
131 page_cache_get(buf->page);
132}
133
Jens Axboe5274f052006-03-30 15:15:30 +0200134static struct pipe_buf_operations page_cache_pipe_buf_ops = {
135 .can_merge = 0,
136 .map = page_cache_pipe_buf_map,
137 .unmap = page_cache_pipe_buf_unmap,
138 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200139 .steal = page_cache_pipe_buf_steal,
Jens Axboe70524492006-04-11 15:51:17 +0200140 .get = page_cache_pipe_buf_get,
Jens Axboe5274f052006-03-30 15:15:30 +0200141};
142
Jens Axboe83f91352006-04-02 23:05:09 +0200143/*
144 * Pipe output worker. This sets up our pipe format with the page cache
145 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
146 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200147static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
Jens Axboe91ad66e2006-04-19 15:55:10 +0200148 int nr_pages, unsigned long len,
149 unsigned int offset, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200150{
Jens Axboe5274f052006-03-30 15:15:30 +0200151 int ret, do_wakeup, i;
152
153 ret = 0;
154 do_wakeup = 0;
155 i = 0;
156
Ingo Molnar3a326a22006-04-10 15:18:35 +0200157 if (pipe->inode)
158 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200159
Jens Axboe5274f052006-03-30 15:15:30 +0200160 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200161 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200162 send_sig(SIGPIPE, current, 0);
163 if (!ret)
164 ret = -EPIPE;
165 break;
166 }
167
Jens Axboe6f767b02006-04-11 13:53:56 +0200168 if (pipe->nrbufs < PIPE_BUFFERS) {
169 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200170 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200171 struct page *page = pages[i++];
172 unsigned long this_len;
173
174 this_len = PAGE_CACHE_SIZE - offset;
175 if (this_len > len)
176 this_len = len;
177
178 buf->page = page;
179 buf->offset = offset;
180 buf->len = this_len;
181 buf->ops = &page_cache_pipe_buf_ops;
Jens Axboe6f767b02006-04-11 13:53:56 +0200182 pipe->nrbufs++;
183 if (pipe->inode)
184 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200185
186 ret += this_len;
187 len -= this_len;
188 offset = 0;
189 if (!--nr_pages)
190 break;
191 if (!len)
192 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200193 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200194 continue;
195
196 break;
197 }
198
Linus Torvalds29e35092006-04-02 12:46:35 -0700199 if (flags & SPLICE_F_NONBLOCK) {
200 if (!ret)
201 ret = -EAGAIN;
202 break;
203 }
204
Jens Axboe5274f052006-03-30 15:15:30 +0200205 if (signal_pending(current)) {
206 if (!ret)
207 ret = -ERESTARTSYS;
208 break;
209 }
210
211 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200212 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200213 if (waitqueue_active(&pipe->wait))
214 wake_up_interruptible_sync(&pipe->wait);
215 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200216 do_wakeup = 0;
217 }
218
Ingo Molnar3a326a22006-04-10 15:18:35 +0200219 pipe->waiting_writers++;
220 pipe_wait(pipe);
221 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200222 }
223
Ingo Molnar3a326a22006-04-10 15:18:35 +0200224 if (pipe->inode)
225 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200226
227 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200228 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200229 if (waitqueue_active(&pipe->wait))
230 wake_up_interruptible(&pipe->wait);
231 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200232 }
233
234 while (i < nr_pages)
235 page_cache_release(pages[i++]);
236
237 return ret;
238}
239
Ingo Molnar3a326a22006-04-10 15:18:35 +0200240static int
Jens Axboecbb7e572006-04-11 14:57:50 +0200241__generic_file_splice_read(struct file *in, loff_t *ppos,
242 struct pipe_inode_info *pipe, size_t len,
243 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200244{
245 struct address_space *mapping = in->f_mapping;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200246 unsigned int loff, offset, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200247 struct page *pages[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200248 struct page *page;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200249 pgoff_t index, end_index;
250 loff_t isize;
251 size_t bytes;
Jens Axboe7480a902006-04-11 13:52:47 +0200252 int i, error;
Jens Axboe5274f052006-03-30 15:15:30 +0200253
Jens Axboecbb7e572006-04-11 14:57:50 +0200254 index = *ppos >> PAGE_CACHE_SHIFT;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200255 loff = offset = *ppos & ~PAGE_CACHE_MASK;
Jens Axboe5274f052006-03-30 15:15:30 +0200256 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
257
258 if (nr_pages > PIPE_BUFFERS)
259 nr_pages = PIPE_BUFFERS;
260
261 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200262 * Initiate read-ahead on this page range. however, don't call into
Jens Axboe0b749ce2006-04-10 09:05:04 +0200263 * read-ahead if this is a non-zero offset (we are likely doing small
264 * chunk splice and the page is already there) for a single page.
Jens Axboe5274f052006-03-30 15:15:30 +0200265 */
Jens Axboe0b749ce2006-04-10 09:05:04 +0200266 if (!offset || nr_pages > 1)
267 do_page_cache_readahead(mapping, in, index, nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200268
269 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200270 * Now fill in the holes:
Jens Axboe5274f052006-03-30 15:15:30 +0200271 */
Jens Axboe7480a902006-04-11 13:52:47 +0200272 error = 0;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200273 bytes = 0;
Jens Axboe16c523d2006-04-10 09:03:58 +0200274 for (i = 0; i < nr_pages; i++, index++) {
Jens Axboe7480a902006-04-11 13:52:47 +0200275find_page:
Jens Axboe5274f052006-03-30 15:15:30 +0200276 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200277 * lookup the page for this index
Jens Axboe5274f052006-03-30 15:15:30 +0200278 */
Jens Axboe7480a902006-04-11 13:52:47 +0200279 page = find_get_page(mapping, index);
280 if (!page) {
281 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200282 * page didn't exist, allocate one
283 */
284 page = page_cache_alloc_cold(mapping);
285 if (!page)
286 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200287
Jens Axboe7480a902006-04-11 13:52:47 +0200288 error = add_to_page_cache_lru(page, mapping, index,
289 mapping_gfp_mask(mapping));
Jens Axboe5274f052006-03-30 15:15:30 +0200290 if (unlikely(error)) {
291 page_cache_release(page);
292 break;
293 }
Jens Axboe7480a902006-04-11 13:52:47 +0200294
295 goto readpage;
Jens Axboe5274f052006-03-30 15:15:30 +0200296 }
Jens Axboe7480a902006-04-11 13:52:47 +0200297
298 /*
299 * If the page isn't uptodate, we may need to start io on it
300 */
301 if (!PageUptodate(page)) {
Jens Axboec4f895c2006-04-19 15:56:12 +0200302 /*
303 * If in nonblock mode then dont block on waiting
304 * for an in-flight io page
305 */
306 if (flags & SPLICE_F_NONBLOCK)
307 break;
308
Jens Axboe7480a902006-04-11 13:52:47 +0200309 lock_page(page);
310
311 /*
312 * page was truncated, stop here. if this isn't the
313 * first page, we'll just complete what we already
314 * added
315 */
316 if (!page->mapping) {
317 unlock_page(page);
318 page_cache_release(page);
319 break;
320 }
321 /*
322 * page was already under io and is now done, great
323 */
324 if (PageUptodate(page)) {
325 unlock_page(page);
326 goto fill_it;
327 }
328
329readpage:
330 /*
331 * need to read in the page
332 */
333 error = mapping->a_ops->readpage(in, page);
334
335 if (unlikely(error)) {
336 page_cache_release(page);
337 if (error == AOP_TRUNCATED_PAGE)
338 goto find_page;
339 break;
340 }
Jens Axboe91ad66e2006-04-19 15:55:10 +0200341
342 /*
343 * i_size must be checked after ->readpage().
344 */
345 isize = i_size_read(mapping->host);
346 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
347 if (unlikely(!isize || index > end_index)) {
348 page_cache_release(page);
349 break;
350 }
351
352 /*
353 * if this is the last page, see if we need to shrink
354 * the length and stop
355 */
356 if (end_index == index) {
357 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
358 if (bytes + loff > isize) {
359 page_cache_release(page);
360 break;
361 }
362 /*
363 * force quit after adding this page
364 */
365 nr_pages = i;
366 }
Jens Axboe7480a902006-04-11 13:52:47 +0200367 }
368fill_it:
Jens Axboe16c523d2006-04-10 09:03:58 +0200369 pages[i] = page;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200370 bytes += PAGE_CACHE_SIZE - loff;
371 loff = 0;
Jens Axboe5274f052006-03-30 15:15:30 +0200372 }
373
Jens Axboe16c523d2006-04-10 09:03:58 +0200374 if (i)
Jens Axboe91ad66e2006-04-19 15:55:10 +0200375 return move_to_pipe(pipe, pages, i, bytes, offset, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200376
Jens Axboe7480a902006-04-11 13:52:47 +0200377 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200378}
379
Jens Axboe83f91352006-04-02 23:05:09 +0200380/**
381 * generic_file_splice_read - splice data from file to a pipe
382 * @in: file to splice from
383 * @pipe: pipe to splice to
384 * @len: number of bytes to splice
385 * @flags: splice modifier flags
386 *
387 * Will read pages from given file and fill them into a pipe.
Jens Axboe83f91352006-04-02 23:05:09 +0200388 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200389ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
390 struct pipe_inode_info *pipe, size_t len,
391 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200392{
393 ssize_t spliced;
394 int ret;
395
396 ret = 0;
397 spliced = 0;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200398
Jens Axboe5274f052006-03-30 15:15:30 +0200399 while (len) {
Jens Axboecbb7e572006-04-11 14:57:50 +0200400 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200401
Jens Axboec4f895c2006-04-19 15:56:12 +0200402 if (ret < 0)
Jens Axboe5274f052006-03-30 15:15:30 +0200403 break;
Jens Axboec4f895c2006-04-19 15:56:12 +0200404 else if (!ret) {
405 if (spliced)
406 break;
407 if (flags & SPLICE_F_NONBLOCK) {
408 ret = -EAGAIN;
409 break;
410 }
411 }
Jens Axboe5274f052006-03-30 15:15:30 +0200412
Jens Axboecbb7e572006-04-11 14:57:50 +0200413 *ppos += ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200414 len -= ret;
415 spliced += ret;
416 }
417
418 if (spliced)
419 return spliced;
420
421 return ret;
422}
423
Jens Axboe059a8f32006-04-02 23:06:05 +0200424EXPORT_SYMBOL(generic_file_splice_read);
425
Jens Axboe5274f052006-03-30 15:15:30 +0200426/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200427 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
428 * using sendpage().
Jens Axboe5274f052006-03-30 15:15:30 +0200429 */
430static int pipe_to_sendpage(struct pipe_inode_info *info,
431 struct pipe_buffer *buf, struct splice_desc *sd)
432{
433 struct file *file = sd->file;
434 loff_t pos = sd->pos;
435 unsigned int offset;
436 ssize_t ret;
437 void *ptr;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200438 int more;
Jens Axboe5274f052006-03-30 15:15:30 +0200439
440 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200441 * Sub-optimal, but we are limited by the pipe ->map. We don't
Jens Axboe5274f052006-03-30 15:15:30 +0200442 * need a kmap'ed buffer here, we just want to make sure we
443 * have the page pinned if the pipe page originates from the
Ingo Molnar73d62d82006-04-11 13:57:21 +0200444 * page cache.
Jens Axboe5274f052006-03-30 15:15:30 +0200445 */
446 ptr = buf->ops->map(file, info, buf);
447 if (IS_ERR(ptr))
448 return PTR_ERR(ptr);
449
450 offset = pos & ~PAGE_CACHE_MASK;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200451 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200452
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200453 ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
Jens Axboe5274f052006-03-30 15:15:30 +0200454
455 buf->ops->unmap(info, buf);
456 if (ret == sd->len)
457 return 0;
458
459 return -EIO;
460}
461
462/*
463 * This is a little more tricky than the file -> pipe splicing. There are
464 * basically three cases:
465 *
466 * - Destination page already exists in the address space and there
467 * are users of it. For that case we have no other option that
468 * copying the data. Tough luck.
469 * - Destination page already exists in the address space, but there
470 * are no users of it. Make sure it's uptodate, then drop it. Fall
471 * through to last case.
472 * - Destination page does not exist, we can add the pipe page to
473 * the page cache and avoid the copy.
474 *
Jens Axboe83f91352006-04-02 23:05:09 +0200475 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
476 * sd->flags), we attempt to migrate pages from the pipe to the output
477 * file address space page cache. This is possible if no one else has
478 * the pipe page referenced outside of the pipe and page cache. If
479 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
480 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200481 */
482static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
483 struct splice_desc *sd)
484{
485 struct file *file = sd->file;
486 struct address_space *mapping = file->f_mapping;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200487 gfp_t gfp_mask = mapping_gfp_mask(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200488 unsigned int offset;
489 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200490 pgoff_t index;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200491 char *src;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200492 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200493
494 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200495 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200496 */
497 src = buf->ops->map(file, info, buf);
498 if (IS_ERR(src))
499 return PTR_ERR(src);
500
501 index = sd->pos >> PAGE_CACHE_SHIFT;
502 offset = sd->pos & ~PAGE_CACHE_MASK;
503
Jens Axboe5274f052006-03-30 15:15:30 +0200504 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200505 * Reuse buf page, if SPLICE_F_MOVE is set.
Jens Axboe5274f052006-03-30 15:15:30 +0200506 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200507 if (sd->flags & SPLICE_F_MOVE) {
Jens Axboe83f91352006-04-02 23:05:09 +0200508 /*
509 * If steal succeeds, buf->page is now pruned from the vm
510 * side (LRU and page cache) and we can reuse it.
511 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200512 if (buf->ops->steal(info, buf))
513 goto find_page;
Jens Axboe5274f052006-03-30 15:15:30 +0200514
Jens Axboe49d0b212006-04-10 09:04:41 +0200515 /*
516 * this will also set the page locked
517 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200518 page = buf->page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200519 if (add_to_page_cache(page, mapping, index, gfp_mask))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200520 goto find_page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200521
522 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
523 lru_cache_add(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200524 } else {
525find_page:
526 ret = -ENOMEM;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200527 page = find_or_create_page(mapping, index, gfp_mask);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200528 if (!page)
Dave Jones9aefe432006-04-10 09:02:40 +0200529 goto out_nomem;
Jens Axboe5274f052006-03-30 15:15:30 +0200530
Jens Axboe5abc97a2006-03-30 15:16:46 +0200531 /*
532 * If the page is uptodate, it is also locked. If it isn't
533 * uptodate, we can mark it uptodate if we are filling the
534 * full page. Otherwise we need to read it in first...
535 */
536 if (!PageUptodate(page)) {
537 if (sd->len < PAGE_CACHE_SIZE) {
538 ret = mapping->a_ops->readpage(file, page);
539 if (unlikely(ret))
540 goto out;
541
542 lock_page(page);
543
544 if (!PageUptodate(page)) {
545 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200546 * Page got invalidated, repeat.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200547 */
548 if (!page->mapping) {
549 unlock_page(page);
550 page_cache_release(page);
551 goto find_page;
552 }
553 ret = -EIO;
554 goto out;
Jens Axboe5274f052006-03-30 15:15:30 +0200555 }
Jens Axboe5abc97a2006-03-30 15:16:46 +0200556 } else {
557 WARN_ON(!PageLocked(page));
558 SetPageUptodate(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200559 }
Jens Axboe5274f052006-03-30 15:15:30 +0200560 }
561 }
562
563 ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200564 if (ret == AOP_TRUNCATED_PAGE) {
565 page_cache_release(page);
566 goto find_page;
567 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200568 goto out;
569
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200570 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200571 char *dst = kmap_atomic(page, KM_USER0);
572
573 memcpy(dst + offset, src + buf->offset, sd->len);
574 flush_dcache_page(page);
575 kunmap_atomic(dst, KM_USER0);
576 }
Jens Axboe5274f052006-03-30 15:15:30 +0200577
578 ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200579 if (ret == AOP_TRUNCATED_PAGE) {
580 page_cache_release(page);
581 goto find_page;
582 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200583 goto out;
584
Jens Axboec7f21e42006-04-10 09:01:01 +0200585 mark_page_accessed(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200586 balance_dirty_pages_ratelimited(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200587out:
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200588 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200589 page_cache_release(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200590 unlock_page(page);
591 }
Dave Jones9aefe432006-04-10 09:02:40 +0200592out_nomem:
Jens Axboe5274f052006-03-30 15:15:30 +0200593 buf->ops->unmap(info, buf);
594 return ret;
595}
596
597typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
598 struct splice_desc *);
599
Jens Axboe83f91352006-04-02 23:05:09 +0200600/*
601 * Pipe input worker. Most of this logic works like a regular pipe, the
602 * key here is the 'actor' worker passed in that actually moves the data
603 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
604 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200605static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200606 loff_t *ppos, size_t len, unsigned int flags,
Jens Axboe5274f052006-03-30 15:15:30 +0200607 splice_actor *actor)
608{
Jens Axboe5274f052006-03-30 15:15:30 +0200609 int ret, do_wakeup, err;
610 struct splice_desc sd;
611
612 ret = 0;
613 do_wakeup = 0;
614
615 sd.total_len = len;
616 sd.flags = flags;
617 sd.file = out;
Jens Axboecbb7e572006-04-11 14:57:50 +0200618 sd.pos = *ppos;
Jens Axboe5274f052006-03-30 15:15:30 +0200619
Ingo Molnar3a326a22006-04-10 15:18:35 +0200620 if (pipe->inode)
621 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200622
Jens Axboe5274f052006-03-30 15:15:30 +0200623 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200624 if (pipe->nrbufs) {
625 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200626 struct pipe_buf_operations *ops = buf->ops;
627
628 sd.len = buf->len;
629 if (sd.len > sd.total_len)
630 sd.len = sd.total_len;
631
Ingo Molnar3a326a22006-04-10 15:18:35 +0200632 err = actor(pipe, buf, &sd);
Jens Axboe5274f052006-03-30 15:15:30 +0200633 if (err) {
634 if (!ret && err != -ENODATA)
635 ret = err;
636
637 break;
638 }
639
640 ret += sd.len;
641 buf->offset += sd.len;
642 buf->len -= sd.len;
Ingo Molnar73d62d82006-04-11 13:57:21 +0200643
Jens Axboe5274f052006-03-30 15:15:30 +0200644 if (!buf->len) {
645 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200646 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200647 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
648 pipe->nrbufs--;
649 if (pipe->inode)
650 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200651 }
652
653 sd.pos += sd.len;
654 sd.total_len -= sd.len;
655 if (!sd.total_len)
656 break;
657 }
658
Jens Axboe6f767b02006-04-11 13:53:56 +0200659 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200660 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200661 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200662 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200663 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200664 if (ret)
665 break;
666 }
667
Linus Torvalds29e35092006-04-02 12:46:35 -0700668 if (flags & SPLICE_F_NONBLOCK) {
669 if (!ret)
670 ret = -EAGAIN;
671 break;
672 }
673
Jens Axboe5274f052006-03-30 15:15:30 +0200674 if (signal_pending(current)) {
675 if (!ret)
676 ret = -ERESTARTSYS;
677 break;
678 }
679
680 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200681 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200682 if (waitqueue_active(&pipe->wait))
683 wake_up_interruptible_sync(&pipe->wait);
684 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200685 do_wakeup = 0;
686 }
687
Ingo Molnar3a326a22006-04-10 15:18:35 +0200688 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200689 }
690
Ingo Molnar3a326a22006-04-10 15:18:35 +0200691 if (pipe->inode)
692 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200693
694 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200695 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200696 if (waitqueue_active(&pipe->wait))
697 wake_up_interruptible(&pipe->wait);
698 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200699 }
700
Jens Axboe5274f052006-03-30 15:15:30 +0200701 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200702}
703
Jens Axboe83f91352006-04-02 23:05:09 +0200704/**
705 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200706 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200707 * @out: file to write to
708 * @len: number of bytes to splice
709 * @flags: splice modifier flags
710 *
711 * Will either move or copy pages (determined by @flags options) from
712 * the given pipe inode to the given file.
713 *
714 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200715ssize_t
716generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200717 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200718{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200719 struct address_space *mapping = out->f_mapping;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200720 ssize_t ret;
721
Jens Axboecbb7e572006-04-11 14:57:50 +0200722 ret = move_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200723
724 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200725 * If file or inode is SYNC and we actually wrote some data, sync it.
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200726 */
727 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
728 && ret > 0) {
729 struct inode *inode = mapping->host;
730 int err;
731
732 mutex_lock(&inode->i_mutex);
733 err = generic_osync_inode(mapping->host, mapping,
Jens Axboe49570e92006-04-11 13:56:09 +0200734 OSYNC_METADATA|OSYNC_DATA);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200735 mutex_unlock(&inode->i_mutex);
736
737 if (err)
738 ret = err;
739 }
740
741 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200742}
743
Jens Axboe059a8f32006-04-02 23:06:05 +0200744EXPORT_SYMBOL(generic_file_splice_write);
745
Jens Axboe83f91352006-04-02 23:05:09 +0200746/**
747 * generic_splice_sendpage - splice data from a pipe to a socket
748 * @inode: pipe inode
749 * @out: socket to write to
750 * @len: number of bytes to splice
751 * @flags: splice modifier flags
752 *
753 * Will send @len bytes from the pipe to a network socket. No data copying
754 * is involved.
755 *
756 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200757ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200758 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200759{
Jens Axboecbb7e572006-04-11 14:57:50 +0200760 return move_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200761}
762
Jens Axboe059a8f32006-04-02 23:06:05 +0200763EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500764
Jens Axboe83f91352006-04-02 23:05:09 +0200765/*
766 * Attempt to initiate a splice from pipe to file.
767 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200768static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200769 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200770{
Jens Axboe5274f052006-03-30 15:15:30 +0200771 int ret;
772
Jens Axboe49570e92006-04-11 13:56:09 +0200773 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200774 return -EINVAL;
775
Jens Axboe49570e92006-04-11 13:56:09 +0200776 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200777 return -EBADF;
778
Jens Axboecbb7e572006-04-11 14:57:50 +0200779 ret = rw_verify_area(WRITE, out, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200780 if (unlikely(ret < 0))
781 return ret;
782
Jens Axboecbb7e572006-04-11 14:57:50 +0200783 return out->f_op->splice_write(pipe, out, ppos, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200784}
785
Jens Axboe83f91352006-04-02 23:05:09 +0200786/*
787 * Attempt to initiate a splice from a file to a pipe.
788 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200789static long do_splice_to(struct file *in, loff_t *ppos,
790 struct pipe_inode_info *pipe, size_t len,
791 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200792{
Jens Axboecbb7e572006-04-11 14:57:50 +0200793 loff_t isize, left;
Jens Axboe5274f052006-03-30 15:15:30 +0200794 int ret;
795
Jens Axboe49570e92006-04-11 13:56:09 +0200796 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200797 return -EINVAL;
798
Jens Axboe49570e92006-04-11 13:56:09 +0200799 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200800 return -EBADF;
801
Jens Axboecbb7e572006-04-11 14:57:50 +0200802 ret = rw_verify_area(READ, in, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200803 if (unlikely(ret < 0))
804 return ret;
805
806 isize = i_size_read(in->f_mapping->host);
Jens Axboecbb7e572006-04-11 14:57:50 +0200807 if (unlikely(*ppos >= isize))
Jens Axboe5274f052006-03-30 15:15:30 +0200808 return 0;
809
Jens Axboecbb7e572006-04-11 14:57:50 +0200810 left = isize - *ppos;
Jens Axboe49570e92006-04-11 13:56:09 +0200811 if (unlikely(left < len))
Jens Axboe5274f052006-03-30 15:15:30 +0200812 len = left;
813
Jens Axboecbb7e572006-04-11 14:57:50 +0200814 return in->f_op->splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200815}
816
Jens Axboecbb7e572006-04-11 14:57:50 +0200817long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
818 size_t len, unsigned int flags)
Jens Axboeb92ce552006-04-11 13:52:07 +0200819{
820 struct pipe_inode_info *pipe;
821 long ret, bytes;
Jens Axboecbb7e572006-04-11 14:57:50 +0200822 loff_t out_off;
Jens Axboeb92ce552006-04-11 13:52:07 +0200823 umode_t i_mode;
824 int i;
825
826 /*
827 * We require the input being a regular file, as we don't want to
828 * randomly drop data for eg socket -> socket splicing. Use the
829 * piped splicing for that!
830 */
831 i_mode = in->f_dentry->d_inode->i_mode;
832 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
833 return -EINVAL;
834
835 /*
836 * neither in nor out is a pipe, setup an internal pipe attached to
837 * 'out' and transfer the wanted data from 'in' to 'out' through that
838 */
839 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +0200840 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +0200841 pipe = alloc_pipe_info(NULL);
842 if (!pipe)
843 return -ENOMEM;
844
845 /*
846 * We don't have an immediate reader, but we'll read the stuff
847 * out of the pipe right after the move_to_pipe(). So set
848 * PIPE_READERS appropriately.
849 */
850 pipe->readers = 1;
851
852 current->splice_pipe = pipe;
853 }
854
855 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200856 * Do the splice.
Jens Axboeb92ce552006-04-11 13:52:07 +0200857 */
858 ret = 0;
859 bytes = 0;
Jens Axboecbb7e572006-04-11 14:57:50 +0200860 out_off = 0;
Jens Axboeb92ce552006-04-11 13:52:07 +0200861
862 while (len) {
863 size_t read_len, max_read_len;
864
865 /*
866 * Do at most PIPE_BUFFERS pages worth of transfer:
867 */
868 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
869
Jens Axboecbb7e572006-04-11 14:57:50 +0200870 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
Jens Axboeb92ce552006-04-11 13:52:07 +0200871 if (unlikely(ret < 0))
872 goto out_release;
873
874 read_len = ret;
875
876 /*
877 * NOTE: nonblocking mode only applies to the input. We
878 * must not do the output in nonblocking mode as then we
879 * could get stuck data in the internal pipe:
880 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200881 ret = do_splice_from(pipe, out, &out_off, read_len,
Jens Axboeb92ce552006-04-11 13:52:07 +0200882 flags & ~SPLICE_F_NONBLOCK);
883 if (unlikely(ret < 0))
884 goto out_release;
885
886 bytes += ret;
887 len -= ret;
888
889 /*
890 * In nonblocking mode, if we got back a short read then
891 * that was due to either an IO error or due to the
892 * pagecache entry not being there. In the IO error case
893 * the _next_ splice attempt will produce a clean IO error
894 * return value (not a short read), so in both cases it's
895 * correct to break out of the loop here:
896 */
897 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
898 break;
899 }
900
901 pipe->nrbufs = pipe->curbuf = 0;
902
903 return bytes;
904
905out_release:
906 /*
907 * If we did an incomplete transfer we must release
908 * the pipe buffers in question:
909 */
910 for (i = 0; i < PIPE_BUFFERS; i++) {
911 struct pipe_buffer *buf = pipe->bufs + i;
912
913 if (buf->ops) {
914 buf->ops->release(pipe, buf);
915 buf->ops = NULL;
916 }
917 }
918 pipe->nrbufs = pipe->curbuf = 0;
919
920 /*
921 * If we transferred some data, return the number of bytes:
922 */
923 if (bytes > 0)
924 return bytes;
925
926 return ret;
927}
928
929EXPORT_SYMBOL(do_splice_direct);
930
Jens Axboe83f91352006-04-02 23:05:09 +0200931/*
932 * Determine where to splice to/from.
933 */
Ingo Molnar529565d2006-04-10 15:18:58 +0200934static long do_splice(struct file *in, loff_t __user *off_in,
935 struct file *out, loff_t __user *off_out,
936 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200937{
Ingo Molnar3a326a22006-04-10 15:18:35 +0200938 struct pipe_inode_info *pipe;
Jens Axboecbb7e572006-04-11 14:57:50 +0200939 loff_t offset, *off;
Jens Axboe5274f052006-03-30 15:15:30 +0200940
Ingo Molnar3a326a22006-04-10 15:18:35 +0200941 pipe = in->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200942 if (pipe) {
943 if (off_in)
944 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200945 if (off_out) {
946 if (out->f_op->llseek == no_llseek)
947 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200948 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200949 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200950 off = &offset;
951 } else
952 off = &out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200953
Jens Axboecbb7e572006-04-11 14:57:50 +0200954 return do_splice_from(pipe, out, off, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200955 }
Jens Axboe5274f052006-03-30 15:15:30 +0200956
Ingo Molnar3a326a22006-04-10 15:18:35 +0200957 pipe = out->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200958 if (pipe) {
959 if (off_out)
960 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200961 if (off_in) {
962 if (in->f_op->llseek == no_llseek)
963 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200964 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200965 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200966 off = &offset;
967 } else
968 off = &in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200969
Jens Axboecbb7e572006-04-11 14:57:50 +0200970 return do_splice_to(in, off, pipe, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200971 }
Jens Axboe5274f052006-03-30 15:15:30 +0200972
973 return -EINVAL;
974}
975
Ingo Molnar529565d2006-04-10 15:18:58 +0200976asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
977 int fd_out, loff_t __user *off_out,
978 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200979{
980 long error;
981 struct file *in, *out;
982 int fput_in, fput_out;
983
984 if (unlikely(!len))
985 return 0;
986
987 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +0200988 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +0200989 if (in) {
990 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +0200991 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +0200992 if (out) {
993 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +0200994 error = do_splice(in, off_in,
995 out, off_out,
996 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200997 fput_light(out, fput_out);
998 }
999 }
1000
1001 fput_light(in, fput_in);
1002 }
1003
1004 return error;
1005}
Jens Axboe70524492006-04-11 15:51:17 +02001006
1007/*
1008 * Link contents of ipipe to opipe.
1009 */
1010static int link_pipe(struct pipe_inode_info *ipipe,
1011 struct pipe_inode_info *opipe,
1012 size_t len, unsigned int flags)
1013{
1014 struct pipe_buffer *ibuf, *obuf;
Jens Axboe2a27250e2006-04-19 15:56:40 +02001015 int ret, do_wakeup, i, ipipe_first;
1016
1017 ret = do_wakeup = ipipe_first = 0;
Jens Axboe70524492006-04-11 15:51:17 +02001018
1019 /*
1020 * Potential ABBA deadlock, work around it by ordering lock
1021 * grabbing by inode address. Otherwise two different processes
1022 * could deadlock (one doing tee from A -> B, the other from B -> A).
1023 */
1024 if (ipipe->inode < opipe->inode) {
Jens Axboe2a27250e2006-04-19 15:56:40 +02001025 ipipe_first = 1;
Jens Axboe70524492006-04-11 15:51:17 +02001026 mutex_lock(&ipipe->inode->i_mutex);
1027 mutex_lock(&opipe->inode->i_mutex);
1028 } else {
1029 mutex_lock(&opipe->inode->i_mutex);
1030 mutex_lock(&ipipe->inode->i_mutex);
1031 }
1032
1033 for (i = 0;; i++) {
1034 if (!opipe->readers) {
1035 send_sig(SIGPIPE, current, 0);
1036 if (!ret)
1037 ret = -EPIPE;
1038 break;
1039 }
1040 if (ipipe->nrbufs - i) {
1041 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1042
1043 /*
1044 * If we have room, fill this buffer
1045 */
1046 if (opipe->nrbufs < PIPE_BUFFERS) {
1047 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1048
1049 /*
1050 * Get a reference to this pipe buffer,
1051 * so we can copy the contents over.
1052 */
1053 ibuf->ops->get(ipipe, ibuf);
1054
1055 obuf = opipe->bufs + nbuf;
1056 *obuf = *ibuf;
1057
1058 if (obuf->len > len)
1059 obuf->len = len;
1060
1061 opipe->nrbufs++;
1062 do_wakeup = 1;
1063 ret += obuf->len;
1064 len -= obuf->len;
1065
1066 if (!len)
1067 break;
1068 if (opipe->nrbufs < PIPE_BUFFERS)
1069 continue;
1070 }
1071
1072 /*
1073 * We have input available, but no output room.
Jens Axboe2a27250e2006-04-19 15:56:40 +02001074 * If we already copied data, return that. If we
1075 * need to drop the opipe lock, it must be ordered
1076 * last to avoid deadlocks.
Jens Axboe70524492006-04-11 15:51:17 +02001077 */
Jens Axboe2a27250e2006-04-19 15:56:40 +02001078 if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001079 if (!ret)
1080 ret = -EAGAIN;
1081 break;
1082 }
1083 if (signal_pending(current)) {
1084 if (!ret)
1085 ret = -ERESTARTSYS;
1086 break;
1087 }
1088 if (do_wakeup) {
1089 smp_mb();
1090 if (waitqueue_active(&opipe->wait))
1091 wake_up_interruptible(&opipe->wait);
1092 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1093 do_wakeup = 0;
1094 }
1095
1096 opipe->waiting_writers++;
1097 pipe_wait(opipe);
1098 opipe->waiting_writers--;
1099 continue;
1100 }
1101
1102 /*
1103 * No input buffers, do the usual checks for available
1104 * writers and blocking and wait if necessary
1105 */
1106 if (!ipipe->writers)
1107 break;
1108 if (!ipipe->waiting_writers) {
1109 if (ret)
1110 break;
1111 }
Jens Axboe2a27250e2006-04-19 15:56:40 +02001112 /*
1113 * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1114 * with another process, we can only safely do that if
1115 * the ipipe lock is ordered last.
1116 */
1117 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001118 if (!ret)
1119 ret = -EAGAIN;
1120 break;
1121 }
1122 if (signal_pending(current)) {
1123 if (!ret)
1124 ret = -ERESTARTSYS;
1125 break;
1126 }
1127
1128 if (waitqueue_active(&ipipe->wait))
1129 wake_up_interruptible_sync(&ipipe->wait);
1130 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1131
1132 pipe_wait(ipipe);
1133 }
1134
1135 mutex_unlock(&ipipe->inode->i_mutex);
1136 mutex_unlock(&opipe->inode->i_mutex);
1137
1138 if (do_wakeup) {
1139 smp_mb();
1140 if (waitqueue_active(&opipe->wait))
1141 wake_up_interruptible(&opipe->wait);
1142 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1143 }
1144
1145 return ret;
1146}
1147
1148/*
1149 * This is a tee(1) implementation that works on pipes. It doesn't copy
1150 * any data, it simply references the 'in' pages on the 'out' pipe.
1151 * The 'flags' used are the SPLICE_F_* variants, currently the only
1152 * applicable one is SPLICE_F_NONBLOCK.
1153 */
1154static long do_tee(struct file *in, struct file *out, size_t len,
1155 unsigned int flags)
1156{
1157 struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1158 struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1159
1160 /*
1161 * Link ipipe to the two output pipes, consuming as we go along.
1162 */
1163 if (ipipe && opipe)
1164 return link_pipe(ipipe, opipe, len, flags);
1165
1166 return -EINVAL;
1167}
1168
1169asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1170{
1171 struct file *in;
1172 int error, fput_in;
1173
1174 if (unlikely(!len))
1175 return 0;
1176
1177 error = -EBADF;
1178 in = fget_light(fdin, &fput_in);
1179 if (in) {
1180 if (in->f_mode & FMODE_READ) {
1181 int fput_out;
1182 struct file *out = fget_light(fdout, &fput_out);
1183
1184 if (out) {
1185 if (out->f_mode & FMODE_WRITE)
1186 error = do_tee(in, out, len, flags);
1187 fput_light(out, fput_out);
1188 }
1189 }
1190 fput_light(in, fput_in);
1191 }
1192
1193 return error;
1194}