blob: 8d57e89924a68990efa6a72d279dd35efa0fc7a7 [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboec2058e02006-04-11 13:56:34 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/pipe_fs_i.h>
24#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe5274f052006-03-30 15:15:30 +020030
31/*
32 * Passed to the actors
33 */
34struct splice_desc {
35 unsigned int len, total_len; /* current and remaining length */
36 unsigned int flags; /* splice flags */
37 struct file *file; /* file to read/write */
38 loff_t pos; /* file position */
39};
40
Jens Axboe83f91352006-04-02 23:05:09 +020041/*
42 * Attempt to steal a page from a pipe buffer. This should perhaps go into
43 * a vm helper function, it's already simplified quite a bit by the
44 * addition of remove_mapping(). If success is returned, the caller may
45 * attempt to reuse this page for another destination.
46 */
Jens Axboe5abc97a2006-03-30 15:16:46 +020047static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
48 struct pipe_buffer *buf)
49{
50 struct page *page = buf->page;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020051 struct address_space *mapping = page_mapping(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020052
53 WARN_ON(!PageLocked(page));
54 WARN_ON(!PageUptodate(page));
55
Jens Axboead8d6f02006-04-02 23:10:32 +020056 /*
57 * At least for ext2 with nobh option, we need to wait on writeback
58 * completing on this page, since we'll remove it from the pagecache.
59 * Otherwise truncate wont wait on the page, allowing the disk
60 * blocks to be reused by someone else before we actually wrote our
61 * data to them. fs corruption ensues.
62 */
63 wait_on_page_writeback(page);
64
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020065 if (PagePrivate(page))
66 try_to_release_page(page, mapping_gfp_mask(mapping));
67
68 if (!remove_mapping(mapping, page))
Jens Axboe5abc97a2006-03-30 15:16:46 +020069 return 1;
70
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020071 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
Jens Axboe5abc97a2006-03-30 15:16:46 +020072 return 0;
73}
74
Jens Axboe5274f052006-03-30 15:15:30 +020075static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
76 struct pipe_buffer *buf)
77{
78 page_cache_release(buf->page);
79 buf->page = NULL;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020080 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
Jens Axboe5274f052006-03-30 15:15:30 +020081}
82
83static void *page_cache_pipe_buf_map(struct file *file,
84 struct pipe_inode_info *info,
85 struct pipe_buffer *buf)
86{
87 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020088 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020089
90 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +020091 lock_page(page);
92
93 /*
94 * Page got truncated/unhashed. This will cause a 0-byte
Ingo Molnar73d62d82006-04-11 13:57:21 +020095 * splice, if this is the first page.
Jens Axboe49d0b212006-04-10 09:04:41 +020096 */
97 if (!page->mapping) {
98 err = -ENODATA;
99 goto error;
100 }
101
102 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200103 * Uh oh, read-error from disk.
Jens Axboe49d0b212006-04-10 09:04:41 +0200104 */
105 if (!PageUptodate(page)) {
106 err = -EIO;
107 goto error;
108 }
109
110 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200111 * Page is ok afterall, fall through to mapping.
Jens Axboe49d0b212006-04-10 09:04:41 +0200112 */
Jens Axboe5274f052006-03-30 15:15:30 +0200113 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200114 }
115
Jens Axboe49d0b212006-04-10 09:04:41 +0200116 return kmap(page);
117error:
118 unlock_page(page);
119 return ERR_PTR(err);
Jens Axboe5274f052006-03-30 15:15:30 +0200120}
121
122static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
123 struct pipe_buffer *buf)
124{
Jens Axboe5274f052006-03-30 15:15:30 +0200125 kunmap(buf->page);
126}
127
Jens Axboe70524492006-04-11 15:51:17 +0200128static void page_cache_pipe_buf_get(struct pipe_inode_info *info,
129 struct pipe_buffer *buf)
130{
131 page_cache_get(buf->page);
132}
133
Jens Axboe5274f052006-03-30 15:15:30 +0200134static struct pipe_buf_operations page_cache_pipe_buf_ops = {
135 .can_merge = 0,
136 .map = page_cache_pipe_buf_map,
137 .unmap = page_cache_pipe_buf_unmap,
138 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200139 .steal = page_cache_pipe_buf_steal,
Jens Axboe70524492006-04-11 15:51:17 +0200140 .get = page_cache_pipe_buf_get,
Jens Axboe5274f052006-03-30 15:15:30 +0200141};
142
Jens Axboe83f91352006-04-02 23:05:09 +0200143/*
144 * Pipe output worker. This sets up our pipe format with the page cache
145 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
146 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200147static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
Jens Axboe5274f052006-03-30 15:15:30 +0200148 int nr_pages, unsigned long offset,
Linus Torvalds29e35092006-04-02 12:46:35 -0700149 unsigned long len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200150{
Jens Axboe5274f052006-03-30 15:15:30 +0200151 int ret, do_wakeup, i;
152
153 ret = 0;
154 do_wakeup = 0;
155 i = 0;
156
Ingo Molnar3a326a22006-04-10 15:18:35 +0200157 if (pipe->inode)
158 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200159
Jens Axboe5274f052006-03-30 15:15:30 +0200160 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200161 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200162 send_sig(SIGPIPE, current, 0);
163 if (!ret)
164 ret = -EPIPE;
165 break;
166 }
167
Jens Axboe6f767b02006-04-11 13:53:56 +0200168 if (pipe->nrbufs < PIPE_BUFFERS) {
169 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200170 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200171 struct page *page = pages[i++];
172 unsigned long this_len;
173
174 this_len = PAGE_CACHE_SIZE - offset;
175 if (this_len > len)
176 this_len = len;
177
178 buf->page = page;
179 buf->offset = offset;
180 buf->len = this_len;
181 buf->ops = &page_cache_pipe_buf_ops;
Jens Axboe6f767b02006-04-11 13:53:56 +0200182 pipe->nrbufs++;
183 if (pipe->inode)
184 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200185
186 ret += this_len;
187 len -= this_len;
188 offset = 0;
189 if (!--nr_pages)
190 break;
191 if (!len)
192 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200193 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200194 continue;
195
196 break;
197 }
198
Linus Torvalds29e35092006-04-02 12:46:35 -0700199 if (flags & SPLICE_F_NONBLOCK) {
200 if (!ret)
201 ret = -EAGAIN;
202 break;
203 }
204
Jens Axboe5274f052006-03-30 15:15:30 +0200205 if (signal_pending(current)) {
206 if (!ret)
207 ret = -ERESTARTSYS;
208 break;
209 }
210
211 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200212 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200213 if (waitqueue_active(&pipe->wait))
214 wake_up_interruptible_sync(&pipe->wait);
215 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200216 do_wakeup = 0;
217 }
218
Ingo Molnar3a326a22006-04-10 15:18:35 +0200219 pipe->waiting_writers++;
220 pipe_wait(pipe);
221 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200222 }
223
Ingo Molnar3a326a22006-04-10 15:18:35 +0200224 if (pipe->inode)
225 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200226
227 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200228 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200229 if (waitqueue_active(&pipe->wait))
230 wake_up_interruptible(&pipe->wait);
231 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200232 }
233
234 while (i < nr_pages)
235 page_cache_release(pages[i++]);
236
237 return ret;
238}
239
Ingo Molnar3a326a22006-04-10 15:18:35 +0200240static int
Jens Axboecbb7e572006-04-11 14:57:50 +0200241__generic_file_splice_read(struct file *in, loff_t *ppos,
242 struct pipe_inode_info *pipe, size_t len,
243 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200244{
245 struct address_space *mapping = in->f_mapping;
246 unsigned int offset, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200247 struct page *pages[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200248 struct page *page;
Jens Axboe16c523d2006-04-10 09:03:58 +0200249 pgoff_t index;
Jens Axboe7480a902006-04-11 13:52:47 +0200250 int i, error;
Jens Axboe5274f052006-03-30 15:15:30 +0200251
Jens Axboecbb7e572006-04-11 14:57:50 +0200252 index = *ppos >> PAGE_CACHE_SHIFT;
253 offset = *ppos & ~PAGE_CACHE_MASK;
Jens Axboe5274f052006-03-30 15:15:30 +0200254 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
255
256 if (nr_pages > PIPE_BUFFERS)
257 nr_pages = PIPE_BUFFERS;
258
259 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200260 * Initiate read-ahead on this page range. however, don't call into
Jens Axboe0b749ce2006-04-10 09:05:04 +0200261 * read-ahead if this is a non-zero offset (we are likely doing small
262 * chunk splice and the page is already there) for a single page.
Jens Axboe5274f052006-03-30 15:15:30 +0200263 */
Jens Axboe0b749ce2006-04-10 09:05:04 +0200264 if (!offset || nr_pages > 1)
265 do_page_cache_readahead(mapping, in, index, nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200266
267 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200268 * Now fill in the holes:
Jens Axboe5274f052006-03-30 15:15:30 +0200269 */
Jens Axboe7480a902006-04-11 13:52:47 +0200270 error = 0;
Jens Axboe16c523d2006-04-10 09:03:58 +0200271 for (i = 0; i < nr_pages; i++, index++) {
Jens Axboe7480a902006-04-11 13:52:47 +0200272find_page:
Jens Axboe5274f052006-03-30 15:15:30 +0200273 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200274 * lookup the page for this index
Jens Axboe5274f052006-03-30 15:15:30 +0200275 */
Jens Axboe7480a902006-04-11 13:52:47 +0200276 page = find_get_page(mapping, index);
277 if (!page) {
278 /*
279 * If in nonblock mode then dont block on
280 * readpage (we've kicked readahead so there
281 * will be asynchronous progress):
282 */
283 if (flags & SPLICE_F_NONBLOCK)
284 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200285
Jens Axboe7480a902006-04-11 13:52:47 +0200286 /*
287 * page didn't exist, allocate one
288 */
289 page = page_cache_alloc_cold(mapping);
290 if (!page)
291 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200292
Jens Axboe7480a902006-04-11 13:52:47 +0200293 error = add_to_page_cache_lru(page, mapping, index,
294 mapping_gfp_mask(mapping));
Jens Axboe5274f052006-03-30 15:15:30 +0200295 if (unlikely(error)) {
296 page_cache_release(page);
297 break;
298 }
Jens Axboe7480a902006-04-11 13:52:47 +0200299
300 goto readpage;
Jens Axboe5274f052006-03-30 15:15:30 +0200301 }
Jens Axboe7480a902006-04-11 13:52:47 +0200302
303 /*
304 * If the page isn't uptodate, we may need to start io on it
305 */
306 if (!PageUptodate(page)) {
307 lock_page(page);
308
309 /*
310 * page was truncated, stop here. if this isn't the
311 * first page, we'll just complete what we already
312 * added
313 */
314 if (!page->mapping) {
315 unlock_page(page);
316 page_cache_release(page);
317 break;
318 }
319 /*
320 * page was already under io and is now done, great
321 */
322 if (PageUptodate(page)) {
323 unlock_page(page);
324 goto fill_it;
325 }
326
327readpage:
328 /*
329 * need to read in the page
330 */
331 error = mapping->a_ops->readpage(in, page);
332
333 if (unlikely(error)) {
334 page_cache_release(page);
335 if (error == AOP_TRUNCATED_PAGE)
336 goto find_page;
337 break;
338 }
339 }
340fill_it:
Jens Axboe16c523d2006-04-10 09:03:58 +0200341 pages[i] = page;
Jens Axboe5274f052006-03-30 15:15:30 +0200342 }
343
Jens Axboe16c523d2006-04-10 09:03:58 +0200344 if (i)
345 return move_to_pipe(pipe, pages, i, offset, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200346
Jens Axboe7480a902006-04-11 13:52:47 +0200347 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200348}
349
Jens Axboe83f91352006-04-02 23:05:09 +0200350/**
351 * generic_file_splice_read - splice data from file to a pipe
352 * @in: file to splice from
353 * @pipe: pipe to splice to
354 * @len: number of bytes to splice
355 * @flags: splice modifier flags
356 *
357 * Will read pages from given file and fill them into a pipe.
Jens Axboe83f91352006-04-02 23:05:09 +0200358 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200359ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
360 struct pipe_inode_info *pipe, size_t len,
361 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200362{
363 ssize_t spliced;
364 int ret;
365
366 ret = 0;
367 spliced = 0;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200368
Jens Axboe5274f052006-03-30 15:15:30 +0200369 while (len) {
Jens Axboecbb7e572006-04-11 14:57:50 +0200370 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200371
372 if (ret <= 0)
373 break;
374
Jens Axboecbb7e572006-04-11 14:57:50 +0200375 *ppos += ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200376 len -= ret;
377 spliced += ret;
Linus Torvalds29e35092006-04-02 12:46:35 -0700378
379 if (!(flags & SPLICE_F_NONBLOCK))
380 continue;
381 ret = -EAGAIN;
382 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200383 }
384
385 if (spliced)
386 return spliced;
387
388 return ret;
389}
390
Jens Axboe059a8f32006-04-02 23:06:05 +0200391EXPORT_SYMBOL(generic_file_splice_read);
392
Jens Axboe5274f052006-03-30 15:15:30 +0200393/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200394 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
395 * using sendpage().
Jens Axboe5274f052006-03-30 15:15:30 +0200396 */
397static int pipe_to_sendpage(struct pipe_inode_info *info,
398 struct pipe_buffer *buf, struct splice_desc *sd)
399{
400 struct file *file = sd->file;
401 loff_t pos = sd->pos;
402 unsigned int offset;
403 ssize_t ret;
404 void *ptr;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200405 int more;
Jens Axboe5274f052006-03-30 15:15:30 +0200406
407 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200408 * Sub-optimal, but we are limited by the pipe ->map. We don't
Jens Axboe5274f052006-03-30 15:15:30 +0200409 * need a kmap'ed buffer here, we just want to make sure we
410 * have the page pinned if the pipe page originates from the
Ingo Molnar73d62d82006-04-11 13:57:21 +0200411 * page cache.
Jens Axboe5274f052006-03-30 15:15:30 +0200412 */
413 ptr = buf->ops->map(file, info, buf);
414 if (IS_ERR(ptr))
415 return PTR_ERR(ptr);
416
417 offset = pos & ~PAGE_CACHE_MASK;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200418 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200419
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200420 ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
Jens Axboe5274f052006-03-30 15:15:30 +0200421
422 buf->ops->unmap(info, buf);
423 if (ret == sd->len)
424 return 0;
425
426 return -EIO;
427}
428
429/*
430 * This is a little more tricky than the file -> pipe splicing. There are
431 * basically three cases:
432 *
433 * - Destination page already exists in the address space and there
434 * are users of it. For that case we have no other option that
435 * copying the data. Tough luck.
436 * - Destination page already exists in the address space, but there
437 * are no users of it. Make sure it's uptodate, then drop it. Fall
438 * through to last case.
439 * - Destination page does not exist, we can add the pipe page to
440 * the page cache and avoid the copy.
441 *
Jens Axboe83f91352006-04-02 23:05:09 +0200442 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
443 * sd->flags), we attempt to migrate pages from the pipe to the output
444 * file address space page cache. This is possible if no one else has
445 * the pipe page referenced outside of the pipe and page cache. If
446 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
447 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200448 */
449static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
450 struct splice_desc *sd)
451{
452 struct file *file = sd->file;
453 struct address_space *mapping = file->f_mapping;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200454 gfp_t gfp_mask = mapping_gfp_mask(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200455 unsigned int offset;
456 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200457 pgoff_t index;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200458 char *src;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200459 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200460
461 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200462 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200463 */
464 src = buf->ops->map(file, info, buf);
465 if (IS_ERR(src))
466 return PTR_ERR(src);
467
468 index = sd->pos >> PAGE_CACHE_SHIFT;
469 offset = sd->pos & ~PAGE_CACHE_MASK;
470
Jens Axboe5274f052006-03-30 15:15:30 +0200471 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200472 * Reuse buf page, if SPLICE_F_MOVE is set.
Jens Axboe5274f052006-03-30 15:15:30 +0200473 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200474 if (sd->flags & SPLICE_F_MOVE) {
Jens Axboe83f91352006-04-02 23:05:09 +0200475 /*
476 * If steal succeeds, buf->page is now pruned from the vm
477 * side (LRU and page cache) and we can reuse it.
478 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200479 if (buf->ops->steal(info, buf))
480 goto find_page;
Jens Axboe5274f052006-03-30 15:15:30 +0200481
Jens Axboe49d0b212006-04-10 09:04:41 +0200482 /*
483 * this will also set the page locked
484 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200485 page = buf->page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200486 if (add_to_page_cache(page, mapping, index, gfp_mask))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200487 goto find_page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200488
489 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
490 lru_cache_add(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200491 } else {
492find_page:
493 ret = -ENOMEM;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200494 page = find_or_create_page(mapping, index, gfp_mask);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200495 if (!page)
Dave Jones9aefe432006-04-10 09:02:40 +0200496 goto out_nomem;
Jens Axboe5274f052006-03-30 15:15:30 +0200497
Jens Axboe5abc97a2006-03-30 15:16:46 +0200498 /*
499 * If the page is uptodate, it is also locked. If it isn't
500 * uptodate, we can mark it uptodate if we are filling the
501 * full page. Otherwise we need to read it in first...
502 */
503 if (!PageUptodate(page)) {
504 if (sd->len < PAGE_CACHE_SIZE) {
505 ret = mapping->a_ops->readpage(file, page);
506 if (unlikely(ret))
507 goto out;
508
509 lock_page(page);
510
511 if (!PageUptodate(page)) {
512 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200513 * Page got invalidated, repeat.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200514 */
515 if (!page->mapping) {
516 unlock_page(page);
517 page_cache_release(page);
518 goto find_page;
519 }
520 ret = -EIO;
521 goto out;
Jens Axboe5274f052006-03-30 15:15:30 +0200522 }
Jens Axboe5abc97a2006-03-30 15:16:46 +0200523 } else {
524 WARN_ON(!PageLocked(page));
525 SetPageUptodate(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200526 }
Jens Axboe5274f052006-03-30 15:15:30 +0200527 }
528 }
529
530 ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200531 if (ret == AOP_TRUNCATED_PAGE) {
532 page_cache_release(page);
533 goto find_page;
534 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200535 goto out;
536
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200537 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200538 char *dst = kmap_atomic(page, KM_USER0);
539
540 memcpy(dst + offset, src + buf->offset, sd->len);
541 flush_dcache_page(page);
542 kunmap_atomic(dst, KM_USER0);
543 }
Jens Axboe5274f052006-03-30 15:15:30 +0200544
545 ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200546 if (ret == AOP_TRUNCATED_PAGE) {
547 page_cache_release(page);
548 goto find_page;
549 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200550 goto out;
551
Jens Axboec7f21e42006-04-10 09:01:01 +0200552 mark_page_accessed(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200553 balance_dirty_pages_ratelimited(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200554out:
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200555 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200556 page_cache_release(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200557 unlock_page(page);
558 }
Dave Jones9aefe432006-04-10 09:02:40 +0200559out_nomem:
Jens Axboe5274f052006-03-30 15:15:30 +0200560 buf->ops->unmap(info, buf);
561 return ret;
562}
563
564typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
565 struct splice_desc *);
566
Jens Axboe83f91352006-04-02 23:05:09 +0200567/*
568 * Pipe input worker. Most of this logic works like a regular pipe, the
569 * key here is the 'actor' worker passed in that actually moves the data
570 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
571 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200572static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200573 loff_t *ppos, size_t len, unsigned int flags,
Jens Axboe5274f052006-03-30 15:15:30 +0200574 splice_actor *actor)
575{
Jens Axboe5274f052006-03-30 15:15:30 +0200576 int ret, do_wakeup, err;
577 struct splice_desc sd;
578
579 ret = 0;
580 do_wakeup = 0;
581
582 sd.total_len = len;
583 sd.flags = flags;
584 sd.file = out;
Jens Axboecbb7e572006-04-11 14:57:50 +0200585 sd.pos = *ppos;
Jens Axboe5274f052006-03-30 15:15:30 +0200586
Ingo Molnar3a326a22006-04-10 15:18:35 +0200587 if (pipe->inode)
588 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200589
Jens Axboe5274f052006-03-30 15:15:30 +0200590 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200591 if (pipe->nrbufs) {
592 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200593 struct pipe_buf_operations *ops = buf->ops;
594
595 sd.len = buf->len;
596 if (sd.len > sd.total_len)
597 sd.len = sd.total_len;
598
Ingo Molnar3a326a22006-04-10 15:18:35 +0200599 err = actor(pipe, buf, &sd);
Jens Axboe5274f052006-03-30 15:15:30 +0200600 if (err) {
601 if (!ret && err != -ENODATA)
602 ret = err;
603
604 break;
605 }
606
607 ret += sd.len;
608 buf->offset += sd.len;
609 buf->len -= sd.len;
Ingo Molnar73d62d82006-04-11 13:57:21 +0200610
Jens Axboe5274f052006-03-30 15:15:30 +0200611 if (!buf->len) {
612 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200613 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200614 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
615 pipe->nrbufs--;
616 if (pipe->inode)
617 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200618 }
619
620 sd.pos += sd.len;
621 sd.total_len -= sd.len;
622 if (!sd.total_len)
623 break;
624 }
625
Jens Axboe6f767b02006-04-11 13:53:56 +0200626 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200627 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200628 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200629 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200630 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200631 if (ret)
632 break;
633 }
634
Linus Torvalds29e35092006-04-02 12:46:35 -0700635 if (flags & SPLICE_F_NONBLOCK) {
636 if (!ret)
637 ret = -EAGAIN;
638 break;
639 }
640
Jens Axboe5274f052006-03-30 15:15:30 +0200641 if (signal_pending(current)) {
642 if (!ret)
643 ret = -ERESTARTSYS;
644 break;
645 }
646
647 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200648 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200649 if (waitqueue_active(&pipe->wait))
650 wake_up_interruptible_sync(&pipe->wait);
651 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200652 do_wakeup = 0;
653 }
654
Ingo Molnar3a326a22006-04-10 15:18:35 +0200655 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200656 }
657
Ingo Molnar3a326a22006-04-10 15:18:35 +0200658 if (pipe->inode)
659 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200660
661 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200662 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200663 if (waitqueue_active(&pipe->wait))
664 wake_up_interruptible(&pipe->wait);
665 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200666 }
667
Jens Axboe5274f052006-03-30 15:15:30 +0200668 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200669}
670
Jens Axboe83f91352006-04-02 23:05:09 +0200671/**
672 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200673 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200674 * @out: file to write to
675 * @len: number of bytes to splice
676 * @flags: splice modifier flags
677 *
678 * Will either move or copy pages (determined by @flags options) from
679 * the given pipe inode to the given file.
680 *
681 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200682ssize_t
683generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200684 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200685{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200686 struct address_space *mapping = out->f_mapping;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200687 ssize_t ret;
688
Jens Axboecbb7e572006-04-11 14:57:50 +0200689 ret = move_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200690
691 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200692 * If file or inode is SYNC and we actually wrote some data, sync it.
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200693 */
694 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
695 && ret > 0) {
696 struct inode *inode = mapping->host;
697 int err;
698
699 mutex_lock(&inode->i_mutex);
700 err = generic_osync_inode(mapping->host, mapping,
Jens Axboe49570e92006-04-11 13:56:09 +0200701 OSYNC_METADATA|OSYNC_DATA);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200702 mutex_unlock(&inode->i_mutex);
703
704 if (err)
705 ret = err;
706 }
707
708 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200709}
710
Jens Axboe059a8f32006-04-02 23:06:05 +0200711EXPORT_SYMBOL(generic_file_splice_write);
712
Jens Axboe83f91352006-04-02 23:05:09 +0200713/**
714 * generic_splice_sendpage - splice data from a pipe to a socket
715 * @inode: pipe inode
716 * @out: socket to write to
717 * @len: number of bytes to splice
718 * @flags: splice modifier flags
719 *
720 * Will send @len bytes from the pipe to a network socket. No data copying
721 * is involved.
722 *
723 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200724ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200725 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200726{
Jens Axboecbb7e572006-04-11 14:57:50 +0200727 return move_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200728}
729
Jens Axboe059a8f32006-04-02 23:06:05 +0200730EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500731
Jens Axboe83f91352006-04-02 23:05:09 +0200732/*
733 * Attempt to initiate a splice from pipe to file.
734 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200735static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200736 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200737{
Jens Axboe5274f052006-03-30 15:15:30 +0200738 int ret;
739
Jens Axboe49570e92006-04-11 13:56:09 +0200740 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200741 return -EINVAL;
742
Jens Axboe49570e92006-04-11 13:56:09 +0200743 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200744 return -EBADF;
745
Jens Axboecbb7e572006-04-11 14:57:50 +0200746 ret = rw_verify_area(WRITE, out, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200747 if (unlikely(ret < 0))
748 return ret;
749
Jens Axboecbb7e572006-04-11 14:57:50 +0200750 return out->f_op->splice_write(pipe, out, ppos, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200751}
752
Jens Axboe83f91352006-04-02 23:05:09 +0200753/*
754 * Attempt to initiate a splice from a file to a pipe.
755 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200756static long do_splice_to(struct file *in, loff_t *ppos,
757 struct pipe_inode_info *pipe, size_t len,
758 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200759{
Jens Axboecbb7e572006-04-11 14:57:50 +0200760 loff_t isize, left;
Jens Axboe5274f052006-03-30 15:15:30 +0200761 int ret;
762
Jens Axboe49570e92006-04-11 13:56:09 +0200763 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200764 return -EINVAL;
765
Jens Axboe49570e92006-04-11 13:56:09 +0200766 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200767 return -EBADF;
768
Jens Axboecbb7e572006-04-11 14:57:50 +0200769 ret = rw_verify_area(READ, in, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200770 if (unlikely(ret < 0))
771 return ret;
772
773 isize = i_size_read(in->f_mapping->host);
Jens Axboecbb7e572006-04-11 14:57:50 +0200774 if (unlikely(*ppos >= isize))
Jens Axboe5274f052006-03-30 15:15:30 +0200775 return 0;
776
Jens Axboecbb7e572006-04-11 14:57:50 +0200777 left = isize - *ppos;
Jens Axboe49570e92006-04-11 13:56:09 +0200778 if (unlikely(left < len))
Jens Axboe5274f052006-03-30 15:15:30 +0200779 len = left;
780
Jens Axboecbb7e572006-04-11 14:57:50 +0200781 return in->f_op->splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200782}
783
Jens Axboecbb7e572006-04-11 14:57:50 +0200784long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
785 size_t len, unsigned int flags)
Jens Axboeb92ce552006-04-11 13:52:07 +0200786{
787 struct pipe_inode_info *pipe;
788 long ret, bytes;
Jens Axboecbb7e572006-04-11 14:57:50 +0200789 loff_t out_off;
Jens Axboeb92ce552006-04-11 13:52:07 +0200790 umode_t i_mode;
791 int i;
792
793 /*
794 * We require the input being a regular file, as we don't want to
795 * randomly drop data for eg socket -> socket splicing. Use the
796 * piped splicing for that!
797 */
798 i_mode = in->f_dentry->d_inode->i_mode;
799 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
800 return -EINVAL;
801
802 /*
803 * neither in nor out is a pipe, setup an internal pipe attached to
804 * 'out' and transfer the wanted data from 'in' to 'out' through that
805 */
806 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +0200807 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +0200808 pipe = alloc_pipe_info(NULL);
809 if (!pipe)
810 return -ENOMEM;
811
812 /*
813 * We don't have an immediate reader, but we'll read the stuff
814 * out of the pipe right after the move_to_pipe(). So set
815 * PIPE_READERS appropriately.
816 */
817 pipe->readers = 1;
818
819 current->splice_pipe = pipe;
820 }
821
822 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200823 * Do the splice.
Jens Axboeb92ce552006-04-11 13:52:07 +0200824 */
825 ret = 0;
826 bytes = 0;
Jens Axboecbb7e572006-04-11 14:57:50 +0200827 out_off = 0;
Jens Axboeb92ce552006-04-11 13:52:07 +0200828
829 while (len) {
830 size_t read_len, max_read_len;
831
832 /*
833 * Do at most PIPE_BUFFERS pages worth of transfer:
834 */
835 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
836
Jens Axboecbb7e572006-04-11 14:57:50 +0200837 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
Jens Axboeb92ce552006-04-11 13:52:07 +0200838 if (unlikely(ret < 0))
839 goto out_release;
840
841 read_len = ret;
842
843 /*
844 * NOTE: nonblocking mode only applies to the input. We
845 * must not do the output in nonblocking mode as then we
846 * could get stuck data in the internal pipe:
847 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200848 ret = do_splice_from(pipe, out, &out_off, read_len,
Jens Axboeb92ce552006-04-11 13:52:07 +0200849 flags & ~SPLICE_F_NONBLOCK);
850 if (unlikely(ret < 0))
851 goto out_release;
852
853 bytes += ret;
854 len -= ret;
855
856 /*
857 * In nonblocking mode, if we got back a short read then
858 * that was due to either an IO error or due to the
859 * pagecache entry not being there. In the IO error case
860 * the _next_ splice attempt will produce a clean IO error
861 * return value (not a short read), so in both cases it's
862 * correct to break out of the loop here:
863 */
864 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
865 break;
866 }
867
868 pipe->nrbufs = pipe->curbuf = 0;
869
870 return bytes;
871
872out_release:
873 /*
874 * If we did an incomplete transfer we must release
875 * the pipe buffers in question:
876 */
877 for (i = 0; i < PIPE_BUFFERS; i++) {
878 struct pipe_buffer *buf = pipe->bufs + i;
879
880 if (buf->ops) {
881 buf->ops->release(pipe, buf);
882 buf->ops = NULL;
883 }
884 }
885 pipe->nrbufs = pipe->curbuf = 0;
886
887 /*
888 * If we transferred some data, return the number of bytes:
889 */
890 if (bytes > 0)
891 return bytes;
892
893 return ret;
894}
895
896EXPORT_SYMBOL(do_splice_direct);
897
Jens Axboe83f91352006-04-02 23:05:09 +0200898/*
899 * Determine where to splice to/from.
900 */
Ingo Molnar529565d2006-04-10 15:18:58 +0200901static long do_splice(struct file *in, loff_t __user *off_in,
902 struct file *out, loff_t __user *off_out,
903 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200904{
Ingo Molnar3a326a22006-04-10 15:18:35 +0200905 struct pipe_inode_info *pipe;
Jens Axboecbb7e572006-04-11 14:57:50 +0200906 loff_t offset, *off;
Jens Axboe5274f052006-03-30 15:15:30 +0200907
Ingo Molnar3a326a22006-04-10 15:18:35 +0200908 pipe = in->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200909 if (pipe) {
910 if (off_in)
911 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200912 if (off_out) {
913 if (out->f_op->llseek == no_llseek)
914 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200915 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200916 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200917 off = &offset;
918 } else
919 off = &out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200920
Jens Axboecbb7e572006-04-11 14:57:50 +0200921 return do_splice_from(pipe, out, off, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200922 }
Jens Axboe5274f052006-03-30 15:15:30 +0200923
Ingo Molnar3a326a22006-04-10 15:18:35 +0200924 pipe = out->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200925 if (pipe) {
926 if (off_out)
927 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200928 if (off_in) {
929 if (in->f_op->llseek == no_llseek)
930 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200931 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200932 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200933 off = &offset;
934 } else
935 off = &in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200936
Jens Axboecbb7e572006-04-11 14:57:50 +0200937 return do_splice_to(in, off, pipe, len, flags);
Ingo Molnar529565d2006-04-10 15:18:58 +0200938 }
Jens Axboe5274f052006-03-30 15:15:30 +0200939
940 return -EINVAL;
941}
942
Ingo Molnar529565d2006-04-10 15:18:58 +0200943asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
944 int fd_out, loff_t __user *off_out,
945 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200946{
947 long error;
948 struct file *in, *out;
949 int fput_in, fput_out;
950
951 if (unlikely(!len))
952 return 0;
953
954 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +0200955 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +0200956 if (in) {
957 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +0200958 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +0200959 if (out) {
960 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +0200961 error = do_splice(in, off_in,
962 out, off_out,
963 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200964 fput_light(out, fput_out);
965 }
966 }
967
968 fput_light(in, fput_in);
969 }
970
971 return error;
972}
Jens Axboe70524492006-04-11 15:51:17 +0200973
974/*
975 * Link contents of ipipe to opipe.
976 */
977static int link_pipe(struct pipe_inode_info *ipipe,
978 struct pipe_inode_info *opipe,
979 size_t len, unsigned int flags)
980{
981 struct pipe_buffer *ibuf, *obuf;
982 int ret = 0, do_wakeup = 0, i;
983
984 /*
985 * Potential ABBA deadlock, work around it by ordering lock
986 * grabbing by inode address. Otherwise two different processes
987 * could deadlock (one doing tee from A -> B, the other from B -> A).
988 */
989 if (ipipe->inode < opipe->inode) {
990 mutex_lock(&ipipe->inode->i_mutex);
991 mutex_lock(&opipe->inode->i_mutex);
992 } else {
993 mutex_lock(&opipe->inode->i_mutex);
994 mutex_lock(&ipipe->inode->i_mutex);
995 }
996
997 for (i = 0;; i++) {
998 if (!opipe->readers) {
999 send_sig(SIGPIPE, current, 0);
1000 if (!ret)
1001 ret = -EPIPE;
1002 break;
1003 }
1004 if (ipipe->nrbufs - i) {
1005 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1006
1007 /*
1008 * If we have room, fill this buffer
1009 */
1010 if (opipe->nrbufs < PIPE_BUFFERS) {
1011 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1012
1013 /*
1014 * Get a reference to this pipe buffer,
1015 * so we can copy the contents over.
1016 */
1017 ibuf->ops->get(ipipe, ibuf);
1018
1019 obuf = opipe->bufs + nbuf;
1020 *obuf = *ibuf;
1021
1022 if (obuf->len > len)
1023 obuf->len = len;
1024
1025 opipe->nrbufs++;
1026 do_wakeup = 1;
1027 ret += obuf->len;
1028 len -= obuf->len;
1029
1030 if (!len)
1031 break;
1032 if (opipe->nrbufs < PIPE_BUFFERS)
1033 continue;
1034 }
1035
1036 /*
1037 * We have input available, but no output room.
1038 * If we already copied data, return that.
1039 */
1040 if (flags & SPLICE_F_NONBLOCK) {
1041 if (!ret)
1042 ret = -EAGAIN;
1043 break;
1044 }
1045 if (signal_pending(current)) {
1046 if (!ret)
1047 ret = -ERESTARTSYS;
1048 break;
1049 }
1050 if (do_wakeup) {
1051 smp_mb();
1052 if (waitqueue_active(&opipe->wait))
1053 wake_up_interruptible(&opipe->wait);
1054 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1055 do_wakeup = 0;
1056 }
1057
1058 opipe->waiting_writers++;
1059 pipe_wait(opipe);
1060 opipe->waiting_writers--;
1061 continue;
1062 }
1063
1064 /*
1065 * No input buffers, do the usual checks for available
1066 * writers and blocking and wait if necessary
1067 */
1068 if (!ipipe->writers)
1069 break;
1070 if (!ipipe->waiting_writers) {
1071 if (ret)
1072 break;
1073 }
1074 if (flags & SPLICE_F_NONBLOCK) {
1075 if (!ret)
1076 ret = -EAGAIN;
1077 break;
1078 }
1079 if (signal_pending(current)) {
1080 if (!ret)
1081 ret = -ERESTARTSYS;
1082 break;
1083 }
1084
1085 if (waitqueue_active(&ipipe->wait))
1086 wake_up_interruptible_sync(&ipipe->wait);
1087 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1088
1089 pipe_wait(ipipe);
1090 }
1091
1092 mutex_unlock(&ipipe->inode->i_mutex);
1093 mutex_unlock(&opipe->inode->i_mutex);
1094
1095 if (do_wakeup) {
1096 smp_mb();
1097 if (waitqueue_active(&opipe->wait))
1098 wake_up_interruptible(&opipe->wait);
1099 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1100 }
1101
1102 return ret;
1103}
1104
1105/*
1106 * This is a tee(1) implementation that works on pipes. It doesn't copy
1107 * any data, it simply references the 'in' pages on the 'out' pipe.
1108 * The 'flags' used are the SPLICE_F_* variants, currently the only
1109 * applicable one is SPLICE_F_NONBLOCK.
1110 */
1111static long do_tee(struct file *in, struct file *out, size_t len,
1112 unsigned int flags)
1113{
1114 struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1115 struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1116
1117 /*
1118 * Link ipipe to the two output pipes, consuming as we go along.
1119 */
1120 if (ipipe && opipe)
1121 return link_pipe(ipipe, opipe, len, flags);
1122
1123 return -EINVAL;
1124}
1125
1126asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1127{
1128 struct file *in;
1129 int error, fput_in;
1130
1131 if (unlikely(!len))
1132 return 0;
1133
1134 error = -EBADF;
1135 in = fget_light(fdin, &fput_in);
1136 if (in) {
1137 if (in->f_mode & FMODE_READ) {
1138 int fput_out;
1139 struct file *out = fget_light(fdout, &fput_out);
1140
1141 if (out) {
1142 if (out->f_mode & FMODE_WRITE)
1143 error = do_tee(in, out, len, flags);
1144 fput_light(out, fput_out);
1145 }
1146 }
1147 fput_light(in, fput_in);
1148 }
1149
1150 return error;
1151}