blob: 0559e7577a04164ececffdcdbd57089d980dd586 [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboec2058e02006-04-11 13:56:34 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/pipe_fs_i.h>
24#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe5274f052006-03-30 15:15:30 +020030
31/*
32 * Passed to the actors
33 */
34struct splice_desc {
35 unsigned int len, total_len; /* current and remaining length */
36 unsigned int flags; /* splice flags */
37 struct file *file; /* file to read/write */
38 loff_t pos; /* file position */
39};
40
Jens Axboe83f91352006-04-02 23:05:09 +020041/*
42 * Attempt to steal a page from a pipe buffer. This should perhaps go into
43 * a vm helper function, it's already simplified quite a bit by the
44 * addition of remove_mapping(). If success is returned, the caller may
45 * attempt to reuse this page for another destination.
46 */
Jens Axboe5abc97a2006-03-30 15:16:46 +020047static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
48 struct pipe_buffer *buf)
49{
50 struct page *page = buf->page;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020051 struct address_space *mapping = page_mapping(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020052
Jens Axboe9e0267c2006-04-19 15:57:31 +020053 lock_page(page);
54
Jens Axboe5abc97a2006-03-30 15:16:46 +020055 WARN_ON(!PageUptodate(page));
56
Jens Axboead8d6f02006-04-02 23:10:32 +020057 /*
58 * At least for ext2 with nobh option, we need to wait on writeback
59 * completing on this page, since we'll remove it from the pagecache.
60 * Otherwise truncate wont wait on the page, allowing the disk
61 * blocks to be reused by someone else before we actually wrote our
62 * data to them. fs corruption ensues.
63 */
64 wait_on_page_writeback(page);
65
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020066 if (PagePrivate(page))
67 try_to_release_page(page, mapping_gfp_mask(mapping));
68
Jens Axboe9e0267c2006-04-19 15:57:31 +020069 if (!remove_mapping(mapping, page)) {
70 unlock_page(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020071 return 1;
Jens Axboe9e0267c2006-04-19 15:57:31 +020072 }
Jens Axboe5abc97a2006-03-30 15:16:46 +020073
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020074 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
Jens Axboe5abc97a2006-03-30 15:16:46 +020075 return 0;
76}
77
Jens Axboe5274f052006-03-30 15:15:30 +020078static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
79 struct pipe_buffer *buf)
80{
81 page_cache_release(buf->page);
82 buf->page = NULL;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020083 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
Jens Axboe5274f052006-03-30 15:15:30 +020084}
85
86static void *page_cache_pipe_buf_map(struct file *file,
87 struct pipe_inode_info *info,
88 struct pipe_buffer *buf)
89{
90 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020091 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020092
93 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +020094 lock_page(page);
95
96 /*
97 * Page got truncated/unhashed. This will cause a 0-byte
Ingo Molnar73d62d82006-04-11 13:57:21 +020098 * splice, if this is the first page.
Jens Axboe49d0b212006-04-10 09:04:41 +020099 */
100 if (!page->mapping) {
101 err = -ENODATA;
102 goto error;
103 }
104
105 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200106 * Uh oh, read-error from disk.
Jens Axboe49d0b212006-04-10 09:04:41 +0200107 */
108 if (!PageUptodate(page)) {
109 err = -EIO;
110 goto error;
111 }
112
113 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200114 * Page is ok afterall, fall through to mapping.
Jens Axboe49d0b212006-04-10 09:04:41 +0200115 */
Jens Axboe5274f052006-03-30 15:15:30 +0200116 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200117 }
118
Jens Axboe49d0b212006-04-10 09:04:41 +0200119 return kmap(page);
120error:
121 unlock_page(page);
122 return ERR_PTR(err);
Jens Axboe5274f052006-03-30 15:15:30 +0200123}
124
125static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
126 struct pipe_buffer *buf)
127{
Jens Axboe5274f052006-03-30 15:15:30 +0200128 kunmap(buf->page);
129}
130
Jens Axboe70524492006-04-11 15:51:17 +0200131static void page_cache_pipe_buf_get(struct pipe_inode_info *info,
132 struct pipe_buffer *buf)
133{
134 page_cache_get(buf->page);
135}
136
Jens Axboe5274f052006-03-30 15:15:30 +0200137static struct pipe_buf_operations page_cache_pipe_buf_ops = {
138 .can_merge = 0,
139 .map = page_cache_pipe_buf_map,
140 .unmap = page_cache_pipe_buf_unmap,
141 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200142 .steal = page_cache_pipe_buf_steal,
Jens Axboe70524492006-04-11 15:51:17 +0200143 .get = page_cache_pipe_buf_get,
Jens Axboe5274f052006-03-30 15:15:30 +0200144};
145
Jens Axboe83f91352006-04-02 23:05:09 +0200146/*
147 * Pipe output worker. This sets up our pipe format with the page cache
148 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
149 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200150static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
Jens Axboe91ad66e2006-04-19 15:55:10 +0200151 int nr_pages, unsigned long len,
152 unsigned int offset, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200153{
Jens Axboe5274f052006-03-30 15:15:30 +0200154 int ret, do_wakeup, i;
155
156 ret = 0;
157 do_wakeup = 0;
158 i = 0;
159
Ingo Molnar3a326a22006-04-10 15:18:35 +0200160 if (pipe->inode)
161 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200162
Jens Axboe5274f052006-03-30 15:15:30 +0200163 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200164 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200165 send_sig(SIGPIPE, current, 0);
166 if (!ret)
167 ret = -EPIPE;
168 break;
169 }
170
Jens Axboe6f767b02006-04-11 13:53:56 +0200171 if (pipe->nrbufs < PIPE_BUFFERS) {
172 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200173 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200174 struct page *page = pages[i++];
175 unsigned long this_len;
176
177 this_len = PAGE_CACHE_SIZE - offset;
178 if (this_len > len)
179 this_len = len;
180
181 buf->page = page;
182 buf->offset = offset;
183 buf->len = this_len;
184 buf->ops = &page_cache_pipe_buf_ops;
Jens Axboe6f767b02006-04-11 13:53:56 +0200185 pipe->nrbufs++;
186 if (pipe->inode)
187 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200188
189 ret += this_len;
190 len -= this_len;
191 offset = 0;
192 if (!--nr_pages)
193 break;
194 if (!len)
195 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200196 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200197 continue;
198
199 break;
200 }
201
Linus Torvalds29e35092006-04-02 12:46:35 -0700202 if (flags & SPLICE_F_NONBLOCK) {
203 if (!ret)
204 ret = -EAGAIN;
205 break;
206 }
207
Jens Axboe5274f052006-03-30 15:15:30 +0200208 if (signal_pending(current)) {
209 if (!ret)
210 ret = -ERESTARTSYS;
211 break;
212 }
213
214 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200215 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200216 if (waitqueue_active(&pipe->wait))
217 wake_up_interruptible_sync(&pipe->wait);
218 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200219 do_wakeup = 0;
220 }
221
Ingo Molnar3a326a22006-04-10 15:18:35 +0200222 pipe->waiting_writers++;
223 pipe_wait(pipe);
224 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200225 }
226
Ingo Molnar3a326a22006-04-10 15:18:35 +0200227 if (pipe->inode)
228 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200229
230 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200231 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200232 if (waitqueue_active(&pipe->wait))
233 wake_up_interruptible(&pipe->wait);
234 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200235 }
236
237 while (i < nr_pages)
238 page_cache_release(pages[i++]);
239
240 return ret;
241}
242
Ingo Molnar3a326a22006-04-10 15:18:35 +0200243static int
Jens Axboecbb7e572006-04-11 14:57:50 +0200244__generic_file_splice_read(struct file *in, loff_t *ppos,
245 struct pipe_inode_info *pipe, size_t len,
246 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200247{
248 struct address_space *mapping = in->f_mapping;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200249 unsigned int loff, offset, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200250 struct page *pages[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200251 struct page *page;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200252 pgoff_t index, end_index;
253 loff_t isize;
254 size_t bytes;
Jens Axboe7480a902006-04-11 13:52:47 +0200255 int i, error;
Jens Axboe5274f052006-03-30 15:15:30 +0200256
Jens Axboecbb7e572006-04-11 14:57:50 +0200257 index = *ppos >> PAGE_CACHE_SHIFT;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200258 loff = offset = *ppos & ~PAGE_CACHE_MASK;
Jens Axboe5274f052006-03-30 15:15:30 +0200259 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
260
261 if (nr_pages > PIPE_BUFFERS)
262 nr_pages = PIPE_BUFFERS;
263
264 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200265 * Initiate read-ahead on this page range. however, don't call into
Jens Axboe0b749ce2006-04-10 09:05:04 +0200266 * read-ahead if this is a non-zero offset (we are likely doing small
267 * chunk splice and the page is already there) for a single page.
Jens Axboe5274f052006-03-30 15:15:30 +0200268 */
Jens Axboe0b749ce2006-04-10 09:05:04 +0200269 if (!offset || nr_pages > 1)
270 do_page_cache_readahead(mapping, in, index, nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200271
272 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200273 * Now fill in the holes:
Jens Axboe5274f052006-03-30 15:15:30 +0200274 */
Jens Axboe7480a902006-04-11 13:52:47 +0200275 error = 0;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200276 bytes = 0;
Jens Axboe16c523d2006-04-10 09:03:58 +0200277 for (i = 0; i < nr_pages; i++, index++) {
Jens Axboe82aa5d62006-04-20 13:05:48 +0200278 unsigned int this_len;
279
280 if (!len)
281 break;
282
283 /*
284 * this_len is the max we'll use from this page
285 */
286 this_len = min(len, PAGE_CACHE_SIZE - loff);
Jens Axboe7480a902006-04-11 13:52:47 +0200287find_page:
Jens Axboe5274f052006-03-30 15:15:30 +0200288 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200289 * lookup the page for this index
Jens Axboe5274f052006-03-30 15:15:30 +0200290 */
Jens Axboe7480a902006-04-11 13:52:47 +0200291 page = find_get_page(mapping, index);
292 if (!page) {
293 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200294 * page didn't exist, allocate one
295 */
296 page = page_cache_alloc_cold(mapping);
297 if (!page)
298 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200299
Jens Axboe7480a902006-04-11 13:52:47 +0200300 error = add_to_page_cache_lru(page, mapping, index,
301 mapping_gfp_mask(mapping));
Jens Axboe5274f052006-03-30 15:15:30 +0200302 if (unlikely(error)) {
303 page_cache_release(page);
304 break;
305 }
Jens Axboe7480a902006-04-11 13:52:47 +0200306
307 goto readpage;
Jens Axboe5274f052006-03-30 15:15:30 +0200308 }
Jens Axboe7480a902006-04-11 13:52:47 +0200309
310 /*
311 * If the page isn't uptodate, we may need to start io on it
312 */
313 if (!PageUptodate(page)) {
Jens Axboec4f895c2006-04-19 15:56:12 +0200314 /*
315 * If in nonblock mode then dont block on waiting
316 * for an in-flight io page
317 */
318 if (flags & SPLICE_F_NONBLOCK)
319 break;
320
Jens Axboe7480a902006-04-11 13:52:47 +0200321 lock_page(page);
322
323 /*
324 * page was truncated, stop here. if this isn't the
325 * first page, we'll just complete what we already
326 * added
327 */
328 if (!page->mapping) {
329 unlock_page(page);
330 page_cache_release(page);
331 break;
332 }
333 /*
334 * page was already under io and is now done, great
335 */
336 if (PageUptodate(page)) {
337 unlock_page(page);
338 goto fill_it;
339 }
340
341readpage:
342 /*
343 * need to read in the page
344 */
345 error = mapping->a_ops->readpage(in, page);
346
347 if (unlikely(error)) {
348 page_cache_release(page);
349 if (error == AOP_TRUNCATED_PAGE)
350 goto find_page;
351 break;
352 }
Jens Axboe91ad66e2006-04-19 15:55:10 +0200353
354 /*
355 * i_size must be checked after ->readpage().
356 */
357 isize = i_size_read(mapping->host);
358 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
359 if (unlikely(!isize || index > end_index)) {
360 page_cache_release(page);
361 break;
362 }
363
364 /*
365 * if this is the last page, see if we need to shrink
366 * the length and stop
367 */
368 if (end_index == index) {
369 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
370 if (bytes + loff > isize) {
371 page_cache_release(page);
372 break;
373 }
374 /*
375 * force quit after adding this page
376 */
377 nr_pages = i;
Jens Axboe82aa5d62006-04-20 13:05:48 +0200378 this_len = min(this_len, loff);
Jens Axboe91ad66e2006-04-19 15:55:10 +0200379 }
Jens Axboe7480a902006-04-11 13:52:47 +0200380 }
381fill_it:
Jens Axboe16c523d2006-04-10 09:03:58 +0200382 pages[i] = page;
Jens Axboe82aa5d62006-04-20 13:05:48 +0200383 bytes += this_len;
384 len -= this_len;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200385 loff = 0;
Jens Axboe5274f052006-03-30 15:15:30 +0200386 }
387
Jens Axboe16c523d2006-04-10 09:03:58 +0200388 if (i)
Jens Axboe91ad66e2006-04-19 15:55:10 +0200389 return move_to_pipe(pipe, pages, i, bytes, offset, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200390
Jens Axboe7480a902006-04-11 13:52:47 +0200391 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200392}
393
Jens Axboe83f91352006-04-02 23:05:09 +0200394/**
395 * generic_file_splice_read - splice data from file to a pipe
396 * @in: file to splice from
397 * @pipe: pipe to splice to
398 * @len: number of bytes to splice
399 * @flags: splice modifier flags
400 *
401 * Will read pages from given file and fill them into a pipe.
Jens Axboe83f91352006-04-02 23:05:09 +0200402 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200403ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
404 struct pipe_inode_info *pipe, size_t len,
405 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200406{
407 ssize_t spliced;
408 int ret;
409
410 ret = 0;
411 spliced = 0;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200412
Jens Axboe5274f052006-03-30 15:15:30 +0200413 while (len) {
Jens Axboecbb7e572006-04-11 14:57:50 +0200414 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200415
Jens Axboec4f895c2006-04-19 15:56:12 +0200416 if (ret < 0)
Jens Axboe5274f052006-03-30 15:15:30 +0200417 break;
Jens Axboec4f895c2006-04-19 15:56:12 +0200418 else if (!ret) {
419 if (spliced)
420 break;
421 if (flags & SPLICE_F_NONBLOCK) {
422 ret = -EAGAIN;
423 break;
424 }
425 }
Jens Axboe5274f052006-03-30 15:15:30 +0200426
Jens Axboecbb7e572006-04-11 14:57:50 +0200427 *ppos += ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200428 len -= ret;
429 spliced += ret;
430 }
431
432 if (spliced)
433 return spliced;
434
435 return ret;
436}
437
Jens Axboe059a8f32006-04-02 23:06:05 +0200438EXPORT_SYMBOL(generic_file_splice_read);
439
Jens Axboe5274f052006-03-30 15:15:30 +0200440/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200441 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
442 * using sendpage().
Jens Axboe5274f052006-03-30 15:15:30 +0200443 */
444static int pipe_to_sendpage(struct pipe_inode_info *info,
445 struct pipe_buffer *buf, struct splice_desc *sd)
446{
447 struct file *file = sd->file;
448 loff_t pos = sd->pos;
449 unsigned int offset;
450 ssize_t ret;
451 void *ptr;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200452 int more;
Jens Axboe5274f052006-03-30 15:15:30 +0200453
454 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200455 * Sub-optimal, but we are limited by the pipe ->map. We don't
Jens Axboe5274f052006-03-30 15:15:30 +0200456 * need a kmap'ed buffer here, we just want to make sure we
457 * have the page pinned if the pipe page originates from the
Ingo Molnar73d62d82006-04-11 13:57:21 +0200458 * page cache.
Jens Axboe5274f052006-03-30 15:15:30 +0200459 */
460 ptr = buf->ops->map(file, info, buf);
461 if (IS_ERR(ptr))
462 return PTR_ERR(ptr);
463
464 offset = pos & ~PAGE_CACHE_MASK;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200465 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200466
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200467 ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
Jens Axboe5274f052006-03-30 15:15:30 +0200468
469 buf->ops->unmap(info, buf);
470 if (ret == sd->len)
471 return 0;
472
473 return -EIO;
474}
475
476/*
477 * This is a little more tricky than the file -> pipe splicing. There are
478 * basically three cases:
479 *
480 * - Destination page already exists in the address space and there
481 * are users of it. For that case we have no other option that
482 * copying the data. Tough luck.
483 * - Destination page already exists in the address space, but there
484 * are no users of it. Make sure it's uptodate, then drop it. Fall
485 * through to last case.
486 * - Destination page does not exist, we can add the pipe page to
487 * the page cache and avoid the copy.
488 *
Jens Axboe83f91352006-04-02 23:05:09 +0200489 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
490 * sd->flags), we attempt to migrate pages from the pipe to the output
491 * file address space page cache. This is possible if no one else has
492 * the pipe page referenced outside of the pipe and page cache. If
493 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
494 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200495 */
496static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
497 struct splice_desc *sd)
498{
499 struct file *file = sd->file;
500 struct address_space *mapping = file->f_mapping;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200501 gfp_t gfp_mask = mapping_gfp_mask(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200502 unsigned int offset;
503 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200504 pgoff_t index;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200505 char *src;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200506 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200507
508 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200509 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200510 */
511 src = buf->ops->map(file, info, buf);
512 if (IS_ERR(src))
513 return PTR_ERR(src);
514
515 index = sd->pos >> PAGE_CACHE_SHIFT;
516 offset = sd->pos & ~PAGE_CACHE_MASK;
517
Jens Axboe5274f052006-03-30 15:15:30 +0200518 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200519 * Reuse buf page, if SPLICE_F_MOVE is set.
Jens Axboe5274f052006-03-30 15:15:30 +0200520 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200521 if (sd->flags & SPLICE_F_MOVE) {
Jens Axboe83f91352006-04-02 23:05:09 +0200522 /*
523 * If steal succeeds, buf->page is now pruned from the vm
Jens Axboe9e0267c2006-04-19 15:57:31 +0200524 * side (LRU and page cache) and we can reuse it. The page
525 * will also be looked on successful return.
Jens Axboe83f91352006-04-02 23:05:09 +0200526 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200527 if (buf->ops->steal(info, buf))
528 goto find_page;
Jens Axboe5274f052006-03-30 15:15:30 +0200529
Jens Axboe5abc97a2006-03-30 15:16:46 +0200530 page = buf->page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200531 if (add_to_page_cache(page, mapping, index, gfp_mask))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200532 goto find_page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200533
534 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
535 lru_cache_add(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200536 } else {
537find_page:
Jens Axboe9e0267c2006-04-19 15:57:31 +0200538 page = find_lock_page(mapping, index);
539 if (!page) {
540 ret = -ENOMEM;
541 page = page_cache_alloc_cold(mapping);
542 if (unlikely(!page))
543 goto out_nomem;
544
545 /*
546 * This will also lock the page
547 */
548 ret = add_to_page_cache_lru(page, mapping, index,
549 gfp_mask);
550 if (unlikely(ret))
551 goto out;
552 }
Jens Axboe5274f052006-03-30 15:15:30 +0200553
Jens Axboe5abc97a2006-03-30 15:16:46 +0200554 /*
Jens Axboe9e0267c2006-04-19 15:57:31 +0200555 * We get here with the page locked. If the page is also
556 * uptodate, we don't need to do more. If it isn't, we
557 * may need to bring it in if we are not going to overwrite
558 * the full page.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200559 */
560 if (!PageUptodate(page)) {
561 if (sd->len < PAGE_CACHE_SIZE) {
562 ret = mapping->a_ops->readpage(file, page);
563 if (unlikely(ret))
564 goto out;
565
566 lock_page(page);
567
568 if (!PageUptodate(page)) {
569 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200570 * Page got invalidated, repeat.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200571 */
572 if (!page->mapping) {
573 unlock_page(page);
574 page_cache_release(page);
575 goto find_page;
576 }
577 ret = -EIO;
578 goto out;
Jens Axboe5274f052006-03-30 15:15:30 +0200579 }
Jens Axboe9e0267c2006-04-19 15:57:31 +0200580 } else
Jens Axboe5abc97a2006-03-30 15:16:46 +0200581 SetPageUptodate(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200582 }
583 }
584
585 ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200586 if (ret == AOP_TRUNCATED_PAGE) {
587 page_cache_release(page);
588 goto find_page;
589 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200590 goto out;
591
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200592 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200593 char *dst = kmap_atomic(page, KM_USER0);
594
595 memcpy(dst + offset, src + buf->offset, sd->len);
596 flush_dcache_page(page);
597 kunmap_atomic(dst, KM_USER0);
598 }
Jens Axboe5274f052006-03-30 15:15:30 +0200599
600 ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200601 if (ret == AOP_TRUNCATED_PAGE) {
602 page_cache_release(page);
603 goto find_page;
604 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200605 goto out;
606
Jens Axboec7f21e42006-04-10 09:01:01 +0200607 mark_page_accessed(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200608 balance_dirty_pages_ratelimited(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200609out:
Jens Axboe9e0267c2006-04-19 15:57:31 +0200610 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200611 page_cache_release(page);
Jens Axboe9e0267c2006-04-19 15:57:31 +0200612
613 unlock_page(page);
Dave Jones9aefe432006-04-10 09:02:40 +0200614out_nomem:
Jens Axboe5274f052006-03-30 15:15:30 +0200615 buf->ops->unmap(info, buf);
616 return ret;
617}
618
619typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
620 struct splice_desc *);
621
Jens Axboe83f91352006-04-02 23:05:09 +0200622/*
623 * Pipe input worker. Most of this logic works like a regular pipe, the
624 * key here is the 'actor' worker passed in that actually moves the data
625 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
626 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200627static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200628 loff_t *ppos, size_t len, unsigned int flags,
Jens Axboe5274f052006-03-30 15:15:30 +0200629 splice_actor *actor)
630{
Jens Axboe5274f052006-03-30 15:15:30 +0200631 int ret, do_wakeup, err;
632 struct splice_desc sd;
633
634 ret = 0;
635 do_wakeup = 0;
636
637 sd.total_len = len;
638 sd.flags = flags;
639 sd.file = out;
Jens Axboecbb7e572006-04-11 14:57:50 +0200640 sd.pos = *ppos;
Jens Axboe5274f052006-03-30 15:15:30 +0200641
Ingo Molnar3a326a22006-04-10 15:18:35 +0200642 if (pipe->inode)
643 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200644
Jens Axboe5274f052006-03-30 15:15:30 +0200645 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200646 if (pipe->nrbufs) {
647 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200648 struct pipe_buf_operations *ops = buf->ops;
649
650 sd.len = buf->len;
651 if (sd.len > sd.total_len)
652 sd.len = sd.total_len;
653
Ingo Molnar3a326a22006-04-10 15:18:35 +0200654 err = actor(pipe, buf, &sd);
Jens Axboe5274f052006-03-30 15:15:30 +0200655 if (err) {
656 if (!ret && err != -ENODATA)
657 ret = err;
658
659 break;
660 }
661
662 ret += sd.len;
663 buf->offset += sd.len;
664 buf->len -= sd.len;
Ingo Molnar73d62d82006-04-11 13:57:21 +0200665
Jens Axboe5274f052006-03-30 15:15:30 +0200666 if (!buf->len) {
667 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200668 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200669 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
670 pipe->nrbufs--;
671 if (pipe->inode)
672 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200673 }
674
675 sd.pos += sd.len;
676 sd.total_len -= sd.len;
677 if (!sd.total_len)
678 break;
679 }
680
Jens Axboe6f767b02006-04-11 13:53:56 +0200681 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200682 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200683 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200684 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200685 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200686 if (ret)
687 break;
688 }
689
Linus Torvalds29e35092006-04-02 12:46:35 -0700690 if (flags & SPLICE_F_NONBLOCK) {
691 if (!ret)
692 ret = -EAGAIN;
693 break;
694 }
695
Jens Axboe5274f052006-03-30 15:15:30 +0200696 if (signal_pending(current)) {
697 if (!ret)
698 ret = -ERESTARTSYS;
699 break;
700 }
701
702 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200703 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200704 if (waitqueue_active(&pipe->wait))
705 wake_up_interruptible_sync(&pipe->wait);
706 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200707 do_wakeup = 0;
708 }
709
Ingo Molnar3a326a22006-04-10 15:18:35 +0200710 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200711 }
712
Ingo Molnar3a326a22006-04-10 15:18:35 +0200713 if (pipe->inode)
714 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200715
716 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200717 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200718 if (waitqueue_active(&pipe->wait))
719 wake_up_interruptible(&pipe->wait);
720 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200721 }
722
Jens Axboe5274f052006-03-30 15:15:30 +0200723 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200724}
725
Jens Axboe83f91352006-04-02 23:05:09 +0200726/**
727 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200728 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200729 * @out: file to write to
730 * @len: number of bytes to splice
731 * @flags: splice modifier flags
732 *
733 * Will either move or copy pages (determined by @flags options) from
734 * the given pipe inode to the given file.
735 *
736 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200737ssize_t
738generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200739 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200740{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200741 struct address_space *mapping = out->f_mapping;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200742 ssize_t ret;
743
Jens Axboecbb7e572006-04-11 14:57:50 +0200744 ret = move_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
Jens Axboea4514eb2006-04-19 15:57:05 +0200745 if (ret > 0) {
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200746 struct inode *inode = mapping->host;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200747
Jens Axboea4514eb2006-04-19 15:57:05 +0200748 *ppos += ret;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200749
Jens Axboea4514eb2006-04-19 15:57:05 +0200750 /*
751 * If file or inode is SYNC and we actually wrote some data,
752 * sync it.
753 */
754 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
755 int err;
756
757 mutex_lock(&inode->i_mutex);
758 err = generic_osync_inode(inode, mapping,
759 OSYNC_METADATA|OSYNC_DATA);
760 mutex_unlock(&inode->i_mutex);
761
762 if (err)
763 ret = err;
764 }
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200765 }
766
767 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200768}
769
Jens Axboe059a8f32006-04-02 23:06:05 +0200770EXPORT_SYMBOL(generic_file_splice_write);
771
Jens Axboe83f91352006-04-02 23:05:09 +0200772/**
773 * generic_splice_sendpage - splice data from a pipe to a socket
774 * @inode: pipe inode
775 * @out: socket to write to
776 * @len: number of bytes to splice
777 * @flags: splice modifier flags
778 *
779 * Will send @len bytes from the pipe to a network socket. No data copying
780 * is involved.
781 *
782 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200783ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200784 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200785{
Jens Axboecbb7e572006-04-11 14:57:50 +0200786 return move_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200787}
788
Jens Axboe059a8f32006-04-02 23:06:05 +0200789EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500790
Jens Axboe83f91352006-04-02 23:05:09 +0200791/*
792 * Attempt to initiate a splice from pipe to file.
793 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200794static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200795 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200796{
Jens Axboe5274f052006-03-30 15:15:30 +0200797 int ret;
798
Jens Axboe49570e92006-04-11 13:56:09 +0200799 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200800 return -EINVAL;
801
Jens Axboe49570e92006-04-11 13:56:09 +0200802 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200803 return -EBADF;
804
Jens Axboecbb7e572006-04-11 14:57:50 +0200805 ret = rw_verify_area(WRITE, out, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200806 if (unlikely(ret < 0))
807 return ret;
808
Jens Axboecbb7e572006-04-11 14:57:50 +0200809 return out->f_op->splice_write(pipe, out, ppos, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200810}
811
Jens Axboe83f91352006-04-02 23:05:09 +0200812/*
813 * Attempt to initiate a splice from a file to a pipe.
814 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200815static long do_splice_to(struct file *in, loff_t *ppos,
816 struct pipe_inode_info *pipe, size_t len,
817 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200818{
Jens Axboecbb7e572006-04-11 14:57:50 +0200819 loff_t isize, left;
Jens Axboe5274f052006-03-30 15:15:30 +0200820 int ret;
821
Jens Axboe49570e92006-04-11 13:56:09 +0200822 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200823 return -EINVAL;
824
Jens Axboe49570e92006-04-11 13:56:09 +0200825 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200826 return -EBADF;
827
Jens Axboecbb7e572006-04-11 14:57:50 +0200828 ret = rw_verify_area(READ, in, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200829 if (unlikely(ret < 0))
830 return ret;
831
832 isize = i_size_read(in->f_mapping->host);
Jens Axboecbb7e572006-04-11 14:57:50 +0200833 if (unlikely(*ppos >= isize))
Jens Axboe5274f052006-03-30 15:15:30 +0200834 return 0;
835
Jens Axboecbb7e572006-04-11 14:57:50 +0200836 left = isize - *ppos;
Jens Axboe49570e92006-04-11 13:56:09 +0200837 if (unlikely(left < len))
Jens Axboe5274f052006-03-30 15:15:30 +0200838 len = left;
839
Jens Axboecbb7e572006-04-11 14:57:50 +0200840 return in->f_op->splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200841}
842
Jens Axboecbb7e572006-04-11 14:57:50 +0200843long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
844 size_t len, unsigned int flags)
Jens Axboeb92ce552006-04-11 13:52:07 +0200845{
846 struct pipe_inode_info *pipe;
847 long ret, bytes;
Jens Axboecbb7e572006-04-11 14:57:50 +0200848 loff_t out_off;
Jens Axboeb92ce552006-04-11 13:52:07 +0200849 umode_t i_mode;
850 int i;
851
852 /*
853 * We require the input being a regular file, as we don't want to
854 * randomly drop data for eg socket -> socket splicing. Use the
855 * piped splicing for that!
856 */
857 i_mode = in->f_dentry->d_inode->i_mode;
858 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
859 return -EINVAL;
860
861 /*
862 * neither in nor out is a pipe, setup an internal pipe attached to
863 * 'out' and transfer the wanted data from 'in' to 'out' through that
864 */
865 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +0200866 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +0200867 pipe = alloc_pipe_info(NULL);
868 if (!pipe)
869 return -ENOMEM;
870
871 /*
872 * We don't have an immediate reader, but we'll read the stuff
873 * out of the pipe right after the move_to_pipe(). So set
874 * PIPE_READERS appropriately.
875 */
876 pipe->readers = 1;
877
878 current->splice_pipe = pipe;
879 }
880
881 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200882 * Do the splice.
Jens Axboeb92ce552006-04-11 13:52:07 +0200883 */
884 ret = 0;
885 bytes = 0;
Jens Axboecbb7e572006-04-11 14:57:50 +0200886 out_off = 0;
Jens Axboeb92ce552006-04-11 13:52:07 +0200887
888 while (len) {
889 size_t read_len, max_read_len;
890
891 /*
892 * Do at most PIPE_BUFFERS pages worth of transfer:
893 */
894 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
895
Jens Axboecbb7e572006-04-11 14:57:50 +0200896 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
Jens Axboeb92ce552006-04-11 13:52:07 +0200897 if (unlikely(ret < 0))
898 goto out_release;
899
900 read_len = ret;
901
902 /*
903 * NOTE: nonblocking mode only applies to the input. We
904 * must not do the output in nonblocking mode as then we
905 * could get stuck data in the internal pipe:
906 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200907 ret = do_splice_from(pipe, out, &out_off, read_len,
Jens Axboeb92ce552006-04-11 13:52:07 +0200908 flags & ~SPLICE_F_NONBLOCK);
909 if (unlikely(ret < 0))
910 goto out_release;
911
912 bytes += ret;
913 len -= ret;
914
915 /*
916 * In nonblocking mode, if we got back a short read then
917 * that was due to either an IO error or due to the
918 * pagecache entry not being there. In the IO error case
919 * the _next_ splice attempt will produce a clean IO error
920 * return value (not a short read), so in both cases it's
921 * correct to break out of the loop here:
922 */
923 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
924 break;
925 }
926
927 pipe->nrbufs = pipe->curbuf = 0;
928
929 return bytes;
930
931out_release:
932 /*
933 * If we did an incomplete transfer we must release
934 * the pipe buffers in question:
935 */
936 for (i = 0; i < PIPE_BUFFERS; i++) {
937 struct pipe_buffer *buf = pipe->bufs + i;
938
939 if (buf->ops) {
940 buf->ops->release(pipe, buf);
941 buf->ops = NULL;
942 }
943 }
944 pipe->nrbufs = pipe->curbuf = 0;
945
946 /*
947 * If we transferred some data, return the number of bytes:
948 */
949 if (bytes > 0)
950 return bytes;
951
952 return ret;
953}
954
955EXPORT_SYMBOL(do_splice_direct);
956
Jens Axboe83f91352006-04-02 23:05:09 +0200957/*
958 * Determine where to splice to/from.
959 */
Ingo Molnar529565d2006-04-10 15:18:58 +0200960static long do_splice(struct file *in, loff_t __user *off_in,
961 struct file *out, loff_t __user *off_out,
962 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200963{
Ingo Molnar3a326a22006-04-10 15:18:35 +0200964 struct pipe_inode_info *pipe;
Jens Axboecbb7e572006-04-11 14:57:50 +0200965 loff_t offset, *off;
Jens Axboea4514eb2006-04-19 15:57:05 +0200966 long ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200967
Ingo Molnar3a326a22006-04-10 15:18:35 +0200968 pipe = in->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200969 if (pipe) {
970 if (off_in)
971 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200972 if (off_out) {
973 if (out->f_op->llseek == no_llseek)
974 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200975 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200976 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200977 off = &offset;
978 } else
979 off = &out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +0200980
Jens Axboea4514eb2006-04-19 15:57:05 +0200981 ret = do_splice_from(pipe, out, off, len, flags);
982
983 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
984 ret = -EFAULT;
985
986 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +0200987 }
Jens Axboe5274f052006-03-30 15:15:30 +0200988
Ingo Molnar3a326a22006-04-10 15:18:35 +0200989 pipe = out->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +0200990 if (pipe) {
991 if (off_out)
992 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +0200993 if (off_in) {
994 if (in->f_op->llseek == no_llseek)
995 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +0200996 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +0200997 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +0200998 off = &offset;
999 } else
1000 off = &in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +02001001
Jens Axboea4514eb2006-04-19 15:57:05 +02001002 ret = do_splice_to(in, off, pipe, len, flags);
1003
1004 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1005 ret = -EFAULT;
1006
1007 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +02001008 }
Jens Axboe5274f052006-03-30 15:15:30 +02001009
1010 return -EINVAL;
1011}
1012
Ingo Molnar529565d2006-04-10 15:18:58 +02001013asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
1014 int fd_out, loff_t __user *off_out,
1015 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +02001016{
1017 long error;
1018 struct file *in, *out;
1019 int fput_in, fput_out;
1020
1021 if (unlikely(!len))
1022 return 0;
1023
1024 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +02001025 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +02001026 if (in) {
1027 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +02001028 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +02001029 if (out) {
1030 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +02001031 error = do_splice(in, off_in,
1032 out, off_out,
1033 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +02001034 fput_light(out, fput_out);
1035 }
1036 }
1037
1038 fput_light(in, fput_in);
1039 }
1040
1041 return error;
1042}
Jens Axboe70524492006-04-11 15:51:17 +02001043
1044/*
1045 * Link contents of ipipe to opipe.
1046 */
1047static int link_pipe(struct pipe_inode_info *ipipe,
1048 struct pipe_inode_info *opipe,
1049 size_t len, unsigned int flags)
1050{
1051 struct pipe_buffer *ibuf, *obuf;
Jens Axboe2a272502006-04-19 15:56:40 +02001052 int ret, do_wakeup, i, ipipe_first;
1053
1054 ret = do_wakeup = ipipe_first = 0;
Jens Axboe70524492006-04-11 15:51:17 +02001055
1056 /*
1057 * Potential ABBA deadlock, work around it by ordering lock
1058 * grabbing by inode address. Otherwise two different processes
1059 * could deadlock (one doing tee from A -> B, the other from B -> A).
1060 */
1061 if (ipipe->inode < opipe->inode) {
Jens Axboe2a272502006-04-19 15:56:40 +02001062 ipipe_first = 1;
Jens Axboe70524492006-04-11 15:51:17 +02001063 mutex_lock(&ipipe->inode->i_mutex);
1064 mutex_lock(&opipe->inode->i_mutex);
1065 } else {
1066 mutex_lock(&opipe->inode->i_mutex);
1067 mutex_lock(&ipipe->inode->i_mutex);
1068 }
1069
1070 for (i = 0;; i++) {
1071 if (!opipe->readers) {
1072 send_sig(SIGPIPE, current, 0);
1073 if (!ret)
1074 ret = -EPIPE;
1075 break;
1076 }
1077 if (ipipe->nrbufs - i) {
1078 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1079
1080 /*
1081 * If we have room, fill this buffer
1082 */
1083 if (opipe->nrbufs < PIPE_BUFFERS) {
1084 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1085
1086 /*
1087 * Get a reference to this pipe buffer,
1088 * so we can copy the contents over.
1089 */
1090 ibuf->ops->get(ipipe, ibuf);
1091
1092 obuf = opipe->bufs + nbuf;
1093 *obuf = *ibuf;
1094
1095 if (obuf->len > len)
1096 obuf->len = len;
1097
1098 opipe->nrbufs++;
1099 do_wakeup = 1;
1100 ret += obuf->len;
1101 len -= obuf->len;
1102
1103 if (!len)
1104 break;
1105 if (opipe->nrbufs < PIPE_BUFFERS)
1106 continue;
1107 }
1108
1109 /*
1110 * We have input available, but no output room.
Jens Axboe2a272502006-04-19 15:56:40 +02001111 * If we already copied data, return that. If we
1112 * need to drop the opipe lock, it must be ordered
1113 * last to avoid deadlocks.
Jens Axboe70524492006-04-11 15:51:17 +02001114 */
Jens Axboe2a272502006-04-19 15:56:40 +02001115 if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001116 if (!ret)
1117 ret = -EAGAIN;
1118 break;
1119 }
1120 if (signal_pending(current)) {
1121 if (!ret)
1122 ret = -ERESTARTSYS;
1123 break;
1124 }
1125 if (do_wakeup) {
1126 smp_mb();
1127 if (waitqueue_active(&opipe->wait))
1128 wake_up_interruptible(&opipe->wait);
1129 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1130 do_wakeup = 0;
1131 }
1132
1133 opipe->waiting_writers++;
1134 pipe_wait(opipe);
1135 opipe->waiting_writers--;
1136 continue;
1137 }
1138
1139 /*
1140 * No input buffers, do the usual checks for available
1141 * writers and blocking and wait if necessary
1142 */
1143 if (!ipipe->writers)
1144 break;
1145 if (!ipipe->waiting_writers) {
1146 if (ret)
1147 break;
1148 }
Jens Axboe2a272502006-04-19 15:56:40 +02001149 /*
1150 * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1151 * with another process, we can only safely do that if
1152 * the ipipe lock is ordered last.
1153 */
1154 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001155 if (!ret)
1156 ret = -EAGAIN;
1157 break;
1158 }
1159 if (signal_pending(current)) {
1160 if (!ret)
1161 ret = -ERESTARTSYS;
1162 break;
1163 }
1164
1165 if (waitqueue_active(&ipipe->wait))
1166 wake_up_interruptible_sync(&ipipe->wait);
1167 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1168
1169 pipe_wait(ipipe);
1170 }
1171
1172 mutex_unlock(&ipipe->inode->i_mutex);
1173 mutex_unlock(&opipe->inode->i_mutex);
1174
1175 if (do_wakeup) {
1176 smp_mb();
1177 if (waitqueue_active(&opipe->wait))
1178 wake_up_interruptible(&opipe->wait);
1179 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1180 }
1181
1182 return ret;
1183}
1184
1185/*
1186 * This is a tee(1) implementation that works on pipes. It doesn't copy
1187 * any data, it simply references the 'in' pages on the 'out' pipe.
1188 * The 'flags' used are the SPLICE_F_* variants, currently the only
1189 * applicable one is SPLICE_F_NONBLOCK.
1190 */
1191static long do_tee(struct file *in, struct file *out, size_t len,
1192 unsigned int flags)
1193{
1194 struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1195 struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1196
1197 /*
1198 * Link ipipe to the two output pipes, consuming as we go along.
1199 */
1200 if (ipipe && opipe)
1201 return link_pipe(ipipe, opipe, len, flags);
1202
1203 return -EINVAL;
1204}
1205
1206asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1207{
1208 struct file *in;
1209 int error, fput_in;
1210
1211 if (unlikely(!len))
1212 return 0;
1213
1214 error = -EBADF;
1215 in = fget_light(fdin, &fput_in);
1216 if (in) {
1217 if (in->f_mode & FMODE_READ) {
1218 int fput_out;
1219 struct file *out = fget_light(fdout, &fput_out);
1220
1221 if (out) {
1222 if (out->f_mode & FMODE_WRITE)
1223 error = do_tee(in, out, len, flags);
1224 fput_light(out, fput_out);
1225 }
1226 }
1227 fput_light(in, fput_in);
1228 }
1229
1230 return error;
1231}