blob: 447ebc0a37f342eea7075ee18abb2f4a7a40714e [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboec2058e02006-04-11 13:56:34 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/pipe_fs_i.h>
24#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe912d35f2006-04-26 10:59:21 +020030#include <linux/uio.h>
Jens Axboe5274f052006-03-30 15:15:30 +020031
Jens Axboe912d35f2006-04-26 10:59:21 +020032struct partial_page {
33 unsigned int offset;
34 unsigned int len;
35};
36
37/*
Jens Axboe00522fb2006-04-26 14:39:29 +020038 * Passed to splice_to_pipe
Jens Axboe912d35f2006-04-26 10:59:21 +020039 */
40struct splice_pipe_desc {
41 struct page **pages; /* page map */
42 struct partial_page *partial; /* pages[] may not be contig */
43 int nr_pages; /* number of pages in map */
44 unsigned int flags; /* splice flags */
45 struct pipe_buf_operations *ops;/* ops associated with output pipe */
46};
47
Jens Axboe83f91352006-04-02 23:05:09 +020048/*
49 * Attempt to steal a page from a pipe buffer. This should perhaps go into
50 * a vm helper function, it's already simplified quite a bit by the
51 * addition of remove_mapping(). If success is returned, the caller may
52 * attempt to reuse this page for another destination.
53 */
Jens Axboe5abc97a2006-03-30 15:16:46 +020054static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
55 struct pipe_buffer *buf)
56{
57 struct page *page = buf->page;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020058 struct address_space *mapping = page_mapping(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020059
Jens Axboe9e0267c2006-04-19 15:57:31 +020060 lock_page(page);
61
Jens Axboe5abc97a2006-03-30 15:16:46 +020062 WARN_ON(!PageUptodate(page));
63
Jens Axboead8d6f02006-04-02 23:10:32 +020064 /*
65 * At least for ext2 with nobh option, we need to wait on writeback
66 * completing on this page, since we'll remove it from the pagecache.
67 * Otherwise truncate wont wait on the page, allowing the disk
68 * blocks to be reused by someone else before we actually wrote our
69 * data to them. fs corruption ensues.
70 */
71 wait_on_page_writeback(page);
72
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020073 if (PagePrivate(page))
74 try_to_release_page(page, mapping_gfp_mask(mapping));
75
Jens Axboe9e0267c2006-04-19 15:57:31 +020076 if (!remove_mapping(mapping, page)) {
77 unlock_page(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +020078 return 1;
Jens Axboe9e0267c2006-04-19 15:57:31 +020079 }
Jens Axboe5abc97a2006-03-30 15:16:46 +020080
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020081 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
Jens Axboe5abc97a2006-03-30 15:16:46 +020082 return 0;
83}
84
Jens Axboe5274f052006-03-30 15:15:30 +020085static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
86 struct pipe_buffer *buf)
87{
88 page_cache_release(buf->page);
89 buf->page = NULL;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +020090 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
Jens Axboe5274f052006-03-30 15:15:30 +020091}
92
93static void *page_cache_pipe_buf_map(struct file *file,
94 struct pipe_inode_info *info,
95 struct pipe_buffer *buf)
96{
97 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020098 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020099
100 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +0200101 lock_page(page);
102
103 /*
104 * Page got truncated/unhashed. This will cause a 0-byte
Ingo Molnar73d62d82006-04-11 13:57:21 +0200105 * splice, if this is the first page.
Jens Axboe49d0b212006-04-10 09:04:41 +0200106 */
107 if (!page->mapping) {
108 err = -ENODATA;
109 goto error;
110 }
111
112 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200113 * Uh oh, read-error from disk.
Jens Axboe49d0b212006-04-10 09:04:41 +0200114 */
115 if (!PageUptodate(page)) {
116 err = -EIO;
117 goto error;
118 }
119
120 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200121 * Page is ok afterall, fall through to mapping.
Jens Axboe49d0b212006-04-10 09:04:41 +0200122 */
Jens Axboe5274f052006-03-30 15:15:30 +0200123 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200124 }
125
Jens Axboe49d0b212006-04-10 09:04:41 +0200126 return kmap(page);
127error:
128 unlock_page(page);
129 return ERR_PTR(err);
Jens Axboe5274f052006-03-30 15:15:30 +0200130}
131
132static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
133 struct pipe_buffer *buf)
134{
Jens Axboe5274f052006-03-30 15:15:30 +0200135 kunmap(buf->page);
136}
137
Jens Axboe912d35f2006-04-26 10:59:21 +0200138static void *user_page_pipe_buf_map(struct file *file,
139 struct pipe_inode_info *pipe,
140 struct pipe_buffer *buf)
141{
142 return kmap(buf->page);
143}
144
145static void user_page_pipe_buf_unmap(struct pipe_inode_info *pipe,
146 struct pipe_buffer *buf)
147{
148 kunmap(buf->page);
149}
150
Jens Axboe70524492006-04-11 15:51:17 +0200151static void page_cache_pipe_buf_get(struct pipe_inode_info *info,
152 struct pipe_buffer *buf)
153{
154 page_cache_get(buf->page);
155}
156
Jens Axboe5274f052006-03-30 15:15:30 +0200157static struct pipe_buf_operations page_cache_pipe_buf_ops = {
158 .can_merge = 0,
159 .map = page_cache_pipe_buf_map,
160 .unmap = page_cache_pipe_buf_unmap,
161 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200162 .steal = page_cache_pipe_buf_steal,
Jens Axboe70524492006-04-11 15:51:17 +0200163 .get = page_cache_pipe_buf_get,
Jens Axboe5274f052006-03-30 15:15:30 +0200164};
165
Jens Axboe912d35f2006-04-26 10:59:21 +0200166static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
167 struct pipe_buffer *buf)
168{
169 return 1;
170}
171
172static struct pipe_buf_operations user_page_pipe_buf_ops = {
173 .can_merge = 0,
174 .map = user_page_pipe_buf_map,
175 .unmap = user_page_pipe_buf_unmap,
176 .release = page_cache_pipe_buf_release,
177 .steal = user_page_pipe_buf_steal,
178 .get = page_cache_pipe_buf_get,
179};
180
Jens Axboe83f91352006-04-02 23:05:09 +0200181/*
182 * Pipe output worker. This sets up our pipe format with the page cache
183 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
184 */
Jens Axboe00522fb2006-04-26 14:39:29 +0200185static ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
186 struct splice_pipe_desc *spd)
Jens Axboe5274f052006-03-30 15:15:30 +0200187{
Jens Axboe912d35f2006-04-26 10:59:21 +0200188 int ret, do_wakeup, page_nr;
Jens Axboe5274f052006-03-30 15:15:30 +0200189
190 ret = 0;
191 do_wakeup = 0;
Jens Axboe912d35f2006-04-26 10:59:21 +0200192 page_nr = 0;
Jens Axboe5274f052006-03-30 15:15:30 +0200193
Ingo Molnar3a326a22006-04-10 15:18:35 +0200194 if (pipe->inode)
195 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200196
Jens Axboe5274f052006-03-30 15:15:30 +0200197 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200198 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200199 send_sig(SIGPIPE, current, 0);
200 if (!ret)
201 ret = -EPIPE;
202 break;
203 }
204
Jens Axboe6f767b02006-04-11 13:53:56 +0200205 if (pipe->nrbufs < PIPE_BUFFERS) {
206 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200207 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200208
Jens Axboe912d35f2006-04-26 10:59:21 +0200209 buf->page = spd->pages[page_nr];
210 buf->offset = spd->partial[page_nr].offset;
211 buf->len = spd->partial[page_nr].len;
212 buf->ops = spd->ops;
Jens Axboe6f767b02006-04-11 13:53:56 +0200213 pipe->nrbufs++;
Jens Axboe912d35f2006-04-26 10:59:21 +0200214 page_nr++;
215 ret += buf->len;
216
Jens Axboe6f767b02006-04-11 13:53:56 +0200217 if (pipe->inode)
218 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200219
Jens Axboe912d35f2006-04-26 10:59:21 +0200220 if (!--spd->nr_pages)
Jens Axboe5274f052006-03-30 15:15:30 +0200221 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200222 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200223 continue;
224
225 break;
226 }
227
Jens Axboe912d35f2006-04-26 10:59:21 +0200228 if (spd->flags & SPLICE_F_NONBLOCK) {
Linus Torvalds29e35092006-04-02 12:46:35 -0700229 if (!ret)
230 ret = -EAGAIN;
231 break;
232 }
233
Jens Axboe5274f052006-03-30 15:15:30 +0200234 if (signal_pending(current)) {
235 if (!ret)
236 ret = -ERESTARTSYS;
237 break;
238 }
239
240 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200241 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200242 if (waitqueue_active(&pipe->wait))
243 wake_up_interruptible_sync(&pipe->wait);
244 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200245 do_wakeup = 0;
246 }
247
Ingo Molnar3a326a22006-04-10 15:18:35 +0200248 pipe->waiting_writers++;
249 pipe_wait(pipe);
250 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200251 }
252
Ingo Molnar3a326a22006-04-10 15:18:35 +0200253 if (pipe->inode)
254 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200255
256 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200257 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200258 if (waitqueue_active(&pipe->wait))
259 wake_up_interruptible(&pipe->wait);
260 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200261 }
262
Jens Axboe912d35f2006-04-26 10:59:21 +0200263 while (page_nr < spd->nr_pages)
264 page_cache_release(spd->pages[page_nr++]);
Jens Axboe5274f052006-03-30 15:15:30 +0200265
266 return ret;
267}
268
Ingo Molnar3a326a22006-04-10 15:18:35 +0200269static int
Jens Axboecbb7e572006-04-11 14:57:50 +0200270__generic_file_splice_read(struct file *in, loff_t *ppos,
271 struct pipe_inode_info *pipe, size_t len,
272 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200273{
274 struct address_space *mapping = in->f_mapping;
Jens Axboe912d35f2006-04-26 10:59:21 +0200275 unsigned int loff, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200276 struct page *pages[PIPE_BUFFERS];
Jens Axboe912d35f2006-04-26 10:59:21 +0200277 struct partial_page partial[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200278 struct page *page;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200279 pgoff_t index, end_index;
280 loff_t isize;
Jens Axboe912d35f2006-04-26 10:59:21 +0200281 size_t total_len;
282 int error;
283 struct splice_pipe_desc spd = {
284 .pages = pages,
285 .partial = partial,
286 .flags = flags,
287 .ops = &page_cache_pipe_buf_ops,
288 };
Jens Axboe5274f052006-03-30 15:15:30 +0200289
Jens Axboecbb7e572006-04-11 14:57:50 +0200290 index = *ppos >> PAGE_CACHE_SHIFT;
Jens Axboe912d35f2006-04-26 10:59:21 +0200291 loff = *ppos & ~PAGE_CACHE_MASK;
292 nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Jens Axboe5274f052006-03-30 15:15:30 +0200293
294 if (nr_pages > PIPE_BUFFERS)
295 nr_pages = PIPE_BUFFERS;
296
297 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200298 * Initiate read-ahead on this page range. however, don't call into
Jens Axboe0b749ce2006-04-10 09:05:04 +0200299 * read-ahead if this is a non-zero offset (we are likely doing small
300 * chunk splice and the page is already there) for a single page.
Jens Axboe5274f052006-03-30 15:15:30 +0200301 */
Jens Axboe912d35f2006-04-26 10:59:21 +0200302 if (!loff || spd.nr_pages > 1)
303 do_page_cache_readahead(mapping, in, index, spd.nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200304
305 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200306 * Now fill in the holes:
Jens Axboe5274f052006-03-30 15:15:30 +0200307 */
Jens Axboe7480a902006-04-11 13:52:47 +0200308 error = 0;
Jens Axboe912d35f2006-04-26 10:59:21 +0200309 total_len = 0;
310 for (spd.nr_pages = 0; spd.nr_pages < nr_pages; spd.nr_pages++, index++) {
Jens Axboe82aa5d62006-04-20 13:05:48 +0200311 unsigned int this_len;
312
313 if (!len)
314 break;
315
316 /*
317 * this_len is the max we'll use from this page
318 */
Andrew Mortonba5f5d92006-04-25 15:33:34 +0200319 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
Jens Axboe7480a902006-04-11 13:52:47 +0200320find_page:
Jens Axboe5274f052006-03-30 15:15:30 +0200321 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200322 * lookup the page for this index
Jens Axboe5274f052006-03-30 15:15:30 +0200323 */
Jens Axboe7480a902006-04-11 13:52:47 +0200324 page = find_get_page(mapping, index);
325 if (!page) {
326 /*
Jens Axboe7480a902006-04-11 13:52:47 +0200327 * page didn't exist, allocate one
328 */
329 page = page_cache_alloc_cold(mapping);
330 if (!page)
331 break;
Jens Axboe5274f052006-03-30 15:15:30 +0200332
Jens Axboe7480a902006-04-11 13:52:47 +0200333 error = add_to_page_cache_lru(page, mapping, index,
334 mapping_gfp_mask(mapping));
Jens Axboe5274f052006-03-30 15:15:30 +0200335 if (unlikely(error)) {
336 page_cache_release(page);
337 break;
338 }
Jens Axboe7480a902006-04-11 13:52:47 +0200339
340 goto readpage;
Jens Axboe5274f052006-03-30 15:15:30 +0200341 }
Jens Axboe7480a902006-04-11 13:52:47 +0200342
343 /*
344 * If the page isn't uptodate, we may need to start io on it
345 */
346 if (!PageUptodate(page)) {
Jens Axboec4f895c2006-04-19 15:56:12 +0200347 /*
348 * If in nonblock mode then dont block on waiting
349 * for an in-flight io page
350 */
351 if (flags & SPLICE_F_NONBLOCK)
352 break;
353
Jens Axboe7480a902006-04-11 13:52:47 +0200354 lock_page(page);
355
356 /*
357 * page was truncated, stop here. if this isn't the
358 * first page, we'll just complete what we already
359 * added
360 */
361 if (!page->mapping) {
362 unlock_page(page);
363 page_cache_release(page);
364 break;
365 }
366 /*
367 * page was already under io and is now done, great
368 */
369 if (PageUptodate(page)) {
370 unlock_page(page);
371 goto fill_it;
372 }
373
374readpage:
375 /*
376 * need to read in the page
377 */
378 error = mapping->a_ops->readpage(in, page);
379
380 if (unlikely(error)) {
381 page_cache_release(page);
382 if (error == AOP_TRUNCATED_PAGE)
383 goto find_page;
384 break;
385 }
Jens Axboe91ad66e2006-04-19 15:55:10 +0200386
387 /*
388 * i_size must be checked after ->readpage().
389 */
390 isize = i_size_read(mapping->host);
391 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
392 if (unlikely(!isize || index > end_index)) {
393 page_cache_release(page);
394 break;
395 }
396
397 /*
398 * if this is the last page, see if we need to shrink
399 * the length and stop
400 */
401 if (end_index == index) {
402 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
Jens Axboe912d35f2006-04-26 10:59:21 +0200403 if (total_len + loff > isize) {
Jens Axboe91ad66e2006-04-19 15:55:10 +0200404 page_cache_release(page);
405 break;
406 }
407 /*
408 * force quit after adding this page
409 */
Jens Axboe912d35f2006-04-26 10:59:21 +0200410 nr_pages = spd.nr_pages;
Jens Axboe82aa5d62006-04-20 13:05:48 +0200411 this_len = min(this_len, loff);
Jens Axboe912d35f2006-04-26 10:59:21 +0200412 loff = 0;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200413 }
Jens Axboe7480a902006-04-11 13:52:47 +0200414 }
415fill_it:
Jens Axboe912d35f2006-04-26 10:59:21 +0200416 pages[spd.nr_pages] = page;
417 partial[spd.nr_pages].offset = loff;
418 partial[spd.nr_pages].len = this_len;
Jens Axboe82aa5d62006-04-20 13:05:48 +0200419 len -= this_len;
Jens Axboe912d35f2006-04-26 10:59:21 +0200420 total_len += this_len;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200421 loff = 0;
Jens Axboe5274f052006-03-30 15:15:30 +0200422 }
423
Jens Axboe912d35f2006-04-26 10:59:21 +0200424 if (spd.nr_pages)
Jens Axboe00522fb2006-04-26 14:39:29 +0200425 return splice_to_pipe(pipe, &spd);
Jens Axboe5274f052006-03-30 15:15:30 +0200426
Jens Axboe7480a902006-04-11 13:52:47 +0200427 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200428}
429
Jens Axboe83f91352006-04-02 23:05:09 +0200430/**
431 * generic_file_splice_read - splice data from file to a pipe
432 * @in: file to splice from
433 * @pipe: pipe to splice to
434 * @len: number of bytes to splice
435 * @flags: splice modifier flags
436 *
437 * Will read pages from given file and fill them into a pipe.
Jens Axboe83f91352006-04-02 23:05:09 +0200438 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200439ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
440 struct pipe_inode_info *pipe, size_t len,
441 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200442{
443 ssize_t spliced;
444 int ret;
445
446 ret = 0;
447 spliced = 0;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200448
Jens Axboe5274f052006-03-30 15:15:30 +0200449 while (len) {
Jens Axboecbb7e572006-04-11 14:57:50 +0200450 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200451
Jens Axboec4f895c2006-04-19 15:56:12 +0200452 if (ret < 0)
Jens Axboe5274f052006-03-30 15:15:30 +0200453 break;
Jens Axboec4f895c2006-04-19 15:56:12 +0200454 else if (!ret) {
455 if (spliced)
456 break;
457 if (flags & SPLICE_F_NONBLOCK) {
458 ret = -EAGAIN;
459 break;
460 }
461 }
Jens Axboe5274f052006-03-30 15:15:30 +0200462
Jens Axboecbb7e572006-04-11 14:57:50 +0200463 *ppos += ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200464 len -= ret;
465 spliced += ret;
466 }
467
468 if (spliced)
469 return spliced;
470
471 return ret;
472}
473
Jens Axboe059a8f32006-04-02 23:06:05 +0200474EXPORT_SYMBOL(generic_file_splice_read);
475
Jens Axboe5274f052006-03-30 15:15:30 +0200476/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200477 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
Jens Axboe016b6612006-04-25 15:42:00 +0200478 * using sendpage(). Return the number of bytes sent.
Jens Axboe5274f052006-03-30 15:15:30 +0200479 */
480static int pipe_to_sendpage(struct pipe_inode_info *info,
481 struct pipe_buffer *buf, struct splice_desc *sd)
482{
483 struct file *file = sd->file;
484 loff_t pos = sd->pos;
Jens Axboe5274f052006-03-30 15:15:30 +0200485 ssize_t ret;
486 void *ptr;
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200487 int more;
Jens Axboe5274f052006-03-30 15:15:30 +0200488
489 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200490 * Sub-optimal, but we are limited by the pipe ->map. We don't
Jens Axboe5274f052006-03-30 15:15:30 +0200491 * need a kmap'ed buffer here, we just want to make sure we
492 * have the page pinned if the pipe page originates from the
Ingo Molnar73d62d82006-04-11 13:57:21 +0200493 * page cache.
Jens Axboe5274f052006-03-30 15:15:30 +0200494 */
495 ptr = buf->ops->map(file, info, buf);
496 if (IS_ERR(ptr))
497 return PTR_ERR(ptr);
498
Jens Axboeb2b39fa2006-04-02 23:05:41 +0200499 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200500
Jens Axboe016b6612006-04-25 15:42:00 +0200501 ret = file->f_op->sendpage(file, buf->page, buf->offset, sd->len,
502 &pos, more);
Jens Axboe5274f052006-03-30 15:15:30 +0200503
504 buf->ops->unmap(info, buf);
Jens Axboe016b6612006-04-25 15:42:00 +0200505 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200506}
507
508/*
509 * This is a little more tricky than the file -> pipe splicing. There are
510 * basically three cases:
511 *
512 * - Destination page already exists in the address space and there
513 * are users of it. For that case we have no other option that
514 * copying the data. Tough luck.
515 * - Destination page already exists in the address space, but there
516 * are no users of it. Make sure it's uptodate, then drop it. Fall
517 * through to last case.
518 * - Destination page does not exist, we can add the pipe page to
519 * the page cache and avoid the copy.
520 *
Jens Axboe83f91352006-04-02 23:05:09 +0200521 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
522 * sd->flags), we attempt to migrate pages from the pipe to the output
523 * file address space page cache. This is possible if no one else has
524 * the pipe page referenced outside of the pipe and page cache. If
525 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
526 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200527 */
528static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
529 struct splice_desc *sd)
530{
531 struct file *file = sd->file;
532 struct address_space *mapping = file->f_mapping;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200533 gfp_t gfp_mask = mapping_gfp_mask(mapping);
Jens Axboe016b6612006-04-25 15:42:00 +0200534 unsigned int offset, this_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200535 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200536 pgoff_t index;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200537 char *src;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200538 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200539
540 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200541 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200542 */
543 src = buf->ops->map(file, info, buf);
544 if (IS_ERR(src))
545 return PTR_ERR(src);
546
547 index = sd->pos >> PAGE_CACHE_SHIFT;
548 offset = sd->pos & ~PAGE_CACHE_MASK;
549
Jens Axboe016b6612006-04-25 15:42:00 +0200550 this_len = sd->len;
551 if (this_len + offset > PAGE_CACHE_SIZE)
552 this_len = PAGE_CACHE_SIZE - offset;
553
Jens Axboe5274f052006-03-30 15:15:30 +0200554 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200555 * Reuse buf page, if SPLICE_F_MOVE is set.
Jens Axboe5274f052006-03-30 15:15:30 +0200556 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200557 if (sd->flags & SPLICE_F_MOVE) {
Jens Axboe83f91352006-04-02 23:05:09 +0200558 /*
559 * If steal succeeds, buf->page is now pruned from the vm
Jens Axboe9e0267c2006-04-19 15:57:31 +0200560 * side (LRU and page cache) and we can reuse it. The page
561 * will also be looked on successful return.
Jens Axboe83f91352006-04-02 23:05:09 +0200562 */
Jens Axboe5abc97a2006-03-30 15:16:46 +0200563 if (buf->ops->steal(info, buf))
564 goto find_page;
Jens Axboe5274f052006-03-30 15:15:30 +0200565
Jens Axboe5abc97a2006-03-30 15:16:46 +0200566 page = buf->page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200567 if (add_to_page_cache(page, mapping, index, gfp_mask))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200568 goto find_page;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200569
570 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
571 lru_cache_add(page);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200572 } else {
573find_page:
Jens Axboe9e0267c2006-04-19 15:57:31 +0200574 page = find_lock_page(mapping, index);
575 if (!page) {
576 ret = -ENOMEM;
577 page = page_cache_alloc_cold(mapping);
578 if (unlikely(!page))
579 goto out_nomem;
580
581 /*
582 * This will also lock the page
583 */
584 ret = add_to_page_cache_lru(page, mapping, index,
585 gfp_mask);
586 if (unlikely(ret))
587 goto out;
588 }
Jens Axboe5274f052006-03-30 15:15:30 +0200589
Jens Axboe5abc97a2006-03-30 15:16:46 +0200590 /*
Jens Axboe9e0267c2006-04-19 15:57:31 +0200591 * We get here with the page locked. If the page is also
592 * uptodate, we don't need to do more. If it isn't, we
593 * may need to bring it in if we are not going to overwrite
594 * the full page.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200595 */
596 if (!PageUptodate(page)) {
Jens Axboe016b6612006-04-25 15:42:00 +0200597 if (this_len < PAGE_CACHE_SIZE) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200598 ret = mapping->a_ops->readpage(file, page);
599 if (unlikely(ret))
600 goto out;
601
602 lock_page(page);
603
604 if (!PageUptodate(page)) {
605 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200606 * Page got invalidated, repeat.
Jens Axboe5abc97a2006-03-30 15:16:46 +0200607 */
608 if (!page->mapping) {
609 unlock_page(page);
610 page_cache_release(page);
611 goto find_page;
612 }
613 ret = -EIO;
614 goto out;
Jens Axboe5274f052006-03-30 15:15:30 +0200615 }
Jens Axboe9e0267c2006-04-19 15:57:31 +0200616 } else
Jens Axboe5abc97a2006-03-30 15:16:46 +0200617 SetPageUptodate(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200618 }
619 }
620
Jens Axboe016b6612006-04-25 15:42:00 +0200621 ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200622 if (ret == AOP_TRUNCATED_PAGE) {
623 page_cache_release(page);
624 goto find_page;
625 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200626 goto out;
627
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200628 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
Jens Axboe5abc97a2006-03-30 15:16:46 +0200629 char *dst = kmap_atomic(page, KM_USER0);
630
Jens Axboe016b6612006-04-25 15:42:00 +0200631 memcpy(dst + offset, src + buf->offset, this_len);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200632 flush_dcache_page(page);
633 kunmap_atomic(dst, KM_USER0);
634 }
Jens Axboe5274f052006-03-30 15:15:30 +0200635
Jens Axboe016b6612006-04-25 15:42:00 +0200636 ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200637 if (ret == AOP_TRUNCATED_PAGE) {
638 page_cache_release(page);
639 goto find_page;
640 } else if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200641 goto out;
642
Jens Axboe016b6612006-04-25 15:42:00 +0200643 /*
644 * Return the number of bytes written.
645 */
646 ret = this_len;
Jens Axboec7f21e42006-04-10 09:01:01 +0200647 mark_page_accessed(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200648 balance_dirty_pages_ratelimited(mapping);
Jens Axboe5274f052006-03-30 15:15:30 +0200649out:
Jens Axboe9e0267c2006-04-19 15:57:31 +0200650 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN))
Jens Axboe5abc97a2006-03-30 15:16:46 +0200651 page_cache_release(page);
Jens Axboe9e0267c2006-04-19 15:57:31 +0200652
653 unlock_page(page);
Dave Jones9aefe432006-04-10 09:02:40 +0200654out_nomem:
Jens Axboe5274f052006-03-30 15:15:30 +0200655 buf->ops->unmap(info, buf);
656 return ret;
657}
658
Jens Axboe83f91352006-04-02 23:05:09 +0200659/*
660 * Pipe input worker. Most of this logic works like a regular pipe, the
661 * key here is the 'actor' worker passed in that actually moves the data
662 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
663 */
Jens Axboe00522fb2006-04-26 14:39:29 +0200664ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
665 loff_t *ppos, size_t len, unsigned int flags,
666 splice_actor *actor)
Jens Axboe5274f052006-03-30 15:15:30 +0200667{
Jens Axboe5274f052006-03-30 15:15:30 +0200668 int ret, do_wakeup, err;
669 struct splice_desc sd;
670
671 ret = 0;
672 do_wakeup = 0;
673
674 sd.total_len = len;
675 sd.flags = flags;
676 sd.file = out;
Jens Axboecbb7e572006-04-11 14:57:50 +0200677 sd.pos = *ppos;
Jens Axboe5274f052006-03-30 15:15:30 +0200678
Ingo Molnar3a326a22006-04-10 15:18:35 +0200679 if (pipe->inode)
680 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200681
Jens Axboe5274f052006-03-30 15:15:30 +0200682 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200683 if (pipe->nrbufs) {
684 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200685 struct pipe_buf_operations *ops = buf->ops;
686
687 sd.len = buf->len;
688 if (sd.len > sd.total_len)
689 sd.len = sd.total_len;
690
Ingo Molnar3a326a22006-04-10 15:18:35 +0200691 err = actor(pipe, buf, &sd);
Jens Axboe016b6612006-04-25 15:42:00 +0200692 if (err <= 0) {
Jens Axboe5274f052006-03-30 15:15:30 +0200693 if (!ret && err != -ENODATA)
694 ret = err;
695
696 break;
697 }
698
Jens Axboe016b6612006-04-25 15:42:00 +0200699 ret += err;
700 buf->offset += err;
701 buf->len -= err;
702
703 sd.len -= err;
704 sd.pos += err;
705 sd.total_len -= err;
706 if (sd.len)
707 continue;
Ingo Molnar73d62d82006-04-11 13:57:21 +0200708
Jens Axboe5274f052006-03-30 15:15:30 +0200709 if (!buf->len) {
710 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200711 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200712 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
713 pipe->nrbufs--;
714 if (pipe->inode)
715 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200716 }
717
Jens Axboe5274f052006-03-30 15:15:30 +0200718 if (!sd.total_len)
719 break;
720 }
721
Jens Axboe6f767b02006-04-11 13:53:56 +0200722 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200723 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200724 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200725 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200726 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200727 if (ret)
728 break;
729 }
730
Linus Torvalds29e35092006-04-02 12:46:35 -0700731 if (flags & SPLICE_F_NONBLOCK) {
732 if (!ret)
733 ret = -EAGAIN;
734 break;
735 }
736
Jens Axboe5274f052006-03-30 15:15:30 +0200737 if (signal_pending(current)) {
738 if (!ret)
739 ret = -ERESTARTSYS;
740 break;
741 }
742
743 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200744 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200745 if (waitqueue_active(&pipe->wait))
746 wake_up_interruptible_sync(&pipe->wait);
747 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200748 do_wakeup = 0;
749 }
750
Ingo Molnar3a326a22006-04-10 15:18:35 +0200751 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200752 }
753
Ingo Molnar3a326a22006-04-10 15:18:35 +0200754 if (pipe->inode)
755 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200756
757 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200758 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200759 if (waitqueue_active(&pipe->wait))
760 wake_up_interruptible(&pipe->wait);
761 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200762 }
763
Jens Axboe5274f052006-03-30 15:15:30 +0200764 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200765}
766
Jens Axboe83f91352006-04-02 23:05:09 +0200767/**
768 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200769 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200770 * @out: file to write to
771 * @len: number of bytes to splice
772 * @flags: splice modifier flags
773 *
774 * Will either move or copy pages (determined by @flags options) from
775 * the given pipe inode to the given file.
776 *
777 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200778ssize_t
779generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200780 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200781{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200782 struct address_space *mapping = out->f_mapping;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200783 ssize_t ret;
784
Jens Axboe00522fb2006-04-26 14:39:29 +0200785 ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
Jens Axboea4514eb2006-04-19 15:57:05 +0200786 if (ret > 0) {
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200787 struct inode *inode = mapping->host;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200788
Jens Axboea4514eb2006-04-19 15:57:05 +0200789 *ppos += ret;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200790
Jens Axboea4514eb2006-04-19 15:57:05 +0200791 /*
792 * If file or inode is SYNC and we actually wrote some data,
793 * sync it.
794 */
795 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
796 int err;
797
798 mutex_lock(&inode->i_mutex);
799 err = generic_osync_inode(inode, mapping,
800 OSYNC_METADATA|OSYNC_DATA);
801 mutex_unlock(&inode->i_mutex);
802
803 if (err)
804 ret = err;
805 }
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200806 }
807
808 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200809}
810
Jens Axboe059a8f32006-04-02 23:06:05 +0200811EXPORT_SYMBOL(generic_file_splice_write);
812
Jens Axboe83f91352006-04-02 23:05:09 +0200813/**
814 * generic_splice_sendpage - splice data from a pipe to a socket
815 * @inode: pipe inode
816 * @out: socket to write to
817 * @len: number of bytes to splice
818 * @flags: splice modifier flags
819 *
820 * Will send @len bytes from the pipe to a network socket. No data copying
821 * is involved.
822 *
823 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200824ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200825 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200826{
Jens Axboe00522fb2006-04-26 14:39:29 +0200827 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200828}
829
Jens Axboe059a8f32006-04-02 23:06:05 +0200830EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500831
Jens Axboe83f91352006-04-02 23:05:09 +0200832/*
833 * Attempt to initiate a splice from pipe to file.
834 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200835static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200836 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200837{
Jens Axboe5274f052006-03-30 15:15:30 +0200838 int ret;
839
Jens Axboe49570e92006-04-11 13:56:09 +0200840 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200841 return -EINVAL;
842
Jens Axboe49570e92006-04-11 13:56:09 +0200843 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200844 return -EBADF;
845
Jens Axboecbb7e572006-04-11 14:57:50 +0200846 ret = rw_verify_area(WRITE, out, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200847 if (unlikely(ret < 0))
848 return ret;
849
Jens Axboecbb7e572006-04-11 14:57:50 +0200850 return out->f_op->splice_write(pipe, out, ppos, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200851}
852
Jens Axboe83f91352006-04-02 23:05:09 +0200853/*
854 * Attempt to initiate a splice from a file to a pipe.
855 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200856static long do_splice_to(struct file *in, loff_t *ppos,
857 struct pipe_inode_info *pipe, size_t len,
858 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200859{
Jens Axboecbb7e572006-04-11 14:57:50 +0200860 loff_t isize, left;
Jens Axboe5274f052006-03-30 15:15:30 +0200861 int ret;
862
Jens Axboe49570e92006-04-11 13:56:09 +0200863 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200864 return -EINVAL;
865
Jens Axboe49570e92006-04-11 13:56:09 +0200866 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200867 return -EBADF;
868
Jens Axboecbb7e572006-04-11 14:57:50 +0200869 ret = rw_verify_area(READ, in, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200870 if (unlikely(ret < 0))
871 return ret;
872
873 isize = i_size_read(in->f_mapping->host);
Jens Axboecbb7e572006-04-11 14:57:50 +0200874 if (unlikely(*ppos >= isize))
Jens Axboe5274f052006-03-30 15:15:30 +0200875 return 0;
876
Jens Axboecbb7e572006-04-11 14:57:50 +0200877 left = isize - *ppos;
Jens Axboe49570e92006-04-11 13:56:09 +0200878 if (unlikely(left < len))
Jens Axboe5274f052006-03-30 15:15:30 +0200879 len = left;
880
Jens Axboecbb7e572006-04-11 14:57:50 +0200881 return in->f_op->splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200882}
883
Jens Axboecbb7e572006-04-11 14:57:50 +0200884long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
885 size_t len, unsigned int flags)
Jens Axboeb92ce552006-04-11 13:52:07 +0200886{
887 struct pipe_inode_info *pipe;
888 long ret, bytes;
Jens Axboecbb7e572006-04-11 14:57:50 +0200889 loff_t out_off;
Jens Axboeb92ce552006-04-11 13:52:07 +0200890 umode_t i_mode;
891 int i;
892
893 /*
894 * We require the input being a regular file, as we don't want to
895 * randomly drop data for eg socket -> socket splicing. Use the
896 * piped splicing for that!
897 */
898 i_mode = in->f_dentry->d_inode->i_mode;
899 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
900 return -EINVAL;
901
902 /*
903 * neither in nor out is a pipe, setup an internal pipe attached to
904 * 'out' and transfer the wanted data from 'in' to 'out' through that
905 */
906 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +0200907 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +0200908 pipe = alloc_pipe_info(NULL);
909 if (!pipe)
910 return -ENOMEM;
911
912 /*
913 * We don't have an immediate reader, but we'll read the stuff
Jens Axboe00522fb2006-04-26 14:39:29 +0200914 * out of the pipe right after the splice_to_pipe(). So set
Jens Axboeb92ce552006-04-11 13:52:07 +0200915 * PIPE_READERS appropriately.
916 */
917 pipe->readers = 1;
918
919 current->splice_pipe = pipe;
920 }
921
922 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200923 * Do the splice.
Jens Axboeb92ce552006-04-11 13:52:07 +0200924 */
925 ret = 0;
926 bytes = 0;
Jens Axboecbb7e572006-04-11 14:57:50 +0200927 out_off = 0;
Jens Axboeb92ce552006-04-11 13:52:07 +0200928
929 while (len) {
930 size_t read_len, max_read_len;
931
932 /*
933 * Do at most PIPE_BUFFERS pages worth of transfer:
934 */
935 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
936
Jens Axboecbb7e572006-04-11 14:57:50 +0200937 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
Jens Axboeb92ce552006-04-11 13:52:07 +0200938 if (unlikely(ret < 0))
939 goto out_release;
940
941 read_len = ret;
942
943 /*
944 * NOTE: nonblocking mode only applies to the input. We
945 * must not do the output in nonblocking mode as then we
946 * could get stuck data in the internal pipe:
947 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200948 ret = do_splice_from(pipe, out, &out_off, read_len,
Jens Axboeb92ce552006-04-11 13:52:07 +0200949 flags & ~SPLICE_F_NONBLOCK);
950 if (unlikely(ret < 0))
951 goto out_release;
952
953 bytes += ret;
954 len -= ret;
955
956 /*
957 * In nonblocking mode, if we got back a short read then
958 * that was due to either an IO error or due to the
959 * pagecache entry not being there. In the IO error case
960 * the _next_ splice attempt will produce a clean IO error
961 * return value (not a short read), so in both cases it's
962 * correct to break out of the loop here:
963 */
964 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
965 break;
966 }
967
968 pipe->nrbufs = pipe->curbuf = 0;
969
970 return bytes;
971
972out_release:
973 /*
974 * If we did an incomplete transfer we must release
975 * the pipe buffers in question:
976 */
977 for (i = 0; i < PIPE_BUFFERS; i++) {
978 struct pipe_buffer *buf = pipe->bufs + i;
979
980 if (buf->ops) {
981 buf->ops->release(pipe, buf);
982 buf->ops = NULL;
983 }
984 }
985 pipe->nrbufs = pipe->curbuf = 0;
986
987 /*
988 * If we transferred some data, return the number of bytes:
989 */
990 if (bytes > 0)
991 return bytes;
992
993 return ret;
994}
995
996EXPORT_SYMBOL(do_splice_direct);
997
Jens Axboe83f91352006-04-02 23:05:09 +0200998/*
999 * Determine where to splice to/from.
1000 */
Ingo Molnar529565d2006-04-10 15:18:58 +02001001static long do_splice(struct file *in, loff_t __user *off_in,
1002 struct file *out, loff_t __user *off_out,
1003 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +02001004{
Ingo Molnar3a326a22006-04-10 15:18:35 +02001005 struct pipe_inode_info *pipe;
Jens Axboecbb7e572006-04-11 14:57:50 +02001006 loff_t offset, *off;
Jens Axboea4514eb2006-04-19 15:57:05 +02001007 long ret;
Jens Axboe5274f052006-03-30 15:15:30 +02001008
Ingo Molnar3a326a22006-04-10 15:18:35 +02001009 pipe = in->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +02001010 if (pipe) {
1011 if (off_in)
1012 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +02001013 if (off_out) {
1014 if (out->f_op->llseek == no_llseek)
1015 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +02001016 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +02001017 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +02001018 off = &offset;
1019 } else
1020 off = &out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +02001021
Jens Axboea4514eb2006-04-19 15:57:05 +02001022 ret = do_splice_from(pipe, out, off, len, flags);
1023
1024 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1025 ret = -EFAULT;
1026
1027 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +02001028 }
Jens Axboe5274f052006-03-30 15:15:30 +02001029
Ingo Molnar3a326a22006-04-10 15:18:35 +02001030 pipe = out->f_dentry->d_inode->i_pipe;
Ingo Molnar529565d2006-04-10 15:18:58 +02001031 if (pipe) {
1032 if (off_out)
1033 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +02001034 if (off_in) {
1035 if (in->f_op->llseek == no_llseek)
1036 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +02001037 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +02001038 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +02001039 off = &offset;
1040 } else
1041 off = &in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +02001042
Jens Axboea4514eb2006-04-19 15:57:05 +02001043 ret = do_splice_to(in, off, pipe, len, flags);
1044
1045 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1046 ret = -EFAULT;
1047
1048 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +02001049 }
Jens Axboe5274f052006-03-30 15:15:30 +02001050
1051 return -EINVAL;
1052}
1053
Jens Axboe912d35f2006-04-26 10:59:21 +02001054/*
1055 * Map an iov into an array of pages and offset/length tupples. With the
1056 * partial_page structure, we can map several non-contiguous ranges into
1057 * our ones pages[] map instead of splitting that operation into pieces.
1058 * Could easily be exported as a generic helper for other users, in which
1059 * case one would probably want to add a 'max_nr_pages' parameter as well.
1060 */
1061static int get_iovec_page_array(const struct iovec __user *iov,
1062 unsigned int nr_vecs, struct page **pages,
1063 struct partial_page *partial)
1064{
1065 int buffers = 0, error = 0;
1066
1067 /*
1068 * It's ok to take the mmap_sem for reading, even
1069 * across a "get_user()".
1070 */
1071 down_read(&current->mm->mmap_sem);
1072
1073 while (nr_vecs) {
1074 unsigned long off, npages;
1075 void __user *base;
1076 size_t len;
1077 int i;
1078
1079 /*
1080 * Get user address base and length for this iovec.
1081 */
1082 error = get_user(base, &iov->iov_base);
1083 if (unlikely(error))
1084 break;
1085 error = get_user(len, &iov->iov_len);
1086 if (unlikely(error))
1087 break;
1088
1089 /*
1090 * Sanity check this iovec. 0 read succeeds.
1091 */
1092 if (unlikely(!len))
1093 break;
1094 error = -EFAULT;
1095 if (unlikely(!base))
1096 break;
1097
1098 /*
1099 * Get this base offset and number of pages, then map
1100 * in the user pages.
1101 */
1102 off = (unsigned long) base & ~PAGE_MASK;
1103 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1104 if (npages > PIPE_BUFFERS - buffers)
1105 npages = PIPE_BUFFERS - buffers;
1106
1107 error = get_user_pages(current, current->mm,
1108 (unsigned long) base, npages, 0, 0,
1109 &pages[buffers], NULL);
1110
1111 if (unlikely(error <= 0))
1112 break;
1113
1114 /*
1115 * Fill this contiguous range into the partial page map.
1116 */
1117 for (i = 0; i < error; i++) {
1118 const int plen = min_t(size_t, len, PAGE_SIZE) - off;
1119
1120 partial[buffers].offset = off;
1121 partial[buffers].len = plen;
1122
1123 off = 0;
1124 len -= plen;
1125 buffers++;
1126 }
1127
1128 /*
1129 * We didn't complete this iov, stop here since it probably
1130 * means we have to move some of this into a pipe to
1131 * be able to continue.
1132 */
1133 if (len)
1134 break;
1135
1136 /*
1137 * Don't continue if we mapped fewer pages than we asked for,
1138 * or if we mapped the max number of pages that we have
1139 * room for.
1140 */
1141 if (error < npages || buffers == PIPE_BUFFERS)
1142 break;
1143
1144 nr_vecs--;
1145 iov++;
1146 }
1147
1148 up_read(&current->mm->mmap_sem);
1149
1150 if (buffers)
1151 return buffers;
1152
1153 return error;
1154}
1155
1156/*
1157 * vmsplice splices a user address range into a pipe. It can be thought of
1158 * as splice-from-memory, where the regular splice is splice-from-file (or
1159 * to file). In both cases the output is a pipe, naturally.
1160 *
1161 * Note that vmsplice only supports splicing _from_ user memory to a pipe,
1162 * not the other way around. Splicing from user memory is a simple operation
1163 * that can be supported without any funky alignment restrictions or nasty
1164 * vm tricks. We simply map in the user memory and fill them into a pipe.
1165 * The reverse isn't quite as easy, though. There are two possible solutions
1166 * for that:
1167 *
1168 * - memcpy() the data internally, at which point we might as well just
1169 * do a regular read() on the buffer anyway.
1170 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1171 * has restriction limitations on both ends of the pipe).
1172 *
1173 * Alas, it isn't here.
1174 *
1175 */
1176static long do_vmsplice(struct file *file, const struct iovec __user *iov,
1177 unsigned long nr_segs, unsigned int flags)
1178{
1179 struct pipe_inode_info *pipe = file->f_dentry->d_inode->i_pipe;
1180 struct page *pages[PIPE_BUFFERS];
1181 struct partial_page partial[PIPE_BUFFERS];
1182 struct splice_pipe_desc spd = {
1183 .pages = pages,
1184 .partial = partial,
1185 .flags = flags,
1186 .ops = &user_page_pipe_buf_ops,
1187 };
1188
1189 if (unlikely(!pipe))
1190 return -EBADF;
1191 if (unlikely(nr_segs > UIO_MAXIOV))
1192 return -EINVAL;
1193 else if (unlikely(!nr_segs))
1194 return 0;
1195
1196 spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial);
1197 if (spd.nr_pages <= 0)
1198 return spd.nr_pages;
1199
Jens Axboe00522fb2006-04-26 14:39:29 +02001200 return splice_to_pipe(pipe, &spd);
Jens Axboe912d35f2006-04-26 10:59:21 +02001201}
1202
1203asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
1204 unsigned long nr_segs, unsigned int flags)
1205{
1206 struct file *file;
1207 long error;
1208 int fput;
1209
1210 error = -EBADF;
1211 file = fget_light(fd, &fput);
1212 if (file) {
1213 if (file->f_mode & FMODE_WRITE)
1214 error = do_vmsplice(file, iov, nr_segs, flags);
1215
1216 fput_light(file, fput);
1217 }
1218
1219 return error;
1220}
1221
Ingo Molnar529565d2006-04-10 15:18:58 +02001222asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
1223 int fd_out, loff_t __user *off_out,
1224 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +02001225{
1226 long error;
1227 struct file *in, *out;
1228 int fput_in, fput_out;
1229
1230 if (unlikely(!len))
1231 return 0;
1232
1233 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +02001234 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +02001235 if (in) {
1236 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +02001237 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +02001238 if (out) {
1239 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +02001240 error = do_splice(in, off_in,
1241 out, off_out,
1242 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +02001243 fput_light(out, fput_out);
1244 }
1245 }
1246
1247 fput_light(in, fput_in);
1248 }
1249
1250 return error;
1251}
Jens Axboe70524492006-04-11 15:51:17 +02001252
1253/*
1254 * Link contents of ipipe to opipe.
1255 */
1256static int link_pipe(struct pipe_inode_info *ipipe,
1257 struct pipe_inode_info *opipe,
1258 size_t len, unsigned int flags)
1259{
1260 struct pipe_buffer *ibuf, *obuf;
Jens Axboe2a272502006-04-19 15:56:40 +02001261 int ret, do_wakeup, i, ipipe_first;
1262
1263 ret = do_wakeup = ipipe_first = 0;
Jens Axboe70524492006-04-11 15:51:17 +02001264
1265 /*
1266 * Potential ABBA deadlock, work around it by ordering lock
1267 * grabbing by inode address. Otherwise two different processes
1268 * could deadlock (one doing tee from A -> B, the other from B -> A).
1269 */
1270 if (ipipe->inode < opipe->inode) {
Jens Axboe2a272502006-04-19 15:56:40 +02001271 ipipe_first = 1;
Jens Axboe70524492006-04-11 15:51:17 +02001272 mutex_lock(&ipipe->inode->i_mutex);
1273 mutex_lock(&opipe->inode->i_mutex);
1274 } else {
1275 mutex_lock(&opipe->inode->i_mutex);
1276 mutex_lock(&ipipe->inode->i_mutex);
1277 }
1278
1279 for (i = 0;; i++) {
1280 if (!opipe->readers) {
1281 send_sig(SIGPIPE, current, 0);
1282 if (!ret)
1283 ret = -EPIPE;
1284 break;
1285 }
1286 if (ipipe->nrbufs - i) {
1287 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1288
1289 /*
1290 * If we have room, fill this buffer
1291 */
1292 if (opipe->nrbufs < PIPE_BUFFERS) {
1293 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1294
1295 /*
1296 * Get a reference to this pipe buffer,
1297 * so we can copy the contents over.
1298 */
1299 ibuf->ops->get(ipipe, ibuf);
1300
1301 obuf = opipe->bufs + nbuf;
1302 *obuf = *ibuf;
1303
1304 if (obuf->len > len)
1305 obuf->len = len;
1306
1307 opipe->nrbufs++;
1308 do_wakeup = 1;
1309 ret += obuf->len;
1310 len -= obuf->len;
1311
1312 if (!len)
1313 break;
1314 if (opipe->nrbufs < PIPE_BUFFERS)
1315 continue;
1316 }
1317
1318 /*
1319 * We have input available, but no output room.
Jens Axboe2a272502006-04-19 15:56:40 +02001320 * If we already copied data, return that. If we
1321 * need to drop the opipe lock, it must be ordered
1322 * last to avoid deadlocks.
Jens Axboe70524492006-04-11 15:51:17 +02001323 */
Jens Axboe2a272502006-04-19 15:56:40 +02001324 if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001325 if (!ret)
1326 ret = -EAGAIN;
1327 break;
1328 }
1329 if (signal_pending(current)) {
1330 if (!ret)
1331 ret = -ERESTARTSYS;
1332 break;
1333 }
1334 if (do_wakeup) {
1335 smp_mb();
1336 if (waitqueue_active(&opipe->wait))
1337 wake_up_interruptible(&opipe->wait);
1338 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1339 do_wakeup = 0;
1340 }
1341
1342 opipe->waiting_writers++;
1343 pipe_wait(opipe);
1344 opipe->waiting_writers--;
1345 continue;
1346 }
1347
1348 /*
1349 * No input buffers, do the usual checks for available
1350 * writers and blocking and wait if necessary
1351 */
1352 if (!ipipe->writers)
1353 break;
1354 if (!ipipe->waiting_writers) {
1355 if (ret)
1356 break;
1357 }
Jens Axboe2a272502006-04-19 15:56:40 +02001358 /*
1359 * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1360 * with another process, we can only safely do that if
1361 * the ipipe lock is ordered last.
1362 */
1363 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
Jens Axboe70524492006-04-11 15:51:17 +02001364 if (!ret)
1365 ret = -EAGAIN;
1366 break;
1367 }
1368 if (signal_pending(current)) {
1369 if (!ret)
1370 ret = -ERESTARTSYS;
1371 break;
1372 }
1373
1374 if (waitqueue_active(&ipipe->wait))
1375 wake_up_interruptible_sync(&ipipe->wait);
1376 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1377
1378 pipe_wait(ipipe);
1379 }
1380
1381 mutex_unlock(&ipipe->inode->i_mutex);
1382 mutex_unlock(&opipe->inode->i_mutex);
1383
1384 if (do_wakeup) {
1385 smp_mb();
1386 if (waitqueue_active(&opipe->wait))
1387 wake_up_interruptible(&opipe->wait);
1388 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1389 }
1390
1391 return ret;
1392}
1393
1394/*
1395 * This is a tee(1) implementation that works on pipes. It doesn't copy
1396 * any data, it simply references the 'in' pages on the 'out' pipe.
1397 * The 'flags' used are the SPLICE_F_* variants, currently the only
1398 * applicable one is SPLICE_F_NONBLOCK.
1399 */
1400static long do_tee(struct file *in, struct file *out, size_t len,
1401 unsigned int flags)
1402{
1403 struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1404 struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1405
1406 /*
1407 * Link ipipe to the two output pipes, consuming as we go along.
1408 */
1409 if (ipipe && opipe)
1410 return link_pipe(ipipe, opipe, len, flags);
1411
1412 return -EINVAL;
1413}
1414
1415asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1416{
1417 struct file *in;
1418 int error, fput_in;
1419
1420 if (unlikely(!len))
1421 return 0;
1422
1423 error = -EBADF;
1424 in = fget_light(fdin, &fput_in);
1425 if (in) {
1426 if (in->f_mode & FMODE_READ) {
1427 int fput_out;
1428 struct file *out = fget_light(fdout, &fput_out);
1429
1430 if (out) {
1431 if (out->f_mode & FMODE_WRITE)
1432 error = do_tee(in, out, len, flags);
1433 fput_light(out, fput_out);
1434 }
1435 }
1436 fput_light(in, fput_in);
1437 }
1438
1439 return error;
1440}