blob: 6c9828651e6f5cdec78ce99d5169bb238c61f0ea [file] [log] [blame]
Jens Axboe5274f052006-03-30 15:15:30 +02001/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
Jens Axboec2058e02006-04-11 13:56:34 +020012 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
Jens Axboe5274f052006-03-30 15:15:30 +020014 *
Jens Axboe0fe23472006-09-04 15:41:16 +020015 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
Jens Axboec2058e02006-04-11 13:56:34 +020016 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
Jens Axboe5274f052006-03-30 15:15:30 +020018 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020023#include <linux/splice.h>
Jens Axboe5274f052006-03-30 15:15:30 +020024#include <linux/mm_inline.h>
Jens Axboe5abc97a2006-03-30 15:16:46 +020025#include <linux/swap.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020026#include <linux/writeback.h>
27#include <linux/buffer_head.h>
Jeff Garzika0f06782006-03-30 23:06:13 -050028#include <linux/module.h>
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020029#include <linux/syscalls.h>
Jens Axboe912d35f2006-04-26 10:59:21 +020030#include <linux/uio.h>
James Morris29ce2052007-07-13 11:44:32 +020031#include <linux/security.h>
Jens Axboe5274f052006-03-30 15:15:30 +020032
Jens Axboe83f91352006-04-02 23:05:09 +020033/*
34 * Attempt to steal a page from a pipe buffer. This should perhaps go into
35 * a vm helper function, it's already simplified quite a bit by the
36 * addition of remove_mapping(). If success is returned, the caller may
37 * attempt to reuse this page for another destination.
38 */
Jens Axboe76ad4d12006-05-03 10:41:33 +020039static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
Jens Axboe5abc97a2006-03-30 15:16:46 +020040 struct pipe_buffer *buf)
41{
42 struct page *page = buf->page;
Jens Axboe9e94cd42006-06-20 15:01:12 +020043 struct address_space *mapping;
Jens Axboe5abc97a2006-03-30 15:16:46 +020044
Jens Axboe9e0267c2006-04-19 15:57:31 +020045 lock_page(page);
46
Jens Axboe9e94cd42006-06-20 15:01:12 +020047 mapping = page_mapping(page);
48 if (mapping) {
49 WARN_ON(!PageUptodate(page));
Jens Axboe5abc97a2006-03-30 15:16:46 +020050
Jens Axboe9e94cd42006-06-20 15:01:12 +020051 /*
52 * At least for ext2 with nobh option, we need to wait on
53 * writeback completing on this page, since we'll remove it
54 * from the pagecache. Otherwise truncate wont wait on the
55 * page, allowing the disk blocks to be reused by someone else
56 * before we actually wrote our data to them. fs corruption
57 * ensues.
58 */
59 wait_on_page_writeback(page);
Jens Axboead8d6f02006-04-02 23:10:32 +020060
Jens Axboe9e94cd42006-06-20 15:01:12 +020061 if (PagePrivate(page))
Nick Piggin2ae88142006-10-28 10:38:23 -070062 try_to_release_page(page, GFP_KERNEL);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +020063
Jens Axboe9e94cd42006-06-20 15:01:12 +020064 /*
65 * If we succeeded in removing the mapping, set LRU flag
66 * and return good.
67 */
68 if (remove_mapping(mapping, page)) {
69 buf->flags |= PIPE_BUF_FLAG_LRU;
70 return 0;
71 }
Jens Axboe9e0267c2006-04-19 15:57:31 +020072 }
Jens Axboe5abc97a2006-03-30 15:16:46 +020073
Jens Axboe9e94cd42006-06-20 15:01:12 +020074 /*
75 * Raced with truncate or failed to remove page from current
76 * address space, unlock and return failure.
77 */
78 unlock_page(page);
79 return 1;
Jens Axboe5abc97a2006-03-30 15:16:46 +020080}
81
Jens Axboe76ad4d12006-05-03 10:41:33 +020082static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
Jens Axboe5274f052006-03-30 15:15:30 +020083 struct pipe_buffer *buf)
84{
85 page_cache_release(buf->page);
Jens Axboe14328732006-05-03 10:35:26 +020086 buf->flags &= ~PIPE_BUF_FLAG_LRU;
Jens Axboe5274f052006-03-30 15:15:30 +020087}
88
Jens Axboe08457182007-06-12 20:51:32 +020089/*
90 * Check whether the contents of buf is OK to access. Since the content
91 * is a page cache page, IO may be in flight.
92 */
Jens Axboecac36bb02007-06-14 13:10:48 +020093static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
94 struct pipe_buffer *buf)
Jens Axboe5274f052006-03-30 15:15:30 +020095{
96 struct page *page = buf->page;
Jens Axboe49d0b212006-04-10 09:04:41 +020097 int err;
Jens Axboe5274f052006-03-30 15:15:30 +020098
99 if (!PageUptodate(page)) {
Jens Axboe49d0b212006-04-10 09:04:41 +0200100 lock_page(page);
101
102 /*
103 * Page got truncated/unhashed. This will cause a 0-byte
Ingo Molnar73d62d82006-04-11 13:57:21 +0200104 * splice, if this is the first page.
Jens Axboe49d0b212006-04-10 09:04:41 +0200105 */
106 if (!page->mapping) {
107 err = -ENODATA;
108 goto error;
109 }
110
111 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +0200112 * Uh oh, read-error from disk.
Jens Axboe49d0b212006-04-10 09:04:41 +0200113 */
114 if (!PageUptodate(page)) {
115 err = -EIO;
116 goto error;
117 }
118
119 /*
Jens Axboef84d7512006-05-01 19:59:03 +0200120 * Page is ok afterall, we are done.
Jens Axboe49d0b212006-04-10 09:04:41 +0200121 */
Jens Axboe5274f052006-03-30 15:15:30 +0200122 unlock_page(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200123 }
124
Jens Axboef84d7512006-05-01 19:59:03 +0200125 return 0;
Jens Axboe49d0b212006-04-10 09:04:41 +0200126error:
127 unlock_page(page);
Jens Axboef84d7512006-05-01 19:59:03 +0200128 return err;
Jens Axboe70524492006-04-11 15:51:17 +0200129}
130
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800131static const struct pipe_buf_operations page_cache_pipe_buf_ops = {
Jens Axboe5274f052006-03-30 15:15:30 +0200132 .can_merge = 0,
Jens Axboef84d7512006-05-01 19:59:03 +0200133 .map = generic_pipe_buf_map,
134 .unmap = generic_pipe_buf_unmap,
Jens Axboecac36bb02007-06-14 13:10:48 +0200135 .confirm = page_cache_pipe_buf_confirm,
Jens Axboe5274f052006-03-30 15:15:30 +0200136 .release = page_cache_pipe_buf_release,
Jens Axboe5abc97a2006-03-30 15:16:46 +0200137 .steal = page_cache_pipe_buf_steal,
Jens Axboef84d7512006-05-01 19:59:03 +0200138 .get = generic_pipe_buf_get,
Jens Axboe5274f052006-03-30 15:15:30 +0200139};
140
Jens Axboe912d35f2006-04-26 10:59:21 +0200141static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
142 struct pipe_buffer *buf)
143{
Jens Axboe7afa6fd2006-05-01 20:02:33 +0200144 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
145 return 1;
146
Jens Axboe14328732006-05-03 10:35:26 +0200147 buf->flags |= PIPE_BUF_FLAG_LRU;
Jens Axboe330ab712006-05-02 15:29:57 +0200148 return generic_pipe_buf_steal(pipe, buf);
Jens Axboe912d35f2006-04-26 10:59:21 +0200149}
150
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800151static const struct pipe_buf_operations user_page_pipe_buf_ops = {
Jens Axboe912d35f2006-04-26 10:59:21 +0200152 .can_merge = 0,
Jens Axboef84d7512006-05-01 19:59:03 +0200153 .map = generic_pipe_buf_map,
154 .unmap = generic_pipe_buf_unmap,
Jens Axboecac36bb02007-06-14 13:10:48 +0200155 .confirm = generic_pipe_buf_confirm,
Jens Axboe912d35f2006-04-26 10:59:21 +0200156 .release = page_cache_pipe_buf_release,
157 .steal = user_page_pipe_buf_steal,
Jens Axboef84d7512006-05-01 19:59:03 +0200158 .get = generic_pipe_buf_get,
Jens Axboe912d35f2006-04-26 10:59:21 +0200159};
160
Jens Axboe932cc6d2007-06-21 13:10:21 +0200161/**
162 * splice_to_pipe - fill passed data into a pipe
163 * @pipe: pipe to fill
164 * @spd: data to fill
165 *
166 * Description:
167 * @spd contains a map of pages and len/offset tupples, a long with
168 * the struct pipe_buf_operations associated with these pages. This
169 * function will link that data to the pipe.
170 *
Jens Axboe83f91352006-04-02 23:05:09 +0200171 */
Jens Axboed6b29d72007-06-04 09:59:47 +0200172ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
173 struct splice_pipe_desc *spd)
Jens Axboe5274f052006-03-30 15:15:30 +0200174{
Jens Axboe00de00b2007-06-15 13:14:22 +0200175 unsigned int spd_pages = spd->nr_pages;
Jens Axboe912d35f2006-04-26 10:59:21 +0200176 int ret, do_wakeup, page_nr;
Jens Axboe5274f052006-03-30 15:15:30 +0200177
178 ret = 0;
179 do_wakeup = 0;
Jens Axboe912d35f2006-04-26 10:59:21 +0200180 page_nr = 0;
Jens Axboe5274f052006-03-30 15:15:30 +0200181
Ingo Molnar3a326a22006-04-10 15:18:35 +0200182 if (pipe->inode)
183 mutex_lock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200184
Jens Axboe5274f052006-03-30 15:15:30 +0200185 for (;;) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200186 if (!pipe->readers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200187 send_sig(SIGPIPE, current, 0);
188 if (!ret)
189 ret = -EPIPE;
190 break;
191 }
192
Jens Axboe6f767b02006-04-11 13:53:56 +0200193 if (pipe->nrbufs < PIPE_BUFFERS) {
194 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200195 struct pipe_buffer *buf = pipe->bufs + newbuf;
Jens Axboe5274f052006-03-30 15:15:30 +0200196
Jens Axboe912d35f2006-04-26 10:59:21 +0200197 buf->page = spd->pages[page_nr];
198 buf->offset = spd->partial[page_nr].offset;
199 buf->len = spd->partial[page_nr].len;
Jens Axboe497f9622007-06-11 12:00:45 +0200200 buf->private = spd->partial[page_nr].private;
Jens Axboe912d35f2006-04-26 10:59:21 +0200201 buf->ops = spd->ops;
Jens Axboe7afa6fd2006-05-01 20:02:33 +0200202 if (spd->flags & SPLICE_F_GIFT)
203 buf->flags |= PIPE_BUF_FLAG_GIFT;
204
Jens Axboe6f767b02006-04-11 13:53:56 +0200205 pipe->nrbufs++;
Jens Axboe912d35f2006-04-26 10:59:21 +0200206 page_nr++;
207 ret += buf->len;
208
Jens Axboe6f767b02006-04-11 13:53:56 +0200209 if (pipe->inode)
210 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200211
Jens Axboe912d35f2006-04-26 10:59:21 +0200212 if (!--spd->nr_pages)
Jens Axboe5274f052006-03-30 15:15:30 +0200213 break;
Jens Axboe6f767b02006-04-11 13:53:56 +0200214 if (pipe->nrbufs < PIPE_BUFFERS)
Jens Axboe5274f052006-03-30 15:15:30 +0200215 continue;
216
217 break;
218 }
219
Jens Axboe912d35f2006-04-26 10:59:21 +0200220 if (spd->flags & SPLICE_F_NONBLOCK) {
Linus Torvalds29e35092006-04-02 12:46:35 -0700221 if (!ret)
222 ret = -EAGAIN;
223 break;
224 }
225
Jens Axboe5274f052006-03-30 15:15:30 +0200226 if (signal_pending(current)) {
227 if (!ret)
228 ret = -ERESTARTSYS;
229 break;
230 }
231
232 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200233 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200234 if (waitqueue_active(&pipe->wait))
235 wake_up_interruptible_sync(&pipe->wait);
236 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Jens Axboe5274f052006-03-30 15:15:30 +0200237 do_wakeup = 0;
238 }
239
Ingo Molnar3a326a22006-04-10 15:18:35 +0200240 pipe->waiting_writers++;
241 pipe_wait(pipe);
242 pipe->waiting_writers--;
Jens Axboe5274f052006-03-30 15:15:30 +0200243 }
244
Jens Axboe02676e52007-06-15 13:16:13 +0200245 if (pipe->inode) {
Ingo Molnar3a326a22006-04-10 15:18:35 +0200246 mutex_unlock(&pipe->inode->i_mutex);
Jens Axboe5274f052006-03-30 15:15:30 +0200247
Jens Axboe02676e52007-06-15 13:16:13 +0200248 if (do_wakeup) {
249 smp_mb();
250 if (waitqueue_active(&pipe->wait))
251 wake_up_interruptible(&pipe->wait);
252 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
253 }
Jens Axboe5274f052006-03-30 15:15:30 +0200254 }
255
Jens Axboe00de00b2007-06-15 13:14:22 +0200256 while (page_nr < spd_pages)
Jens Axboe912d35f2006-04-26 10:59:21 +0200257 page_cache_release(spd->pages[page_nr++]);
Jens Axboe5274f052006-03-30 15:15:30 +0200258
259 return ret;
260}
261
Ingo Molnar3a326a22006-04-10 15:18:35 +0200262static int
Jens Axboecbb7e572006-04-11 14:57:50 +0200263__generic_file_splice_read(struct file *in, loff_t *ppos,
264 struct pipe_inode_info *pipe, size_t len,
265 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200266{
267 struct address_space *mapping = in->f_mapping;
Jens Axboe912d35f2006-04-26 10:59:21 +0200268 unsigned int loff, nr_pages;
Jens Axboe16c523d2006-04-10 09:03:58 +0200269 struct page *pages[PIPE_BUFFERS];
Jens Axboe912d35f2006-04-26 10:59:21 +0200270 struct partial_page partial[PIPE_BUFFERS];
Jens Axboe5274f052006-03-30 15:15:30 +0200271 struct page *page;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200272 pgoff_t index, end_index;
273 loff_t isize;
Jens Axboeeb207962006-04-27 11:05:22 +0200274 int error, page_nr;
Jens Axboe912d35f2006-04-26 10:59:21 +0200275 struct splice_pipe_desc spd = {
276 .pages = pages,
277 .partial = partial,
278 .flags = flags,
279 .ops = &page_cache_pipe_buf_ops,
280 };
Jens Axboe5274f052006-03-30 15:15:30 +0200281
Jens Axboecbb7e572006-04-11 14:57:50 +0200282 index = *ppos >> PAGE_CACHE_SHIFT;
Jens Axboe912d35f2006-04-26 10:59:21 +0200283 loff = *ppos & ~PAGE_CACHE_MASK;
284 nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Jens Axboe5274f052006-03-30 15:15:30 +0200285
286 if (nr_pages > PIPE_BUFFERS)
287 nr_pages = PIPE_BUFFERS;
288
289 /*
Jens Axboe86aa5ac2007-05-08 08:46:19 +0200290 * Don't try to 2nd guess the read-ahead logic, call into
291 * page_cache_readahead() like the page cache reads would do.
Jens Axboe5274f052006-03-30 15:15:30 +0200292 */
Jens Axboe86aa5ac2007-05-08 08:46:19 +0200293 page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
Jens Axboe5274f052006-03-30 15:15:30 +0200294
295 /*
Jens Axboeeb207962006-04-27 11:05:22 +0200296 * Lookup the (hopefully) full range of pages we need.
297 */
298 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
299
300 /*
301 * If find_get_pages_contig() returned fewer pages than we needed,
Jens Axboe932cc6d2007-06-21 13:10:21 +0200302 * allocate the rest and fill in the holes.
Jens Axboeeb207962006-04-27 11:05:22 +0200303 */
Jens Axboe932cc6d2007-06-21 13:10:21 +0200304 error = 0;
Jens Axboeeb207962006-04-27 11:05:22 +0200305 index += spd.nr_pages;
306 while (spd.nr_pages < nr_pages) {
307 /*
308 * Page could be there, find_get_pages_contig() breaks on
309 * the first hole.
310 */
311 page = find_get_page(mapping, index);
312 if (!page) {
313 /*
Jens Axboee27dedd2006-05-01 19:59:54 +0200314 * Make sure the read-ahead engine is notified
315 * about this failure.
316 */
317 handle_ra_miss(mapping, &in->f_ra, index);
318
319 /*
Jens Axboeeb207962006-04-27 11:05:22 +0200320 * page didn't exist, allocate one.
321 */
322 page = page_cache_alloc_cold(mapping);
323 if (!page)
324 break;
325
326 error = add_to_page_cache_lru(page, mapping, index,
Nick Piggin2ae88142006-10-28 10:38:23 -0700327 GFP_KERNEL);
Jens Axboeeb207962006-04-27 11:05:22 +0200328 if (unlikely(error)) {
329 page_cache_release(page);
Jens Axboea0548872006-05-03 10:58:22 +0200330 if (error == -EEXIST)
331 continue;
Jens Axboeeb207962006-04-27 11:05:22 +0200332 break;
333 }
334 /*
335 * add_to_page_cache() locks the page, unlock it
336 * to avoid convoluting the logic below even more.
337 */
338 unlock_page(page);
339 }
340
341 pages[spd.nr_pages++] = page;
342 index++;
343 }
344
345 /*
346 * Now loop over the map and see if we need to start IO on any
347 * pages, fill in the partial map, etc.
348 */
349 index = *ppos >> PAGE_CACHE_SHIFT;
350 nr_pages = spd.nr_pages;
351 spd.nr_pages = 0;
352 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
Jens Axboe82aa5d62006-04-20 13:05:48 +0200353 unsigned int this_len;
354
355 if (!len)
356 break;
357
358 /*
359 * this_len is the max we'll use from this page
360 */
Andrew Mortonba5f5d92006-04-25 15:33:34 +0200361 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
Jens Axboeeb207962006-04-27 11:05:22 +0200362 page = pages[page_nr];
Jens Axboe7480a902006-04-11 13:52:47 +0200363
364 /*
365 * If the page isn't uptodate, we may need to start io on it
366 */
367 if (!PageUptodate(page)) {
Jens Axboec4f895c2006-04-19 15:56:12 +0200368 /*
369 * If in nonblock mode then dont block on waiting
370 * for an in-flight io page
371 */
Fengguang Wu9ae9d682007-05-08 08:44:36 +0200372 if (flags & SPLICE_F_NONBLOCK) {
373 if (TestSetPageLocked(page))
374 break;
375 } else
376 lock_page(page);
Jens Axboe7480a902006-04-11 13:52:47 +0200377
378 /*
379 * page was truncated, stop here. if this isn't the
380 * first page, we'll just complete what we already
381 * added
382 */
383 if (!page->mapping) {
384 unlock_page(page);
Jens Axboe7480a902006-04-11 13:52:47 +0200385 break;
386 }
387 /*
388 * page was already under io and is now done, great
389 */
390 if (PageUptodate(page)) {
391 unlock_page(page);
392 goto fill_it;
393 }
394
Jens Axboe7480a902006-04-11 13:52:47 +0200395 /*
396 * need to read in the page
397 */
398 error = mapping->a_ops->readpage(in, page);
Jens Axboe7480a902006-04-11 13:52:47 +0200399 if (unlikely(error)) {
Jens Axboeeb207962006-04-27 11:05:22 +0200400 /*
401 * We really should re-lookup the page here,
402 * but it complicates things a lot. Instead
403 * lets just do what we already stored, and
404 * we'll get it the next time we are called.
405 */
Jens Axboe7480a902006-04-11 13:52:47 +0200406 if (error == AOP_TRUNCATED_PAGE)
Jens Axboeeb207962006-04-27 11:05:22 +0200407 error = 0;
408
Jens Axboe7480a902006-04-11 13:52:47 +0200409 break;
410 }
Jens Axboe620a3242007-06-07 09:39:42 +0200411 }
412fill_it:
413 /*
414 * i_size must be checked after PageUptodate.
415 */
416 isize = i_size_read(mapping->host);
417 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
418 if (unlikely(!isize || index > end_index))
419 break;
420
421 /*
422 * if this is the last page, see if we need to shrink
423 * the length and stop
424 */
425 if (end_index == index) {
426 unsigned int plen;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200427
428 /*
Jens Axboe620a3242007-06-07 09:39:42 +0200429 * max good bytes in this page
Jens Axboe91ad66e2006-04-19 15:55:10 +0200430 */
Jens Axboe620a3242007-06-07 09:39:42 +0200431 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
432 if (plen <= loff)
Jens Axboe91ad66e2006-04-19 15:55:10 +0200433 break;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200434
435 /*
Jens Axboe620a3242007-06-07 09:39:42 +0200436 * force quit after adding this page
Jens Axboe91ad66e2006-04-19 15:55:10 +0200437 */
Jens Axboe620a3242007-06-07 09:39:42 +0200438 this_len = min(this_len, plen - loff);
439 len = this_len;
Jens Axboe7480a902006-04-11 13:52:47 +0200440 }
Jens Axboe620a3242007-06-07 09:39:42 +0200441
Jens Axboeeb207962006-04-27 11:05:22 +0200442 partial[page_nr].offset = loff;
443 partial[page_nr].len = this_len;
Jens Axboe82aa5d62006-04-20 13:05:48 +0200444 len -= this_len;
Jens Axboe91ad66e2006-04-19 15:55:10 +0200445 loff = 0;
Jens Axboeeb207962006-04-27 11:05:22 +0200446 spd.nr_pages++;
447 index++;
Jens Axboe5274f052006-03-30 15:15:30 +0200448 }
449
Jens Axboeeb207962006-04-27 11:05:22 +0200450 /*
Hugh Dickins475ecad2007-06-07 09:36:00 +0200451 * Release any pages at the end, if we quit early. 'page_nr' is how far
Jens Axboeeb207962006-04-27 11:05:22 +0200452 * we got, 'nr_pages' is how many pages are in the map.
453 */
454 while (page_nr < nr_pages)
455 page_cache_release(pages[page_nr++]);
456
Jens Axboe912d35f2006-04-26 10:59:21 +0200457 if (spd.nr_pages)
Jens Axboe00522fb2006-04-26 14:39:29 +0200458 return splice_to_pipe(pipe, &spd);
Jens Axboe5274f052006-03-30 15:15:30 +0200459
Jens Axboe7480a902006-04-11 13:52:47 +0200460 return error;
Jens Axboe5274f052006-03-30 15:15:30 +0200461}
462
Jens Axboe83f91352006-04-02 23:05:09 +0200463/**
464 * generic_file_splice_read - splice data from file to a pipe
465 * @in: file to splice from
Jens Axboe932cc6d2007-06-21 13:10:21 +0200466 * @ppos: position in @in
Jens Axboe83f91352006-04-02 23:05:09 +0200467 * @pipe: pipe to splice to
468 * @len: number of bytes to splice
469 * @flags: splice modifier flags
470 *
Jens Axboe932cc6d2007-06-21 13:10:21 +0200471 * Description:
472 * Will read pages from given file and fill them into a pipe. Can be
473 * used as long as the address_space operations for the source implements
474 * a readpage() hook.
475 *
Jens Axboe83f91352006-04-02 23:05:09 +0200476 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200477ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
478 struct pipe_inode_info *pipe, size_t len,
479 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200480{
481 ssize_t spliced;
482 int ret;
Jens Axboed366d3982007-06-01 14:54:11 +0200483 loff_t isize, left;
484
485 isize = i_size_read(in->f_mapping->host);
486 if (unlikely(*ppos >= isize))
487 return 0;
488
489 left = isize - *ppos;
490 if (unlikely(left < len))
491 len = left;
Jens Axboe5274f052006-03-30 15:15:30 +0200492
493 ret = 0;
494 spliced = 0;
Jens Axboe51a92c02007-07-13 14:11:43 +0200495 while (len && !spliced) {
Jens Axboecbb7e572006-04-11 14:57:50 +0200496 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200497
Jens Axboec4f895c2006-04-19 15:56:12 +0200498 if (ret < 0)
Jens Axboe5274f052006-03-30 15:15:30 +0200499 break;
Jens Axboec4f895c2006-04-19 15:56:12 +0200500 else if (!ret) {
501 if (spliced)
502 break;
503 if (flags & SPLICE_F_NONBLOCK) {
504 ret = -EAGAIN;
505 break;
506 }
507 }
Jens Axboe5274f052006-03-30 15:15:30 +0200508
Jens Axboecbb7e572006-04-11 14:57:50 +0200509 *ppos += ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200510 len -= ret;
511 spliced += ret;
512 }
513
514 if (spliced)
515 return spliced;
516
517 return ret;
518}
519
Jens Axboe059a8f32006-04-02 23:06:05 +0200520EXPORT_SYMBOL(generic_file_splice_read);
521
Jens Axboe5274f052006-03-30 15:15:30 +0200522/*
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200523 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
Jens Axboe016b6612006-04-25 15:42:00 +0200524 * using sendpage(). Return the number of bytes sent.
Jens Axboe5274f052006-03-30 15:15:30 +0200525 */
Jens Axboe76ad4d12006-05-03 10:41:33 +0200526static int pipe_to_sendpage(struct pipe_inode_info *pipe,
Jens Axboe5274f052006-03-30 15:15:30 +0200527 struct pipe_buffer *buf, struct splice_desc *sd)
528{
Jens Axboe6a14b902007-06-14 13:08:55 +0200529 struct file *file = sd->u.file;
Jens Axboe5274f052006-03-30 15:15:30 +0200530 loff_t pos = sd->pos;
Jens Axboef84d7512006-05-01 19:59:03 +0200531 int ret, more;
Jens Axboe5274f052006-03-30 15:15:30 +0200532
Jens Axboecac36bb02007-06-14 13:10:48 +0200533 ret = buf->ops->confirm(pipe, buf);
Jens Axboef84d7512006-05-01 19:59:03 +0200534 if (!ret) {
535 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200536
Jens Axboef84d7512006-05-01 19:59:03 +0200537 ret = file->f_op->sendpage(file, buf->page, buf->offset,
538 sd->len, &pos, more);
539 }
Jens Axboe5274f052006-03-30 15:15:30 +0200540
Jens Axboe016b6612006-04-25 15:42:00 +0200541 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200542}
543
544/*
545 * This is a little more tricky than the file -> pipe splicing. There are
546 * basically three cases:
547 *
548 * - Destination page already exists in the address space and there
549 * are users of it. For that case we have no other option that
550 * copying the data. Tough luck.
551 * - Destination page already exists in the address space, but there
552 * are no users of it. Make sure it's uptodate, then drop it. Fall
553 * through to last case.
554 * - Destination page does not exist, we can add the pipe page to
555 * the page cache and avoid the copy.
556 *
Jens Axboe83f91352006-04-02 23:05:09 +0200557 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
558 * sd->flags), we attempt to migrate pages from the pipe to the output
559 * file address space page cache. This is possible if no one else has
560 * the pipe page referenced outside of the pipe and page cache. If
561 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
562 * a new page in the output file page cache and fill/dirty that.
Jens Axboe5274f052006-03-30 15:15:30 +0200563 */
Jens Axboe76ad4d12006-05-03 10:41:33 +0200564static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
Jens Axboe5274f052006-03-30 15:15:30 +0200565 struct splice_desc *sd)
566{
Jens Axboe6a14b902007-06-14 13:08:55 +0200567 struct file *file = sd->u.file;
Jens Axboe5274f052006-03-30 15:15:30 +0200568 struct address_space *mapping = file->f_mapping;
Jens Axboe016b6612006-04-25 15:42:00 +0200569 unsigned int offset, this_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200570 struct page *page;
Jens Axboe5274f052006-03-30 15:15:30 +0200571 pgoff_t index;
Jens Axboe3e7ee3e2006-04-02 23:11:04 +0200572 int ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200573
574 /*
Jens Axboe49d0b212006-04-10 09:04:41 +0200575 * make sure the data in this buffer is uptodate
Jens Axboe5274f052006-03-30 15:15:30 +0200576 */
Jens Axboecac36bb02007-06-14 13:10:48 +0200577 ret = buf->ops->confirm(pipe, buf);
Jens Axboef84d7512006-05-01 19:59:03 +0200578 if (unlikely(ret))
579 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200580
581 index = sd->pos >> PAGE_CACHE_SHIFT;
582 offset = sd->pos & ~PAGE_CACHE_MASK;
583
Jens Axboe016b6612006-04-25 15:42:00 +0200584 this_len = sd->len;
585 if (this_len + offset > PAGE_CACHE_SIZE)
586 this_len = PAGE_CACHE_SIZE - offset;
587
Jens Axboe5abc97a2006-03-30 15:16:46 +0200588find_page:
Nick Piggin485ddb42007-03-27 08:55:08 +0200589 page = find_lock_page(mapping, index);
590 if (!page) {
591 ret = -ENOMEM;
592 page = page_cache_alloc_cold(mapping);
593 if (unlikely(!page))
594 goto out_ret;
Jens Axboe9e0267c2006-04-19 15:57:31 +0200595
Nick Piggin485ddb42007-03-27 08:55:08 +0200596 /*
597 * This will also lock the page
598 */
599 ret = add_to_page_cache_lru(page, mapping, index,
600 GFP_KERNEL);
601 if (unlikely(ret))
602 goto out;
603 }
604
Jens Axboe016b6612006-04-25 15:42:00 +0200605 ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
Jens Axboebfc4ee32006-05-03 10:35:10 +0200606 if (unlikely(ret)) {
607 loff_t isize = i_size_read(mapping->host);
608
609 if (ret != AOP_TRUNCATED_PAGE)
610 unlock_page(page);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200611 page_cache_release(page);
Jens Axboebfc4ee32006-05-03 10:35:10 +0200612 if (ret == AOP_TRUNCATED_PAGE)
613 goto find_page;
614
615 /*
616 * prepare_write() may have instantiated a few blocks
617 * outside i_size. Trim these off again.
618 */
619 if (sd->pos + this_len > isize)
620 vmtruncate(mapping->host, isize);
621
Jens Axboee6e80f22006-10-11 10:03:09 +0200622 goto out_ret;
Jens Axboebfc4ee32006-05-03 10:35:10 +0200623 }
Jens Axboe5274f052006-03-30 15:15:30 +0200624
Jens Axboe0568b402006-05-01 19:50:48 +0200625 if (buf->page != page) {
Jens Axboef84d7512006-05-01 19:59:03 +0200626 /*
627 * Careful, ->map() uses KM_USER0!
628 */
Jens Axboe76ad4d12006-05-03 10:41:33 +0200629 char *src = buf->ops->map(pipe, buf, 1);
Jens Axboef84d7512006-05-01 19:59:03 +0200630 char *dst = kmap_atomic(page, KM_USER1);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200631
Jens Axboe016b6612006-04-25 15:42:00 +0200632 memcpy(dst + offset, src + buf->offset, this_len);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200633 flush_dcache_page(page);
Jens Axboef84d7512006-05-01 19:59:03 +0200634 kunmap_atomic(dst, KM_USER1);
Jens Axboe76ad4d12006-05-03 10:41:33 +0200635 buf->ops->unmap(pipe, buf, src);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200636 }
Jens Axboe5274f052006-03-30 15:15:30 +0200637
Jens Axboe016b6612006-04-25 15:42:00 +0200638 ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
Dmitriy Monakhovd9993c32007-03-29 14:24:09 +0200639 if (ret) {
640 if (ret == AOP_TRUNCATED_PAGE) {
641 page_cache_release(page);
642 goto find_page;
643 }
644 if (ret < 0)
645 goto out;
Jens Axboe0568b402006-05-01 19:50:48 +0200646 /*
Dmitriy Monakhovd9993c32007-03-29 14:24:09 +0200647 * Partial write has happened, so 'ret' already initialized by
648 * number of bytes written, Where is nothing we have to do here.
Jens Axboe0568b402006-05-01 19:50:48 +0200649 */
Dmitriy Monakhovd9993c32007-03-29 14:24:09 +0200650 } else
Jens Axboe0568b402006-05-01 19:50:48 +0200651 ret = this_len;
Dmitriy Monakhovd9993c32007-03-29 14:24:09 +0200652 /*
653 * Return the number of bytes written and mark page as
654 * accessed, we are now done!
655 */
656 mark_page_accessed(page);
Jens Axboe5274f052006-03-30 15:15:30 +0200657out:
Jens Axboe0568b402006-05-01 19:50:48 +0200658 page_cache_release(page);
Jens Axboe9e0267c2006-04-19 15:57:31 +0200659 unlock_page(page);
Jens Axboee6e80f22006-10-11 10:03:09 +0200660out_ret:
Jens Axboe5274f052006-03-30 15:15:30 +0200661 return ret;
662}
663
Jens Axboe932cc6d2007-06-21 13:10:21 +0200664/**
665 * __splice_from_pipe - splice data from a pipe to given actor
666 * @pipe: pipe to splice from
667 * @sd: information to @actor
668 * @actor: handler that splices the data
669 *
670 * Description:
671 * This function does little more than loop over the pipe and call
672 * @actor to do the actual moving of a single struct pipe_buffer to
673 * the desired destination. See pipe_to_file, pipe_to_sendpage, or
674 * pipe_to_user.
675 *
Jens Axboe83f91352006-04-02 23:05:09 +0200676 */
Jens Axboec66ab6f2007-06-12 21:17:17 +0200677ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
678 splice_actor *actor)
Jens Axboe5274f052006-03-30 15:15:30 +0200679{
Jens Axboe5274f052006-03-30 15:15:30 +0200680 int ret, do_wakeup, err;
Jens Axboe5274f052006-03-30 15:15:30 +0200681
682 ret = 0;
683 do_wakeup = 0;
684
Jens Axboe5274f052006-03-30 15:15:30 +0200685 for (;;) {
Jens Axboe6f767b02006-04-11 13:53:56 +0200686 if (pipe->nrbufs) {
687 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800688 const struct pipe_buf_operations *ops = buf->ops;
Jens Axboe5274f052006-03-30 15:15:30 +0200689
Jens Axboec66ab6f2007-06-12 21:17:17 +0200690 sd->len = buf->len;
691 if (sd->len > sd->total_len)
692 sd->len = sd->total_len;
Jens Axboe5274f052006-03-30 15:15:30 +0200693
Jens Axboec66ab6f2007-06-12 21:17:17 +0200694 err = actor(pipe, buf, sd);
Jens Axboe016b6612006-04-25 15:42:00 +0200695 if (err <= 0) {
Jens Axboe5274f052006-03-30 15:15:30 +0200696 if (!ret && err != -ENODATA)
697 ret = err;
698
699 break;
700 }
701
Jens Axboe016b6612006-04-25 15:42:00 +0200702 ret += err;
703 buf->offset += err;
704 buf->len -= err;
705
Jens Axboec66ab6f2007-06-12 21:17:17 +0200706 sd->len -= err;
707 sd->pos += err;
708 sd->total_len -= err;
709 if (sd->len)
Jens Axboe016b6612006-04-25 15:42:00 +0200710 continue;
Ingo Molnar73d62d82006-04-11 13:57:21 +0200711
Jens Axboe5274f052006-03-30 15:15:30 +0200712 if (!buf->len) {
713 buf->ops = NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200714 ops->release(pipe, buf);
Jens Axboe6f767b02006-04-11 13:53:56 +0200715 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
716 pipe->nrbufs--;
717 if (pipe->inode)
718 do_wakeup = 1;
Jens Axboe5274f052006-03-30 15:15:30 +0200719 }
720
Jens Axboec66ab6f2007-06-12 21:17:17 +0200721 if (!sd->total_len)
Jens Axboe5274f052006-03-30 15:15:30 +0200722 break;
723 }
724
Jens Axboe6f767b02006-04-11 13:53:56 +0200725 if (pipe->nrbufs)
Jens Axboe5274f052006-03-30 15:15:30 +0200726 continue;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200727 if (!pipe->writers)
Jens Axboe5274f052006-03-30 15:15:30 +0200728 break;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200729 if (!pipe->waiting_writers) {
Jens Axboe5274f052006-03-30 15:15:30 +0200730 if (ret)
731 break;
732 }
733
Jens Axboec66ab6f2007-06-12 21:17:17 +0200734 if (sd->flags & SPLICE_F_NONBLOCK) {
Linus Torvalds29e35092006-04-02 12:46:35 -0700735 if (!ret)
736 ret = -EAGAIN;
737 break;
738 }
739
Jens Axboe5274f052006-03-30 15:15:30 +0200740 if (signal_pending(current)) {
741 if (!ret)
742 ret = -ERESTARTSYS;
743 break;
744 }
745
746 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200747 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200748 if (waitqueue_active(&pipe->wait))
749 wake_up_interruptible_sync(&pipe->wait);
750 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200751 do_wakeup = 0;
752 }
753
Ingo Molnar3a326a22006-04-10 15:18:35 +0200754 pipe_wait(pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200755 }
756
Jens Axboe5274f052006-03-30 15:15:30 +0200757 if (do_wakeup) {
Jens Axboec0bd1f62006-04-10 09:03:32 +0200758 smp_mb();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200759 if (waitqueue_active(&pipe->wait))
760 wake_up_interruptible(&pipe->wait);
761 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Jens Axboe5274f052006-03-30 15:15:30 +0200762 }
763
Jens Axboe5274f052006-03-30 15:15:30 +0200764 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200765}
Mark Fasheh40bee44e2007-03-21 13:11:02 +0100766EXPORT_SYMBOL(__splice_from_pipe);
Jens Axboe5274f052006-03-30 15:15:30 +0200767
Jens Axboe932cc6d2007-06-21 13:10:21 +0200768/**
769 * splice_from_pipe - splice data from a pipe to a file
770 * @pipe: pipe to splice from
771 * @out: file to splice to
772 * @ppos: position in @out
773 * @len: how many bytes to splice
774 * @flags: splice modifier flags
775 * @actor: handler that splices the data
776 *
777 * Description:
778 * See __splice_from_pipe. This function locks the input and output inodes,
779 * otherwise it's identical to __splice_from_pipe().
780 *
781 */
Mark Fasheh6da61802006-10-17 18:43:07 +0200782ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
783 loff_t *ppos, size_t len, unsigned int flags,
784 splice_actor *actor)
785{
786 ssize_t ret;
787 struct inode *inode = out->f_mapping->host;
Jens Axboec66ab6f2007-06-12 21:17:17 +0200788 struct splice_desc sd = {
789 .total_len = len,
790 .flags = flags,
791 .pos = *ppos,
Jens Axboe6a14b902007-06-14 13:08:55 +0200792 .u.file = out,
Jens Axboec66ab6f2007-06-12 21:17:17 +0200793 };
Mark Fasheh6da61802006-10-17 18:43:07 +0200794
795 /*
796 * The actor worker might be calling ->prepare_write and
797 * ->commit_write. Most of the time, these expect i_mutex to
798 * be held. Since this may result in an ABBA deadlock with
799 * pipe->inode, we have to order lock acquiry here.
800 */
801 inode_double_lock(inode, pipe->inode);
Jens Axboec66ab6f2007-06-12 21:17:17 +0200802 ret = __splice_from_pipe(pipe, &sd, actor);
Mark Fasheh6da61802006-10-17 18:43:07 +0200803 inode_double_unlock(inode, pipe->inode);
804
805 return ret;
806}
807
808/**
809 * generic_file_splice_write_nolock - generic_file_splice_write without mutexes
810 * @pipe: pipe info
811 * @out: file to write to
Jens Axboe932cc6d2007-06-21 13:10:21 +0200812 * @ppos: position in @out
Mark Fasheh6da61802006-10-17 18:43:07 +0200813 * @len: number of bytes to splice
814 * @flags: splice modifier flags
815 *
Jens Axboe932cc6d2007-06-21 13:10:21 +0200816 * Description:
817 * Will either move or copy pages (determined by @flags options) from
818 * the given pipe inode to the given file. The caller is responsible
819 * for acquiring i_mutex on both inodes.
Mark Fasheh6da61802006-10-17 18:43:07 +0200820 *
821 */
822ssize_t
823generic_file_splice_write_nolock(struct pipe_inode_info *pipe, struct file *out,
824 loff_t *ppos, size_t len, unsigned int flags)
825{
826 struct address_space *mapping = out->f_mapping;
827 struct inode *inode = mapping->host;
Jens Axboec66ab6f2007-06-12 21:17:17 +0200828 struct splice_desc sd = {
829 .total_len = len,
830 .flags = flags,
831 .pos = *ppos,
Jens Axboe6a14b902007-06-14 13:08:55 +0200832 .u.file = out,
Jens Axboec66ab6f2007-06-12 21:17:17 +0200833 };
Mark Fasheh6da61802006-10-17 18:43:07 +0200834 ssize_t ret;
835 int err;
836
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800837 err = remove_suid(out->f_path.dentry);
Jens Axboe8c34e2d2006-10-17 19:43:22 +0200838 if (unlikely(err))
839 return err;
840
Jens Axboec66ab6f2007-06-12 21:17:17 +0200841 ret = __splice_from_pipe(pipe, &sd, pipe_to_file);
Mark Fasheh6da61802006-10-17 18:43:07 +0200842 if (ret > 0) {
Jens Axboe17ee4f42007-06-15 13:10:37 +0200843 unsigned long nr_pages;
844
Mark Fasheh6da61802006-10-17 18:43:07 +0200845 *ppos += ret;
Jens Axboe17ee4f42007-06-15 13:10:37 +0200846 nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Mark Fasheh6da61802006-10-17 18:43:07 +0200847
848 /*
849 * If file or inode is SYNC and we actually wrote some data,
850 * sync it.
851 */
852 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
853 err = generic_osync_inode(inode, mapping,
854 OSYNC_METADATA|OSYNC_DATA);
855
856 if (err)
857 ret = err;
858 }
Jens Axboe17ee4f42007-06-15 13:10:37 +0200859 balance_dirty_pages_ratelimited_nr(mapping, nr_pages);
Mark Fasheh6da61802006-10-17 18:43:07 +0200860 }
861
862 return ret;
863}
864
865EXPORT_SYMBOL(generic_file_splice_write_nolock);
866
Jens Axboe83f91352006-04-02 23:05:09 +0200867/**
868 * generic_file_splice_write - splice data from a pipe to a file
Ingo Molnar3a326a22006-04-10 15:18:35 +0200869 * @pipe: pipe info
Jens Axboe83f91352006-04-02 23:05:09 +0200870 * @out: file to write to
Jens Axboe932cc6d2007-06-21 13:10:21 +0200871 * @ppos: position in @out
Jens Axboe83f91352006-04-02 23:05:09 +0200872 * @len: number of bytes to splice
873 * @flags: splice modifier flags
874 *
Jens Axboe932cc6d2007-06-21 13:10:21 +0200875 * Description:
876 * Will either move or copy pages (determined by @flags options) from
877 * the given pipe inode to the given file.
Jens Axboe83f91352006-04-02 23:05:09 +0200878 *
879 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200880ssize_t
881generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200882 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200883{
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200884 struct address_space *mapping = out->f_mapping;
Jens Axboe8c34e2d2006-10-17 19:43:22 +0200885 struct inode *inode = mapping->host;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200886 ssize_t ret;
Jens Axboe8c34e2d2006-10-17 19:43:22 +0200887 int err;
888
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800889 err = should_remove_suid(out->f_path.dentry);
Jens Axboe8c34e2d2006-10-17 19:43:22 +0200890 if (unlikely(err)) {
891 mutex_lock(&inode->i_mutex);
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800892 err = __remove_suid(out->f_path.dentry, err);
Jens Axboe8c34e2d2006-10-17 19:43:22 +0200893 mutex_unlock(&inode->i_mutex);
894 if (err)
895 return err;
896 }
Ingo Molnar3a326a22006-04-10 15:18:35 +0200897
Jens Axboe00522fb2006-04-26 14:39:29 +0200898 ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
Jens Axboea4514eb2006-04-19 15:57:05 +0200899 if (ret > 0) {
Jens Axboe17ee4f42007-06-15 13:10:37 +0200900 unsigned long nr_pages;
901
Jens Axboea4514eb2006-04-19 15:57:05 +0200902 *ppos += ret;
Jens Axboe17ee4f42007-06-15 13:10:37 +0200903 nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200904
Jens Axboea4514eb2006-04-19 15:57:05 +0200905 /*
906 * If file or inode is SYNC and we actually wrote some data,
907 * sync it.
908 */
909 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
Jens Axboea4514eb2006-04-19 15:57:05 +0200910 mutex_lock(&inode->i_mutex);
911 err = generic_osync_inode(inode, mapping,
912 OSYNC_METADATA|OSYNC_DATA);
913 mutex_unlock(&inode->i_mutex);
914
915 if (err)
916 ret = err;
917 }
Jens Axboe17ee4f42007-06-15 13:10:37 +0200918 balance_dirty_pages_ratelimited_nr(mapping, nr_pages);
Jens Axboe4f6f0bd2006-04-02 23:04:46 +0200919 }
920
921 return ret;
Jens Axboe5274f052006-03-30 15:15:30 +0200922}
923
Jens Axboe059a8f32006-04-02 23:06:05 +0200924EXPORT_SYMBOL(generic_file_splice_write);
925
Jens Axboe83f91352006-04-02 23:05:09 +0200926/**
927 * generic_splice_sendpage - splice data from a pipe to a socket
Jens Axboe932cc6d2007-06-21 13:10:21 +0200928 * @pipe: pipe to splice from
Jens Axboe83f91352006-04-02 23:05:09 +0200929 * @out: socket to write to
Jens Axboe932cc6d2007-06-21 13:10:21 +0200930 * @ppos: position in @out
Jens Axboe83f91352006-04-02 23:05:09 +0200931 * @len: number of bytes to splice
932 * @flags: splice modifier flags
933 *
Jens Axboe932cc6d2007-06-21 13:10:21 +0200934 * Description:
935 * Will send @len bytes from the pipe to a network socket. No data copying
936 * is involved.
Jens Axboe83f91352006-04-02 23:05:09 +0200937 *
938 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200939ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200940 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200941{
Jens Axboe00522fb2006-04-26 14:39:29 +0200942 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
Jens Axboe5274f052006-03-30 15:15:30 +0200943}
944
Jens Axboe059a8f32006-04-02 23:06:05 +0200945EXPORT_SYMBOL(generic_splice_sendpage);
Jeff Garzika0f06782006-03-30 23:06:13 -0500946
Jens Axboe83f91352006-04-02 23:05:09 +0200947/*
948 * Attempt to initiate a splice from pipe to file.
949 */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200950static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
Jens Axboecbb7e572006-04-11 14:57:50 +0200951 loff_t *ppos, size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200952{
Jens Axboe5274f052006-03-30 15:15:30 +0200953 int ret;
954
Jens Axboe49570e92006-04-11 13:56:09 +0200955 if (unlikely(!out->f_op || !out->f_op->splice_write))
Jens Axboe5274f052006-03-30 15:15:30 +0200956 return -EINVAL;
957
Jens Axboe49570e92006-04-11 13:56:09 +0200958 if (unlikely(!(out->f_mode & FMODE_WRITE)))
Jens Axboe5274f052006-03-30 15:15:30 +0200959 return -EBADF;
960
Jens Axboecbb7e572006-04-11 14:57:50 +0200961 ret = rw_verify_area(WRITE, out, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200962 if (unlikely(ret < 0))
963 return ret;
964
James Morris29ce2052007-07-13 11:44:32 +0200965 ret = security_file_permission(out, MAY_WRITE);
966 if (unlikely(ret < 0))
967 return ret;
968
Jens Axboecbb7e572006-04-11 14:57:50 +0200969 return out->f_op->splice_write(pipe, out, ppos, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200970}
971
Jens Axboe83f91352006-04-02 23:05:09 +0200972/*
973 * Attempt to initiate a splice from a file to a pipe.
974 */
Jens Axboecbb7e572006-04-11 14:57:50 +0200975static long do_splice_to(struct file *in, loff_t *ppos,
976 struct pipe_inode_info *pipe, size_t len,
977 unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +0200978{
Jens Axboe5274f052006-03-30 15:15:30 +0200979 int ret;
980
Jens Axboe49570e92006-04-11 13:56:09 +0200981 if (unlikely(!in->f_op || !in->f_op->splice_read))
Jens Axboe5274f052006-03-30 15:15:30 +0200982 return -EINVAL;
983
Jens Axboe49570e92006-04-11 13:56:09 +0200984 if (unlikely(!(in->f_mode & FMODE_READ)))
Jens Axboe5274f052006-03-30 15:15:30 +0200985 return -EBADF;
986
Jens Axboecbb7e572006-04-11 14:57:50 +0200987 ret = rw_verify_area(READ, in, ppos, len);
Jens Axboe5274f052006-03-30 15:15:30 +0200988 if (unlikely(ret < 0))
989 return ret;
990
James Morris29ce2052007-07-13 11:44:32 +0200991 ret = security_file_permission(in, MAY_READ);
992 if (unlikely(ret < 0))
993 return ret;
994
Jens Axboecbb7e572006-04-11 14:57:50 +0200995 return in->f_op->splice_read(in, ppos, pipe, len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +0200996}
997
Jens Axboe932cc6d2007-06-21 13:10:21 +0200998/**
999 * splice_direct_to_actor - splices data directly between two non-pipes
1000 * @in: file to splice from
1001 * @sd: actor information on where to splice to
1002 * @actor: handles the data splicing
1003 *
1004 * Description:
1005 * This is a special case helper to splice directly between two
1006 * points, without requiring an explicit pipe. Internally an allocated
1007 * pipe is cached in the process, and reused during the life time of
1008 * that process.
1009 *
Jens Axboec66ab6f2007-06-12 21:17:17 +02001010 */
1011ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1012 splice_direct_actor *actor)
Jens Axboeb92ce552006-04-11 13:52:07 +02001013{
1014 struct pipe_inode_info *pipe;
1015 long ret, bytes;
1016 umode_t i_mode;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001017 size_t len;
1018 int i, flags;
Jens Axboeb92ce552006-04-11 13:52:07 +02001019
1020 /*
1021 * We require the input being a regular file, as we don't want to
1022 * randomly drop data for eg socket -> socket splicing. Use the
1023 * piped splicing for that!
1024 */
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -08001025 i_mode = in->f_path.dentry->d_inode->i_mode;
Jens Axboeb92ce552006-04-11 13:52:07 +02001026 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1027 return -EINVAL;
1028
1029 /*
1030 * neither in nor out is a pipe, setup an internal pipe attached to
1031 * 'out' and transfer the wanted data from 'in' to 'out' through that
1032 */
1033 pipe = current->splice_pipe;
Jens Axboe49570e92006-04-11 13:56:09 +02001034 if (unlikely(!pipe)) {
Jens Axboeb92ce552006-04-11 13:52:07 +02001035 pipe = alloc_pipe_info(NULL);
1036 if (!pipe)
1037 return -ENOMEM;
1038
1039 /*
1040 * We don't have an immediate reader, but we'll read the stuff
Jens Axboe00522fb2006-04-26 14:39:29 +02001041 * out of the pipe right after the splice_to_pipe(). So set
Jens Axboeb92ce552006-04-11 13:52:07 +02001042 * PIPE_READERS appropriately.
1043 */
1044 pipe->readers = 1;
1045
1046 current->splice_pipe = pipe;
1047 }
1048
1049 /*
Ingo Molnar73d62d82006-04-11 13:57:21 +02001050 * Do the splice.
Jens Axboeb92ce552006-04-11 13:52:07 +02001051 */
1052 ret = 0;
1053 bytes = 0;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001054 len = sd->total_len;
1055 flags = sd->flags;
1056
1057 /*
1058 * Don't block on output, we have to drain the direct pipe.
1059 */
1060 sd->flags &= ~SPLICE_F_NONBLOCK;
Jens Axboeb92ce552006-04-11 13:52:07 +02001061
1062 while (len) {
Jens Axboe51a92c02007-07-13 14:11:43 +02001063 size_t read_len;
Jens Axboeb92ce552006-04-11 13:52:07 +02001064
Jens Axboe51a92c02007-07-13 14:11:43 +02001065 ret = do_splice_to(in, &sd->pos, pipe, len, flags);
1066 if (unlikely(ret <= 0))
Jens Axboeb92ce552006-04-11 13:52:07 +02001067 goto out_release;
1068
1069 read_len = ret;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001070 sd->total_len = read_len;
Jens Axboeb92ce552006-04-11 13:52:07 +02001071
1072 /*
1073 * NOTE: nonblocking mode only applies to the input. We
1074 * must not do the output in nonblocking mode as then we
1075 * could get stuck data in the internal pipe:
1076 */
Jens Axboec66ab6f2007-06-12 21:17:17 +02001077 ret = actor(pipe, sd);
Jens Axboe51a92c02007-07-13 14:11:43 +02001078 if (unlikely(ret <= 0))
Jens Axboeb92ce552006-04-11 13:52:07 +02001079 goto out_release;
1080
1081 bytes += ret;
1082 len -= ret;
1083
Jens Axboe51a92c02007-07-13 14:11:43 +02001084 if (ret < read_len)
1085 goto out_release;
Jens Axboeb92ce552006-04-11 13:52:07 +02001086 }
1087
1088 pipe->nrbufs = pipe->curbuf = 0;
Jens Axboeb92ce552006-04-11 13:52:07 +02001089 return bytes;
1090
1091out_release:
1092 /*
1093 * If we did an incomplete transfer we must release
1094 * the pipe buffers in question:
1095 */
1096 for (i = 0; i < PIPE_BUFFERS; i++) {
1097 struct pipe_buffer *buf = pipe->bufs + i;
1098
1099 if (buf->ops) {
1100 buf->ops->release(pipe, buf);
1101 buf->ops = NULL;
1102 }
1103 }
1104 pipe->nrbufs = pipe->curbuf = 0;
1105
1106 /*
1107 * If we transferred some data, return the number of bytes:
1108 */
1109 if (bytes > 0)
1110 return bytes;
1111
1112 return ret;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001113
1114}
1115EXPORT_SYMBOL(splice_direct_to_actor);
1116
1117static int direct_splice_actor(struct pipe_inode_info *pipe,
1118 struct splice_desc *sd)
1119{
Jens Axboe6a14b902007-06-14 13:08:55 +02001120 struct file *file = sd->u.file;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001121
1122 return do_splice_from(pipe, file, &sd->pos, sd->total_len, sd->flags);
1123}
1124
Jens Axboe932cc6d2007-06-21 13:10:21 +02001125/**
1126 * do_splice_direct - splices data directly between two files
1127 * @in: file to splice from
1128 * @ppos: input file offset
1129 * @out: file to splice to
1130 * @len: number of bytes to splice
1131 * @flags: splice modifier flags
1132 *
1133 * Description:
1134 * For use by do_sendfile(). splice can easily emulate sendfile, but
1135 * doing it in the application would incur an extra system call
1136 * (splice in + splice out, as compared to just sendfile()). So this helper
1137 * can splice directly through a process-private pipe.
1138 *
1139 */
Jens Axboec66ab6f2007-06-12 21:17:17 +02001140long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1141 size_t len, unsigned int flags)
1142{
1143 struct splice_desc sd = {
1144 .len = len,
1145 .total_len = len,
1146 .flags = flags,
1147 .pos = *ppos,
Jens Axboe6a14b902007-06-14 13:08:55 +02001148 .u.file = out,
Jens Axboec66ab6f2007-06-12 21:17:17 +02001149 };
Jens Axboe51a92c02007-07-13 14:11:43 +02001150 long ret;
Jens Axboec66ab6f2007-06-12 21:17:17 +02001151
1152 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
Jens Axboe51a92c02007-07-13 14:11:43 +02001153 if (ret > 0)
1154 *ppos += ret;
1155
Jens Axboec66ab6f2007-06-12 21:17:17 +02001156 return ret;
Jens Axboeb92ce552006-04-11 13:52:07 +02001157}
1158
Jens Axboe83f91352006-04-02 23:05:09 +02001159/*
Jens Axboeddac0d32006-11-04 12:49:32 +01001160 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
1161 * location, so checking ->i_pipe is not enough to verify that this is a
1162 * pipe.
1163 */
1164static inline struct pipe_inode_info *pipe_info(struct inode *inode)
1165{
1166 if (S_ISFIFO(inode->i_mode))
1167 return inode->i_pipe;
1168
1169 return NULL;
1170}
1171
1172/*
Jens Axboe83f91352006-04-02 23:05:09 +02001173 * Determine where to splice to/from.
1174 */
Ingo Molnar529565d2006-04-10 15:18:58 +02001175static long do_splice(struct file *in, loff_t __user *off_in,
1176 struct file *out, loff_t __user *off_out,
1177 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +02001178{
Ingo Molnar3a326a22006-04-10 15:18:35 +02001179 struct pipe_inode_info *pipe;
Jens Axboecbb7e572006-04-11 14:57:50 +02001180 loff_t offset, *off;
Jens Axboea4514eb2006-04-19 15:57:05 +02001181 long ret;
Jens Axboe5274f052006-03-30 15:15:30 +02001182
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -08001183 pipe = pipe_info(in->f_path.dentry->d_inode);
Ingo Molnar529565d2006-04-10 15:18:58 +02001184 if (pipe) {
1185 if (off_in)
1186 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +02001187 if (off_out) {
1188 if (out->f_op->llseek == no_llseek)
1189 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +02001190 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +02001191 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +02001192 off = &offset;
1193 } else
1194 off = &out->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +02001195
Jens Axboea4514eb2006-04-19 15:57:05 +02001196 ret = do_splice_from(pipe, out, off, len, flags);
1197
1198 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1199 ret = -EFAULT;
1200
1201 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +02001202 }
Jens Axboe5274f052006-03-30 15:15:30 +02001203
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -08001204 pipe = pipe_info(out->f_path.dentry->d_inode);
Ingo Molnar529565d2006-04-10 15:18:58 +02001205 if (pipe) {
1206 if (off_out)
1207 return -ESPIPE;
Jens Axboeb92ce552006-04-11 13:52:07 +02001208 if (off_in) {
1209 if (in->f_op->llseek == no_llseek)
1210 return -EINVAL;
Jens Axboecbb7e572006-04-11 14:57:50 +02001211 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
Jens Axboeb92ce552006-04-11 13:52:07 +02001212 return -EFAULT;
Jens Axboecbb7e572006-04-11 14:57:50 +02001213 off = &offset;
1214 } else
1215 off = &in->f_pos;
Ingo Molnar529565d2006-04-10 15:18:58 +02001216
Jens Axboea4514eb2006-04-19 15:57:05 +02001217 ret = do_splice_to(in, off, pipe, len, flags);
1218
1219 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1220 ret = -EFAULT;
1221
1222 return ret;
Ingo Molnar529565d2006-04-10 15:18:58 +02001223 }
Jens Axboe5274f052006-03-30 15:15:30 +02001224
1225 return -EINVAL;
1226}
1227
Jens Axboe912d35f2006-04-26 10:59:21 +02001228/*
1229 * Map an iov into an array of pages and offset/length tupples. With the
1230 * partial_page structure, we can map several non-contiguous ranges into
1231 * our ones pages[] map instead of splitting that operation into pieces.
1232 * Could easily be exported as a generic helper for other users, in which
1233 * case one would probably want to add a 'max_nr_pages' parameter as well.
1234 */
1235static int get_iovec_page_array(const struct iovec __user *iov,
1236 unsigned int nr_vecs, struct page **pages,
Jens Axboe7afa6fd2006-05-01 20:02:33 +02001237 struct partial_page *partial, int aligned)
Jens Axboe912d35f2006-04-26 10:59:21 +02001238{
1239 int buffers = 0, error = 0;
1240
1241 /*
1242 * It's ok to take the mmap_sem for reading, even
1243 * across a "get_user()".
1244 */
1245 down_read(&current->mm->mmap_sem);
1246
1247 while (nr_vecs) {
1248 unsigned long off, npages;
1249 void __user *base;
1250 size_t len;
1251 int i;
1252
1253 /*
1254 * Get user address base and length for this iovec.
1255 */
1256 error = get_user(base, &iov->iov_base);
1257 if (unlikely(error))
1258 break;
1259 error = get_user(len, &iov->iov_len);
1260 if (unlikely(error))
1261 break;
1262
1263 /*
1264 * Sanity check this iovec. 0 read succeeds.
1265 */
1266 if (unlikely(!len))
1267 break;
1268 error = -EFAULT;
1269 if (unlikely(!base))
1270 break;
1271
1272 /*
1273 * Get this base offset and number of pages, then map
1274 * in the user pages.
1275 */
1276 off = (unsigned long) base & ~PAGE_MASK;
Jens Axboe7afa6fd2006-05-01 20:02:33 +02001277
1278 /*
1279 * If asked for alignment, the offset must be zero and the
1280 * length a multiple of the PAGE_SIZE.
1281 */
1282 error = -EINVAL;
1283 if (aligned && (off || len & ~PAGE_MASK))
1284 break;
1285
Jens Axboe912d35f2006-04-26 10:59:21 +02001286 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1287 if (npages > PIPE_BUFFERS - buffers)
1288 npages = PIPE_BUFFERS - buffers;
1289
1290 error = get_user_pages(current, current->mm,
1291 (unsigned long) base, npages, 0, 0,
1292 &pages[buffers], NULL);
1293
1294 if (unlikely(error <= 0))
1295 break;
1296
1297 /*
1298 * Fill this contiguous range into the partial page map.
1299 */
1300 for (i = 0; i < error; i++) {
Jens Axboe75914892006-05-02 12:57:18 +02001301 const int plen = min_t(size_t, len, PAGE_SIZE - off);
Jens Axboe912d35f2006-04-26 10:59:21 +02001302
1303 partial[buffers].offset = off;
1304 partial[buffers].len = plen;
1305
1306 off = 0;
1307 len -= plen;
1308 buffers++;
1309 }
1310
1311 /*
1312 * We didn't complete this iov, stop here since it probably
1313 * means we have to move some of this into a pipe to
1314 * be able to continue.
1315 */
1316 if (len)
1317 break;
1318
1319 /*
1320 * Don't continue if we mapped fewer pages than we asked for,
1321 * or if we mapped the max number of pages that we have
1322 * room for.
1323 */
1324 if (error < npages || buffers == PIPE_BUFFERS)
1325 break;
1326
1327 nr_vecs--;
1328 iov++;
1329 }
1330
1331 up_read(&current->mm->mmap_sem);
1332
1333 if (buffers)
1334 return buffers;
1335
1336 return error;
1337}
1338
Jens Axboe6a14b902007-06-14 13:08:55 +02001339static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1340 struct splice_desc *sd)
1341{
1342 char *src;
1343 int ret;
1344
Jens Axboecac36bb02007-06-14 13:10:48 +02001345 ret = buf->ops->confirm(pipe, buf);
Jens Axboe6a14b902007-06-14 13:08:55 +02001346 if (unlikely(ret))
1347 return ret;
1348
1349 /*
1350 * See if we can use the atomic maps, by prefaulting in the
1351 * pages and doing an atomic copy
1352 */
1353 if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) {
1354 src = buf->ops->map(pipe, buf, 1);
1355 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset,
1356 sd->len);
1357 buf->ops->unmap(pipe, buf, src);
1358 if (!ret) {
1359 ret = sd->len;
1360 goto out;
1361 }
1362 }
1363
1364 /*
1365 * No dice, use slow non-atomic map and copy
1366 */
1367 src = buf->ops->map(pipe, buf, 0);
1368
1369 ret = sd->len;
1370 if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len))
1371 ret = -EFAULT;
1372
1373out:
1374 if (ret > 0)
1375 sd->u.userptr += ret;
1376 buf->ops->unmap(pipe, buf, src);
1377 return ret;
1378}
1379
1380/*
1381 * For lack of a better implementation, implement vmsplice() to userspace
1382 * as a simple copy of the pipes pages to the user iov.
1383 */
1384static long vmsplice_to_user(struct file *file, const struct iovec __user *iov,
1385 unsigned long nr_segs, unsigned int flags)
1386{
1387 struct pipe_inode_info *pipe;
1388 struct splice_desc sd;
1389 ssize_t size;
1390 int error;
1391 long ret;
1392
1393 pipe = pipe_info(file->f_path.dentry->d_inode);
1394 if (!pipe)
1395 return -EBADF;
1396
1397 if (pipe->inode)
1398 mutex_lock(&pipe->inode->i_mutex);
1399
1400 error = ret = 0;
1401 while (nr_segs) {
1402 void __user *base;
1403 size_t len;
1404
1405 /*
1406 * Get user address base and length for this iovec.
1407 */
1408 error = get_user(base, &iov->iov_base);
1409 if (unlikely(error))
1410 break;
1411 error = get_user(len, &iov->iov_len);
1412 if (unlikely(error))
1413 break;
1414
1415 /*
1416 * Sanity check this iovec. 0 read succeeds.
1417 */
1418 if (unlikely(!len))
1419 break;
1420 if (unlikely(!base)) {
1421 error = -EFAULT;
1422 break;
1423 }
1424
1425 sd.len = 0;
1426 sd.total_len = len;
1427 sd.flags = flags;
1428 sd.u.userptr = base;
1429 sd.pos = 0;
1430
1431 size = __splice_from_pipe(pipe, &sd, pipe_to_user);
1432 if (size < 0) {
1433 if (!ret)
1434 ret = size;
1435
1436 break;
1437 }
1438
1439 ret += size;
1440
1441 if (size < len)
1442 break;
1443
1444 nr_segs--;
1445 iov++;
1446 }
1447
1448 if (pipe->inode)
1449 mutex_unlock(&pipe->inode->i_mutex);
1450
1451 if (!ret)
1452 ret = error;
1453
1454 return ret;
1455}
1456
Jens Axboe912d35f2006-04-26 10:59:21 +02001457/*
1458 * vmsplice splices a user address range into a pipe. It can be thought of
1459 * as splice-from-memory, where the regular splice is splice-from-file (or
1460 * to file). In both cases the output is a pipe, naturally.
Jens Axboe912d35f2006-04-26 10:59:21 +02001461 */
Jens Axboe6a14b902007-06-14 13:08:55 +02001462static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1463 unsigned long nr_segs, unsigned int flags)
Jens Axboe912d35f2006-04-26 10:59:21 +02001464{
Jens Axboeddac0d32006-11-04 12:49:32 +01001465 struct pipe_inode_info *pipe;
Jens Axboe912d35f2006-04-26 10:59:21 +02001466 struct page *pages[PIPE_BUFFERS];
1467 struct partial_page partial[PIPE_BUFFERS];
1468 struct splice_pipe_desc spd = {
1469 .pages = pages,
1470 .partial = partial,
1471 .flags = flags,
1472 .ops = &user_page_pipe_buf_ops,
1473 };
1474
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -08001475 pipe = pipe_info(file->f_path.dentry->d_inode);
Jens Axboeddac0d32006-11-04 12:49:32 +01001476 if (!pipe)
Jens Axboe912d35f2006-04-26 10:59:21 +02001477 return -EBADF;
Jens Axboe912d35f2006-04-26 10:59:21 +02001478
Jens Axboe7afa6fd2006-05-01 20:02:33 +02001479 spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
1480 flags & SPLICE_F_GIFT);
Jens Axboe912d35f2006-04-26 10:59:21 +02001481 if (spd.nr_pages <= 0)
1482 return spd.nr_pages;
1483
Jens Axboe00522fb2006-04-26 14:39:29 +02001484 return splice_to_pipe(pipe, &spd);
Jens Axboe912d35f2006-04-26 10:59:21 +02001485}
1486
Jens Axboe6a14b902007-06-14 13:08:55 +02001487/*
1488 * Note that vmsplice only really supports true splicing _from_ user memory
1489 * to a pipe, not the other way around. Splicing from user memory is a simple
1490 * operation that can be supported without any funky alignment restrictions
1491 * or nasty vm tricks. We simply map in the user memory and fill them into
1492 * a pipe. The reverse isn't quite as easy, though. There are two possible
1493 * solutions for that:
1494 *
1495 * - memcpy() the data internally, at which point we might as well just
1496 * do a regular read() on the buffer anyway.
1497 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1498 * has restriction limitations on both ends of the pipe).
1499 *
1500 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1501 *
1502 */
Jens Axboe912d35f2006-04-26 10:59:21 +02001503asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
1504 unsigned long nr_segs, unsigned int flags)
1505{
1506 struct file *file;
1507 long error;
1508 int fput;
1509
Jens Axboe6a14b902007-06-14 13:08:55 +02001510 if (unlikely(nr_segs > UIO_MAXIOV))
1511 return -EINVAL;
1512 else if (unlikely(!nr_segs))
1513 return 0;
1514
Jens Axboe912d35f2006-04-26 10:59:21 +02001515 error = -EBADF;
1516 file = fget_light(fd, &fput);
1517 if (file) {
1518 if (file->f_mode & FMODE_WRITE)
Jens Axboe6a14b902007-06-14 13:08:55 +02001519 error = vmsplice_to_pipe(file, iov, nr_segs, flags);
1520 else if (file->f_mode & FMODE_READ)
1521 error = vmsplice_to_user(file, iov, nr_segs, flags);
Jens Axboe912d35f2006-04-26 10:59:21 +02001522
1523 fput_light(file, fput);
1524 }
1525
1526 return error;
1527}
1528
Ingo Molnar529565d2006-04-10 15:18:58 +02001529asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
1530 int fd_out, loff_t __user *off_out,
1531 size_t len, unsigned int flags)
Jens Axboe5274f052006-03-30 15:15:30 +02001532{
1533 long error;
1534 struct file *in, *out;
1535 int fput_in, fput_out;
1536
1537 if (unlikely(!len))
1538 return 0;
1539
1540 error = -EBADF;
Ingo Molnar529565d2006-04-10 15:18:58 +02001541 in = fget_light(fd_in, &fput_in);
Jens Axboe5274f052006-03-30 15:15:30 +02001542 if (in) {
1543 if (in->f_mode & FMODE_READ) {
Ingo Molnar529565d2006-04-10 15:18:58 +02001544 out = fget_light(fd_out, &fput_out);
Jens Axboe5274f052006-03-30 15:15:30 +02001545 if (out) {
1546 if (out->f_mode & FMODE_WRITE)
Ingo Molnar529565d2006-04-10 15:18:58 +02001547 error = do_splice(in, off_in,
1548 out, off_out,
1549 len, flags);
Jens Axboe5274f052006-03-30 15:15:30 +02001550 fput_light(out, fput_out);
1551 }
1552 }
1553
1554 fput_light(in, fput_in);
1555 }
1556
1557 return error;
1558}
Jens Axboe70524492006-04-11 15:51:17 +02001559
1560/*
Jens Axboeaadd06e2006-07-10 11:00:01 +02001561 * Make sure there's data to read. Wait for input if we can, otherwise
1562 * return an appropriate error.
1563 */
1564static int link_ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1565{
1566 int ret;
1567
1568 /*
1569 * Check ->nrbufs without the inode lock first. This function
1570 * is speculative anyways, so missing one is ok.
1571 */
1572 if (pipe->nrbufs)
1573 return 0;
1574
1575 ret = 0;
1576 mutex_lock(&pipe->inode->i_mutex);
1577
1578 while (!pipe->nrbufs) {
1579 if (signal_pending(current)) {
1580 ret = -ERESTARTSYS;
1581 break;
1582 }
1583 if (!pipe->writers)
1584 break;
1585 if (!pipe->waiting_writers) {
1586 if (flags & SPLICE_F_NONBLOCK) {
1587 ret = -EAGAIN;
1588 break;
1589 }
1590 }
1591 pipe_wait(pipe);
1592 }
1593
1594 mutex_unlock(&pipe->inode->i_mutex);
1595 return ret;
1596}
1597
1598/*
1599 * Make sure there's writeable room. Wait for room if we can, otherwise
1600 * return an appropriate error.
1601 */
1602static int link_opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1603{
1604 int ret;
1605
1606 /*
1607 * Check ->nrbufs without the inode lock first. This function
1608 * is speculative anyways, so missing one is ok.
1609 */
1610 if (pipe->nrbufs < PIPE_BUFFERS)
1611 return 0;
1612
1613 ret = 0;
1614 mutex_lock(&pipe->inode->i_mutex);
1615
1616 while (pipe->nrbufs >= PIPE_BUFFERS) {
1617 if (!pipe->readers) {
1618 send_sig(SIGPIPE, current, 0);
1619 ret = -EPIPE;
1620 break;
1621 }
1622 if (flags & SPLICE_F_NONBLOCK) {
1623 ret = -EAGAIN;
1624 break;
1625 }
1626 if (signal_pending(current)) {
1627 ret = -ERESTARTSYS;
1628 break;
1629 }
1630 pipe->waiting_writers++;
1631 pipe_wait(pipe);
1632 pipe->waiting_writers--;
1633 }
1634
1635 mutex_unlock(&pipe->inode->i_mutex);
1636 return ret;
1637}
1638
1639/*
Jens Axboe70524492006-04-11 15:51:17 +02001640 * Link contents of ipipe to opipe.
1641 */
1642static int link_pipe(struct pipe_inode_info *ipipe,
1643 struct pipe_inode_info *opipe,
1644 size_t len, unsigned int flags)
1645{
1646 struct pipe_buffer *ibuf, *obuf;
Jens Axboeaadd06e2006-07-10 11:00:01 +02001647 int ret = 0, i = 0, nbuf;
Jens Axboe70524492006-04-11 15:51:17 +02001648
1649 /*
1650 * Potential ABBA deadlock, work around it by ordering lock
1651 * grabbing by inode address. Otherwise two different processes
1652 * could deadlock (one doing tee from A -> B, the other from B -> A).
1653 */
Mark Fasheh62752ee2006-10-17 10:31:38 +02001654 inode_double_lock(ipipe->inode, opipe->inode);
Jens Axboe70524492006-04-11 15:51:17 +02001655
Jens Axboeaadd06e2006-07-10 11:00:01 +02001656 do {
Jens Axboe70524492006-04-11 15:51:17 +02001657 if (!opipe->readers) {
1658 send_sig(SIGPIPE, current, 0);
1659 if (!ret)
1660 ret = -EPIPE;
1661 break;
1662 }
Jens Axboe70524492006-04-11 15:51:17 +02001663
1664 /*
Jens Axboeaadd06e2006-07-10 11:00:01 +02001665 * If we have iterated all input buffers or ran out of
1666 * output room, break.
Jens Axboe70524492006-04-11 15:51:17 +02001667 */
Jens Axboeaadd06e2006-07-10 11:00:01 +02001668 if (i >= ipipe->nrbufs || opipe->nrbufs >= PIPE_BUFFERS)
Jens Axboe70524492006-04-11 15:51:17 +02001669 break;
Jens Axboeaadd06e2006-07-10 11:00:01 +02001670
1671 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1672 nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1673
Jens Axboe2a27250e2006-04-19 15:56:40 +02001674 /*
Jens Axboeaadd06e2006-07-10 11:00:01 +02001675 * Get a reference to this pipe buffer,
1676 * so we can copy the contents over.
Jens Axboe2a27250e2006-04-19 15:56:40 +02001677 */
Jens Axboeaadd06e2006-07-10 11:00:01 +02001678 ibuf->ops->get(ipipe, ibuf);
Jens Axboe70524492006-04-11 15:51:17 +02001679
Jens Axboeaadd06e2006-07-10 11:00:01 +02001680 obuf = opipe->bufs + nbuf;
1681 *obuf = *ibuf;
Jens Axboe70524492006-04-11 15:51:17 +02001682
Jens Axboeaadd06e2006-07-10 11:00:01 +02001683 /*
1684 * Don't inherit the gift flag, we need to
1685 * prevent multiple steals of this page.
1686 */
1687 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1688
1689 if (obuf->len > len)
1690 obuf->len = len;
1691
1692 opipe->nrbufs++;
1693 ret += obuf->len;
1694 len -= obuf->len;
1695 i++;
1696 } while (len);
Jens Axboe70524492006-04-11 15:51:17 +02001697
Mark Fasheh62752ee2006-10-17 10:31:38 +02001698 inode_double_unlock(ipipe->inode, opipe->inode);
Jens Axboe70524492006-04-11 15:51:17 +02001699
Jens Axboeaadd06e2006-07-10 11:00:01 +02001700 /*
1701 * If we put data in the output pipe, wakeup any potential readers.
1702 */
1703 if (ret > 0) {
Jens Axboe70524492006-04-11 15:51:17 +02001704 smp_mb();
1705 if (waitqueue_active(&opipe->wait))
1706 wake_up_interruptible(&opipe->wait);
1707 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1708 }
1709
1710 return ret;
1711}
1712
1713/*
1714 * This is a tee(1) implementation that works on pipes. It doesn't copy
1715 * any data, it simply references the 'in' pages on the 'out' pipe.
1716 * The 'flags' used are the SPLICE_F_* variants, currently the only
1717 * applicable one is SPLICE_F_NONBLOCK.
1718 */
1719static long do_tee(struct file *in, struct file *out, size_t len,
1720 unsigned int flags)
1721{
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -08001722 struct pipe_inode_info *ipipe = pipe_info(in->f_path.dentry->d_inode);
1723 struct pipe_inode_info *opipe = pipe_info(out->f_path.dentry->d_inode);
Jens Axboeaadd06e2006-07-10 11:00:01 +02001724 int ret = -EINVAL;
Jens Axboe70524492006-04-11 15:51:17 +02001725
1726 /*
Jens Axboeaadd06e2006-07-10 11:00:01 +02001727 * Duplicate the contents of ipipe to opipe without actually
1728 * copying the data.
Jens Axboe70524492006-04-11 15:51:17 +02001729 */
Jens Axboeaadd06e2006-07-10 11:00:01 +02001730 if (ipipe && opipe && ipipe != opipe) {
1731 /*
1732 * Keep going, unless we encounter an error. The ipipe/opipe
1733 * ordering doesn't really matter.
1734 */
1735 ret = link_ipipe_prep(ipipe, flags);
1736 if (!ret) {
1737 ret = link_opipe_prep(opipe, flags);
1738 if (!ret) {
1739 ret = link_pipe(ipipe, opipe, len, flags);
1740 if (!ret && (flags & SPLICE_F_NONBLOCK))
1741 ret = -EAGAIN;
1742 }
1743 }
1744 }
Jens Axboe70524492006-04-11 15:51:17 +02001745
Jens Axboeaadd06e2006-07-10 11:00:01 +02001746 return ret;
Jens Axboe70524492006-04-11 15:51:17 +02001747}
1748
1749asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1750{
1751 struct file *in;
1752 int error, fput_in;
1753
1754 if (unlikely(!len))
1755 return 0;
1756
1757 error = -EBADF;
1758 in = fget_light(fdin, &fput_in);
1759 if (in) {
1760 if (in->f_mode & FMODE_READ) {
1761 int fput_out;
1762 struct file *out = fget_light(fdout, &fput_out);
1763
1764 if (out) {
1765 if (out->f_mode & FMODE_WRITE)
1766 error = do_tee(in, out, len, flags);
1767 fput_light(out, fput_out);
1768 }
1769 }
1770 fput_light(in, fput_in);
1771 }
1772
1773 return error;
1774}