blob: ab8dad3ccb6a8bac13a2eab25eeb2b68791d60f5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/pipe.c
3 *
4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
5 */
6
7#include <linux/mm.h>
8#include <linux/file.h>
9#include <linux/poll.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/init.h>
13#include <linux/fs.h>
Jens Axboe35f3d142010-05-20 10:43:18 +020014#include <linux/log2.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/mount.h>
Muthu Kumarb502bd12012-03-23 15:01:50 -070016#include <linux/magic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/pipe_fs_i.h>
18#include <linux/uio.h>
19#include <linux/highmem.h>
Jens Axboe5274f052006-03-30 15:15:30 +020020#include <linux/pagemap.h>
Al Virodb349502007-02-07 01:48:00 -050021#include <linux/audit.h>
Ulrich Drepperba719ba2008-05-06 20:42:38 -070022#include <linux/syscalls.h>
Jens Axboeb492e952010-05-19 21:03:16 +020023#include <linux/fcntl.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#include <asm/uaccess.h>
26#include <asm/ioctls.h>
27
Al Viro599a0ac2013-03-12 09:58:10 -040028#include "internal.h"
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030/*
Jens Axboeb492e952010-05-19 21:03:16 +020031 * The max size that a non-root user is allowed to grow the pipe. Can
Jens Axboeff9da692010-06-03 14:54:39 +020032 * be set by root in /proc/sys/fs/pipe-max-size
Jens Axboeb492e952010-05-19 21:03:16 +020033 */
Jens Axboeff9da692010-06-03 14:54:39 +020034unsigned int pipe_max_size = 1048576;
35
36/*
37 * Minimum pipe size, as required by POSIX
38 */
39unsigned int pipe_min_size = PAGE_SIZE;
Jens Axboeb492e952010-05-19 21:03:16 +020040
Willy Tarreau759c0112016-01-18 16:36:09 +010041/* Maximum allocatable pages per user. Hard limit is unset by default, soft
42 * matches default values.
43 */
44unsigned long pipe_user_pages_hard;
45unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
46
Jens Axboeb492e952010-05-19 21:03:16 +020047/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * We use a start+len construction, which provides full use of the
49 * allocated memory.
50 * -- Florian Coosmann (FGC)
51 *
52 * Reads with count = 0 should always return 0.
53 * -- Julian Bradfield 1999-06-07.
54 *
55 * FIFOs and Pipes now generate SIGIO for both readers and writers.
56 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
57 *
58 * pipe_read & write cleanup
59 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
60 */
61
Miklos Szeredi61e0d472009-04-14 19:48:41 +020062static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
63{
Al Viro6447a3c2013-03-21 11:01:38 -040064 if (pipe->files)
Al Viro72b0d9a2013-03-21 02:32:24 -040065 mutex_lock_nested(&pipe->mutex, subclass);
Miklos Szeredi61e0d472009-04-14 19:48:41 +020066}
67
68void pipe_lock(struct pipe_inode_info *pipe)
69{
70 /*
71 * pipe_lock() nests non-pipe inode locks (for writing to a file)
72 */
73 pipe_lock_nested(pipe, I_MUTEX_PARENT);
74}
75EXPORT_SYMBOL(pipe_lock);
76
77void pipe_unlock(struct pipe_inode_info *pipe)
78{
Al Viro6447a3c2013-03-21 11:01:38 -040079 if (pipe->files)
Al Viro72b0d9a2013-03-21 02:32:24 -040080 mutex_unlock(&pipe->mutex);
Miklos Szeredi61e0d472009-04-14 19:48:41 +020081}
82EXPORT_SYMBOL(pipe_unlock);
83
Al Viroebec73f2013-03-21 12:24:01 -040084static inline void __pipe_lock(struct pipe_inode_info *pipe)
85{
86 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
87}
88
89static inline void __pipe_unlock(struct pipe_inode_info *pipe)
90{
91 mutex_unlock(&pipe->mutex);
92}
93
Miklos Szeredi61e0d472009-04-14 19:48:41 +020094void pipe_double_lock(struct pipe_inode_info *pipe1,
95 struct pipe_inode_info *pipe2)
96{
97 BUG_ON(pipe1 == pipe2);
98
99 if (pipe1 < pipe2) {
100 pipe_lock_nested(pipe1, I_MUTEX_PARENT);
101 pipe_lock_nested(pipe2, I_MUTEX_CHILD);
102 } else {
Peter Zijlstra023d43c2009-07-21 10:09:23 +0200103 pipe_lock_nested(pipe2, I_MUTEX_PARENT);
104 pipe_lock_nested(pipe1, I_MUTEX_CHILD);
Miklos Szeredi61e0d472009-04-14 19:48:41 +0200105 }
106}
107
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108/* Drop the inode semaphore and wait for a pipe event, atomically */
Ingo Molnar3a326a22006-04-10 15:18:35 +0200109void pipe_wait(struct pipe_inode_info *pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110{
111 DEFINE_WAIT(wait);
112
Ingo Molnard79fc0f2005-09-10 00:26:12 -0700113 /*
114 * Pipes are system-local resources, so sleeping on them
115 * is considered a noninteractive wait:
116 */
Mike Galbraithaf927232007-10-15 17:00:13 +0200117 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
Miklos Szeredi61e0d472009-04-14 19:48:41 +0200118 pipe_unlock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 schedule();
Ingo Molnar3a326a22006-04-10 15:18:35 +0200120 finish_wait(&pipe->wait, &wait);
Miklos Szeredi61e0d472009-04-14 19:48:41 +0200121 pipe_lock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122}
123
Ingo Molnar341b4462006-04-11 13:57:45 +0200124static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
125 struct pipe_buffer *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
127 struct page *page = buf->page;
128
Jens Axboe5274f052006-03-30 15:15:30 +0200129 /*
130 * If nobody else uses this page, and we don't already have a
131 * temporary page, let's keep track of it as a one-deep
Ingo Molnar341b4462006-04-11 13:57:45 +0200132 * allocation cache. (Otherwise just release our reference to it)
Jens Axboe5274f052006-03-30 15:15:30 +0200133 */
Ingo Molnar341b4462006-04-11 13:57:45 +0200134 if (page_count(page) == 1 && !pipe->tmp_page)
Ingo Molnar923f4f22006-04-11 13:53:33 +0200135 pipe->tmp_page = page;
Ingo Molnar341b4462006-04-11 13:57:45 +0200136 else
137 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138}
139
Jens Axboe08457182007-06-12 20:51:32 +0200140/**
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800141 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
Jens Axboe08457182007-06-12 20:51:32 +0200142 * @pipe: the pipe that the buffer belongs to
143 * @buf: the buffer to attempt to steal
144 *
145 * Description:
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800146 * This function attempts to steal the &struct page attached to
Jens Axboe08457182007-06-12 20:51:32 +0200147 * @buf. If successful, this function returns 0 and returns with
148 * the page locked. The caller may then reuse the page for whatever
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800149 * he wishes; the typical use is insertion into a different file
Jens Axboe08457182007-06-12 20:51:32 +0200150 * page cache.
151 */
Jens Axboe330ab712006-05-02 15:29:57 +0200152int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
153 struct pipe_buffer *buf)
Jens Axboe5abc97a2006-03-30 15:16:46 +0200154{
Jens Axboe46e678c2006-04-30 16:36:32 +0200155 struct page *page = buf->page;
156
Jens Axboe08457182007-06-12 20:51:32 +0200157 /*
158 * A reference of one is golden, that means that the owner of this
159 * page is the only one holding a reference to it. lock the page
160 * and return OK.
161 */
Jens Axboe46e678c2006-04-30 16:36:32 +0200162 if (page_count(page) == 1) {
Jens Axboe46e678c2006-04-30 16:36:32 +0200163 lock_page(page);
164 return 0;
165 }
166
167 return 1;
Jens Axboe5abc97a2006-03-30 15:16:46 +0200168}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200169EXPORT_SYMBOL(generic_pipe_buf_steal);
Jens Axboe5abc97a2006-03-30 15:16:46 +0200170
Jens Axboe08457182007-06-12 20:51:32 +0200171/**
Randy Dunlapb51d63c2008-02-13 15:03:22 -0800172 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
Jens Axboe08457182007-06-12 20:51:32 +0200173 * @pipe: the pipe that the buffer belongs to
174 * @buf: the buffer to get a reference to
175 *
176 * Description:
177 * This function grabs an extra reference to @buf. It's used in
178 * in the tee() system call, when we duplicate the buffers in one
179 * pipe into another.
180 */
181void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
Jens Axboe70524492006-04-11 15:51:17 +0200182{
183 page_cache_get(buf->page);
184}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200185EXPORT_SYMBOL(generic_pipe_buf_get);
Jens Axboe70524492006-04-11 15:51:17 +0200186
Jens Axboe08457182007-06-12 20:51:32 +0200187/**
188 * generic_pipe_buf_confirm - verify contents of the pipe buffer
Randy Dunlap79685b82007-07-27 08:08:51 +0200189 * @info: the pipe that the buffer belongs to
Jens Axboe08457182007-06-12 20:51:32 +0200190 * @buf: the buffer to confirm
191 *
192 * Description:
193 * This function does nothing, because the generic pipe code uses
194 * pages that are always good when inserted into the pipe.
195 */
Jens Axboecac36bb02007-06-14 13:10:48 +0200196int generic_pipe_buf_confirm(struct pipe_inode_info *info,
197 struct pipe_buffer *buf)
Jens Axboef84d7512006-05-01 19:59:03 +0200198{
199 return 0;
200}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200201EXPORT_SYMBOL(generic_pipe_buf_confirm);
Jens Axboef84d7512006-05-01 19:59:03 +0200202
Miklos Szeredi68181732009-05-07 15:37:36 +0200203/**
204 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
205 * @pipe: the pipe that the buffer belongs to
206 * @buf: the buffer to put a reference to
207 *
208 * Description:
209 * This function releases a reference to @buf.
210 */
211void generic_pipe_buf_release(struct pipe_inode_info *pipe,
212 struct pipe_buffer *buf)
213{
214 page_cache_release(buf->page);
215}
Miklos Szeredi51921cb2010-05-26 08:44:22 +0200216EXPORT_SYMBOL(generic_pipe_buf_release);
Miklos Szeredi68181732009-05-07 15:37:36 +0200217
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800218static const struct pipe_buf_operations anon_pipe_buf_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 .can_merge = 1,
Jens Axboecac36bb02007-06-14 13:10:48 +0200220 .confirm = generic_pipe_buf_confirm,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 .release = anon_pipe_buf_release,
Jens Axboe330ab712006-05-02 15:29:57 +0200222 .steal = generic_pipe_buf_steal,
Jens Axboef84d7512006-05-01 19:59:03 +0200223 .get = generic_pipe_buf_get,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224};
225
Linus Torvalds98830352012-04-29 13:12:42 -0700226static const struct pipe_buf_operations packet_pipe_buf_ops = {
227 .can_merge = 0,
Linus Torvalds98830352012-04-29 13:12:42 -0700228 .confirm = generic_pipe_buf_confirm,
229 .release = anon_pipe_buf_release,
230 .steal = generic_pipe_buf_steal,
231 .get = generic_pipe_buf_get,
232};
233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234static ssize_t
Al Virofb9096a2014-04-02 19:56:54 -0400235pipe_read(struct kiocb *iocb, struct iov_iter *to)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236{
Al Virofb9096a2014-04-02 19:56:54 -0400237 size_t total_len = iov_iter_count(to);
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700238 struct file *filp = iocb->ki_filp;
Al Virode32ec42013-03-21 11:16:56 -0400239 struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 int do_wakeup;
241 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 /* Null read succeeds. */
244 if (unlikely(total_len == 0))
245 return 0;
246
247 do_wakeup = 0;
248 ret = 0;
Al Viroebec73f2013-03-21 12:24:01 -0400249 __pipe_lock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 for (;;) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200251 int bufs = pipe->nrbufs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 if (bufs) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200253 int curbuf = pipe->curbuf;
254 struct pipe_buffer *buf = pipe->bufs + curbuf;
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800255 const struct pipe_buf_operations *ops = buf->ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 size_t chars = buf->len;
Al Viro637b58c2014-02-03 19:11:42 -0500257 size_t written;
258 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260 if (chars > total_len)
261 chars = total_len;
262
Jens Axboecac36bb02007-06-14 13:10:48 +0200263 error = ops->confirm(pipe, buf);
Jens Axboef84d7512006-05-01 19:59:03 +0200264 if (error) {
Jens Axboe5274f052006-03-30 15:15:30 +0200265 if (!ret)
Nicolas Kaisere5953cb2010-10-21 14:56:00 +0200266 ret = error;
Jens Axboe5274f052006-03-30 15:15:30 +0200267 break;
268 }
Jens Axboef84d7512006-05-01 19:59:03 +0200269
Al Virofb9096a2014-04-02 19:56:54 -0400270 written = copy_page_to_iter(buf->page, buf->offset, chars, to);
Al Viro637b58c2014-02-03 19:11:42 -0500271 if (unlikely(written < chars)) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200272 if (!ret)
Al Viro637b58c2014-02-03 19:11:42 -0500273 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 break;
275 }
276 ret += chars;
277 buf->offset += chars;
278 buf->len -= chars;
Linus Torvalds98830352012-04-29 13:12:42 -0700279
280 /* Was it a packet buffer? Clean up and exit */
281 if (buf->flags & PIPE_BUF_FLAG_PACKET) {
282 total_len = chars;
283 buf->len = 0;
284 }
285
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 if (!buf->len) {
287 buf->ops = NULL;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200288 ops->release(pipe, buf);
Jens Axboe35f3d142010-05-20 10:43:18 +0200289 curbuf = (curbuf + 1) & (pipe->buffers - 1);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200290 pipe->curbuf = curbuf;
291 pipe->nrbufs = --bufs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 do_wakeup = 1;
293 }
294 total_len -= chars;
295 if (!total_len)
296 break; /* common path: read succeeded */
297 }
298 if (bufs) /* More to do? */
299 continue;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200300 if (!pipe->writers)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 break;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200302 if (!pipe->waiting_writers) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 /* syscall merging: Usually we must not sleep
304 * if O_NONBLOCK is set, or if we got some data.
305 * But if a writer sleeps in kernel space, then
306 * we can wait for that data without violating POSIX.
307 */
308 if (ret)
309 break;
310 if (filp->f_flags & O_NONBLOCK) {
311 ret = -EAGAIN;
312 break;
313 }
314 }
315 if (signal_pending(current)) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200316 if (!ret)
317 ret = -ERESTARTSYS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 break;
319 }
320 if (do_wakeup) {
Linus Torvalds28e58ee2011-01-20 16:21:59 -0800321 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200322 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200324 pipe_wait(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 }
Al Viroebec73f2013-03-21 12:24:01 -0400326 __pipe_unlock(pipe);
Ingo Molnar341b4462006-04-11 13:57:45 +0200327
328 /* Signal writers asynchronously that there is more room. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 if (do_wakeup) {
Linus Torvalds28e58ee2011-01-20 16:21:59 -0800330 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200331 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 }
333 if (ret > 0)
334 file_accessed(filp);
335 return ret;
336}
337
Linus Torvalds98830352012-04-29 13:12:42 -0700338static inline int is_packetized(struct file *file)
339{
340 return (file->f_flags & O_DIRECT) != 0;
341}
342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343static ssize_t
Al Virof0d1bec2014-04-03 15:05:18 -0400344pipe_write(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345{
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700346 struct file *filp = iocb->ki_filp;
Al Virode32ec42013-03-21 11:16:56 -0400347 struct pipe_inode_info *pipe = filp->private_data;
Al Virof0d1bec2014-04-03 15:05:18 -0400348 ssize_t ret = 0;
349 int do_wakeup = 0;
350 size_t total_len = iov_iter_count(from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 ssize_t chars;
352
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 /* Null write succeeds. */
354 if (unlikely(total_len == 0))
355 return 0;
356
Al Viroebec73f2013-03-21 12:24:01 -0400357 __pipe_lock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358
Ingo Molnar923f4f22006-04-11 13:53:33 +0200359 if (!pipe->readers) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 send_sig(SIGPIPE, current, 0);
361 ret = -EPIPE;
362 goto out;
363 }
364
365 /* We try to merge small writes */
366 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
Ingo Molnar923f4f22006-04-11 13:53:33 +0200367 if (pipe->nrbufs && chars != 0) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200368 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
Jens Axboe35f3d142010-05-20 10:43:18 +0200369 (pipe->buffers - 1);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200370 struct pipe_buffer *buf = pipe->bufs + lastbuf;
Eric Dumazetd4c3cca2006-12-13 00:34:04 -0800371 const struct pipe_buf_operations *ops = buf->ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 int offset = buf->offset + buf->len;
Ingo Molnar341b4462006-04-11 13:57:45 +0200373
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 if (ops->can_merge && offset + chars <= PAGE_SIZE) {
Eric Biggers6ae08062015-10-17 16:26:09 -0500375 ret = ops->confirm(pipe, buf);
376 if (ret)
Jens Axboe5274f052006-03-30 15:15:30 +0200377 goto out;
Jens Axboef84d7512006-05-01 19:59:03 +0200378
Al Virof0d1bec2014-04-03 15:05:18 -0400379 ret = copy_page_from_iter(buf->page, offset, chars, from);
380 if (unlikely(ret < chars)) {
Eric Biggers6ae08062015-10-17 16:26:09 -0500381 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 goto out;
Jens Axboef6762b72006-05-01 20:02:05 +0200383 }
Al Virof0d1bec2014-04-03 15:05:18 -0400384 do_wakeup = 1;
Eric Biggers6ae08062015-10-17 16:26:09 -0500385 buf->len += ret;
Al Virof0d1bec2014-04-03 15:05:18 -0400386 if (!iov_iter_count(from))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 goto out;
388 }
389 }
390
391 for (;;) {
392 int bufs;
Ingo Molnar341b4462006-04-11 13:57:45 +0200393
Ingo Molnar923f4f22006-04-11 13:53:33 +0200394 if (!pipe->readers) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 send_sig(SIGPIPE, current, 0);
Ingo Molnar341b4462006-04-11 13:57:45 +0200396 if (!ret)
397 ret = -EPIPE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 break;
399 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200400 bufs = pipe->nrbufs;
Jens Axboe35f3d142010-05-20 10:43:18 +0200401 if (bufs < pipe->buffers) {
402 int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200403 struct pipe_buffer *buf = pipe->bufs + newbuf;
404 struct page *page = pipe->tmp_page;
Al Virof0d1bec2014-04-03 15:05:18 -0400405 int copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
407 if (!page) {
408 page = alloc_page(GFP_HIGHUSER);
409 if (unlikely(!page)) {
410 ret = ret ? : -ENOMEM;
411 break;
412 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200413 pipe->tmp_page = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 }
Ingo Molnar341b4462006-04-11 13:57:45 +0200415 /* Always wake up, even if the copy fails. Otherwise
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 * we lock up (O_NONBLOCK-)readers that sleep due to
417 * syscall merging.
418 * FIXME! Is this really true?
419 */
420 do_wakeup = 1;
Al Virof0d1bec2014-04-03 15:05:18 -0400421 copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
422 if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200423 if (!ret)
Al Virof0d1bec2014-04-03 15:05:18 -0400424 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 break;
426 }
Al Virof0d1bec2014-04-03 15:05:18 -0400427 ret += copied;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428
429 /* Insert it into the buffer array */
430 buf->page = page;
431 buf->ops = &anon_pipe_buf_ops;
432 buf->offset = 0;
Al Virof0d1bec2014-04-03 15:05:18 -0400433 buf->len = copied;
Linus Torvalds98830352012-04-29 13:12:42 -0700434 buf->flags = 0;
435 if (is_packetized(filp)) {
436 buf->ops = &packet_pipe_buf_ops;
437 buf->flags = PIPE_BUF_FLAG_PACKET;
438 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200439 pipe->nrbufs = ++bufs;
440 pipe->tmp_page = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
Al Virof0d1bec2014-04-03 15:05:18 -0400442 if (!iov_iter_count(from))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 break;
444 }
Jens Axboe35f3d142010-05-20 10:43:18 +0200445 if (bufs < pipe->buffers)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 continue;
447 if (filp->f_flags & O_NONBLOCK) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200448 if (!ret)
449 ret = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 break;
451 }
452 if (signal_pending(current)) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200453 if (!ret)
454 ret = -ERESTARTSYS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 break;
456 }
457 if (do_wakeup) {
Linus Torvalds28e58ee2011-01-20 16:21:59 -0800458 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200459 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 do_wakeup = 0;
461 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200462 pipe->waiting_writers++;
463 pipe_wait(pipe);
464 pipe->waiting_writers--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 }
466out:
Al Viroebec73f2013-03-21 12:24:01 -0400467 __pipe_unlock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 if (do_wakeup) {
Linus Torvalds28e58ee2011-01-20 16:21:59 -0800469 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200470 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 }
Dmitry Monakhov7e775f42014-01-23 15:55:21 -0800472 if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
Josef Bacikc3b2da32012-03-26 09:59:21 -0400473 int err = file_update_time(filp);
474 if (err)
475 ret = err;
Dmitry Monakhov7e775f42014-01-23 15:55:21 -0800476 sb_end_write(file_inode(filp)->i_sb);
Josef Bacikc3b2da32012-03-26 09:59:21 -0400477 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 return ret;
479}
480
Andi Kleend59d0b12008-02-08 04:21:23 -0800481static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482{
Al Virode32ec42013-03-21 11:16:56 -0400483 struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 int count, buf, nrbufs;
485
486 switch (cmd) {
487 case FIONREAD:
Al Viroebec73f2013-03-21 12:24:01 -0400488 __pipe_lock(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 count = 0;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200490 buf = pipe->curbuf;
491 nrbufs = pipe->nrbufs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 while (--nrbufs >= 0) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200493 count += pipe->bufs[buf].len;
Jens Axboe35f3d142010-05-20 10:43:18 +0200494 buf = (buf+1) & (pipe->buffers - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 }
Al Viroebec73f2013-03-21 12:24:01 -0400496 __pipe_unlock(pipe);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200497
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 return put_user(count, (int __user *)arg);
499 default:
Will Deacon46ce3412012-05-25 11:39:13 +0100500 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 }
502}
503
504/* No kernel lock held - fine */
505static unsigned int
506pipe_poll(struct file *filp, poll_table *wait)
507{
508 unsigned int mask;
Al Virode32ec42013-03-21 11:16:56 -0400509 struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 int nrbufs;
511
Ingo Molnar923f4f22006-04-11 13:53:33 +0200512 poll_wait(filp, &pipe->wait, wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
514 /* Reading only -- no need for acquiring the semaphore. */
Ingo Molnar923f4f22006-04-11 13:53:33 +0200515 nrbufs = pipe->nrbufs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 mask = 0;
517 if (filp->f_mode & FMODE_READ) {
518 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200519 if (!pipe->writers && filp->f_version != pipe->w_counter)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 mask |= POLLHUP;
521 }
522
523 if (filp->f_mode & FMODE_WRITE) {
Jens Axboe35f3d142010-05-20 10:43:18 +0200524 mask |= (nrbufs < pipe->buffers) ? POLLOUT | POLLWRNORM : 0;
Pekka Enberg5e5d7a22005-09-06 15:17:48 -0700525 /*
526 * Most Unices do not set POLLERR for FIFOs but on Linux they
527 * behave exactly like pipes for poll().
528 */
Ingo Molnar923f4f22006-04-11 13:53:33 +0200529 if (!pipe->readers)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 mask |= POLLERR;
531 }
532
533 return mask;
534}
535
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800536static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
537{
538 int kill = 0;
539
540 spin_lock(&inode->i_lock);
541 if (!--pipe->files) {
542 inode->i_pipe = NULL;
543 kill = 1;
544 }
545 spin_unlock(&inode->i_lock);
546
547 if (kill)
548 free_pipe_info(pipe);
549}
550
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551static int
Al Viro599a0ac2013-03-12 09:58:10 -0400552pipe_release(struct inode *inode, struct file *file)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553{
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800554 struct pipe_inode_info *pipe = file->private_data;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200555
Al Viroebec73f2013-03-21 12:24:01 -0400556 __pipe_lock(pipe);
Al Viro599a0ac2013-03-12 09:58:10 -0400557 if (file->f_mode & FMODE_READ)
558 pipe->readers--;
559 if (file->f_mode & FMODE_WRITE)
560 pipe->writers--;
Ingo Molnar341b4462006-04-11 13:57:45 +0200561
Al Viroba5bb142013-03-21 02:21:19 -0400562 if (pipe->readers || pipe->writers) {
Linus Torvalds28e58ee2011-01-20 16:21:59 -0800563 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200564 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
565 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 }
Al Viroebec73f2013-03-21 12:24:01 -0400567 __pipe_unlock(pipe);
Al Viroba5bb142013-03-21 02:21:19 -0400568
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800569 put_pipe_info(inode, pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 return 0;
571}
572
573static int
Al Viro599a0ac2013-03-12 09:58:10 -0400574pipe_fasync(int fd, struct file *filp, int on)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575{
Al Virode32ec42013-03-21 11:16:56 -0400576 struct pipe_inode_info *pipe = filp->private_data;
Al Viro599a0ac2013-03-12 09:58:10 -0400577 int retval = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578
Al Viroebec73f2013-03-21 12:24:01 -0400579 __pipe_lock(pipe);
Al Viro599a0ac2013-03-12 09:58:10 -0400580 if (filp->f_mode & FMODE_READ)
581 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
582 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
Ingo Molnar341b4462006-04-11 13:57:45 +0200583 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
Al Viro599a0ac2013-03-12 09:58:10 -0400584 if (retval < 0 && (filp->f_mode & FMODE_READ))
585 /* this can happen only if on == T */
Oleg Nesterove5bc49b2009-03-12 14:31:28 -0700586 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
587 }
Al Viroebec73f2013-03-21 12:24:01 -0400588 __pipe_unlock(pipe);
Jonathan Corbet60aa4922009-02-01 14:52:56 -0700589 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590}
591
Willy Tarreau759c0112016-01-18 16:36:09 +0100592static void account_pipe_buffers(struct pipe_inode_info *pipe,
593 unsigned long old, unsigned long new)
594{
595 atomic_long_add(new - old, &pipe->user->pipe_bufs);
596}
597
598static bool too_many_pipe_buffers_soft(struct user_struct *user)
599{
600 return pipe_user_pages_soft &&
601 atomic_long_read(&user->pipe_bufs) >= pipe_user_pages_soft;
602}
603
604static bool too_many_pipe_buffers_hard(struct user_struct *user)
605{
606 return pipe_user_pages_hard &&
607 atomic_long_read(&user->pipe_bufs) >= pipe_user_pages_hard;
608}
609
Al Viro7bee1302013-03-21 11:04:15 -0400610struct pipe_inode_info *alloc_pipe_info(void)
Ingo Molnar3a326a22006-04-10 15:18:35 +0200611{
Ingo Molnar923f4f22006-04-11 13:53:33 +0200612 struct pipe_inode_info *pipe;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200613
Ingo Molnar923f4f22006-04-11 13:53:33 +0200614 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
615 if (pipe) {
Willy Tarreau759c0112016-01-18 16:36:09 +0100616 unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
617 struct user_struct *user = get_current_user();
618
619 if (!too_many_pipe_buffers_hard(user)) {
620 if (too_many_pipe_buffers_soft(user))
621 pipe_bufs = 1;
622 pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * pipe_bufs, GFP_KERNEL);
623 }
624
Jens Axboe35f3d142010-05-20 10:43:18 +0200625 if (pipe->bufs) {
626 init_waitqueue_head(&pipe->wait);
627 pipe->r_counter = pipe->w_counter = 1;
Willy Tarreau759c0112016-01-18 16:36:09 +0100628 pipe->buffers = pipe_bufs;
629 pipe->user = user;
630 account_pipe_buffers(pipe, 0, pipe_bufs);
Al Viro72b0d9a2013-03-21 02:32:24 -0400631 mutex_init(&pipe->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +0200632 return pipe;
633 }
Willy Tarreau759c0112016-01-18 16:36:09 +0100634 free_uid(user);
Jens Axboe35f3d142010-05-20 10:43:18 +0200635 kfree(pipe);
Ingo Molnar3a326a22006-04-10 15:18:35 +0200636 }
637
Jens Axboe35f3d142010-05-20 10:43:18 +0200638 return NULL;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200639}
640
Al Viro4b8a8f12013-03-21 11:06:46 -0400641void free_pipe_info(struct pipe_inode_info *pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642{
643 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Willy Tarreau759c0112016-01-18 16:36:09 +0100645 account_pipe_buffers(pipe, pipe->buffers, 0);
646 free_uid(pipe->user);
Jens Axboe35f3d142010-05-20 10:43:18 +0200647 for (i = 0; i < pipe->buffers; i++) {
Ingo Molnar923f4f22006-04-11 13:53:33 +0200648 struct pipe_buffer *buf = pipe->bufs + i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 if (buf->ops)
Ingo Molnar923f4f22006-04-11 13:53:33 +0200650 buf->ops->release(pipe, buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 }
Ingo Molnar923f4f22006-04-11 13:53:33 +0200652 if (pipe->tmp_page)
653 __free_page(pipe->tmp_page);
Jens Axboe35f3d142010-05-20 10:43:18 +0200654 kfree(pipe->bufs);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200655 kfree(pipe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656}
657
Eric Dumazetfa3536c2006-03-26 01:37:24 -0800658static struct vfsmount *pipe_mnt __read_mostly;
Ingo Molnar341b4462006-04-11 13:57:45 +0200659
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700660/*
661 * pipefs_dname() is called from d_path().
662 */
663static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
664{
665 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
David Howells75c3cfa2015-03-17 22:26:12 +0000666 d_inode(dentry)->i_ino);
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700667}
668
Al Viro3ba13d12009-02-20 06:02:22 +0000669static const struct dentry_operations pipefs_dentry_operations = {
Eric Dumazetc23fbb62007-05-08 00:26:18 -0700670 .d_dname = pipefs_dname,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671};
672
673static struct inode * get_pipe_inode(void)
674{
Eric Dumazeta209dfc2011-07-26 11:36:34 +0200675 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
Ingo Molnar923f4f22006-04-11 13:53:33 +0200676 struct pipe_inode_info *pipe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677
678 if (!inode)
679 goto fail_inode;
680
Christoph Hellwig85fe4022010-10-23 11:19:54 -0400681 inode->i_ino = get_next_ino();
682
Al Viro7bee1302013-03-21 11:04:15 -0400683 pipe = alloc_pipe_info();
Ingo Molnar923f4f22006-04-11 13:53:33 +0200684 if (!pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 goto fail_iput;
Ingo Molnar3a326a22006-04-10 15:18:35 +0200686
Al Viroba5bb142013-03-21 02:21:19 -0400687 inode->i_pipe = pipe;
688 pipe->files = 2;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200689 pipe->readers = pipe->writers = 1;
Al Viro599a0ac2013-03-12 09:58:10 -0400690 inode->i_fop = &pipefifo_fops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
692 /*
693 * Mark the inode dirty from the very beginning,
694 * that way it will never be moved to the dirty
695 * list because "mark_inode_dirty()" will think
696 * that it already _is_ on the dirty list.
697 */
698 inode->i_state = I_DIRTY;
699 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
David Howellsda9592e2008-11-14 10:39:05 +1100700 inode->i_uid = current_fsuid();
701 inode->i_gid = current_fsgid();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
Ingo Molnar923f4f22006-04-11 13:53:33 +0200703
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 return inode;
705
706fail_iput:
707 iput(inode);
Ingo Molnar341b4462006-04-11 13:57:45 +0200708
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709fail_inode:
710 return NULL;
711}
712
Al Viroe4fad8e2012-07-21 15:33:25 +0400713int create_pipe_files(struct file **res, int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714{
Andi Kleend6cbd282006-09-30 23:29:26 -0700715 int err;
Al Viroe4fad8e2012-07-21 15:33:25 +0400716 struct inode *inode = get_pipe_inode();
Andi Kleend6cbd282006-09-30 23:29:26 -0700717 struct file *f;
Al Viro2c48b9c2009-08-09 00:52:35 +0400718 struct path path;
Al Viroe4fad8e2012-07-21 15:33:25 +0400719 static struct qstr name = { .name = "" };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 if (!inode)
Al Viroe4fad8e2012-07-21 15:33:25 +0400722 return -ENFILE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723
Andi Kleend6cbd282006-09-30 23:29:26 -0700724 err = -ENOMEM;
Nick Piggin4b936882011-01-07 17:50:07 +1100725 path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &name);
Al Viro2c48b9c2009-08-09 00:52:35 +0400726 if (!path.dentry)
Andi Kleend6cbd282006-09-30 23:29:26 -0700727 goto err_inode;
Al Viro2c48b9c2009-08-09 00:52:35 +0400728 path.mnt = mntget(pipe_mnt);
Ingo Molnar341b4462006-04-11 13:57:45 +0200729
Al Viro2c48b9c2009-08-09 00:52:35 +0400730 d_instantiate(path.dentry, inode);
Dave Hansen430e2852008-02-15 14:37:26 -0800731
Al Viro599a0ac2013-03-12 09:58:10 -0400732 f = alloc_file(&path, FMODE_WRITE, &pipefifo_fops);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500733 if (IS_ERR(f)) {
734 err = PTR_ERR(f);
Dave Hansen430e2852008-02-15 14:37:26 -0800735 goto err_dentry;
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500736 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
Linus Torvalds98830352012-04-29 13:12:42 -0700738 f->f_flags = O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT));
Al Virode32ec42013-03-21 11:16:56 -0400739 f->private_data = inode->i_pipe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
Al Viro599a0ac2013-03-12 09:58:10 -0400741 res[0] = alloc_file(&path, FMODE_READ, &pipefifo_fops);
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500742 if (IS_ERR(res[0])) {
743 err = PTR_ERR(res[0]);
Al Viroe4fad8e2012-07-21 15:33:25 +0400744 goto err_file;
Eric Biggerse9bb1f92015-10-17 16:26:08 -0500745 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Al Viroe4fad8e2012-07-21 15:33:25 +0400747 path_get(&path);
Al Virode32ec42013-03-21 11:16:56 -0400748 res[0]->private_data = inode->i_pipe;
Al Viroe4fad8e2012-07-21 15:33:25 +0400749 res[0]->f_flags = O_RDONLY | (flags & O_NONBLOCK);
750 res[1] = f;
751 return 0;
752
753err_file:
754 put_filp(f);
755err_dentry:
Al Viro4b8a8f12013-03-21 11:06:46 -0400756 free_pipe_info(inode->i_pipe);
Al Viro2c48b9c2009-08-09 00:52:35 +0400757 path_put(&path);
Al Viroe4fad8e2012-07-21 15:33:25 +0400758 return err;
Al Viroed152432008-04-22 19:51:27 -0400759
Al Viroe4fad8e2012-07-21 15:33:25 +0400760err_inode:
Al Viro4b8a8f12013-03-21 11:06:46 -0400761 free_pipe_info(inode->i_pipe);
Andi Kleend6cbd282006-09-30 23:29:26 -0700762 iput(inode);
Al Viroe4fad8e2012-07-21 15:33:25 +0400763 return err;
Andi Kleend6cbd282006-09-30 23:29:26 -0700764}
765
Al Viro5b249b12012-08-19 12:17:29 -0400766static int __do_pipe_flags(int *fd, struct file **files, int flags)
Andi Kleend6cbd282006-09-30 23:29:26 -0700767{
Andi Kleend6cbd282006-09-30 23:29:26 -0700768 int error;
769 int fdw, fdr;
770
Linus Torvalds98830352012-04-29 13:12:42 -0700771 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT))
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700772 return -EINVAL;
773
Al Viroe4fad8e2012-07-21 15:33:25 +0400774 error = create_pipe_files(files, flags);
775 if (error)
776 return error;
Andi Kleend6cbd282006-09-30 23:29:26 -0700777
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700778 error = get_unused_fd_flags(flags);
Andi Kleend6cbd282006-09-30 23:29:26 -0700779 if (error < 0)
780 goto err_read_pipe;
781 fdr = error;
782
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700783 error = get_unused_fd_flags(flags);
Andi Kleend6cbd282006-09-30 23:29:26 -0700784 if (error < 0)
785 goto err_fdr;
786 fdw = error;
787
Al Viro157cf642008-12-14 04:57:47 -0500788 audit_fd_pair(fdr, fdw);
Andi Kleend6cbd282006-09-30 23:29:26 -0700789 fd[0] = fdr;
790 fd[1] = fdw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 return 0;
792
Andi Kleend6cbd282006-09-30 23:29:26 -0700793 err_fdr:
794 put_unused_fd(fdr);
795 err_read_pipe:
Al Viroe4fad8e2012-07-21 15:33:25 +0400796 fput(files[0]);
797 fput(files[1]);
Andi Kleend6cbd282006-09-30 23:29:26 -0700798 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799}
800
Al Viro5b249b12012-08-19 12:17:29 -0400801int do_pipe_flags(int *fd, int flags)
802{
803 struct file *files[2];
804 int error = __do_pipe_flags(fd, files, flags);
805 if (!error) {
806 fd_install(fd[0], files[0]);
807 fd_install(fd[1], files[1]);
808 }
809 return error;
810}
811
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812/*
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400813 * sys_pipe() is the normal C calling standard for creating
814 * a pipe. It's not the way Unix traditionally does this, though.
815 */
Heiko Carstensd4e82042009-01-14 14:14:34 +0100816SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400817{
Al Viro5b249b12012-08-19 12:17:29 -0400818 struct file *files[2];
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400819 int fd[2];
820 int error;
821
Al Viro5b249b12012-08-19 12:17:29 -0400822 error = __do_pipe_flags(fd, files, flags);
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400823 if (!error) {
Al Viro5b249b12012-08-19 12:17:29 -0400824 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
825 fput(files[0]);
826 fput(files[1]);
827 put_unused_fd(fd[0]);
828 put_unused_fd(fd[1]);
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400829 error = -EFAULT;
Al Viro5b249b12012-08-19 12:17:29 -0400830 } else {
831 fd_install(fd[0], files[0]);
832 fd_install(fd[1], files[1]);
Ulrich Drepperba719ba2008-05-06 20:42:38 -0700833 }
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400834 }
835 return error;
836}
837
Heiko Carstens2b664212009-01-14 14:14:35 +0100838SYSCALL_DEFINE1(pipe, int __user *, fildes)
Ulrich Dreppered8cae82008-07-23 21:29:30 -0700839{
840 return sys_pipe2(fildes, 0);
841}
842
Al Virofc7478a2013-03-21 02:07:59 -0400843static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
Al Virof776c732013-03-12 09:46:27 -0400844{
845 int cur = *cnt;
846
847 while (cur == *cnt) {
Al Virofc7478a2013-03-21 02:07:59 -0400848 pipe_wait(pipe);
Al Virof776c732013-03-12 09:46:27 -0400849 if (signal_pending(current))
850 break;
851 }
852 return cur == *cnt ? -ERESTARTSYS : 0;
853}
854
Al Virofc7478a2013-03-21 02:07:59 -0400855static void wake_up_partner(struct pipe_inode_info *pipe)
Al Virof776c732013-03-12 09:46:27 -0400856{
Al Virofc7478a2013-03-21 02:07:59 -0400857 wake_up_interruptible(&pipe->wait);
Al Virof776c732013-03-12 09:46:27 -0400858}
859
860static int fifo_open(struct inode *inode, struct file *filp)
861{
862 struct pipe_inode_info *pipe;
Al Viro599a0ac2013-03-12 09:58:10 -0400863 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
Al Virof776c732013-03-12 09:46:27 -0400864 int ret;
865
Al Viroba5bb142013-03-21 02:21:19 -0400866 filp->f_version = 0;
867
868 spin_lock(&inode->i_lock);
869 if (inode->i_pipe) {
870 pipe = inode->i_pipe;
871 pipe->files++;
872 spin_unlock(&inode->i_lock);
873 } else {
874 spin_unlock(&inode->i_lock);
Al Viro7bee1302013-03-21 11:04:15 -0400875 pipe = alloc_pipe_info();
Al Virof776c732013-03-12 09:46:27 -0400876 if (!pipe)
Al Viroba5bb142013-03-21 02:21:19 -0400877 return -ENOMEM;
878 pipe->files = 1;
879 spin_lock(&inode->i_lock);
880 if (unlikely(inode->i_pipe)) {
881 inode->i_pipe->files++;
882 spin_unlock(&inode->i_lock);
Al Viro4b8a8f12013-03-21 11:06:46 -0400883 free_pipe_info(pipe);
Al Viroba5bb142013-03-21 02:21:19 -0400884 pipe = inode->i_pipe;
885 } else {
886 inode->i_pipe = pipe;
887 spin_unlock(&inode->i_lock);
888 }
Al Virof776c732013-03-12 09:46:27 -0400889 }
Al Virode32ec42013-03-21 11:16:56 -0400890 filp->private_data = pipe;
Al Viroba5bb142013-03-21 02:21:19 -0400891 /* OK, we have a pipe and it's pinned down */
892
Al Viroebec73f2013-03-21 12:24:01 -0400893 __pipe_lock(pipe);
Al Virof776c732013-03-12 09:46:27 -0400894
895 /* We can only do regular read/write on fifos */
896 filp->f_mode &= (FMODE_READ | FMODE_WRITE);
897
898 switch (filp->f_mode) {
899 case FMODE_READ:
900 /*
901 * O_RDONLY
902 * POSIX.1 says that O_NONBLOCK means return with the FIFO
903 * opened, even when there is no process writing the FIFO.
904 */
Al Virof776c732013-03-12 09:46:27 -0400905 pipe->r_counter++;
906 if (pipe->readers++ == 0)
Al Virofc7478a2013-03-21 02:07:59 -0400907 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -0400908
Al Viro599a0ac2013-03-12 09:58:10 -0400909 if (!is_pipe && !pipe->writers) {
Al Virof776c732013-03-12 09:46:27 -0400910 if ((filp->f_flags & O_NONBLOCK)) {
911 /* suppress POLLHUP until we have
912 * seen a writer */
913 filp->f_version = pipe->w_counter;
914 } else {
Al Virofc7478a2013-03-21 02:07:59 -0400915 if (wait_for_partner(pipe, &pipe->w_counter))
Al Virof776c732013-03-12 09:46:27 -0400916 goto err_rd;
917 }
918 }
919 break;
920
921 case FMODE_WRITE:
922 /*
923 * O_WRONLY
924 * POSIX.1 says that O_NONBLOCK means return -1 with
925 * errno=ENXIO when there is no process reading the FIFO.
926 */
927 ret = -ENXIO;
Al Viro599a0ac2013-03-12 09:58:10 -0400928 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
Al Virof776c732013-03-12 09:46:27 -0400929 goto err;
930
Al Virof776c732013-03-12 09:46:27 -0400931 pipe->w_counter++;
932 if (!pipe->writers++)
Al Virofc7478a2013-03-21 02:07:59 -0400933 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -0400934
Al Viro599a0ac2013-03-12 09:58:10 -0400935 if (!is_pipe && !pipe->readers) {
Al Virofc7478a2013-03-21 02:07:59 -0400936 if (wait_for_partner(pipe, &pipe->r_counter))
Al Virof776c732013-03-12 09:46:27 -0400937 goto err_wr;
938 }
939 break;
940
941 case FMODE_READ | FMODE_WRITE:
942 /*
943 * O_RDWR
944 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
945 * This implementation will NEVER block on a O_RDWR open, since
946 * the process can at least talk to itself.
947 */
Al Virof776c732013-03-12 09:46:27 -0400948
949 pipe->readers++;
950 pipe->writers++;
951 pipe->r_counter++;
952 pipe->w_counter++;
953 if (pipe->readers == 1 || pipe->writers == 1)
Al Virofc7478a2013-03-21 02:07:59 -0400954 wake_up_partner(pipe);
Al Virof776c732013-03-12 09:46:27 -0400955 break;
956
957 default:
958 ret = -EINVAL;
959 goto err;
960 }
961
962 /* Ok! */
Al Viroebec73f2013-03-21 12:24:01 -0400963 __pipe_unlock(pipe);
Al Virof776c732013-03-12 09:46:27 -0400964 return 0;
965
966err_rd:
967 if (!--pipe->readers)
968 wake_up_interruptible(&pipe->wait);
969 ret = -ERESTARTSYS;
970 goto err;
971
972err_wr:
973 if (!--pipe->writers)
974 wake_up_interruptible(&pipe->wait);
975 ret = -ERESTARTSYS;
976 goto err;
977
978err:
Al Viroebec73f2013-03-21 12:24:01 -0400979 __pipe_unlock(pipe);
Linus Torvaldsb0d8d222013-12-02 09:44:51 -0800980
981 put_pipe_info(inode, pipe);
Al Virof776c732013-03-12 09:46:27 -0400982 return ret;
983}
984
Al Viro599a0ac2013-03-12 09:58:10 -0400985const struct file_operations pipefifo_fops = {
986 .open = fifo_open,
987 .llseek = no_llseek,
Al Virofb9096a2014-04-02 19:56:54 -0400988 .read_iter = pipe_read,
Al Virof0d1bec2014-04-03 15:05:18 -0400989 .write_iter = pipe_write,
Al Viro599a0ac2013-03-12 09:58:10 -0400990 .poll = pipe_poll,
991 .unlocked_ioctl = pipe_ioctl,
992 .release = pipe_release,
993 .fasync = pipe_fasync,
Al Virof776c732013-03-12 09:46:27 -0400994};
995
Ulrich Drepperd35c7b02008-05-03 15:10:37 -0400996/*
Jens Axboe35f3d142010-05-20 10:43:18 +0200997 * Allocate a new array of pipe buffers and copy the info over. Returns the
998 * pipe size if successful, or return -ERROR on error.
999 */
Jens Axboeb9598db2010-05-24 19:34:43 +02001000static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long nr_pages)
Jens Axboe35f3d142010-05-20 10:43:18 +02001001{
1002 struct pipe_buffer *bufs;
1003
1004 /*
Jens Axboe35f3d142010-05-20 10:43:18 +02001005 * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't
1006 * expect a lot of shrink+grow operations, just free and allocate
1007 * again like we would do for growing. If the pipe currently
1008 * contains more buffers than arg, then return busy.
1009 */
Jens Axboeb9598db2010-05-24 19:34:43 +02001010 if (nr_pages < pipe->nrbufs)
Jens Axboe35f3d142010-05-20 10:43:18 +02001011 return -EBUSY;
1012
Sasha Levin2ccd4f42012-01-12 17:17:40 -08001013 bufs = kcalloc(nr_pages, sizeof(*bufs), GFP_KERNEL | __GFP_NOWARN);
Jens Axboe35f3d142010-05-20 10:43:18 +02001014 if (unlikely(!bufs))
1015 return -ENOMEM;
1016
1017 /*
1018 * The pipe array wraps around, so just start the new one at zero
1019 * and adjust the indexes.
1020 */
1021 if (pipe->nrbufs) {
Miklos Szeredi1d862f42010-06-08 16:28:45 +02001022 unsigned int tail;
1023 unsigned int head;
Jens Axboe35f3d142010-05-20 10:43:18 +02001024
Miklos Szeredi1d862f42010-06-08 16:28:45 +02001025 tail = pipe->curbuf + pipe->nrbufs;
1026 if (tail < pipe->buffers)
1027 tail = 0;
1028 else
1029 tail &= (pipe->buffers - 1);
1030
1031 head = pipe->nrbufs - tail;
Jens Axboe35f3d142010-05-20 10:43:18 +02001032 if (head)
1033 memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer));
1034 if (tail)
Miklos Szeredi1d862f42010-06-08 16:28:45 +02001035 memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer));
Jens Axboe35f3d142010-05-20 10:43:18 +02001036 }
1037
Willy Tarreau759c0112016-01-18 16:36:09 +01001038 account_pipe_buffers(pipe, pipe->buffers, nr_pages);
Jens Axboe35f3d142010-05-20 10:43:18 +02001039 pipe->curbuf = 0;
1040 kfree(pipe->bufs);
1041 pipe->bufs = bufs;
Jens Axboeb9598db2010-05-24 19:34:43 +02001042 pipe->buffers = nr_pages;
1043 return nr_pages * PAGE_SIZE;
Jens Axboe35f3d142010-05-20 10:43:18 +02001044}
1045
Jens Axboeff9da692010-06-03 14:54:39 +02001046/*
1047 * Currently we rely on the pipe array holding a power-of-2 number
1048 * of pages.
1049 */
1050static inline unsigned int round_pipe_size(unsigned int size)
1051{
1052 unsigned long nr_pages;
1053
1054 nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1055 return roundup_pow_of_two(nr_pages) << PAGE_SHIFT;
1056}
1057
1058/*
1059 * This should work even if CONFIG_PROC_FS isn't set, as proc_dointvec_minmax
1060 * will return an error.
1061 */
1062int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf,
1063 size_t *lenp, loff_t *ppos)
1064{
1065 int ret;
1066
1067 ret = proc_dointvec_minmax(table, write, buf, lenp, ppos);
1068 if (ret < 0 || !write)
1069 return ret;
1070
1071 pipe_max_size = round_pipe_size(pipe_max_size);
1072 return ret;
1073}
1074
Linus Torvalds72083642010-11-28 16:27:19 -08001075/*
1076 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
1077 * location, so checking ->i_pipe is not enough to verify that this is a
1078 * pipe.
1079 */
1080struct pipe_inode_info *get_pipe_info(struct file *file)
1081{
Al Virode32ec42013-03-21 11:16:56 -04001082 return file->f_op == &pipefifo_fops ? file->private_data : NULL;
Linus Torvalds72083642010-11-28 16:27:19 -08001083}
1084
Jens Axboe35f3d142010-05-20 10:43:18 +02001085long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
1086{
1087 struct pipe_inode_info *pipe;
1088 long ret;
1089
Linus Torvaldsc66fb342010-11-28 14:09:57 -08001090 pipe = get_pipe_info(file);
Jens Axboe35f3d142010-05-20 10:43:18 +02001091 if (!pipe)
1092 return -EBADF;
1093
Al Viroebec73f2013-03-21 12:24:01 -04001094 __pipe_lock(pipe);
Jens Axboe35f3d142010-05-20 10:43:18 +02001095
1096 switch (cmd) {
Jens Axboeb9598db2010-05-24 19:34:43 +02001097 case F_SETPIPE_SZ: {
Jens Axboeff9da692010-06-03 14:54:39 +02001098 unsigned int size, nr_pages;
Jens Axboeb9598db2010-05-24 19:34:43 +02001099
Jens Axboeff9da692010-06-03 14:54:39 +02001100 size = round_pipe_size(arg);
1101 nr_pages = size >> PAGE_SHIFT;
Jens Axboeb9598db2010-05-24 19:34:43 +02001102
Miklos Szeredi6db40cf2010-06-09 09:27:57 +02001103 ret = -EINVAL;
1104 if (!nr_pages)
1105 goto out;
1106
Jens Axboeff9da692010-06-03 14:54:39 +02001107 if (!capable(CAP_SYS_RESOURCE) && size > pipe_max_size) {
Jens Axboeb4ca7612010-06-01 12:42:12 +02001108 ret = -EPERM;
Julia Lawallcc967be2010-05-26 17:54:39 +02001109 goto out;
Willy Tarreau759c0112016-01-18 16:36:09 +01001110 } else if ((too_many_pipe_buffers_hard(pipe->user) ||
1111 too_many_pipe_buffers_soft(pipe->user)) &&
1112 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
1113 ret = -EPERM;
1114 goto out;
Julia Lawallcc967be2010-05-26 17:54:39 +02001115 }
Jens Axboeff9da692010-06-03 14:54:39 +02001116 ret = pipe_set_size(pipe, nr_pages);
Jens Axboe35f3d142010-05-20 10:43:18 +02001117 break;
Jens Axboeb9598db2010-05-24 19:34:43 +02001118 }
Jens Axboe35f3d142010-05-20 10:43:18 +02001119 case F_GETPIPE_SZ:
Jens Axboeb9598db2010-05-24 19:34:43 +02001120 ret = pipe->buffers * PAGE_SIZE;
Jens Axboe35f3d142010-05-20 10:43:18 +02001121 break;
1122 default:
1123 ret = -EINVAL;
1124 break;
1125 }
1126
Julia Lawallcc967be2010-05-26 17:54:39 +02001127out:
Al Viroebec73f2013-03-21 12:24:01 -04001128 __pipe_unlock(pipe);
Jens Axboe35f3d142010-05-20 10:43:18 +02001129 return ret;
1130}
1131
Nick Pigginff0c7d12011-01-07 17:49:50 +11001132static const struct super_operations pipefs_ops = {
1133 .destroy_inode = free_inode_nonrcu,
Pavel Emelyanovd70ef972011-10-31 17:10:04 -07001134 .statfs = simple_statfs,
Nick Pigginff0c7d12011-01-07 17:49:50 +11001135};
1136
Jens Axboe35f3d142010-05-20 10:43:18 +02001137/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 * pipefs should _never_ be mounted by userland - too much of security hassle,
1139 * no real gain from having the whole whorehouse mounted. So we don't need
1140 * any operations on the root directory. However, we need a non-trivial
1141 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1142 */
Al Viro51139ad2010-07-25 23:47:46 +04001143static struct dentry *pipefs_mount(struct file_system_type *fs_type,
1144 int flags, const char *dev_name, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145{
Al Viroc74a1cb2011-01-12 16:59:34 -05001146 return mount_pseudo(fs_type, "pipe:", &pipefs_ops,
1147 &pipefs_dentry_operations, PIPEFS_MAGIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148}
1149
1150static struct file_system_type pipe_fs_type = {
1151 .name = "pipefs",
Al Viro51139ad2010-07-25 23:47:46 +04001152 .mount = pipefs_mount,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 .kill_sb = kill_anon_super,
1154};
1155
1156static int __init init_pipe_fs(void)
1157{
1158 int err = register_filesystem(&pipe_fs_type);
Ingo Molnar341b4462006-04-11 13:57:45 +02001159
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 if (!err) {
1161 pipe_mnt = kern_mount(&pipe_fs_type);
1162 if (IS_ERR(pipe_mnt)) {
1163 err = PTR_ERR(pipe_mnt);
1164 unregister_filesystem(&pipe_fs_type);
1165 }
1166 }
1167 return err;
1168}
1169
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170fs_initcall(init_pipe_fs);