blob: de3d273c9366093e03c1a626535fe488991a312b [file] [log] [blame]
Miklos Szeredi334f4852005-09-09 13:10:27 -07001/*
2 FUSE: Filesystem in Userspace
Miklos Szeredi1729a162008-11-26 12:03:54 +01003 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
Miklos Szeredi334f4852005-09-09 13:10:27 -07004
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7*/
8
9#include "fuse_i.h"
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/poll.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010014#include <linux/sched/signal.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070015#include <linux/uio.h>
16#include <linux/miscdevice.h>
17#include <linux/pagemap.h>
18#include <linux/file.h>
19#include <linux/slab.h>
Miklos Szeredidd3bb142010-05-25 15:06:06 +020020#include <linux/pipe_fs_i.h>
Miklos Szeredice534fb2010-05-25 15:06:07 +020021#include <linux/swap.h>
22#include <linux/splice.h>
Seth Forshee0b6e9ea2014-07-02 16:29:19 -050023#include <linux/sched.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070024
25MODULE_ALIAS_MISCDEV(FUSE_MINOR);
Kay Sievers578454f2010-05-20 18:07:20 +020026MODULE_ALIAS("devname:fuse");
Miklos Szeredi334f4852005-09-09 13:10:27 -070027
Christoph Lametere18b8902006-12-06 20:33:20 -080028static struct kmem_cache *fuse_req_cachep;
Miklos Szeredi334f4852005-09-09 13:10:27 -070029
Miklos Szeredicc080e92015-07-01 16:26:08 +020030static struct fuse_dev *fuse_get_dev(struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -070031{
Miklos Szeredi0720b312006-04-10 22:54:55 -070032 /*
33 * Lockless access is OK, because file->private data is set
34 * once during mount and is valid until the file is released.
35 */
Mark Rutland6aa7de02017-10-23 14:07:29 -070036 return READ_ONCE(file->private_data);
Miklos Szeredi334f4852005-09-09 13:10:27 -070037}
38
Maxim Patlasov4250c062012-10-26 19:48:07 +040039static void fuse_request_init(struct fuse_req *req, struct page **pages,
Maxim Patlasovb2430d72012-10-26 19:49:24 +040040 struct fuse_page_desc *page_descs,
Maxim Patlasov4250c062012-10-26 19:48:07 +040041 unsigned npages)
Miklos Szeredi334f4852005-09-09 13:10:27 -070042{
43 memset(req, 0, sizeof(*req));
Maxim Patlasov4250c062012-10-26 19:48:07 +040044 memset(pages, 0, sizeof(*pages) * npages);
Maxim Patlasovb2430d72012-10-26 19:49:24 +040045 memset(page_descs, 0, sizeof(*page_descs) * npages);
Miklos Szeredi334f4852005-09-09 13:10:27 -070046 INIT_LIST_HEAD(&req->list);
Miklos Szeredia4d27e72006-06-25 05:48:54 -070047 INIT_LIST_HEAD(&req->intr_entry);
Miklos Szeredi334f4852005-09-09 13:10:27 -070048 init_waitqueue_head(&req->waitq);
Elena Reshetovaec99f6d2017-03-03 11:04:04 +020049 refcount_set(&req->count, 1);
Maxim Patlasov4250c062012-10-26 19:48:07 +040050 req->pages = pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040051 req->page_descs = page_descs;
Maxim Patlasov4250c062012-10-26 19:48:07 +040052 req->max_pages = npages;
Miklos Szeredi33e14b42015-07-01 16:26:01 +020053 __set_bit(FR_PENDING, &req->flags);
Miklos Szeredi334f4852005-09-09 13:10:27 -070054}
55
Maxim Patlasov4250c062012-10-26 19:48:07 +040056static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
Miklos Szeredi334f4852005-09-09 13:10:27 -070057{
Maxim Patlasov4250c062012-10-26 19:48:07 +040058 struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
59 if (req) {
60 struct page **pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040061 struct fuse_page_desc *page_descs;
Maxim Patlasov4250c062012-10-26 19:48:07 +040062
Maxim Patlasovb2430d72012-10-26 19:49:24 +040063 if (npages <= FUSE_REQ_INLINE_PAGES) {
Maxim Patlasov4250c062012-10-26 19:48:07 +040064 pages = req->inline_pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040065 page_descs = req->inline_page_descs;
66 } else {
Kees Cook6da2ec52018-06-12 13:55:00 -070067 pages = kmalloc_array(npages, sizeof(struct page *),
68 flags);
69 page_descs =
70 kmalloc_array(npages,
71 sizeof(struct fuse_page_desc),
72 flags);
Maxim Patlasovb2430d72012-10-26 19:49:24 +040073 }
Maxim Patlasov4250c062012-10-26 19:48:07 +040074
Maxim Patlasovb2430d72012-10-26 19:49:24 +040075 if (!pages || !page_descs) {
76 kfree(pages);
77 kfree(page_descs);
Maxim Patlasov4250c062012-10-26 19:48:07 +040078 kmem_cache_free(fuse_req_cachep, req);
79 return NULL;
80 }
81
Maxim Patlasovb2430d72012-10-26 19:49:24 +040082 fuse_request_init(req, pages, page_descs, npages);
Maxim Patlasov4250c062012-10-26 19:48:07 +040083 }
Miklos Szeredi334f4852005-09-09 13:10:27 -070084 return req;
85}
Maxim Patlasov4250c062012-10-26 19:48:07 +040086
87struct fuse_req *fuse_request_alloc(unsigned npages)
88{
89 return __fuse_request_alloc(npages, GFP_KERNEL);
90}
Tejun Heo08cbf542009-04-14 10:54:53 +090091EXPORT_SYMBOL_GPL(fuse_request_alloc);
Miklos Szeredi334f4852005-09-09 13:10:27 -070092
Maxim Patlasov4250c062012-10-26 19:48:07 +040093struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
Miklos Szeredi3be5a522008-04-30 00:54:41 -070094{
Maxim Patlasov4250c062012-10-26 19:48:07 +040095 return __fuse_request_alloc(npages, GFP_NOFS);
Miklos Szeredi3be5a522008-04-30 00:54:41 -070096}
97
Miklos Szeredi334f4852005-09-09 13:10:27 -070098void fuse_request_free(struct fuse_req *req)
99{
Maxim Patlasovb2430d72012-10-26 19:49:24 +0400100 if (req->pages != req->inline_pages) {
Maxim Patlasov4250c062012-10-26 19:48:07 +0400101 kfree(req->pages);
Maxim Patlasovb2430d72012-10-26 19:49:24 +0400102 kfree(req->page_descs);
103 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700104 kmem_cache_free(fuse_req_cachep, req);
105}
106
Maxim Patlasov36cf66e2012-12-14 19:20:51 +0400107void __fuse_get_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700108{
Elena Reshetovaec99f6d2017-03-03 11:04:04 +0200109 refcount_inc(&req->count);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700110}
111
112/* Must be called with > 1 refcount */
113static void __fuse_put_request(struct fuse_req *req)
114{
Elena Reshetovaec99f6d2017-03-03 11:04:04 +0200115 refcount_dec(&req->count);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700116}
117
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100118void fuse_set_initialized(struct fuse_conn *fc)
119{
120 /* Make sure stores before this are seen on another CPU */
121 smp_wmb();
122 fc->initialized = 1;
123}
124
Maxim Patlasov0aada882013-03-21 18:02:28 +0400125static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
126{
127 return !fc->initialized || (for_background && fc->blocked);
128}
129
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200130static void fuse_drop_waiting(struct fuse_conn *fc)
131{
132 if (fc->connected) {
133 atomic_dec(&fc->num_waiting);
134 } else if (atomic_dec_and_test(&fc->num_waiting)) {
135 /* wake up aborters */
136 wake_up_all(&fc->blocked_waitq);
137 }
138}
139
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400140static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
141 bool for_background)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700142{
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700143 struct fuse_req *req;
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700144 int err;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200145 atomic_inc(&fc->num_waiting);
Maxim Patlasov0aada882013-03-21 18:02:28 +0400146
147 if (fuse_block_alloc(fc, for_background)) {
Maxim Patlasov0aada882013-03-21 18:02:28 +0400148 err = -EINTR;
Al Viro7d3a07f2016-07-19 03:08:27 -0400149 if (wait_event_killable_exclusive(fc->blocked_waitq,
150 !fuse_block_alloc(fc, for_background)))
Maxim Patlasov0aada882013-03-21 18:02:28 +0400151 goto out;
152 }
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100153 /* Matches smp_wmb() in fuse_set_initialized() */
154 smp_rmb();
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700155
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700156 err = -ENOTCONN;
157 if (!fc->connected)
158 goto out;
159
Miklos Szeredide155222015-07-01 16:25:57 +0200160 err = -ECONNREFUSED;
161 if (fc->conn_error)
162 goto out;
163
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400164 req = fuse_request_alloc(npages);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200165 err = -ENOMEM;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400166 if (!req) {
167 if (for_background)
168 wake_up(&fc->blocked_waitq);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200169 goto out;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400170 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700171
Eric W. Biederman8cb08322018-02-21 11:18:07 -0600172 req->in.h.uid = from_kuid(fc->user_ns, current_fsuid());
173 req->in.h.gid = from_kgid(fc->user_ns, current_fsgid());
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600174 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
175
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200176 __set_bit(FR_WAITING, &req->flags);
177 if (for_background)
178 __set_bit(FR_BACKGROUND, &req->flags);
179
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600180 if (unlikely(req->in.h.uid == ((uid_t)-1) ||
181 req->in.h.gid == ((gid_t)-1))) {
182 fuse_put_request(fc, req);
183 return ERR_PTR(-EOVERFLOW);
184 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700185 return req;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200186
187 out:
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200188 fuse_drop_waiting(fc);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200189 return ERR_PTR(err);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700190}
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400191
192struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
193{
194 return __fuse_get_req(fc, npages, false);
195}
Tejun Heo08cbf542009-04-14 10:54:53 +0900196EXPORT_SYMBOL_GPL(fuse_get_req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700197
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400198struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
199 unsigned npages)
200{
201 return __fuse_get_req(fc, npages, true);
202}
203EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
204
Miklos Szeredi33649c92006-06-25 05:48:52 -0700205/*
206 * Return request in fuse_file->reserved_req. However that may
207 * currently be in use. If that is the case, wait for it to become
208 * available.
209 */
210static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
211 struct file *file)
212{
213 struct fuse_req *req = NULL;
214 struct fuse_file *ff = file->private_data;
215
216 do {
Miklos Szeredide5e3de2007-10-16 23:31:00 -0700217 wait_event(fc->reserved_req_waitq, ff->reserved_req);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700218 spin_lock(&fc->lock);
219 if (ff->reserved_req) {
220 req = ff->reserved_req;
221 ff->reserved_req = NULL;
Al Virocb0942b2012-08-27 14:48:26 -0400222 req->stolen_file = get_file(file);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700223 }
224 spin_unlock(&fc->lock);
225 } while (!req);
226
227 return req;
228}
229
230/*
231 * Put stolen request back into fuse_file->reserved_req
232 */
233static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
234{
235 struct file *file = req->stolen_file;
236 struct fuse_file *ff = file->private_data;
237
238 spin_lock(&fc->lock);
Maxim Patlasovb2430d72012-10-26 19:49:24 +0400239 fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700240 BUG_ON(ff->reserved_req);
241 ff->reserved_req = req;
Miklos Szeredide5e3de2007-10-16 23:31:00 -0700242 wake_up_all(&fc->reserved_req_waitq);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700243 spin_unlock(&fc->lock);
244 fput(file);
245}
246
247/*
248 * Gets a requests for a file operation, always succeeds
249 *
250 * This is used for sending the FLUSH request, which must get to
251 * userspace, due to POSIX locks which may need to be unlocked.
252 *
253 * If allocation fails due to OOM, use the reserved request in
254 * fuse_file.
255 *
256 * This is very unlikely to deadlock accidentally, since the
257 * filesystem should not have it's own file open. If deadlock is
258 * intentional, it can still be broken by "aborting" the filesystem.
259 */
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400260struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
261 struct file *file)
Miklos Szeredi33649c92006-06-25 05:48:52 -0700262{
263 struct fuse_req *req;
264
265 atomic_inc(&fc->num_waiting);
Maxim Patlasov0aada882013-03-21 18:02:28 +0400266 wait_event(fc->blocked_waitq, fc->initialized);
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100267 /* Matches smp_wmb() in fuse_set_initialized() */
268 smp_rmb();
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400269 req = fuse_request_alloc(0);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700270 if (!req)
271 req = get_reserved_req(fc, file);
272
Eric W. Biederman8cb08322018-02-21 11:18:07 -0600273 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
274 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
Eric W. Biedermanc9582eb2018-02-21 10:52:06 -0600275 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
276
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200277 __set_bit(FR_WAITING, &req->flags);
278 __clear_bit(FR_BACKGROUND, &req->flags);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700279 return req;
280}
281
Miklos Szeredi334f4852005-09-09 13:10:27 -0700282void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
283{
Elena Reshetovaec99f6d2017-03-03 11:04:04 +0200284 if (refcount_dec_and_test(&req->count)) {
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200285 if (test_bit(FR_BACKGROUND, &req->flags)) {
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400286 /*
287 * We get here in the unlikely case that a background
288 * request was allocated but not sent
289 */
290 spin_lock(&fc->lock);
291 if (!fc->blocked)
292 wake_up(&fc->blocked_waitq);
293 spin_unlock(&fc->lock);
294 }
295
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200296 if (test_bit(FR_WAITING, &req->flags)) {
297 __clear_bit(FR_WAITING, &req->flags);
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200298 fuse_drop_waiting(fc);
Miklos Szeredi73e0e732015-07-01 16:25:56 +0200299 }
Miklos Szeredi33649c92006-06-25 05:48:52 -0700300
301 if (req->stolen_file)
302 put_reserved_req(fc, req);
303 else
304 fuse_request_free(req);
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800305 }
306}
Tejun Heo08cbf542009-04-14 10:54:53 +0900307EXPORT_SYMBOL_GPL(fuse_put_request);
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800308
Miklos Szeredid12def12008-02-06 01:38:39 -0800309static unsigned len_args(unsigned numargs, struct fuse_arg *args)
310{
311 unsigned nbytes = 0;
312 unsigned i;
313
314 for (i = 0; i < numargs; i++)
315 nbytes += args[i].size;
316
317 return nbytes;
318}
319
Miklos Szeredif88996a2015-07-01 16:26:01 +0200320static u64 fuse_get_unique(struct fuse_iqueue *fiq)
Miklos Szeredid12def12008-02-06 01:38:39 -0800321{
Miklos Szeredif88996a2015-07-01 16:26:01 +0200322 return ++fiq->reqctr;
Miklos Szeredid12def12008-02-06 01:38:39 -0800323}
324
Miklos Szeredif88996a2015-07-01 16:26:01 +0200325static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
Miklos Szeredid12def12008-02-06 01:38:39 -0800326{
Miklos Szeredid12def12008-02-06 01:38:39 -0800327 req->in.h.len = sizeof(struct fuse_in_header) +
328 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200329 list_add_tail(&req->list, &fiq->pending);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200330 wake_up_locked(&fiq->waitq);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200331 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredid12def12008-02-06 01:38:39 -0800332}
333
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100334void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
335 u64 nodeid, u64 nlookup)
336{
Miklos Szeredif88996a2015-07-01 16:26:01 +0200337 struct fuse_iqueue *fiq = &fc->iq;
338
Miklos Szeredi02c048b2010-12-07 20:16:56 +0100339 forget->forget_one.nodeid = nodeid;
340 forget->forget_one.nlookup = nlookup;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100341
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200342 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200343 if (fiq->connected) {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200344 fiq->forget_list_tail->next = forget;
345 fiq->forget_list_tail = forget;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200346 wake_up_locked(&fiq->waitq);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200347 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredi5dfcc872011-09-12 09:38:03 +0200348 } else {
349 kfree(forget);
350 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200351 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100352}
353
Miklos Szeredid12def12008-02-06 01:38:39 -0800354static void flush_bg_queue(struct fuse_conn *fc)
355{
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700356 while (fc->active_background < fc->max_background &&
Miklos Szeredid12def12008-02-06 01:38:39 -0800357 !list_empty(&fc->bg_queue)) {
358 struct fuse_req *req;
Miklos Szeredif88996a2015-07-01 16:26:01 +0200359 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredid12def12008-02-06 01:38:39 -0800360
361 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
362 list_del(&req->list);
363 fc->active_background++;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200364 spin_lock(&fiq->waitq.lock);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200365 req->in.h.unique = fuse_get_unique(fiq);
366 queue_request(fiq, req);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200367 spin_unlock(&fiq->waitq.lock);
Miklos Szeredid12def12008-02-06 01:38:39 -0800368 }
369}
370
Miklos Szeredi6dbbcb12006-04-26 10:49:06 +0200371/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700372 * This function is called when a request is finished. Either a reply
Miklos Szeredif9a28422006-06-25 05:48:53 -0700373 * has arrived or it was aborted (and not yet sent) or some error
Miklos Szeredif43b1552006-01-16 22:14:26 -0800374 * occurred during communication with userspace, or the device file
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700375 * was closed. The requester thread is woken up (if still waiting),
376 * the 'end' callback is called if given, else the reference to the
377 * request is released
Miklos Szeredi334f4852005-09-09 13:10:27 -0700378 */
379static void request_end(struct fuse_conn *fc, struct fuse_req *req)
380{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200381 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi365ae712015-07-01 16:26:06 +0200382
Miklos Szerediefe28002015-07-01 16:26:07 +0200383 if (test_and_set_bit(FR_FINISHED, &req->flags))
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200384 goto put_request;
Miklos Szeredi365ae712015-07-01 16:26:06 +0200385
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200386 spin_lock(&fiq->waitq.lock);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200387 list_del_init(&req->intr_entry);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200388 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200389 WARN_ON(test_bit(FR_PENDING, &req->flags));
390 WARN_ON(test_bit(FR_SENT, &req->flags));
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200391 if (test_bit(FR_BACKGROUND, &req->flags)) {
Miklos Szerediefe28002015-07-01 16:26:07 +0200392 spin_lock(&fc->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200393 clear_bit(FR_BACKGROUND, &req->flags);
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400394 if (fc->num_background == fc->max_background)
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700395 fc->blocked = 0;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400396
397 /* Wake up next waiter, if any */
Miklos Szeredi3c18ef82013-04-17 21:50:58 +0200398 if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400399 wake_up(&fc->blocked_waitq);
400
Tejun Heo8a301eb2018-02-02 09:54:14 -0800401 if (fc->num_background == fc->congestion_threshold && fc->sb) {
Jan Kara5f7f7542017-04-12 12:24:40 +0200402 clear_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
403 clear_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
Miklos Szeredif92b99b2007-10-16 23:30:59 -0700404 }
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700405 fc->num_background--;
Miklos Szeredid12def12008-02-06 01:38:39 -0800406 fc->active_background--;
407 flush_bg_queue(fc);
Miklos Szerediefe28002015-07-01 16:26:07 +0200408 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700409 }
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700410 wake_up(&req->waitq);
Miklos Szeredi1e6881c2015-07-01 16:26:07 +0200411 if (req->end)
412 req->end(fc, req);
Miklos Szeredib8f95e52018-07-26 16:13:11 +0200413put_request:
Tejun Heoe9bb09d2008-11-26 12:03:54 +0100414 fuse_put_request(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700415}
416
Miklos Szeredif88996a2015-07-01 16:26:01 +0200417static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700418{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200419 spin_lock(&fiq->waitq.lock);
Sahitya Tummala6ba4d272017-02-08 20:30:56 +0530420 if (test_bit(FR_FINISHED, &req->flags)) {
421 spin_unlock(&fiq->waitq.lock);
422 return;
423 }
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200424 if (list_empty(&req->intr_entry)) {
425 list_add_tail(&req->intr_entry, &fiq->interrupts);
426 wake_up_locked(&fiq->waitq);
427 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200428 spin_unlock(&fiq->waitq.lock);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200429 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700430}
431
Miklos Szeredi7c352bd2005-09-09 13:10:39 -0700432static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700433{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200434 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic4775262015-07-01 16:26:00 +0200435 int err;
436
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700437 if (!fc->no_interrupt) {
438 /* Any signal may interrupt this */
Miklos Szeredic4775262015-07-01 16:26:00 +0200439 err = wait_event_interruptible(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200440 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200441 if (!err)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700442 return;
443
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200444 set_bit(FR_INTERRUPTED, &req->flags);
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200445 /* matches barrier in fuse_dev_do_read() */
446 smp_mb__after_atomic();
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200447 if (test_bit(FR_SENT, &req->flags))
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200448 queue_interrupt(fiq, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700449 }
450
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200451 if (!test_bit(FR_FORCE, &req->flags)) {
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700452 /* Only fatal signals may interrupt this */
Al Viro7d3a07f2016-07-19 03:08:27 -0400453 err = wait_event_killable(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200454 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200455 if (!err)
Miklos Szeredia131de02007-10-16 23:31:04 -0700456 return;
457
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200458 spin_lock(&fiq->waitq.lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700459 /* Request is not yet in userspace, bail out */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200460 if (test_bit(FR_PENDING, &req->flags)) {
Miklos Szeredia131de02007-10-16 23:31:04 -0700461 list_del(&req->list);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200462 spin_unlock(&fiq->waitq.lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700463 __fuse_put_request(req);
464 req->out.h.error = -EINTR;
465 return;
466 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200467 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700468 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700469
Miklos Szeredia131de02007-10-16 23:31:04 -0700470 /*
471 * Either request is already in userspace, or it was forced.
472 * Wait it out.
473 */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200474 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
Miklos Szeredi334f4852005-09-09 13:10:27 -0700475}
476
Eric Wong6a4e9222013-02-04 13:04:44 +0000477static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700478{
Miklos Szeredie16714d2015-07-01 16:26:01 +0200479 struct fuse_iqueue *fiq = &fc->iq;
480
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200481 BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200482 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200483 if (!fiq->connected) {
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200484 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700485 req->out.h.error = -ENOTCONN;
Miklos Szeredic4775262015-07-01 16:26:00 +0200486 } else {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200487 req->in.h.unique = fuse_get_unique(fiq);
488 queue_request(fiq, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700489 /* acquire extra reference, since request is still needed
490 after request_end() */
491 __fuse_get_request(req);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200492 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700493
Miklos Szeredi7c352bd2005-09-09 13:10:39 -0700494 request_wait_answer(fc, req);
Miklos Szeredic4775262015-07-01 16:26:00 +0200495 /* Pairs with smp_wmb() in request_end() */
496 smp_rmb();
Miklos Szeredi334f4852005-09-09 13:10:27 -0700497 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700498}
Eric Wong6a4e9222013-02-04 13:04:44 +0000499
500void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
501{
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200502 __set_bit(FR_ISREPLY, &req->flags);
503 if (!test_bit(FR_WAITING, &req->flags)) {
504 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredi5437f242015-07-01 16:25:56 +0200505 atomic_inc(&fc->num_waiting);
506 }
Eric Wong6a4e9222013-02-04 13:04:44 +0000507 __fuse_request_send(fc, req);
508}
Tejun Heo08cbf542009-04-14 10:54:53 +0900509EXPORT_SYMBOL_GPL(fuse_request_send);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700510
Miklos Szeredi21f62172015-01-06 10:45:35 +0100511static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
512{
513 if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
514 args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;
515
516 if (fc->minor < 9) {
517 switch (args->in.h.opcode) {
518 case FUSE_LOOKUP:
519 case FUSE_CREATE:
520 case FUSE_MKNOD:
521 case FUSE_MKDIR:
522 case FUSE_SYMLINK:
523 case FUSE_LINK:
524 args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
525 break;
526 case FUSE_GETATTR:
527 case FUSE_SETATTR:
528 args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
529 break;
530 }
531 }
532 if (fc->minor < 12) {
533 switch (args->in.h.opcode) {
534 case FUSE_CREATE:
535 args->in.args[0].size = sizeof(struct fuse_open_in);
536 break;
537 case FUSE_MKNOD:
538 args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
539 break;
540 }
541 }
542}
543
Miklos Szeredi70781872014-12-12 09:49:05 +0100544ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
545{
546 struct fuse_req *req;
547 ssize_t ret;
548
549 req = fuse_get_req(fc, 0);
550 if (IS_ERR(req))
551 return PTR_ERR(req);
552
Miklos Szeredi21f62172015-01-06 10:45:35 +0100553 /* Needs to be done after fuse_get_req() so that fc->minor is valid */
554 fuse_adjust_compat(fc, args);
555
Miklos Szeredi70781872014-12-12 09:49:05 +0100556 req->in.h.opcode = args->in.h.opcode;
557 req->in.h.nodeid = args->in.h.nodeid;
558 req->in.numargs = args->in.numargs;
559 memcpy(req->in.args, args->in.args,
560 args->in.numargs * sizeof(struct fuse_in_arg));
561 req->out.argvar = args->out.argvar;
562 req->out.numargs = args->out.numargs;
563 memcpy(req->out.args, args->out.args,
564 args->out.numargs * sizeof(struct fuse_arg));
565 fuse_request_send(fc, req);
566 ret = req->out.h.error;
567 if (!ret && args->out.argvar) {
568 BUG_ON(args->out.numargs != 1);
569 ret = req->out.args[0].size;
570 }
571 fuse_put_request(fc, req);
572
573 return ret;
574}
575
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200576/*
577 * Called under fc->lock
578 *
579 * fc->connected must have been checked previously
580 */
581void fuse_request_send_background_locked(struct fuse_conn *fc,
582 struct fuse_req *req)
Miklos Szeredid12def12008-02-06 01:38:39 -0800583{
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200584 BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
585 if (!test_bit(FR_WAITING, &req->flags)) {
586 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredi5437f242015-07-01 16:25:56 +0200587 atomic_inc(&fc->num_waiting);
588 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200589 __set_bit(FR_ISREPLY, &req->flags);
Miklos Szeredid12def12008-02-06 01:38:39 -0800590 fc->num_background++;
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700591 if (fc->num_background == fc->max_background)
Miklos Szeredid12def12008-02-06 01:38:39 -0800592 fc->blocked = 1;
Jan Kara7fbbe972017-04-12 12:24:41 +0200593 if (fc->num_background == fc->congestion_threshold && fc->sb) {
Jan Kara5f7f7542017-04-12 12:24:40 +0200594 set_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
595 set_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
Miklos Szeredid12def12008-02-06 01:38:39 -0800596 }
597 list_add_tail(&req->list, &fc->bg_queue);
598 flush_bg_queue(fc);
599}
600
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200601void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700602{
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200603 BUG_ON(!req->end);
Miklos Szeredid7133112006-04-10 22:54:55 -0700604 spin_lock(&fc->lock);
Miklos Szeredi1e9a4ed2005-09-09 13:10:31 -0700605 if (fc->connected) {
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200606 fuse_request_send_background_locked(fc, req);
Miklos Szeredid7133112006-04-10 22:54:55 -0700607 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700608 } else {
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200609 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700610 req->out.h.error = -ENOTCONN;
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200611 req->end(fc, req);
612 fuse_put_request(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700613 }
614}
Tejun Heo08cbf542009-04-14 10:54:53 +0900615EXPORT_SYMBOL_GPL(fuse_request_send_background);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700616
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200617static int fuse_request_send_notify_reply(struct fuse_conn *fc,
618 struct fuse_req *req, u64 unique)
619{
620 int err = -ENODEV;
Miklos Szeredif88996a2015-07-01 16:26:01 +0200621 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200622
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200623 __clear_bit(FR_ISREPLY, &req->flags);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200624 req->in.h.unique = unique;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200625 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200626 if (fiq->connected) {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200627 queue_request(fiq, req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200628 err = 0;
629 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200630 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200631
632 return err;
633}
634
Anand V. Avati0b05b182012-08-19 08:53:23 -0400635void fuse_force_forget(struct file *file, u64 nodeid)
636{
Al Viro6131ffa2013-02-27 16:59:05 -0500637 struct inode *inode = file_inode(file);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400638 struct fuse_conn *fc = get_fuse_conn(inode);
639 struct fuse_req *req;
640 struct fuse_forget_in inarg;
641
642 memset(&inarg, 0, sizeof(inarg));
643 inarg.nlookup = 1;
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400644 req = fuse_get_req_nofail_nopages(fc, file);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400645 req->in.h.opcode = FUSE_FORGET;
646 req->in.h.nodeid = nodeid;
647 req->in.numargs = 1;
648 req->in.args[0].size = sizeof(inarg);
649 req->in.args[0].value = &inarg;
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200650 __clear_bit(FR_ISREPLY, &req->flags);
Eric Wong6a4e9222013-02-04 13:04:44 +0000651 __fuse_request_send(fc, req);
652 /* ignore errors */
653 fuse_put_request(fc, req);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400654}
655
Miklos Szeredi3be5a522008-04-30 00:54:41 -0700656/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700657 * Lock the request. Up to the next unlock_request() there mustn't be
658 * anything that could cause a page-fault. If the request was already
Miklos Szeredif9a28422006-06-25 05:48:53 -0700659 * aborted bail out.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700660 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200661static int lock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700662{
663 int err = 0;
664 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200665 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200666 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi334f4852005-09-09 13:10:27 -0700667 err = -ENOENT;
668 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200669 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200670 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700671 }
672 return err;
673}
674
675/*
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200676 * Unlock request. If it was aborted while locked, caller is responsible
677 * for unlocking and ending the request.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700678 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200679static int unlock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700680{
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200681 int err = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700682 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200683 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200684 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200685 err = -ENOENT;
686 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200687 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200688 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700689 }
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200690 return err;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700691}
692
693struct fuse_copy_state {
694 int write;
695 struct fuse_req *req;
Al Viro6c09e942015-04-03 22:06:08 -0400696 struct iov_iter *iter;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200697 struct pipe_buffer *pipebufs;
698 struct pipe_buffer *currbuf;
699 struct pipe_inode_info *pipe;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700700 unsigned long nr_segs;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700701 struct page *pg;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700702 unsigned len;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200703 unsigned offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200704 unsigned move_pages:1;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700705};
706
Miklos Szeredidc008092015-07-01 16:25:58 +0200707static void fuse_copy_init(struct fuse_copy_state *cs, int write,
Al Viro6c09e942015-04-03 22:06:08 -0400708 struct iov_iter *iter)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700709{
710 memset(cs, 0, sizeof(*cs));
711 cs->write = write;
Al Viro6c09e942015-04-03 22:06:08 -0400712 cs->iter = iter;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700713}
714
715/* Unmap and put previous page of userspace buffer */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800716static void fuse_copy_finish(struct fuse_copy_state *cs)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700717{
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200718 if (cs->currbuf) {
719 struct pipe_buffer *buf = cs->currbuf;
720
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200721 if (cs->write)
Miklos Szeredic3021622010-05-25 15:06:07 +0200722 buf->len = PAGE_SIZE - cs->len;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200723 cs->currbuf = NULL;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200724 } else if (cs->pg) {
Miklos Szeredi334f4852005-09-09 13:10:27 -0700725 if (cs->write) {
726 flush_dcache_page(cs->pg);
727 set_page_dirty_lock(cs->pg);
728 }
729 put_page(cs->pg);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700730 }
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200731 cs->pg = NULL;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700732}
733
734/*
735 * Get another pagefull of userspace buffer, and map it to kernel
736 * address space, and lock request
737 */
738static int fuse_copy_fill(struct fuse_copy_state *cs)
739{
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200740 struct page *page;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700741 int err;
742
Miklos Szeredidc008092015-07-01 16:25:58 +0200743 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200744 if (err)
745 return err;
746
Miklos Szeredi334f4852005-09-09 13:10:27 -0700747 fuse_copy_finish(cs);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200748 if (cs->pipebufs) {
749 struct pipe_buffer *buf = cs->pipebufs;
750
Miklos Szeredic3021622010-05-25 15:06:07 +0200751 if (!cs->write) {
Miklos Szeredifba597d2016-09-27 10:45:12 +0200752 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredic3021622010-05-25 15:06:07 +0200753 if (err)
754 return err;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200755
Miklos Szeredic3021622010-05-25 15:06:07 +0200756 BUG_ON(!cs->nr_segs);
757 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200758 cs->pg = buf->page;
759 cs->offset = buf->offset;
Miklos Szeredic3021622010-05-25 15:06:07 +0200760 cs->len = buf->len;
Miklos Szeredic3021622010-05-25 15:06:07 +0200761 cs->pipebufs++;
762 cs->nr_segs--;
763 } else {
Miklos Szeredic3021622010-05-25 15:06:07 +0200764 if (cs->nr_segs == cs->pipe->buffers)
765 return -EIO;
766
767 page = alloc_page(GFP_HIGHUSER);
768 if (!page)
769 return -ENOMEM;
770
771 buf->page = page;
772 buf->offset = 0;
773 buf->len = 0;
774
775 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200776 cs->pg = page;
777 cs->offset = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +0200778 cs->len = PAGE_SIZE;
779 cs->pipebufs++;
780 cs->nr_segs++;
781 }
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200782 } else {
Al Viro6c09e942015-04-03 22:06:08 -0400783 size_t off;
784 err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200785 if (err < 0)
786 return err;
Al Viro6c09e942015-04-03 22:06:08 -0400787 BUG_ON(!err);
788 cs->len = err;
789 cs->offset = off;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200790 cs->pg = page;
Al Viro6c09e942015-04-03 22:06:08 -0400791 iov_iter_advance(cs->iter, err);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700792 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700793
Miklos Szeredidc008092015-07-01 16:25:58 +0200794 return lock_request(cs->req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700795}
796
797/* Do as much copy to/from userspace buffer as we can */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800798static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700799{
800 unsigned ncpy = min(*size, cs->len);
801 if (val) {
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200802 void *pgaddr = kmap_atomic(cs->pg);
803 void *buf = pgaddr + cs->offset;
804
Miklos Szeredi334f4852005-09-09 13:10:27 -0700805 if (cs->write)
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200806 memcpy(buf, *val, ncpy);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700807 else
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200808 memcpy(*val, buf, ncpy);
809
810 kunmap_atomic(pgaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700811 *val += ncpy;
812 }
813 *size -= ncpy;
814 cs->len -= ncpy;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200815 cs->offset += ncpy;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700816 return ncpy;
817}
818
Miklos Szeredice534fb2010-05-25 15:06:07 +0200819static int fuse_check_page(struct page *page)
820{
821 if (page_mapcount(page) ||
822 page->mapping != NULL ||
823 page_count(page) != 1 ||
824 (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
825 ~(1 << PG_locked |
826 1 << PG_referenced |
827 1 << PG_uptodate |
828 1 << PG_lru |
829 1 << PG_active |
830 1 << PG_reclaim))) {
831 printk(KERN_WARNING "fuse: trying to steal weird page\n");
832 printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
833 return 1;
834 }
835 return 0;
836}
837
838static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
839{
840 int err;
841 struct page *oldpage = *pagep;
842 struct page *newpage;
843 struct pipe_buffer *buf = cs->pipebufs;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200844
Miklos Szeredidc008092015-07-01 16:25:58 +0200845 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200846 if (err)
847 return err;
848
Miklos Szeredice534fb2010-05-25 15:06:07 +0200849 fuse_copy_finish(cs);
850
Miklos Szeredifba597d2016-09-27 10:45:12 +0200851 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200852 if (err)
853 return err;
854
855 BUG_ON(!cs->nr_segs);
856 cs->currbuf = buf;
857 cs->len = buf->len;
858 cs->pipebufs++;
859 cs->nr_segs--;
860
861 if (cs->len != PAGE_SIZE)
862 goto out_fallback;
863
Miklos Szeredica76f5b2016-09-27 10:45:12 +0200864 if (pipe_buf_steal(cs->pipe, buf) != 0)
Miklos Szeredice534fb2010-05-25 15:06:07 +0200865 goto out_fallback;
866
867 newpage = buf->page;
868
Miklos Szerediaa991b32015-02-26 11:45:47 +0100869 if (!PageUptodate(newpage))
870 SetPageUptodate(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200871
872 ClearPageMappedToDisk(newpage);
873
874 if (fuse_check_page(newpage) != 0)
875 goto out_fallback_unlock;
876
Miklos Szeredice534fb2010-05-25 15:06:07 +0200877 /*
878 * This is a new and locked page, it shouldn't be mapped or
879 * have any special flags on it
880 */
881 if (WARN_ON(page_mapped(oldpage)))
882 goto out_fallback_unlock;
883 if (WARN_ON(page_has_private(oldpage)))
884 goto out_fallback_unlock;
885 if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
886 goto out_fallback_unlock;
887 if (WARN_ON(PageMlocked(oldpage)))
888 goto out_fallback_unlock;
889
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700890 err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200891 if (err) {
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700892 unlock_page(newpage);
893 return err;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200894 }
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700895
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300896 get_page(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200897
898 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
899 lru_cache_add_file(newpage);
900
901 err = 0;
Miklos Szeredidc008092015-07-01 16:25:58 +0200902 spin_lock(&cs->req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200903 if (test_bit(FR_ABORTED, &cs->req->flags))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200904 err = -ENOENT;
905 else
906 *pagep = newpage;
Miklos Szeredidc008092015-07-01 16:25:58 +0200907 spin_unlock(&cs->req->waitq.lock);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200908
909 if (err) {
910 unlock_page(newpage);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300911 put_page(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200912 return err;
913 }
914
915 unlock_page(oldpage);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300916 put_page(oldpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200917 cs->len = 0;
918
919 return 0;
920
921out_fallback_unlock:
922 unlock_page(newpage);
923out_fallback:
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200924 cs->pg = buf->page;
925 cs->offset = buf->offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200926
Miklos Szeredidc008092015-07-01 16:25:58 +0200927 err = lock_request(cs->req);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200928 if (err)
929 return err;
930
931 return 1;
932}
933
Miklos Szeredic3021622010-05-25 15:06:07 +0200934static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
935 unsigned offset, unsigned count)
936{
937 struct pipe_buffer *buf;
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200938 int err;
Miklos Szeredic3021622010-05-25 15:06:07 +0200939
940 if (cs->nr_segs == cs->pipe->buffers)
941 return -EIO;
942
Miklos Szeredidc008092015-07-01 16:25:58 +0200943 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200944 if (err)
945 return err;
946
Miklos Szeredic3021622010-05-25 15:06:07 +0200947 fuse_copy_finish(cs);
948
949 buf = cs->pipebufs;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300950 get_page(page);
Miklos Szeredic3021622010-05-25 15:06:07 +0200951 buf->page = page;
952 buf->offset = offset;
953 buf->len = count;
954
955 cs->pipebufs++;
956 cs->nr_segs++;
957 cs->len = 0;
958
959 return 0;
960}
961
Miklos Szeredi334f4852005-09-09 13:10:27 -0700962/*
963 * Copy a page in the request to/from the userspace buffer. Must be
964 * done atomically
965 */
Miklos Szeredice534fb2010-05-25 15:06:07 +0200966static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800967 unsigned offset, unsigned count, int zeroing)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700968{
Miklos Szeredice534fb2010-05-25 15:06:07 +0200969 int err;
970 struct page *page = *pagep;
971
Miklos Szeredib6777c42010-10-26 14:22:27 -0700972 if (page && zeroing && count < PAGE_SIZE)
973 clear_highpage(page);
974
Miklos Szeredi334f4852005-09-09 13:10:27 -0700975 while (count) {
Miklos Szeredic3021622010-05-25 15:06:07 +0200976 if (cs->write && cs->pipebufs && page) {
977 return fuse_ref_page(cs, page, offset, count);
978 } else if (!cs->len) {
Miklos Szeredice534fb2010-05-25 15:06:07 +0200979 if (cs->move_pages && page &&
980 offset == 0 && count == PAGE_SIZE) {
981 err = fuse_try_move_page(cs, pagep);
982 if (err <= 0)
983 return err;
984 } else {
985 err = fuse_copy_fill(cs);
986 if (err)
987 return err;
988 }
Miklos Szeredi1729a162008-11-26 12:03:54 +0100989 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700990 if (page) {
Cong Wang2408f6e2011-11-25 23:14:30 +0800991 void *mapaddr = kmap_atomic(page);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700992 void *buf = mapaddr + offset;
993 offset += fuse_copy_do(cs, &buf, &count);
Cong Wang2408f6e2011-11-25 23:14:30 +0800994 kunmap_atomic(mapaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700995 } else
996 offset += fuse_copy_do(cs, NULL, &count);
997 }
998 if (page && !cs->write)
999 flush_dcache_page(page);
1000 return 0;
1001}
1002
1003/* Copy pages in the request to/from userspace buffer */
1004static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
1005 int zeroing)
1006{
1007 unsigned i;
1008 struct fuse_req *req = cs->req;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001009
1010 for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
Miklos Szeredice534fb2010-05-25 15:06:07 +02001011 int err;
Maxim Patlasov85f40ae2012-10-26 19:49:33 +04001012 unsigned offset = req->page_descs[i].offset;
1013 unsigned count = min(nbytes, req->page_descs[i].length);
Miklos Szeredice534fb2010-05-25 15:06:07 +02001014
1015 err = fuse_copy_page(cs, &req->pages[i], offset, count,
1016 zeroing);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001017 if (err)
1018 return err;
1019
1020 nbytes -= count;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001021 }
1022 return 0;
1023}
1024
1025/* Copy a single argument in the request to/from userspace buffer */
1026static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1027{
1028 while (size) {
Miklos Szeredi1729a162008-11-26 12:03:54 +01001029 if (!cs->len) {
1030 int err = fuse_copy_fill(cs);
1031 if (err)
1032 return err;
1033 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001034 fuse_copy_do(cs, &val, &size);
1035 }
1036 return 0;
1037}
1038
1039/* Copy request arguments to/from userspace buffer */
1040static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1041 unsigned argpages, struct fuse_arg *args,
1042 int zeroing)
1043{
1044 int err = 0;
1045 unsigned i;
1046
1047 for (i = 0; !err && i < numargs; i++) {
1048 struct fuse_arg *arg = &args[i];
1049 if (i == numargs - 1 && argpages)
1050 err = fuse_copy_pages(cs, arg->size, zeroing);
1051 else
1052 err = fuse_copy_one(cs, arg->value, arg->size);
1053 }
1054 return err;
1055}
1056
Miklos Szeredif88996a2015-07-01 16:26:01 +02001057static int forget_pending(struct fuse_iqueue *fiq)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001058{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001059 return fiq->forget_list_head.next != NULL;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001060}
1061
Miklos Szeredif88996a2015-07-01 16:26:01 +02001062static int request_pending(struct fuse_iqueue *fiq)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001063{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001064 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1065 forget_pending(fiq);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001066}
1067
Miklos Szeredi334f4852005-09-09 13:10:27 -07001068/*
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001069 * Transfer an interrupt request to userspace
1070 *
1071 * Unlike other requests this is assembled on demand, without a need
1072 * to allocate a separate fuse_req structure.
1073 *
Miklos Szeredifd22d622015-07-01 16:26:03 +02001074 * Called with fiq->waitq.lock held, releases it
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001075 */
Miklos Szeredifd22d622015-07-01 16:26:03 +02001076static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1077 struct fuse_copy_state *cs,
Miklos Szeredic3021622010-05-25 15:06:07 +02001078 size_t nbytes, struct fuse_req *req)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001079__releases(fiq->waitq.lock)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001080{
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001081 struct fuse_in_header ih;
1082 struct fuse_interrupt_in arg;
1083 unsigned reqsize = sizeof(ih) + sizeof(arg);
1084 int err;
1085
1086 list_del_init(&req->intr_entry);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001087 req->intr_unique = fuse_get_unique(fiq);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001088 memset(&ih, 0, sizeof(ih));
1089 memset(&arg, 0, sizeof(arg));
1090 ih.len = reqsize;
1091 ih.opcode = FUSE_INTERRUPT;
1092 ih.unique = req->intr_unique;
1093 arg.unique = req->in.h.unique;
1094
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001095 spin_unlock(&fiq->waitq.lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001096 if (nbytes < reqsize)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001097 return -EINVAL;
1098
Miklos Szeredic3021622010-05-25 15:06:07 +02001099 err = fuse_copy_one(cs, &ih, sizeof(ih));
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001100 if (!err)
Miklos Szeredic3021622010-05-25 15:06:07 +02001101 err = fuse_copy_one(cs, &arg, sizeof(arg));
1102 fuse_copy_finish(cs);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001103
1104 return err ? err : reqsize;
1105}
1106
Miklos Szeredif88996a2015-07-01 16:26:01 +02001107static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001108 unsigned max,
1109 unsigned *countp)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001110{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001111 struct fuse_forget_link *head = fiq->forget_list_head.next;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001112 struct fuse_forget_link **newhead = &head;
1113 unsigned count;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001114
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001115 for (count = 0; *newhead != NULL && count < max; count++)
1116 newhead = &(*newhead)->next;
1117
Miklos Szeredif88996a2015-07-01 16:26:01 +02001118 fiq->forget_list_head.next = *newhead;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001119 *newhead = NULL;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001120 if (fiq->forget_list_head.next == NULL)
1121 fiq->forget_list_tail = &fiq->forget_list_head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001122
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001123 if (countp != NULL)
1124 *countp = count;
1125
1126 return head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001127}
1128
Miklos Szeredifd22d622015-07-01 16:26:03 +02001129static int fuse_read_single_forget(struct fuse_iqueue *fiq,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001130 struct fuse_copy_state *cs,
1131 size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001132__releases(fiq->waitq.lock)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001133{
1134 int err;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001135 struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001136 struct fuse_forget_in arg = {
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001137 .nlookup = forget->forget_one.nlookup,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001138 };
1139 struct fuse_in_header ih = {
1140 .opcode = FUSE_FORGET,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001141 .nodeid = forget->forget_one.nodeid,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001142 .unique = fuse_get_unique(fiq),
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001143 .len = sizeof(ih) + sizeof(arg),
1144 };
1145
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001146 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001147 kfree(forget);
1148 if (nbytes < ih.len)
1149 return -EINVAL;
1150
1151 err = fuse_copy_one(cs, &ih, sizeof(ih));
1152 if (!err)
1153 err = fuse_copy_one(cs, &arg, sizeof(arg));
1154 fuse_copy_finish(cs);
1155
1156 if (err)
1157 return err;
1158
1159 return ih.len;
1160}
1161
Miklos Szeredifd22d622015-07-01 16:26:03 +02001162static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001163 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001164__releases(fiq->waitq.lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001165{
1166 int err;
1167 unsigned max_forgets;
1168 unsigned count;
1169 struct fuse_forget_link *head;
1170 struct fuse_batch_forget_in arg = { .count = 0 };
1171 struct fuse_in_header ih = {
1172 .opcode = FUSE_BATCH_FORGET,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001173 .unique = fuse_get_unique(fiq),
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001174 .len = sizeof(ih) + sizeof(arg),
1175 };
1176
1177 if (nbytes < ih.len) {
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001178 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001179 return -EINVAL;
1180 }
1181
1182 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
Miklos Szeredif88996a2015-07-01 16:26:01 +02001183 head = dequeue_forget(fiq, max_forgets, &count);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001184 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001185
1186 arg.count = count;
1187 ih.len += count * sizeof(struct fuse_forget_one);
1188 err = fuse_copy_one(cs, &ih, sizeof(ih));
1189 if (!err)
1190 err = fuse_copy_one(cs, &arg, sizeof(arg));
1191
1192 while (head) {
1193 struct fuse_forget_link *forget = head;
1194
1195 if (!err) {
1196 err = fuse_copy_one(cs, &forget->forget_one,
1197 sizeof(forget->forget_one));
1198 }
1199 head = forget->next;
1200 kfree(forget);
1201 }
1202
1203 fuse_copy_finish(cs);
1204
1205 if (err)
1206 return err;
1207
1208 return ih.len;
1209}
1210
Miklos Szeredifd22d622015-07-01 16:26:03 +02001211static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1212 struct fuse_copy_state *cs,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001213 size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001214__releases(fiq->waitq.lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001215{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001216 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001217 return fuse_read_single_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001218 else
Miklos Szeredifd22d622015-07-01 16:26:03 +02001219 return fuse_read_batch_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001220}
1221
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001222/*
Miklos Szeredi334f4852005-09-09 13:10:27 -07001223 * Read a single request into the userspace filesystem's buffer. This
1224 * function waits until a request is available, then removes it from
1225 * the pending list and copies request data to userspace buffer. If
Miklos Szeredif9a28422006-06-25 05:48:53 -07001226 * no reply is needed (FORGET) or request has been aborted or there
1227 * was an error during the copying then it's finished by calling
Miklos Szeredi334f4852005-09-09 13:10:27 -07001228 * request_end(). Otherwise add it to the processing list, and set
1229 * the 'sent' flag.
1230 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001231static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
Miklos Szeredic3021622010-05-25 15:06:07 +02001232 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001233{
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001234 ssize_t err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001235 struct fuse_conn *fc = fud->fc;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001236 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic36960462015-07-01 16:26:09 +02001237 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001238 struct fuse_req *req;
1239 struct fuse_in *in;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001240 unsigned reqsize;
1241
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001242 restart:
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001243 spin_lock(&fiq->waitq.lock);
Jeff Dikee5ac1d12006-04-10 22:54:53 -07001244 err = -EAGAIN;
Miklos Szeredie16714d2015-07-01 16:26:01 +02001245 if ((file->f_flags & O_NONBLOCK) && fiq->connected &&
Miklos Szeredif88996a2015-07-01 16:26:01 +02001246 !request_pending(fiq))
Jeff Dikee5ac1d12006-04-10 22:54:53 -07001247 goto err_unlock;
1248
Miklos Szeredi52509212015-07-01 16:26:03 +02001249 err = wait_event_interruptible_exclusive_locked(fiq->waitq,
1250 !fiq->connected || request_pending(fiq));
1251 if (err)
1252 goto err_unlock;
1253
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01001254 if (!fiq->connected) {
1255 err = (fc->aborted && fc->abort_err) ? -ECONNABORTED : -ENODEV;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001256 goto err_unlock;
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01001257 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001258
Miklos Szeredif88996a2015-07-01 16:26:01 +02001259 if (!list_empty(&fiq->interrupts)) {
1260 req = list_entry(fiq->interrupts.next, struct fuse_req,
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001261 intr_entry);
Miklos Szeredifd22d622015-07-01 16:26:03 +02001262 return fuse_read_interrupt(fiq, cs, nbytes, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001263 }
1264
Miklos Szeredif88996a2015-07-01 16:26:01 +02001265 if (forget_pending(fiq)) {
1266 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001267 return fuse_read_forget(fc, fiq, cs, nbytes);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001268
Miklos Szeredif88996a2015-07-01 16:26:01 +02001269 if (fiq->forget_batch <= -8)
1270 fiq->forget_batch = 16;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001271 }
1272
Miklos Szeredif88996a2015-07-01 16:26:01 +02001273 req = list_entry(fiq->pending.next, struct fuse_req, list);
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001274 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredief759252015-07-01 16:26:02 +02001275 list_del_init(&req->list);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001276 spin_unlock(&fiq->waitq.lock);
1277
Miklos Szeredi334f4852005-09-09 13:10:27 -07001278 in = &req->in;
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001279 reqsize = in->h.len;
Miklos Szeredi5d6d3a32017-09-12 16:57:53 +02001280
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001281 /* If request is too large, reply with an error and restart the read */
Miklos Szeredic3021622010-05-25 15:06:07 +02001282 if (nbytes < reqsize) {
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001283 req->out.h.error = -EIO;
1284 /* SETXATTR is special, since it may contain too large data */
1285 if (in->h.opcode == FUSE_SETXATTR)
1286 req->out.h.error = -E2BIG;
1287 request_end(fc, req);
1288 goto restart;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001289 }
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001290 spin_lock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001291 list_add(&req->list, &fpq->io);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001292 spin_unlock(&fpq->lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001293 cs->req = req;
1294 err = fuse_copy_one(cs, &in->h, sizeof(in->h));
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001295 if (!err)
Miklos Szeredic3021622010-05-25 15:06:07 +02001296 err = fuse_copy_args(cs, in->numargs, in->argpages,
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001297 (struct fuse_arg *) in->args, 0);
Miklos Szeredic3021622010-05-25 15:06:07 +02001298 fuse_copy_finish(cs);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001299 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001300 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001301 if (!fpq->connected) {
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01001302 err = (fc->aborted && fc->abort_err) ? -ECONNABORTED : -ENODEV;
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001303 goto out_end;
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001304 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001305 if (err) {
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001306 req->out.h.error = -EIO;
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001307 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001308 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001309 if (!test_bit(FR_ISREPLY, &req->flags)) {
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001310 err = reqsize;
1311 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001312 }
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001313 list_move_tail(&req->list, &fpq->processing);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001314 spin_unlock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001315 set_bit(FR_SENT, &req->flags);
1316 /* matches barrier in request_wait_answer() */
1317 smp_mb__after_atomic();
1318 if (test_bit(FR_INTERRUPTED, &req->flags))
1319 queue_interrupt(fiq, req);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001320
Miklos Szeredi334f4852005-09-09 13:10:27 -07001321 return reqsize;
1322
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001323out_end:
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001324 if (!test_bit(FR_PRIVATE, &req->flags))
1325 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001326 spin_unlock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001327 request_end(fc, req);
1328 return err;
1329
Miklos Szeredi334f4852005-09-09 13:10:27 -07001330 err_unlock:
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001331 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001332 return err;
1333}
1334
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01001335static int fuse_dev_open(struct inode *inode, struct file *file)
1336{
1337 /*
1338 * The fuse device's file's private_data is used to hold
1339 * the fuse_conn(ection) when it is mounted, and is used to
1340 * keep track of whether the file has been mounted already.
1341 */
1342 file->private_data = NULL;
1343 return 0;
1344}
1345
Al Virofbdbacc2015-04-03 21:53:39 -04001346static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
Miklos Szeredic3021622010-05-25 15:06:07 +02001347{
1348 struct fuse_copy_state cs;
1349 struct file *file = iocb->ki_filp;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001350 struct fuse_dev *fud = fuse_get_dev(file);
1351
1352 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001353 return -EPERM;
1354
Al Virofbdbacc2015-04-03 21:53:39 -04001355 if (!iter_is_iovec(to))
1356 return -EINVAL;
Miklos Szeredic3021622010-05-25 15:06:07 +02001357
Miklos Szeredidc008092015-07-01 16:25:58 +02001358 fuse_copy_init(&cs, 1, to);
Al Virofbdbacc2015-04-03 21:53:39 -04001359
Miklos Szeredic36960462015-07-01 16:26:09 +02001360 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
Miklos Szeredic3021622010-05-25 15:06:07 +02001361}
1362
Miklos Szeredic3021622010-05-25 15:06:07 +02001363static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1364 struct pipe_inode_info *pipe,
1365 size_t len, unsigned int flags)
1366{
Al Virod82718e2016-09-17 22:56:25 -04001367 int total, ret;
Miklos Szeredic3021622010-05-25 15:06:07 +02001368 int page_nr = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +02001369 struct pipe_buffer *bufs;
1370 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001371 struct fuse_dev *fud = fuse_get_dev(in);
1372
1373 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001374 return -EPERM;
1375
Kees Cook6da2ec52018-06-12 13:55:00 -07001376 bufs = kmalloc_array(pipe->buffers, sizeof(struct pipe_buffer),
1377 GFP_KERNEL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001378 if (!bufs)
1379 return -ENOMEM;
1380
Miklos Szeredidc008092015-07-01 16:25:58 +02001381 fuse_copy_init(&cs, 1, NULL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001382 cs.pipebufs = bufs;
1383 cs.pipe = pipe;
Miklos Szeredic36960462015-07-01 16:26:09 +02001384 ret = fuse_dev_do_read(fud, in, &cs, len);
Miklos Szeredic3021622010-05-25 15:06:07 +02001385 if (ret < 0)
1386 goto out;
1387
Miklos Szeredic3021622010-05-25 15:06:07 +02001388 if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1389 ret = -EIO;
Al Virod82718e2016-09-17 22:56:25 -04001390 goto out;
Miklos Szeredic3021622010-05-25 15:06:07 +02001391 }
1392
Al Virod82718e2016-09-17 22:56:25 -04001393 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
Miklos Szeredi28a625c2014-01-22 19:36:57 +01001394 /*
1395 * Need to be careful about this. Having buf->ops in module
1396 * code can Oops if the buffer persists after module unload.
1397 */
Al Virod82718e2016-09-17 22:56:25 -04001398 bufs[page_nr].ops = &nosteal_pipe_buf_ops;
Miklos Szeredi84588a92017-02-16 15:08:20 +01001399 bufs[page_nr].flags = 0;
Al Virod82718e2016-09-17 22:56:25 -04001400 ret = add_to_pipe(pipe, &bufs[page_nr++]);
1401 if (unlikely(ret < 0))
1402 break;
Miklos Szeredic3021622010-05-25 15:06:07 +02001403 }
Al Virod82718e2016-09-17 22:56:25 -04001404 if (total)
1405 ret = total;
Miklos Szeredic3021622010-05-25 15:06:07 +02001406out:
1407 for (; page_nr < cs.nr_segs; page_nr++)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001408 put_page(bufs[page_nr].page);
Miklos Szeredic3021622010-05-25 15:06:07 +02001409
1410 kfree(bufs);
1411 return ret;
1412}
1413
Tejun Heo95668a62008-11-26 12:03:55 +01001414static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1415 struct fuse_copy_state *cs)
1416{
1417 struct fuse_notify_poll_wakeup_out outarg;
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001418 int err = -EINVAL;
Tejun Heo95668a62008-11-26 12:03:55 +01001419
1420 if (size != sizeof(outarg))
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001421 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001422
1423 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1424 if (err)
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001425 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001426
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001427 fuse_copy_finish(cs);
Tejun Heo95668a62008-11-26 12:03:55 +01001428 return fuse_notify_poll_wakeup(fc, &outarg);
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001429
1430err:
1431 fuse_copy_finish(cs);
1432 return err;
Tejun Heo95668a62008-11-26 12:03:55 +01001433}
1434
John Muir3b463ae2009-05-31 11:13:57 -04001435static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1436 struct fuse_copy_state *cs)
1437{
1438 struct fuse_notify_inval_inode_out outarg;
1439 int err = -EINVAL;
1440
1441 if (size != sizeof(outarg))
1442 goto err;
1443
1444 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1445 if (err)
1446 goto err;
1447 fuse_copy_finish(cs);
1448
1449 down_read(&fc->killsb);
1450 err = -ENOENT;
Miklos Szeredib21dda42010-02-05 12:08:31 +01001451 if (fc->sb) {
1452 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1453 outarg.off, outarg.len);
1454 }
John Muir3b463ae2009-05-31 11:13:57 -04001455 up_read(&fc->killsb);
1456 return err;
1457
1458err:
1459 fuse_copy_finish(cs);
1460 return err;
1461}
1462
1463static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1464 struct fuse_copy_state *cs)
1465{
1466 struct fuse_notify_inval_entry_out outarg;
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001467 int err = -ENOMEM;
1468 char *buf;
John Muir3b463ae2009-05-31 11:13:57 -04001469 struct qstr name;
1470
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001471 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1472 if (!buf)
1473 goto err;
1474
1475 err = -EINVAL;
John Muir3b463ae2009-05-31 11:13:57 -04001476 if (size < sizeof(outarg))
1477 goto err;
1478
1479 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1480 if (err)
1481 goto err;
1482
1483 err = -ENAMETOOLONG;
1484 if (outarg.namelen > FUSE_NAME_MAX)
1485 goto err;
1486
Miklos Szeredic2183d12011-08-24 10:20:17 +02001487 err = -EINVAL;
1488 if (size != sizeof(outarg) + outarg.namelen + 1)
1489 goto err;
1490
John Muir3b463ae2009-05-31 11:13:57 -04001491 name.name = buf;
1492 name.len = outarg.namelen;
1493 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1494 if (err)
1495 goto err;
1496 fuse_copy_finish(cs);
1497 buf[outarg.namelen] = 0;
John Muir3b463ae2009-05-31 11:13:57 -04001498
1499 down_read(&fc->killsb);
1500 err = -ENOENT;
Miklos Szeredib21dda42010-02-05 12:08:31 +01001501 if (fc->sb)
John Muir451d0f52011-12-06 21:50:06 +01001502 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1503 up_read(&fc->killsb);
1504 kfree(buf);
1505 return err;
1506
1507err:
1508 kfree(buf);
1509 fuse_copy_finish(cs);
1510 return err;
1511}
1512
1513static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1514 struct fuse_copy_state *cs)
1515{
1516 struct fuse_notify_delete_out outarg;
1517 int err = -ENOMEM;
1518 char *buf;
1519 struct qstr name;
1520
1521 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1522 if (!buf)
1523 goto err;
1524
1525 err = -EINVAL;
1526 if (size < sizeof(outarg))
1527 goto err;
1528
1529 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1530 if (err)
1531 goto err;
1532
1533 err = -ENAMETOOLONG;
1534 if (outarg.namelen > FUSE_NAME_MAX)
1535 goto err;
1536
1537 err = -EINVAL;
1538 if (size != sizeof(outarg) + outarg.namelen + 1)
1539 goto err;
1540
1541 name.name = buf;
1542 name.len = outarg.namelen;
1543 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1544 if (err)
1545 goto err;
1546 fuse_copy_finish(cs);
1547 buf[outarg.namelen] = 0;
John Muir451d0f52011-12-06 21:50:06 +01001548
1549 down_read(&fc->killsb);
1550 err = -ENOENT;
1551 if (fc->sb)
1552 err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1553 outarg.child, &name);
John Muir3b463ae2009-05-31 11:13:57 -04001554 up_read(&fc->killsb);
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001555 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001556 return err;
1557
1558err:
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001559 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001560 fuse_copy_finish(cs);
1561 return err;
1562}
1563
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001564static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1565 struct fuse_copy_state *cs)
1566{
1567 struct fuse_notify_store_out outarg;
1568 struct inode *inode;
1569 struct address_space *mapping;
1570 u64 nodeid;
1571 int err;
1572 pgoff_t index;
1573 unsigned int offset;
1574 unsigned int num;
1575 loff_t file_size;
1576 loff_t end;
1577
1578 err = -EINVAL;
1579 if (size < sizeof(outarg))
1580 goto out_finish;
1581
1582 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1583 if (err)
1584 goto out_finish;
1585
1586 err = -EINVAL;
1587 if (size - sizeof(outarg) != outarg.size)
1588 goto out_finish;
1589
1590 nodeid = outarg.nodeid;
1591
1592 down_read(&fc->killsb);
1593
1594 err = -ENOENT;
1595 if (!fc->sb)
1596 goto out_up_killsb;
1597
1598 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1599 if (!inode)
1600 goto out_up_killsb;
1601
1602 mapping = inode->i_mapping;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001603 index = outarg.offset >> PAGE_SHIFT;
1604 offset = outarg.offset & ~PAGE_MASK;
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001605 file_size = i_size_read(inode);
1606 end = outarg.offset + outarg.size;
1607 if (end > file_size) {
1608 file_size = end;
1609 fuse_write_update_size(inode, file_size);
1610 }
1611
1612 num = outarg.size;
1613 while (num) {
1614 struct page *page;
1615 unsigned int this_num;
1616
1617 err = -ENOMEM;
1618 page = find_or_create_page(mapping, index,
1619 mapping_gfp_mask(mapping));
1620 if (!page)
1621 goto out_iput;
1622
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001623 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001624 err = fuse_copy_page(cs, &page, offset, this_num, 0);
Miklos Szeredi063ec1e2014-01-22 19:36:58 +01001625 if (!err && offset == 0 &&
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001626 (this_num == PAGE_SIZE || file_size == end))
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001627 SetPageUptodate(page);
1628 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001629 put_page(page);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001630
1631 if (err)
1632 goto out_iput;
1633
1634 num -= this_num;
1635 offset = 0;
1636 index++;
1637 }
1638
1639 err = 0;
1640
1641out_iput:
1642 iput(inode);
1643out_up_killsb:
1644 up_read(&fc->killsb);
1645out_finish:
1646 fuse_copy_finish(cs);
1647 return err;
1648}
1649
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001650static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1651{
Mel Gormanc6f92f92017-11-15 17:37:55 -08001652 release_pages(req->pages, req->num_pages);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001653}
1654
1655static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1656 struct fuse_notify_retrieve_out *outarg)
1657{
1658 int err;
1659 struct address_space *mapping = inode->i_mapping;
1660 struct fuse_req *req;
1661 pgoff_t index;
1662 loff_t file_size;
1663 unsigned int num;
1664 unsigned int offset;
Geert Uytterhoeven01574432010-09-30 22:06:21 +02001665 size_t total_len = 0;
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001666 int num_pages;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001667
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001668 offset = outarg->offset & ~PAGE_MASK;
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001669 file_size = i_size_read(inode);
1670
1671 num = outarg->size;
1672 if (outarg->offset > file_size)
1673 num = 0;
1674 else if (outarg->offset + num > file_size)
1675 num = file_size - outarg->offset;
1676
1677 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1678 num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1679
1680 req = fuse_get_req(fc, num_pages);
1681 if (IS_ERR(req))
1682 return PTR_ERR(req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001683
1684 req->in.h.opcode = FUSE_NOTIFY_REPLY;
1685 req->in.h.nodeid = outarg->nodeid;
1686 req->in.numargs = 2;
1687 req->in.argpages = 1;
Maxim Patlasovb2430d72012-10-26 19:49:24 +04001688 req->page_descs[0].offset = offset;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001689 req->end = fuse_retrieve_end;
1690
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001691 index = outarg->offset >> PAGE_SHIFT;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001692
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001693 while (num && req->num_pages < num_pages) {
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001694 struct page *page;
1695 unsigned int this_num;
1696
1697 page = find_get_page(mapping, index);
1698 if (!page)
1699 break;
1700
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001701 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001702 req->pages[req->num_pages] = page;
Maxim Patlasov85f40ae2012-10-26 19:49:33 +04001703 req->page_descs[req->num_pages].length = this_num;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001704 req->num_pages++;
1705
Miklos Szeredic9e67d42012-09-04 18:45:54 +02001706 offset = 0;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001707 num -= this_num;
1708 total_len += this_num;
Miklos Szeredi48706d02011-12-13 10:36:59 +01001709 index++;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001710 }
1711 req->misc.retrieve_in.offset = outarg->offset;
1712 req->misc.retrieve_in.size = total_len;
1713 req->in.args[0].size = sizeof(req->misc.retrieve_in);
1714 req->in.args[0].value = &req->misc.retrieve_in;
1715 req->in.args[1].size = total_len;
1716
1717 err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1718 if (err)
1719 fuse_retrieve_end(fc, req);
1720
1721 return err;
1722}
1723
1724static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1725 struct fuse_copy_state *cs)
1726{
1727 struct fuse_notify_retrieve_out outarg;
1728 struct inode *inode;
1729 int err;
1730
1731 err = -EINVAL;
1732 if (size != sizeof(outarg))
1733 goto copy_finish;
1734
1735 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1736 if (err)
1737 goto copy_finish;
1738
1739 fuse_copy_finish(cs);
1740
1741 down_read(&fc->killsb);
1742 err = -ENOENT;
1743 if (fc->sb) {
1744 u64 nodeid = outarg.nodeid;
1745
1746 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1747 if (inode) {
1748 err = fuse_retrieve(fc, inode, &outarg);
1749 iput(inode);
1750 }
1751 }
1752 up_read(&fc->killsb);
1753
1754 return err;
1755
1756copy_finish:
1757 fuse_copy_finish(cs);
1758 return err;
1759}
1760
Tejun Heo85993962008-11-26 12:03:55 +01001761static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1762 unsigned int size, struct fuse_copy_state *cs)
1763{
Miklos Szeredi0d278362015-02-26 11:45:47 +01001764 /* Don't try to move pages (yet) */
1765 cs->move_pages = 0;
1766
Tejun Heo85993962008-11-26 12:03:55 +01001767 switch (code) {
Tejun Heo95668a62008-11-26 12:03:55 +01001768 case FUSE_NOTIFY_POLL:
1769 return fuse_notify_poll(fc, size, cs);
1770
John Muir3b463ae2009-05-31 11:13:57 -04001771 case FUSE_NOTIFY_INVAL_INODE:
1772 return fuse_notify_inval_inode(fc, size, cs);
1773
1774 case FUSE_NOTIFY_INVAL_ENTRY:
1775 return fuse_notify_inval_entry(fc, size, cs);
1776
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001777 case FUSE_NOTIFY_STORE:
1778 return fuse_notify_store(fc, size, cs);
1779
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001780 case FUSE_NOTIFY_RETRIEVE:
1781 return fuse_notify_retrieve(fc, size, cs);
1782
John Muir451d0f52011-12-06 21:50:06 +01001783 case FUSE_NOTIFY_DELETE:
1784 return fuse_notify_delete(fc, size, cs);
1785
Tejun Heo85993962008-11-26 12:03:55 +01001786 default:
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001787 fuse_copy_finish(cs);
Tejun Heo85993962008-11-26 12:03:55 +01001788 return -EINVAL;
1789 }
1790}
1791
Miklos Szeredi334f4852005-09-09 13:10:27 -07001792/* Look up request on processing list by unique ID */
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001793static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001794{
Dong Fang05726ac2013-07-30 22:50:01 -04001795 struct fuse_req *req;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001796
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001797 list_for_each_entry(req, &fpq->processing, list) {
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001798 if (req->in.h.unique == unique || req->intr_unique == unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001799 return req;
1800 }
1801 return NULL;
1802}
1803
1804static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1805 unsigned nbytes)
1806{
1807 unsigned reqsize = sizeof(struct fuse_out_header);
1808
1809 if (out->h.error)
1810 return nbytes != reqsize ? -EINVAL : 0;
1811
1812 reqsize += len_args(out->numargs, out->args);
1813
1814 if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1815 return -EINVAL;
1816 else if (reqsize > nbytes) {
1817 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1818 unsigned diffsize = reqsize - nbytes;
1819 if (diffsize > lastarg->size)
1820 return -EINVAL;
1821 lastarg->size -= diffsize;
1822 }
1823 return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1824 out->page_zeroing);
1825}
1826
1827/*
1828 * Write a single reply to a request. First the header is copied from
1829 * the write buffer. The request is then searched on the processing
1830 * list by the unique ID found in the header. If found, then remove
1831 * it from the list and copy the rest of the buffer to the request.
1832 * The request is finished by calling request_end()
1833 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001834static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001835 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001836{
1837 int err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001838 struct fuse_conn *fc = fud->fc;
1839 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001840 struct fuse_req *req;
1841 struct fuse_out_header oh;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001842
Miklos Szeredi334f4852005-09-09 13:10:27 -07001843 if (nbytes < sizeof(struct fuse_out_header))
1844 return -EINVAL;
1845
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001846 err = fuse_copy_one(cs, &oh, sizeof(oh));
Miklos Szeredi334f4852005-09-09 13:10:27 -07001847 if (err)
1848 goto err_finish;
Tejun Heo85993962008-11-26 12:03:55 +01001849
Miklos Szeredi334f4852005-09-09 13:10:27 -07001850 err = -EINVAL;
Tejun Heo85993962008-11-26 12:03:55 +01001851 if (oh.len != nbytes)
1852 goto err_finish;
1853
1854 /*
1855 * Zero oh.unique indicates unsolicited notification message
1856 * and error contains notification code.
1857 */
1858 if (!oh.unique) {
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001859 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
Tejun Heo85993962008-11-26 12:03:55 +01001860 return err ? err : nbytes;
1861 }
1862
1863 err = -EINVAL;
1864 if (oh.error <= -1000 || oh.error > 0)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001865 goto err_finish;
1866
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001867 spin_lock(&fpq->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08001868 err = -ENOENT;
Miklos Szeredie96edd92015-07-01 16:26:04 +02001869 if (!fpq->connected)
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001870 goto err_unlock_pq;
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08001871
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001872 req = request_find(fpq, oh.unique);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001873 if (!req)
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001874 goto err_unlock_pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001875
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001876 /* Is it an interrupt reply? */
1877 if (req->intr_unique == oh.unique) {
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001878 spin_unlock(&fpq->lock);
1879
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001880 err = -EINVAL;
1881 if (nbytes != sizeof(struct fuse_out_header))
Miklos Szeredi46c34a32015-07-01 16:26:07 +02001882 goto err_finish;
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001883
1884 if (oh.error == -ENOSYS)
1885 fc->no_interrupt = 1;
1886 else if (oh.error == -EAGAIN)
Miklos Szeredif88996a2015-07-01 16:26:01 +02001887 queue_interrupt(&fc->iq, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001888
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001889 fuse_copy_finish(cs);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001890 return nbytes;
1891 }
1892
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001893 clear_bit(FR_SENT, &req->flags);
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001894 list_move(&req->list, &fpq->io);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001895 req->out.h = oh;
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001896 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001897 spin_unlock(&fpq->lock);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001898 cs->req = req;
Miklos Szeredice534fb2010-05-25 15:06:07 +02001899 if (!req->out.page_replace)
1900 cs->move_pages = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001901
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001902 err = copy_out_args(cs, &req->out, nbytes);
1903 fuse_copy_finish(cs);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001904
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001905 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001906 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001907 if (!fpq->connected)
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +02001908 err = -ENOENT;
1909 else if (err)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001910 req->out.h.error = -EIO;
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001911 if (!test_bit(FR_PRIVATE, &req->flags))
1912 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001913 spin_unlock(&fpq->lock);
Miklos Szeredi46c34a32015-07-01 16:26:07 +02001914
Miklos Szeredi334f4852005-09-09 13:10:27 -07001915 request_end(fc, req);
1916
1917 return err ? err : nbytes;
1918
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001919 err_unlock_pq:
1920 spin_unlock(&fpq->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001921 err_finish:
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001922 fuse_copy_finish(cs);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001923 return err;
1924}
1925
Al Virofbdbacc2015-04-03 21:53:39 -04001926static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001927{
1928 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001929 struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
1930
1931 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001932 return -EPERM;
1933
Al Virofbdbacc2015-04-03 21:53:39 -04001934 if (!iter_is_iovec(from))
1935 return -EINVAL;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001936
Miklos Szeredidc008092015-07-01 16:25:58 +02001937 fuse_copy_init(&cs, 0, from);
Al Virofbdbacc2015-04-03 21:53:39 -04001938
Miklos Szeredic36960462015-07-01 16:26:09 +02001939 return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001940}
1941
1942static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1943 struct file *out, loff_t *ppos,
1944 size_t len, unsigned int flags)
1945{
1946 unsigned nbuf;
1947 unsigned idx;
1948 struct pipe_buffer *bufs;
1949 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001950 struct fuse_dev *fud;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001951 size_t rem;
1952 ssize_t ret;
1953
Miklos Szeredicc080e92015-07-01 16:26:08 +02001954 fud = fuse_get_dev(out);
1955 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001956 return -EPERM;
1957
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001958 pipe_lock(pipe);
1959
Kees Cook6da2ec52018-06-12 13:55:00 -07001960 bufs = kmalloc_array(pipe->buffers, sizeof(struct pipe_buffer),
1961 GFP_KERNEL);
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001962 if (!bufs) {
1963 pipe_unlock(pipe);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001964 return -ENOMEM;
Andrey Ryabinina2477b02018-07-17 19:00:33 +03001965 }
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001966
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001967 nbuf = 0;
1968 rem = 0;
1969 for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1970 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1971
1972 ret = -EINVAL;
1973 if (rem < len) {
1974 pipe_unlock(pipe);
1975 goto out;
1976 }
1977
1978 rem = len;
1979 while (rem) {
1980 struct pipe_buffer *ibuf;
1981 struct pipe_buffer *obuf;
1982
1983 BUG_ON(nbuf >= pipe->buffers);
1984 BUG_ON(!pipe->nrbufs);
1985 ibuf = &pipe->bufs[pipe->curbuf];
1986 obuf = &bufs[nbuf];
1987
1988 if (rem >= ibuf->len) {
1989 *obuf = *ibuf;
1990 ibuf->ops = NULL;
1991 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1992 pipe->nrbufs--;
1993 } else {
Miklos Szeredi7bf2d1d2016-09-27 10:45:12 +02001994 pipe_buf_get(pipe, ibuf);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001995 *obuf = *ibuf;
1996 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1997 obuf->len = rem;
1998 ibuf->offset += obuf->len;
1999 ibuf->len -= obuf->len;
2000 }
2001 nbuf++;
2002 rem -= obuf->len;
2003 }
2004 pipe_unlock(pipe);
2005
Miklos Szeredidc008092015-07-01 16:25:58 +02002006 fuse_copy_init(&cs, 0, NULL);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002007 cs.pipebufs = bufs;
Al Viro6c09e942015-04-03 22:06:08 -04002008 cs.nr_segs = nbuf;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002009 cs.pipe = pipe;
2010
Miklos Szeredice534fb2010-05-25 15:06:07 +02002011 if (flags & SPLICE_F_MOVE)
2012 cs.move_pages = 1;
2013
Miklos Szeredic36960462015-07-01 16:26:09 +02002014 ret = fuse_dev_do_write(fud, &cs, len);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002015
Miklos Szeredia7796382016-09-27 10:45:12 +02002016 for (idx = 0; idx < nbuf; idx++)
2017 pipe_buf_release(pipe, &bufs[idx]);
2018
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002019out:
2020 kfree(bufs);
2021 return ret;
2022}
2023
Al Viro076ccb72017-07-03 01:02:18 -04002024static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002025{
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002026 __poll_t mask = EPOLLOUT | EPOLLWRNORM;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002027 struct fuse_iqueue *fiq;
Miklos Szeredicc080e92015-07-01 16:26:08 +02002028 struct fuse_dev *fud = fuse_get_dev(file);
2029
2030 if (!fud)
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002031 return EPOLLERR;
Miklos Szeredi334f4852005-09-09 13:10:27 -07002032
Miklos Szeredicc080e92015-07-01 16:26:08 +02002033 fiq = &fud->fc->iq;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002034 poll_wait(file, &fiq->waitq, wait);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002035
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002036 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +02002037 if (!fiq->connected)
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002038 mask = EPOLLERR;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002039 else if (request_pending(fiq))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08002040 mask |= EPOLLIN | EPOLLRDNORM;
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002041 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002042
2043 return mask;
2044}
2045
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002046/*
2047 * Abort all requests on the given list (pending or processing)
2048 *
Miklos Szeredid7133112006-04-10 22:54:55 -07002049 * This function releases and reacquires fc->lock
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002050 */
Miklos Szeredi334f4852005-09-09 13:10:27 -07002051static void end_requests(struct fuse_conn *fc, struct list_head *head)
2052{
2053 while (!list_empty(head)) {
2054 struct fuse_req *req;
2055 req = list_entry(head->next, struct fuse_req, list);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002056 req->out.h.error = -ECONNABORTED;
Miklos Szeredi33e14b42015-07-01 16:26:01 +02002057 clear_bit(FR_SENT, &req->flags);
Miklos Szeredif377cb72015-07-01 16:26:04 +02002058 list_del_init(&req->list);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002059 request_end(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002060 }
2061}
2062
Bryan Green357ccf22011-03-01 16:43:52 -08002063static void end_polls(struct fuse_conn *fc)
2064{
2065 struct rb_node *p;
2066
2067 p = rb_first(&fc->polled_files);
2068
2069 while (p) {
2070 struct fuse_file *ff;
2071 ff = rb_entry(p, struct fuse_file, polled_node);
2072 wake_up_interruptible_all(&ff->poll_wait);
2073
2074 p = rb_next(p);
2075 }
2076}
2077
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002078/*
2079 * Abort all requests.
2080 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002081 * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2082 * filesystem.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002083 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002084 * The same effect is usually achievable through killing the filesystem daemon
2085 * and all users of the filesystem. The exception is the combination of an
2086 * asynchronous request and the tricky deadlock (see
2087 * Documentation/filesystems/fuse.txt).
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002088 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002089 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2090 * requests, they should be finished off immediately. Locked requests will be
2091 * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2092 * requests. It is possible that some request will finish before we can. This
2093 * is OK, the request will in that case be removed from the list before we touch
2094 * it.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002095 */
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01002096void fuse_abort_conn(struct fuse_conn *fc, bool is_abort)
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002097{
Miklos Szeredif88996a2015-07-01 16:26:01 +02002098 struct fuse_iqueue *fiq = &fc->iq;
2099
Miklos Szeredid7133112006-04-10 22:54:55 -07002100 spin_lock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002101 if (fc->connected) {
Miklos Szeredic36960462015-07-01 16:26:09 +02002102 struct fuse_dev *fud;
Miklos Szeredib716d422015-07-01 16:25:59 +02002103 struct fuse_req *req, *next;
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002104 LIST_HEAD(to_end);
Miklos Szeredib716d422015-07-01 16:25:59 +02002105
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002106 fc->connected = 0;
Miklos Szeredi51eb01e2006-06-25 05:48:50 -07002107 fc->blocked = 0;
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01002108 fc->aborted = is_abort;
Miklos Szeredi9759bd512015-01-06 10:45:35 +01002109 fuse_set_initialized(fc);
Miklos Szeredic36960462015-07-01 16:26:09 +02002110 list_for_each_entry(fud, &fc->devices, entry) {
2111 struct fuse_pqueue *fpq = &fud->pq;
2112
2113 spin_lock(&fpq->lock);
2114 fpq->connected = 0;
2115 list_for_each_entry_safe(req, next, &fpq->io, list) {
2116 req->out.h.error = -ECONNABORTED;
2117 spin_lock(&req->waitq.lock);
2118 set_bit(FR_ABORTED, &req->flags);
2119 if (!test_bit(FR_LOCKED, &req->flags)) {
2120 set_bit(FR_PRIVATE, &req->flags);
Miklos Szeredi87114372018-07-26 16:13:11 +02002121 __fuse_get_request(req);
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002122 list_move(&req->list, &to_end);
Miklos Szeredic36960462015-07-01 16:26:09 +02002123 }
2124 spin_unlock(&req->waitq.lock);
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02002125 }
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002126 list_splice_tail_init(&fpq->processing, &to_end);
Miklos Szeredic36960462015-07-01 16:26:09 +02002127 spin_unlock(&fpq->lock);
Miklos Szeredib716d422015-07-01 16:25:59 +02002128 }
Miklos Szeredi41f98272015-07-01 16:25:59 +02002129 fc->max_background = UINT_MAX;
2130 flush_bg_queue(fc);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002131
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002132 spin_lock(&fiq->waitq.lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002133 fiq->connected = 0;
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002134 list_for_each_entry(req, &fiq->pending, list)
Tahsin Erdogana8a86d72017-01-12 12:04:04 -08002135 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002136 list_splice_tail_init(&fiq->pending, &to_end);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002137 while (forget_pending(fiq))
2138 kfree(dequeue_forget(fiq, 1, NULL));
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002139 wake_up_all_locked(&fiq->waitq);
2140 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002141 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szerediee314a82015-07-01 16:26:08 +02002142 end_polls(fc);
2143 wake_up_all(&fc->blocked_waitq);
2144 spin_unlock(&fc->lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002145
Miklos Szeredi75f3ee42018-07-26 16:13:12 +02002146 end_requests(fc, &to_end);
Miklos Szerediee314a82015-07-01 16:26:08 +02002147 } else {
2148 spin_unlock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002149 }
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002150}
Tejun Heo08cbf542009-04-14 10:54:53 +09002151EXPORT_SYMBOL_GPL(fuse_abort_conn);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002152
Miklos Szeredib8f95e52018-07-26 16:13:11 +02002153void fuse_wait_aborted(struct fuse_conn *fc)
2154{
2155 wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
2156}
2157
Tejun Heo08cbf542009-04-14 10:54:53 +09002158int fuse_dev_release(struct inode *inode, struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002159{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002160 struct fuse_dev *fud = fuse_get_dev(file);
2161
2162 if (fud) {
2163 struct fuse_conn *fc = fud->fc;
Miklos Szeredic36960462015-07-01 16:26:09 +02002164 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002165 LIST_HEAD(to_end);
Miklos Szeredicc080e92015-07-01 16:26:08 +02002166
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002167 spin_lock(&fpq->lock);
Miklos Szeredic36960462015-07-01 16:26:09 +02002168 WARN_ON(!list_empty(&fpq->io));
Miklos Szeredi45ff3502018-07-26 16:13:11 +02002169 list_splice_init(&fpq->processing, &to_end);
2170 spin_unlock(&fpq->lock);
2171
2172 end_requests(fc, &to_end);
2173
Miklos Szeredic36960462015-07-01 16:26:09 +02002174 /* Are we the last open device? */
2175 if (atomic_dec_and_test(&fc->dev_count)) {
2176 WARN_ON(fc->iq.fasync != NULL);
Szymon Lukasz3b7008b2017-11-09 21:23:35 +01002177 fuse_abort_conn(fc, false);
Miklos Szeredic36960462015-07-01 16:26:09 +02002178 }
Miklos Szeredicc080e92015-07-01 16:26:08 +02002179 fuse_dev_free(fud);
Jeff Dike385a17b2006-04-10 22:54:52 -07002180 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07002181 return 0;
2182}
Tejun Heo08cbf542009-04-14 10:54:53 +09002183EXPORT_SYMBOL_GPL(fuse_dev_release);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002184
Jeff Dike385a17b2006-04-10 22:54:52 -07002185static int fuse_dev_fasync(int fd, struct file *file, int on)
2186{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002187 struct fuse_dev *fud = fuse_get_dev(file);
2188
2189 if (!fud)
Miklos Szeredia87046d2006-04-10 22:54:56 -07002190 return -EPERM;
Jeff Dike385a17b2006-04-10 22:54:52 -07002191
2192 /* No locking - fasync_helper does its own locking */
Miklos Szeredicc080e92015-07-01 16:26:08 +02002193 return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
Jeff Dike385a17b2006-04-10 22:54:52 -07002194}
2195
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002196static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2197{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002198 struct fuse_dev *fud;
2199
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002200 if (new->private_data)
2201 return -EINVAL;
2202
Miklos Szeredicc080e92015-07-01 16:26:08 +02002203 fud = fuse_dev_alloc(fc);
2204 if (!fud)
2205 return -ENOMEM;
2206
2207 new->private_data = fud;
Miklos Szeredic36960462015-07-01 16:26:09 +02002208 atomic_inc(&fc->dev_count);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002209
2210 return 0;
2211}
2212
2213static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2214 unsigned long arg)
2215{
2216 int err = -ENOTTY;
2217
2218 if (cmd == FUSE_DEV_IOC_CLONE) {
2219 int oldfd;
2220
2221 err = -EFAULT;
2222 if (!get_user(oldfd, (__u32 __user *) arg)) {
2223 struct file *old = fget(oldfd);
2224
2225 err = -EINVAL;
2226 if (old) {
Jann Horn8ed1f0e2015-08-16 20:27:01 +02002227 struct fuse_dev *fud = NULL;
2228
2229 /*
2230 * Check against file->f_op because CUSE
2231 * uses the same ioctl handler.
2232 */
2233 if (old->f_op == file->f_op &&
2234 old->f_cred->user_ns == file->f_cred->user_ns)
2235 fud = fuse_get_dev(old);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002236
Miklos Szeredicc080e92015-07-01 16:26:08 +02002237 if (fud) {
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002238 mutex_lock(&fuse_mutex);
Miklos Szeredicc080e92015-07-01 16:26:08 +02002239 err = fuse_device_clone(fud->fc, file);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002240 mutex_unlock(&fuse_mutex);
2241 }
2242 fput(old);
2243 }
2244 }
2245 }
2246 return err;
2247}
2248
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -08002249const struct file_operations fuse_dev_operations = {
Miklos Szeredi334f4852005-09-09 13:10:27 -07002250 .owner = THIS_MODULE,
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01002251 .open = fuse_dev_open,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002252 .llseek = no_llseek,
Al Virofbdbacc2015-04-03 21:53:39 -04002253 .read_iter = fuse_dev_read,
Miklos Szeredic3021622010-05-25 15:06:07 +02002254 .splice_read = fuse_dev_splice_read,
Al Virofbdbacc2015-04-03 21:53:39 -04002255 .write_iter = fuse_dev_write,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002256 .splice_write = fuse_dev_splice_write,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002257 .poll = fuse_dev_poll,
2258 .release = fuse_dev_release,
Jeff Dike385a17b2006-04-10 22:54:52 -07002259 .fasync = fuse_dev_fasync,
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002260 .unlocked_ioctl = fuse_dev_ioctl,
2261 .compat_ioctl = fuse_dev_ioctl,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002262};
Tejun Heo08cbf542009-04-14 10:54:53 +09002263EXPORT_SYMBOL_GPL(fuse_dev_operations);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002264
2265static struct miscdevice fuse_miscdevice = {
2266 .minor = FUSE_MINOR,
2267 .name = "fuse",
2268 .fops = &fuse_dev_operations,
2269};
2270
2271int __init fuse_dev_init(void)
2272{
2273 int err = -ENOMEM;
2274 fuse_req_cachep = kmem_cache_create("fuse_request",
2275 sizeof(struct fuse_req),
Paul Mundt20c2df82007-07-20 10:11:58 +09002276 0, 0, NULL);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002277 if (!fuse_req_cachep)
2278 goto out;
2279
2280 err = misc_register(&fuse_miscdevice);
2281 if (err)
2282 goto out_cache_clean;
2283
2284 return 0;
2285
2286 out_cache_clean:
2287 kmem_cache_destroy(fuse_req_cachep);
2288 out:
2289 return err;
2290}
2291
2292void fuse_dev_cleanup(void)
2293{
2294 misc_deregister(&fuse_miscdevice);
2295 kmem_cache_destroy(fuse_req_cachep);
2296}