blob: b681b43c766e11daf45814fc694b4b988f218a43 [file] [log] [blame]
Miklos Szeredi334f4852005-09-09 13:10:27 -07001/*
2 FUSE: Filesystem in Userspace
Miklos Szeredi1729a162008-11-26 12:03:54 +01003 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
Miklos Szeredi334f4852005-09-09 13:10:27 -07004
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7*/
8
9#include "fuse_i.h"
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/poll.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010014#include <linux/sched/signal.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070015#include <linux/uio.h>
16#include <linux/miscdevice.h>
17#include <linux/pagemap.h>
18#include <linux/file.h>
19#include <linux/slab.h>
Miklos Szeredidd3bb142010-05-25 15:06:06 +020020#include <linux/pipe_fs_i.h>
Miklos Szeredice534fb2010-05-25 15:06:07 +020021#include <linux/swap.h>
22#include <linux/splice.h>
Miklos Szeredi334f4852005-09-09 13:10:27 -070023
24MODULE_ALIAS_MISCDEV(FUSE_MINOR);
Kay Sievers578454f2010-05-20 18:07:20 +020025MODULE_ALIAS("devname:fuse");
Miklos Szeredi334f4852005-09-09 13:10:27 -070026
Christoph Lametere18b8902006-12-06 20:33:20 -080027static struct kmem_cache *fuse_req_cachep;
Miklos Szeredi334f4852005-09-09 13:10:27 -070028
Miklos Szeredicc080e92015-07-01 16:26:08 +020029static struct fuse_dev *fuse_get_dev(struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -070030{
Miklos Szeredi0720b312006-04-10 22:54:55 -070031 /*
32 * Lockless access is OK, because file->private data is set
33 * once during mount and is valid until the file is released.
34 */
Miklos Szeredicc080e92015-07-01 16:26:08 +020035 return ACCESS_ONCE(file->private_data);
Miklos Szeredi334f4852005-09-09 13:10:27 -070036}
37
Maxim Patlasov4250c062012-10-26 19:48:07 +040038static void fuse_request_init(struct fuse_req *req, struct page **pages,
Maxim Patlasovb2430d72012-10-26 19:49:24 +040039 struct fuse_page_desc *page_descs,
Maxim Patlasov4250c062012-10-26 19:48:07 +040040 unsigned npages)
Miklos Szeredi334f4852005-09-09 13:10:27 -070041{
42 memset(req, 0, sizeof(*req));
Maxim Patlasov4250c062012-10-26 19:48:07 +040043 memset(pages, 0, sizeof(*pages) * npages);
Maxim Patlasovb2430d72012-10-26 19:49:24 +040044 memset(page_descs, 0, sizeof(*page_descs) * npages);
Miklos Szeredi334f4852005-09-09 13:10:27 -070045 INIT_LIST_HEAD(&req->list);
Miklos Szeredia4d27e72006-06-25 05:48:54 -070046 INIT_LIST_HEAD(&req->intr_entry);
Miklos Szeredi334f4852005-09-09 13:10:27 -070047 init_waitqueue_head(&req->waitq);
48 atomic_set(&req->count, 1);
Maxim Patlasov4250c062012-10-26 19:48:07 +040049 req->pages = pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040050 req->page_descs = page_descs;
Maxim Patlasov4250c062012-10-26 19:48:07 +040051 req->max_pages = npages;
Miklos Szeredi33e14b42015-07-01 16:26:01 +020052 __set_bit(FR_PENDING, &req->flags);
Miklos Szeredi334f4852005-09-09 13:10:27 -070053}
54
Maxim Patlasov4250c062012-10-26 19:48:07 +040055static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
Miklos Szeredi334f4852005-09-09 13:10:27 -070056{
Maxim Patlasov4250c062012-10-26 19:48:07 +040057 struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
58 if (req) {
59 struct page **pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040060 struct fuse_page_desc *page_descs;
Maxim Patlasov4250c062012-10-26 19:48:07 +040061
Maxim Patlasovb2430d72012-10-26 19:49:24 +040062 if (npages <= FUSE_REQ_INLINE_PAGES) {
Maxim Patlasov4250c062012-10-26 19:48:07 +040063 pages = req->inline_pages;
Maxim Patlasovb2430d72012-10-26 19:49:24 +040064 page_descs = req->inline_page_descs;
65 } else {
Maxim Patlasov4250c062012-10-26 19:48:07 +040066 pages = kmalloc(sizeof(struct page *) * npages, flags);
Maxim Patlasovb2430d72012-10-26 19:49:24 +040067 page_descs = kmalloc(sizeof(struct fuse_page_desc) *
68 npages, flags);
69 }
Maxim Patlasov4250c062012-10-26 19:48:07 +040070
Maxim Patlasovb2430d72012-10-26 19:49:24 +040071 if (!pages || !page_descs) {
72 kfree(pages);
73 kfree(page_descs);
Maxim Patlasov4250c062012-10-26 19:48:07 +040074 kmem_cache_free(fuse_req_cachep, req);
75 return NULL;
76 }
77
Maxim Patlasovb2430d72012-10-26 19:49:24 +040078 fuse_request_init(req, pages, page_descs, npages);
Maxim Patlasov4250c062012-10-26 19:48:07 +040079 }
Miklos Szeredi334f4852005-09-09 13:10:27 -070080 return req;
81}
Maxim Patlasov4250c062012-10-26 19:48:07 +040082
83struct fuse_req *fuse_request_alloc(unsigned npages)
84{
85 return __fuse_request_alloc(npages, GFP_KERNEL);
86}
Tejun Heo08cbf542009-04-14 10:54:53 +090087EXPORT_SYMBOL_GPL(fuse_request_alloc);
Miklos Szeredi334f4852005-09-09 13:10:27 -070088
Maxim Patlasov4250c062012-10-26 19:48:07 +040089struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
Miklos Szeredi3be5a522008-04-30 00:54:41 -070090{
Maxim Patlasov4250c062012-10-26 19:48:07 +040091 return __fuse_request_alloc(npages, GFP_NOFS);
Miklos Szeredi3be5a522008-04-30 00:54:41 -070092}
93
Miklos Szeredi334f4852005-09-09 13:10:27 -070094void fuse_request_free(struct fuse_req *req)
95{
Maxim Patlasovb2430d72012-10-26 19:49:24 +040096 if (req->pages != req->inline_pages) {
Maxim Patlasov4250c062012-10-26 19:48:07 +040097 kfree(req->pages);
Maxim Patlasovb2430d72012-10-26 19:49:24 +040098 kfree(req->page_descs);
99 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700100 kmem_cache_free(fuse_req_cachep, req);
101}
102
Maxim Patlasov36cf66e2012-12-14 19:20:51 +0400103void __fuse_get_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700104{
105 atomic_inc(&req->count);
106}
107
108/* Must be called with > 1 refcount */
109static void __fuse_put_request(struct fuse_req *req)
110{
111 BUG_ON(atomic_read(&req->count) < 2);
112 atomic_dec(&req->count);
113}
114
Miklos Szeredi33649c92006-06-25 05:48:52 -0700115static void fuse_req_init_context(struct fuse_req *req)
116{
Eric W. Biederman499dcf22012-02-07 16:26:03 -0800117 req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
118 req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
Miklos Szeredi33649c92006-06-25 05:48:52 -0700119 req->in.h.pid = current->pid;
120}
121
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100122void fuse_set_initialized(struct fuse_conn *fc)
123{
124 /* Make sure stores before this are seen on another CPU */
125 smp_wmb();
126 fc->initialized = 1;
127}
128
Maxim Patlasov0aada882013-03-21 18:02:28 +0400129static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
130{
131 return !fc->initialized || (for_background && fc->blocked);
132}
133
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400134static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
135 bool for_background)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700136{
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700137 struct fuse_req *req;
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700138 int err;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200139 atomic_inc(&fc->num_waiting);
Maxim Patlasov0aada882013-03-21 18:02:28 +0400140
141 if (fuse_block_alloc(fc, for_background)) {
Maxim Patlasov0aada882013-03-21 18:02:28 +0400142 err = -EINTR;
Al Viro7d3a07f2016-07-19 03:08:27 -0400143 if (wait_event_killable_exclusive(fc->blocked_waitq,
144 !fuse_block_alloc(fc, for_background)))
Maxim Patlasov0aada882013-03-21 18:02:28 +0400145 goto out;
146 }
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100147 /* Matches smp_wmb() in fuse_set_initialized() */
148 smp_rmb();
Miklos Szeredi08a53cd2006-04-10 22:54:59 -0700149
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700150 err = -ENOTCONN;
151 if (!fc->connected)
152 goto out;
153
Miklos Szeredide155222015-07-01 16:25:57 +0200154 err = -ECONNREFUSED;
155 if (fc->conn_error)
156 goto out;
157
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400158 req = fuse_request_alloc(npages);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200159 err = -ENOMEM;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400160 if (!req) {
161 if (for_background)
162 wake_up(&fc->blocked_waitq);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200163 goto out;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400164 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700165
Miklos Szeredi33649c92006-06-25 05:48:52 -0700166 fuse_req_init_context(req);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200167 __set_bit(FR_WAITING, &req->flags);
168 if (for_background)
169 __set_bit(FR_BACKGROUND, &req->flags);
170
Miklos Szeredi334f4852005-09-09 13:10:27 -0700171 return req;
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200172
173 out:
174 atomic_dec(&fc->num_waiting);
175 return ERR_PTR(err);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700176}
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400177
178struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
179{
180 return __fuse_get_req(fc, npages, false);
181}
Tejun Heo08cbf542009-04-14 10:54:53 +0900182EXPORT_SYMBOL_GPL(fuse_get_req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700183
Maxim Patlasov8b41e672013-03-21 18:02:04 +0400184struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
185 unsigned npages)
186{
187 return __fuse_get_req(fc, npages, true);
188}
189EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
190
Miklos Szeredi33649c92006-06-25 05:48:52 -0700191/*
192 * Return request in fuse_file->reserved_req. However that may
193 * currently be in use. If that is the case, wait for it to become
194 * available.
195 */
196static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
197 struct file *file)
198{
199 struct fuse_req *req = NULL;
200 struct fuse_file *ff = file->private_data;
201
202 do {
Miklos Szeredide5e3de2007-10-16 23:31:00 -0700203 wait_event(fc->reserved_req_waitq, ff->reserved_req);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700204 spin_lock(&fc->lock);
205 if (ff->reserved_req) {
206 req = ff->reserved_req;
207 ff->reserved_req = NULL;
Al Virocb0942b2012-08-27 14:48:26 -0400208 req->stolen_file = get_file(file);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700209 }
210 spin_unlock(&fc->lock);
211 } while (!req);
212
213 return req;
214}
215
216/*
217 * Put stolen request back into fuse_file->reserved_req
218 */
219static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
220{
221 struct file *file = req->stolen_file;
222 struct fuse_file *ff = file->private_data;
223
224 spin_lock(&fc->lock);
Maxim Patlasovb2430d72012-10-26 19:49:24 +0400225 fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700226 BUG_ON(ff->reserved_req);
227 ff->reserved_req = req;
Miklos Szeredide5e3de2007-10-16 23:31:00 -0700228 wake_up_all(&fc->reserved_req_waitq);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700229 spin_unlock(&fc->lock);
230 fput(file);
231}
232
233/*
234 * Gets a requests for a file operation, always succeeds
235 *
236 * This is used for sending the FLUSH request, which must get to
237 * userspace, due to POSIX locks which may need to be unlocked.
238 *
239 * If allocation fails due to OOM, use the reserved request in
240 * fuse_file.
241 *
242 * This is very unlikely to deadlock accidentally, since the
243 * filesystem should not have it's own file open. If deadlock is
244 * intentional, it can still be broken by "aborting" the filesystem.
245 */
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400246struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
247 struct file *file)
Miklos Szeredi33649c92006-06-25 05:48:52 -0700248{
249 struct fuse_req *req;
250
251 atomic_inc(&fc->num_waiting);
Maxim Patlasov0aada882013-03-21 18:02:28 +0400252 wait_event(fc->blocked_waitq, fc->initialized);
Miklos Szeredi9759bd512015-01-06 10:45:35 +0100253 /* Matches smp_wmb() in fuse_set_initialized() */
254 smp_rmb();
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400255 req = fuse_request_alloc(0);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700256 if (!req)
257 req = get_reserved_req(fc, file);
258
259 fuse_req_init_context(req);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200260 __set_bit(FR_WAITING, &req->flags);
261 __clear_bit(FR_BACKGROUND, &req->flags);
Miklos Szeredi33649c92006-06-25 05:48:52 -0700262 return req;
263}
264
Miklos Szeredi334f4852005-09-09 13:10:27 -0700265void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
266{
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800267 if (atomic_dec_and_test(&req->count)) {
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200268 if (test_bit(FR_BACKGROUND, &req->flags)) {
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400269 /*
270 * We get here in the unlikely case that a background
271 * request was allocated but not sent
272 */
273 spin_lock(&fc->lock);
274 if (!fc->blocked)
275 wake_up(&fc->blocked_waitq);
276 spin_unlock(&fc->lock);
277 }
278
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200279 if (test_bit(FR_WAITING, &req->flags)) {
280 __clear_bit(FR_WAITING, &req->flags);
Miklos Szeredi9bc5ddd2006-04-11 21:16:09 +0200281 atomic_dec(&fc->num_waiting);
Miklos Szeredi73e0e732015-07-01 16:25:56 +0200282 }
Miklos Szeredi33649c92006-06-25 05:48:52 -0700283
284 if (req->stolen_file)
285 put_reserved_req(fc, req);
286 else
287 fuse_request_free(req);
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800288 }
289}
Tejun Heo08cbf542009-04-14 10:54:53 +0900290EXPORT_SYMBOL_GPL(fuse_put_request);
Miklos Szeredi7128ec22006-02-04 23:27:40 -0800291
Miklos Szeredid12def12008-02-06 01:38:39 -0800292static unsigned len_args(unsigned numargs, struct fuse_arg *args)
293{
294 unsigned nbytes = 0;
295 unsigned i;
296
297 for (i = 0; i < numargs; i++)
298 nbytes += args[i].size;
299
300 return nbytes;
301}
302
Miklos Szeredif88996a2015-07-01 16:26:01 +0200303static u64 fuse_get_unique(struct fuse_iqueue *fiq)
Miklos Szeredid12def12008-02-06 01:38:39 -0800304{
Miklos Szeredif88996a2015-07-01 16:26:01 +0200305 return ++fiq->reqctr;
Miklos Szeredid12def12008-02-06 01:38:39 -0800306}
307
Miklos Szeredif88996a2015-07-01 16:26:01 +0200308static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
Miklos Szeredid12def12008-02-06 01:38:39 -0800309{
Miklos Szeredid12def12008-02-06 01:38:39 -0800310 req->in.h.len = sizeof(struct fuse_in_header) +
311 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200312 list_add_tail(&req->list, &fiq->pending);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200313 wake_up_locked(&fiq->waitq);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200314 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredid12def12008-02-06 01:38:39 -0800315}
316
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100317void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
318 u64 nodeid, u64 nlookup)
319{
Miklos Szeredif88996a2015-07-01 16:26:01 +0200320 struct fuse_iqueue *fiq = &fc->iq;
321
Miklos Szeredi02c048b2010-12-07 20:16:56 +0100322 forget->forget_one.nodeid = nodeid;
323 forget->forget_one.nlookup = nlookup;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100324
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200325 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200326 if (fiq->connected) {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200327 fiq->forget_list_tail->next = forget;
328 fiq->forget_list_tail = forget;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200329 wake_up_locked(&fiq->waitq);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200330 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredi5dfcc872011-09-12 09:38:03 +0200331 } else {
332 kfree(forget);
333 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200334 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +0100335}
336
Miklos Szeredid12def12008-02-06 01:38:39 -0800337static void flush_bg_queue(struct fuse_conn *fc)
338{
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700339 while (fc->active_background < fc->max_background &&
Miklos Szeredid12def12008-02-06 01:38:39 -0800340 !list_empty(&fc->bg_queue)) {
341 struct fuse_req *req;
Miklos Szeredif88996a2015-07-01 16:26:01 +0200342 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredid12def12008-02-06 01:38:39 -0800343
344 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
345 list_del(&req->list);
346 fc->active_background++;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200347 spin_lock(&fiq->waitq.lock);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200348 req->in.h.unique = fuse_get_unique(fiq);
349 queue_request(fiq, req);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200350 spin_unlock(&fiq->waitq.lock);
Miklos Szeredid12def12008-02-06 01:38:39 -0800351 }
352}
353
Miklos Szeredi6dbbcb12006-04-26 10:49:06 +0200354/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700355 * This function is called when a request is finished. Either a reply
Miklos Szeredif9a28422006-06-25 05:48:53 -0700356 * has arrived or it was aborted (and not yet sent) or some error
Miklos Szeredif43b1552006-01-16 22:14:26 -0800357 * occurred during communication with userspace, or the device file
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700358 * was closed. The requester thread is woken up (if still waiting),
359 * the 'end' callback is called if given, else the reference to the
360 * request is released
Miklos Szeredi334f4852005-09-09 13:10:27 -0700361 */
362static void request_end(struct fuse_conn *fc, struct fuse_req *req)
363{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200364 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi365ae712015-07-01 16:26:06 +0200365
Miklos Szerediefe28002015-07-01 16:26:07 +0200366 if (test_and_set_bit(FR_FINISHED, &req->flags))
Miklos Szeredi365ae712015-07-01 16:26:06 +0200367 return;
Miklos Szeredi365ae712015-07-01 16:26:06 +0200368
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200369 spin_lock(&fiq->waitq.lock);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200370 list_del_init(&req->intr_entry);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200371 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200372 WARN_ON(test_bit(FR_PENDING, &req->flags));
373 WARN_ON(test_bit(FR_SENT, &req->flags));
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200374 if (test_bit(FR_BACKGROUND, &req->flags)) {
Miklos Szerediefe28002015-07-01 16:26:07 +0200375 spin_lock(&fc->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200376 clear_bit(FR_BACKGROUND, &req->flags);
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400377 if (fc->num_background == fc->max_background)
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700378 fc->blocked = 0;
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400379
380 /* Wake up next waiter, if any */
Miklos Szeredi3c18ef82013-04-17 21:50:58 +0200381 if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
Maxim Patlasov722d2be2013-03-21 18:02:36 +0400382 wake_up(&fc->blocked_waitq);
383
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700384 if (fc->num_background == fc->congestion_threshold &&
Tejun Heoa325f9b2009-04-14 10:54:52 +0900385 fc->connected && fc->bdi_initialized) {
Jens Axboe8aa7e842009-07-09 14:52:32 +0200386 clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
387 clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
Miklos Szeredif92b99b2007-10-16 23:30:59 -0700388 }
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700389 fc->num_background--;
Miklos Szeredid12def12008-02-06 01:38:39 -0800390 fc->active_background--;
391 flush_bg_queue(fc);
Miklos Szerediefe28002015-07-01 16:26:07 +0200392 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700393 }
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700394 wake_up(&req->waitq);
Miklos Szeredi1e6881c2015-07-01 16:26:07 +0200395 if (req->end)
396 req->end(fc, req);
Tejun Heoe9bb09d2008-11-26 12:03:54 +0100397 fuse_put_request(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700398}
399
Miklos Szeredif88996a2015-07-01 16:26:01 +0200400static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700401{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200402 spin_lock(&fiq->waitq.lock);
Sahitya Tummala6ba4d272017-02-08 20:30:56 +0530403 if (test_bit(FR_FINISHED, &req->flags)) {
404 spin_unlock(&fiq->waitq.lock);
405 return;
406 }
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200407 if (list_empty(&req->intr_entry)) {
408 list_add_tail(&req->intr_entry, &fiq->interrupts);
409 wake_up_locked(&fiq->waitq);
410 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200411 spin_unlock(&fiq->waitq.lock);
Miklos Szeredif88996a2015-07-01 16:26:01 +0200412 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700413}
414
Miklos Szeredi7c352bd2005-09-09 13:10:39 -0700415static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700416{
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200417 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic4775262015-07-01 16:26:00 +0200418 int err;
419
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700420 if (!fc->no_interrupt) {
421 /* Any signal may interrupt this */
Miklos Szeredic4775262015-07-01 16:26:00 +0200422 err = wait_event_interruptible(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200423 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200424 if (!err)
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700425 return;
426
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200427 set_bit(FR_INTERRUPTED, &req->flags);
Miklos Szeredi8f7bb362015-07-01 16:26:03 +0200428 /* matches barrier in fuse_dev_do_read() */
429 smp_mb__after_atomic();
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200430 if (test_bit(FR_SENT, &req->flags))
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200431 queue_interrupt(fiq, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700432 }
433
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200434 if (!test_bit(FR_FORCE, &req->flags)) {
Miklos Szeredia4d27e72006-06-25 05:48:54 -0700435 /* Only fatal signals may interrupt this */
Al Viro7d3a07f2016-07-19 03:08:27 -0400436 err = wait_event_killable(req->waitq,
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200437 test_bit(FR_FINISHED, &req->flags));
Miklos Szeredic4775262015-07-01 16:26:00 +0200438 if (!err)
Miklos Szeredia131de02007-10-16 23:31:04 -0700439 return;
440
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200441 spin_lock(&fiq->waitq.lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700442 /* Request is not yet in userspace, bail out */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200443 if (test_bit(FR_PENDING, &req->flags)) {
Miklos Szeredia131de02007-10-16 23:31:04 -0700444 list_del(&req->list);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200445 spin_unlock(&fiq->waitq.lock);
Miklos Szeredia131de02007-10-16 23:31:04 -0700446 __fuse_put_request(req);
447 req->out.h.error = -EINTR;
448 return;
449 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200450 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi51eb01e2006-06-25 05:48:50 -0700451 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700452
Miklos Szeredia131de02007-10-16 23:31:04 -0700453 /*
454 * Either request is already in userspace, or it was forced.
455 * Wait it out.
456 */
Miklos Szeredi33e14b42015-07-01 16:26:01 +0200457 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
Miklos Szeredi334f4852005-09-09 13:10:27 -0700458}
459
Eric Wong6a4e9222013-02-04 13:04:44 +0000460static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700461{
Miklos Szeredie16714d2015-07-01 16:26:01 +0200462 struct fuse_iqueue *fiq = &fc->iq;
463
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200464 BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200465 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200466 if (!fiq->connected) {
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200467 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700468 req->out.h.error = -ENOTCONN;
Miklos Szeredic4775262015-07-01 16:26:00 +0200469 } else {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200470 req->in.h.unique = fuse_get_unique(fiq);
471 queue_request(fiq, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700472 /* acquire extra reference, since request is still needed
473 after request_end() */
474 __fuse_get_request(req);
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200475 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700476
Miklos Szeredi7c352bd2005-09-09 13:10:39 -0700477 request_wait_answer(fc, req);
Miklos Szeredic4775262015-07-01 16:26:00 +0200478 /* Pairs with smp_wmb() in request_end() */
479 smp_rmb();
Miklos Szeredi334f4852005-09-09 13:10:27 -0700480 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700481}
Eric Wong6a4e9222013-02-04 13:04:44 +0000482
483void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
484{
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200485 __set_bit(FR_ISREPLY, &req->flags);
486 if (!test_bit(FR_WAITING, &req->flags)) {
487 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredi5437f242015-07-01 16:25:56 +0200488 atomic_inc(&fc->num_waiting);
489 }
Eric Wong6a4e9222013-02-04 13:04:44 +0000490 __fuse_request_send(fc, req);
491}
Tejun Heo08cbf542009-04-14 10:54:53 +0900492EXPORT_SYMBOL_GPL(fuse_request_send);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700493
Miklos Szeredi21f62172015-01-06 10:45:35 +0100494static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
495{
496 if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
497 args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;
498
499 if (fc->minor < 9) {
500 switch (args->in.h.opcode) {
501 case FUSE_LOOKUP:
502 case FUSE_CREATE:
503 case FUSE_MKNOD:
504 case FUSE_MKDIR:
505 case FUSE_SYMLINK:
506 case FUSE_LINK:
507 args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
508 break;
509 case FUSE_GETATTR:
510 case FUSE_SETATTR:
511 args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
512 break;
513 }
514 }
515 if (fc->minor < 12) {
516 switch (args->in.h.opcode) {
517 case FUSE_CREATE:
518 args->in.args[0].size = sizeof(struct fuse_open_in);
519 break;
520 case FUSE_MKNOD:
521 args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
522 break;
523 }
524 }
525}
526
Miklos Szeredi70781872014-12-12 09:49:05 +0100527ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
528{
529 struct fuse_req *req;
530 ssize_t ret;
531
532 req = fuse_get_req(fc, 0);
533 if (IS_ERR(req))
534 return PTR_ERR(req);
535
Miklos Szeredi21f62172015-01-06 10:45:35 +0100536 /* Needs to be done after fuse_get_req() so that fc->minor is valid */
537 fuse_adjust_compat(fc, args);
538
Miklos Szeredi70781872014-12-12 09:49:05 +0100539 req->in.h.opcode = args->in.h.opcode;
540 req->in.h.nodeid = args->in.h.nodeid;
541 req->in.numargs = args->in.numargs;
542 memcpy(req->in.args, args->in.args,
543 args->in.numargs * sizeof(struct fuse_in_arg));
544 req->out.argvar = args->out.argvar;
545 req->out.numargs = args->out.numargs;
546 memcpy(req->out.args, args->out.args,
547 args->out.numargs * sizeof(struct fuse_arg));
548 fuse_request_send(fc, req);
549 ret = req->out.h.error;
550 if (!ret && args->out.argvar) {
551 BUG_ON(args->out.numargs != 1);
552 ret = req->out.args[0].size;
553 }
554 fuse_put_request(fc, req);
555
556 return ret;
557}
558
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200559/*
560 * Called under fc->lock
561 *
562 * fc->connected must have been checked previously
563 */
564void fuse_request_send_background_locked(struct fuse_conn *fc,
565 struct fuse_req *req)
Miklos Szeredid12def12008-02-06 01:38:39 -0800566{
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200567 BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
568 if (!test_bit(FR_WAITING, &req->flags)) {
569 __set_bit(FR_WAITING, &req->flags);
Miklos Szeredi5437f242015-07-01 16:25:56 +0200570 atomic_inc(&fc->num_waiting);
571 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200572 __set_bit(FR_ISREPLY, &req->flags);
Miklos Szeredid12def12008-02-06 01:38:39 -0800573 fc->num_background++;
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700574 if (fc->num_background == fc->max_background)
Miklos Szeredid12def12008-02-06 01:38:39 -0800575 fc->blocked = 1;
Csaba Henk7a6d3c82009-07-01 17:28:41 -0700576 if (fc->num_background == fc->congestion_threshold &&
Tejun Heoa325f9b2009-04-14 10:54:52 +0900577 fc->bdi_initialized) {
Jens Axboe8aa7e842009-07-09 14:52:32 +0200578 set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
579 set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
Miklos Szeredid12def12008-02-06 01:38:39 -0800580 }
581 list_add_tail(&req->list, &fc->bg_queue);
582 flush_bg_queue(fc);
583}
584
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200585void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700586{
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200587 BUG_ON(!req->end);
Miklos Szeredid7133112006-04-10 22:54:55 -0700588 spin_lock(&fc->lock);
Miklos Szeredi1e9a4ed2005-09-09 13:10:31 -0700589 if (fc->connected) {
Miklos Szeredif0139aa2015-07-01 16:25:57 +0200590 fuse_request_send_background_locked(fc, req);
Miklos Szeredid7133112006-04-10 22:54:55 -0700591 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700592 } else {
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200593 spin_unlock(&fc->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700594 req->out.h.error = -ENOTCONN;
Miklos Szeredi42dc6212015-07-01 16:25:56 +0200595 req->end(fc, req);
596 fuse_put_request(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700597 }
598}
Tejun Heo08cbf542009-04-14 10:54:53 +0900599EXPORT_SYMBOL_GPL(fuse_request_send_background);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700600
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200601static int fuse_request_send_notify_reply(struct fuse_conn *fc,
602 struct fuse_req *req, u64 unique)
603{
604 int err = -ENODEV;
Miklos Szeredif88996a2015-07-01 16:26:01 +0200605 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200606
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200607 __clear_bit(FR_ISREPLY, &req->flags);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200608 req->in.h.unique = unique;
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200609 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +0200610 if (fiq->connected) {
Miklos Szeredif88996a2015-07-01 16:26:01 +0200611 queue_request(fiq, req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200612 err = 0;
613 }
Miklos Szeredi4ce60812015-07-01 16:26:02 +0200614 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +0200615
616 return err;
617}
618
Anand V. Avati0b05b182012-08-19 08:53:23 -0400619void fuse_force_forget(struct file *file, u64 nodeid)
620{
Al Viro6131ffa2013-02-27 16:59:05 -0500621 struct inode *inode = file_inode(file);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400622 struct fuse_conn *fc = get_fuse_conn(inode);
623 struct fuse_req *req;
624 struct fuse_forget_in inarg;
625
626 memset(&inarg, 0, sizeof(inarg));
627 inarg.nlookup = 1;
Maxim Patlasovb111c8c2012-10-26 19:48:30 +0400628 req = fuse_get_req_nofail_nopages(fc, file);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400629 req->in.h.opcode = FUSE_FORGET;
630 req->in.h.nodeid = nodeid;
631 req->in.numargs = 1;
632 req->in.args[0].size = sizeof(inarg);
633 req->in.args[0].value = &inarg;
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200634 __clear_bit(FR_ISREPLY, &req->flags);
Eric Wong6a4e9222013-02-04 13:04:44 +0000635 __fuse_request_send(fc, req);
636 /* ignore errors */
637 fuse_put_request(fc, req);
Anand V. Avati0b05b182012-08-19 08:53:23 -0400638}
639
Miklos Szeredi3be5a522008-04-30 00:54:41 -0700640/*
Miklos Szeredi334f4852005-09-09 13:10:27 -0700641 * Lock the request. Up to the next unlock_request() there mustn't be
642 * anything that could cause a page-fault. If the request was already
Miklos Szeredif9a28422006-06-25 05:48:53 -0700643 * aborted bail out.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700644 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200645static int lock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700646{
647 int err = 0;
648 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200649 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200650 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi334f4852005-09-09 13:10:27 -0700651 err = -ENOENT;
652 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200653 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200654 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700655 }
656 return err;
657}
658
659/*
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200660 * Unlock request. If it was aborted while locked, caller is responsible
661 * for unlocking and ending the request.
Miklos Szeredi334f4852005-09-09 13:10:27 -0700662 */
Miklos Szeredidc008092015-07-01 16:25:58 +0200663static int unlock_request(struct fuse_req *req)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700664{
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200665 int err = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700666 if (req) {
Miklos Szeredidc008092015-07-01 16:25:58 +0200667 spin_lock(&req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200668 if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200669 err = -ENOENT;
670 else
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200671 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredidc008092015-07-01 16:25:58 +0200672 spin_unlock(&req->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700673 }
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200674 return err;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700675}
676
677struct fuse_copy_state {
678 int write;
679 struct fuse_req *req;
Al Viro6c09e942015-04-03 22:06:08 -0400680 struct iov_iter *iter;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200681 struct pipe_buffer *pipebufs;
682 struct pipe_buffer *currbuf;
683 struct pipe_inode_info *pipe;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700684 unsigned long nr_segs;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700685 struct page *pg;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700686 unsigned len;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200687 unsigned offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200688 unsigned move_pages:1;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700689};
690
Miklos Szeredidc008092015-07-01 16:25:58 +0200691static void fuse_copy_init(struct fuse_copy_state *cs, int write,
Al Viro6c09e942015-04-03 22:06:08 -0400692 struct iov_iter *iter)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700693{
694 memset(cs, 0, sizeof(*cs));
695 cs->write = write;
Al Viro6c09e942015-04-03 22:06:08 -0400696 cs->iter = iter;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700697}
698
699/* Unmap and put previous page of userspace buffer */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800700static void fuse_copy_finish(struct fuse_copy_state *cs)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700701{
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200702 if (cs->currbuf) {
703 struct pipe_buffer *buf = cs->currbuf;
704
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200705 if (cs->write)
Miklos Szeredic3021622010-05-25 15:06:07 +0200706 buf->len = PAGE_SIZE - cs->len;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200707 cs->currbuf = NULL;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200708 } else if (cs->pg) {
Miklos Szeredi334f4852005-09-09 13:10:27 -0700709 if (cs->write) {
710 flush_dcache_page(cs->pg);
711 set_page_dirty_lock(cs->pg);
712 }
713 put_page(cs->pg);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700714 }
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200715 cs->pg = NULL;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700716}
717
718/*
719 * Get another pagefull of userspace buffer, and map it to kernel
720 * address space, and lock request
721 */
722static int fuse_copy_fill(struct fuse_copy_state *cs)
723{
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200724 struct page *page;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700725 int err;
726
Miklos Szeredidc008092015-07-01 16:25:58 +0200727 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200728 if (err)
729 return err;
730
Miklos Szeredi334f4852005-09-09 13:10:27 -0700731 fuse_copy_finish(cs);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200732 if (cs->pipebufs) {
733 struct pipe_buffer *buf = cs->pipebufs;
734
Miklos Szeredic3021622010-05-25 15:06:07 +0200735 if (!cs->write) {
Miklos Szeredifba597d2016-09-27 10:45:12 +0200736 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredic3021622010-05-25 15:06:07 +0200737 if (err)
738 return err;
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200739
Miklos Szeredic3021622010-05-25 15:06:07 +0200740 BUG_ON(!cs->nr_segs);
741 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200742 cs->pg = buf->page;
743 cs->offset = buf->offset;
Miklos Szeredic3021622010-05-25 15:06:07 +0200744 cs->len = buf->len;
Miklos Szeredic3021622010-05-25 15:06:07 +0200745 cs->pipebufs++;
746 cs->nr_segs--;
747 } else {
Miklos Szeredic3021622010-05-25 15:06:07 +0200748 if (cs->nr_segs == cs->pipe->buffers)
749 return -EIO;
750
751 page = alloc_page(GFP_HIGHUSER);
752 if (!page)
753 return -ENOMEM;
754
755 buf->page = page;
756 buf->offset = 0;
757 buf->len = 0;
758
759 cs->currbuf = buf;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200760 cs->pg = page;
761 cs->offset = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +0200762 cs->len = PAGE_SIZE;
763 cs->pipebufs++;
764 cs->nr_segs++;
765 }
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200766 } else {
Al Viro6c09e942015-04-03 22:06:08 -0400767 size_t off;
768 err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
Miklos Szeredidd3bb142010-05-25 15:06:06 +0200769 if (err < 0)
770 return err;
Al Viro6c09e942015-04-03 22:06:08 -0400771 BUG_ON(!err);
772 cs->len = err;
773 cs->offset = off;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200774 cs->pg = page;
Al Viro6c09e942015-04-03 22:06:08 -0400775 iov_iter_advance(cs->iter, err);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700776 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700777
Miklos Szeredidc008092015-07-01 16:25:58 +0200778 return lock_request(cs->req);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700779}
780
781/* Do as much copy to/from userspace buffer as we can */
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800782static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700783{
784 unsigned ncpy = min(*size, cs->len);
785 if (val) {
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200786 void *pgaddr = kmap_atomic(cs->pg);
787 void *buf = pgaddr + cs->offset;
788
Miklos Szeredi334f4852005-09-09 13:10:27 -0700789 if (cs->write)
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200790 memcpy(buf, *val, ncpy);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700791 else
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200792 memcpy(*val, buf, ncpy);
793
794 kunmap_atomic(pgaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700795 *val += ncpy;
796 }
797 *size -= ncpy;
798 cs->len -= ncpy;
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200799 cs->offset += ncpy;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700800 return ncpy;
801}
802
Miklos Szeredice534fb2010-05-25 15:06:07 +0200803static int fuse_check_page(struct page *page)
804{
805 if (page_mapcount(page) ||
806 page->mapping != NULL ||
807 page_count(page) != 1 ||
808 (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
809 ~(1 << PG_locked |
810 1 << PG_referenced |
811 1 << PG_uptodate |
812 1 << PG_lru |
813 1 << PG_active |
814 1 << PG_reclaim))) {
815 printk(KERN_WARNING "fuse: trying to steal weird page\n");
816 printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
817 return 1;
818 }
819 return 0;
820}
821
822static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
823{
824 int err;
825 struct page *oldpage = *pagep;
826 struct page *newpage;
827 struct pipe_buffer *buf = cs->pipebufs;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200828
Miklos Szeredidc008092015-07-01 16:25:58 +0200829 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200830 if (err)
831 return err;
832
Miklos Szeredice534fb2010-05-25 15:06:07 +0200833 fuse_copy_finish(cs);
834
Miklos Szeredifba597d2016-09-27 10:45:12 +0200835 err = pipe_buf_confirm(cs->pipe, buf);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200836 if (err)
837 return err;
838
839 BUG_ON(!cs->nr_segs);
840 cs->currbuf = buf;
841 cs->len = buf->len;
842 cs->pipebufs++;
843 cs->nr_segs--;
844
845 if (cs->len != PAGE_SIZE)
846 goto out_fallback;
847
Miklos Szeredica76f5b2016-09-27 10:45:12 +0200848 if (pipe_buf_steal(cs->pipe, buf) != 0)
Miklos Szeredice534fb2010-05-25 15:06:07 +0200849 goto out_fallback;
850
851 newpage = buf->page;
852
Miklos Szerediaa991b32015-02-26 11:45:47 +0100853 if (!PageUptodate(newpage))
854 SetPageUptodate(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200855
856 ClearPageMappedToDisk(newpage);
857
858 if (fuse_check_page(newpage) != 0)
859 goto out_fallback_unlock;
860
Miklos Szeredice534fb2010-05-25 15:06:07 +0200861 /*
862 * This is a new and locked page, it shouldn't be mapped or
863 * have any special flags on it
864 */
865 if (WARN_ON(page_mapped(oldpage)))
866 goto out_fallback_unlock;
867 if (WARN_ON(page_has_private(oldpage)))
868 goto out_fallback_unlock;
869 if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
870 goto out_fallback_unlock;
871 if (WARN_ON(PageMlocked(oldpage)))
872 goto out_fallback_unlock;
873
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700874 err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200875 if (err) {
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700876 unlock_page(newpage);
877 return err;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200878 }
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700879
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300880 get_page(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200881
882 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
883 lru_cache_add_file(newpage);
884
885 err = 0;
Miklos Szeredidc008092015-07-01 16:25:58 +0200886 spin_lock(&cs->req->waitq.lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +0200887 if (test_bit(FR_ABORTED, &cs->req->flags))
Miklos Szeredice534fb2010-05-25 15:06:07 +0200888 err = -ENOENT;
889 else
890 *pagep = newpage;
Miklos Szeredidc008092015-07-01 16:25:58 +0200891 spin_unlock(&cs->req->waitq.lock);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200892
893 if (err) {
894 unlock_page(newpage);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300895 put_page(newpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200896 return err;
897 }
898
899 unlock_page(oldpage);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300900 put_page(oldpage);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200901 cs->len = 0;
902
903 return 0;
904
905out_fallback_unlock:
906 unlock_page(newpage);
907out_fallback:
Miklos Szeredic55a01d2014-07-07 15:28:51 +0200908 cs->pg = buf->page;
909 cs->offset = buf->offset;
Miklos Szeredice534fb2010-05-25 15:06:07 +0200910
Miklos Szeredidc008092015-07-01 16:25:58 +0200911 err = lock_request(cs->req);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200912 if (err)
913 return err;
914
915 return 1;
916}
917
Miklos Szeredic3021622010-05-25 15:06:07 +0200918static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
919 unsigned offset, unsigned count)
920{
921 struct pipe_buffer *buf;
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200922 int err;
Miklos Szeredic3021622010-05-25 15:06:07 +0200923
924 if (cs->nr_segs == cs->pipe->buffers)
925 return -EIO;
926
Miklos Szeredidc008092015-07-01 16:25:58 +0200927 err = unlock_request(cs->req);
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +0200928 if (err)
929 return err;
930
Miklos Szeredic3021622010-05-25 15:06:07 +0200931 fuse_copy_finish(cs);
932
933 buf = cs->pipebufs;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300934 get_page(page);
Miklos Szeredic3021622010-05-25 15:06:07 +0200935 buf->page = page;
936 buf->offset = offset;
937 buf->len = count;
938
939 cs->pipebufs++;
940 cs->nr_segs++;
941 cs->len = 0;
942
943 return 0;
944}
945
Miklos Szeredi334f4852005-09-09 13:10:27 -0700946/*
947 * Copy a page in the request to/from the userspace buffer. Must be
948 * done atomically
949 */
Miklos Szeredice534fb2010-05-25 15:06:07 +0200950static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
Miklos Szeredi8bfc0162006-01-16 22:14:28 -0800951 unsigned offset, unsigned count, int zeroing)
Miklos Szeredi334f4852005-09-09 13:10:27 -0700952{
Miklos Szeredice534fb2010-05-25 15:06:07 +0200953 int err;
954 struct page *page = *pagep;
955
Miklos Szeredib6777c42010-10-26 14:22:27 -0700956 if (page && zeroing && count < PAGE_SIZE)
957 clear_highpage(page);
958
Miklos Szeredi334f4852005-09-09 13:10:27 -0700959 while (count) {
Miklos Szeredic3021622010-05-25 15:06:07 +0200960 if (cs->write && cs->pipebufs && page) {
961 return fuse_ref_page(cs, page, offset, count);
962 } else if (!cs->len) {
Miklos Szeredice534fb2010-05-25 15:06:07 +0200963 if (cs->move_pages && page &&
964 offset == 0 && count == PAGE_SIZE) {
965 err = fuse_try_move_page(cs, pagep);
966 if (err <= 0)
967 return err;
968 } else {
969 err = fuse_copy_fill(cs);
970 if (err)
971 return err;
972 }
Miklos Szeredi1729a162008-11-26 12:03:54 +0100973 }
Miklos Szeredi334f4852005-09-09 13:10:27 -0700974 if (page) {
Cong Wang2408f6e2011-11-25 23:14:30 +0800975 void *mapaddr = kmap_atomic(page);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700976 void *buf = mapaddr + offset;
977 offset += fuse_copy_do(cs, &buf, &count);
Cong Wang2408f6e2011-11-25 23:14:30 +0800978 kunmap_atomic(mapaddr);
Miklos Szeredi334f4852005-09-09 13:10:27 -0700979 } else
980 offset += fuse_copy_do(cs, NULL, &count);
981 }
982 if (page && !cs->write)
983 flush_dcache_page(page);
984 return 0;
985}
986
987/* Copy pages in the request to/from userspace buffer */
988static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
989 int zeroing)
990{
991 unsigned i;
992 struct fuse_req *req = cs->req;
Miklos Szeredi334f4852005-09-09 13:10:27 -0700993
994 for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
Miklos Szeredice534fb2010-05-25 15:06:07 +0200995 int err;
Maxim Patlasov85f40ae2012-10-26 19:49:33 +0400996 unsigned offset = req->page_descs[i].offset;
997 unsigned count = min(nbytes, req->page_descs[i].length);
Miklos Szeredice534fb2010-05-25 15:06:07 +0200998
999 err = fuse_copy_page(cs, &req->pages[i], offset, count,
1000 zeroing);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001001 if (err)
1002 return err;
1003
1004 nbytes -= count;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001005 }
1006 return 0;
1007}
1008
1009/* Copy a single argument in the request to/from userspace buffer */
1010static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1011{
1012 while (size) {
Miklos Szeredi1729a162008-11-26 12:03:54 +01001013 if (!cs->len) {
1014 int err = fuse_copy_fill(cs);
1015 if (err)
1016 return err;
1017 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001018 fuse_copy_do(cs, &val, &size);
1019 }
1020 return 0;
1021}
1022
1023/* Copy request arguments to/from userspace buffer */
1024static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1025 unsigned argpages, struct fuse_arg *args,
1026 int zeroing)
1027{
1028 int err = 0;
1029 unsigned i;
1030
1031 for (i = 0; !err && i < numargs; i++) {
1032 struct fuse_arg *arg = &args[i];
1033 if (i == numargs - 1 && argpages)
1034 err = fuse_copy_pages(cs, arg->size, zeroing);
1035 else
1036 err = fuse_copy_one(cs, arg->value, arg->size);
1037 }
1038 return err;
1039}
1040
Miklos Szeredif88996a2015-07-01 16:26:01 +02001041static int forget_pending(struct fuse_iqueue *fiq)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001042{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001043 return fiq->forget_list_head.next != NULL;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001044}
1045
Miklos Szeredif88996a2015-07-01 16:26:01 +02001046static int request_pending(struct fuse_iqueue *fiq)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001047{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001048 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1049 forget_pending(fiq);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001050}
1051
Miklos Szeredi334f4852005-09-09 13:10:27 -07001052/*
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001053 * Transfer an interrupt request to userspace
1054 *
1055 * Unlike other requests this is assembled on demand, without a need
1056 * to allocate a separate fuse_req structure.
1057 *
Miklos Szeredifd22d622015-07-01 16:26:03 +02001058 * Called with fiq->waitq.lock held, releases it
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001059 */
Miklos Szeredifd22d622015-07-01 16:26:03 +02001060static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1061 struct fuse_copy_state *cs,
Miklos Szeredic3021622010-05-25 15:06:07 +02001062 size_t nbytes, struct fuse_req *req)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001063__releases(fiq->waitq.lock)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001064{
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001065 struct fuse_in_header ih;
1066 struct fuse_interrupt_in arg;
1067 unsigned reqsize = sizeof(ih) + sizeof(arg);
1068 int err;
1069
1070 list_del_init(&req->intr_entry);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001071 req->intr_unique = fuse_get_unique(fiq);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001072 memset(&ih, 0, sizeof(ih));
1073 memset(&arg, 0, sizeof(arg));
1074 ih.len = reqsize;
1075 ih.opcode = FUSE_INTERRUPT;
1076 ih.unique = req->intr_unique;
1077 arg.unique = req->in.h.unique;
1078
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001079 spin_unlock(&fiq->waitq.lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001080 if (nbytes < reqsize)
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001081 return -EINVAL;
1082
Miklos Szeredic3021622010-05-25 15:06:07 +02001083 err = fuse_copy_one(cs, &ih, sizeof(ih));
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001084 if (!err)
Miklos Szeredic3021622010-05-25 15:06:07 +02001085 err = fuse_copy_one(cs, &arg, sizeof(arg));
1086 fuse_copy_finish(cs);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001087
1088 return err ? err : reqsize;
1089}
1090
Miklos Szeredif88996a2015-07-01 16:26:01 +02001091static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001092 unsigned max,
1093 unsigned *countp)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001094{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001095 struct fuse_forget_link *head = fiq->forget_list_head.next;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001096 struct fuse_forget_link **newhead = &head;
1097 unsigned count;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001098
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001099 for (count = 0; *newhead != NULL && count < max; count++)
1100 newhead = &(*newhead)->next;
1101
Miklos Szeredif88996a2015-07-01 16:26:01 +02001102 fiq->forget_list_head.next = *newhead;
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001103 *newhead = NULL;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001104 if (fiq->forget_list_head.next == NULL)
1105 fiq->forget_list_tail = &fiq->forget_list_head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001106
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001107 if (countp != NULL)
1108 *countp = count;
1109
1110 return head;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001111}
1112
Miklos Szeredifd22d622015-07-01 16:26:03 +02001113static int fuse_read_single_forget(struct fuse_iqueue *fiq,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001114 struct fuse_copy_state *cs,
1115 size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001116__releases(fiq->waitq.lock)
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001117{
1118 int err;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001119 struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001120 struct fuse_forget_in arg = {
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001121 .nlookup = forget->forget_one.nlookup,
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001122 };
1123 struct fuse_in_header ih = {
1124 .opcode = FUSE_FORGET,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001125 .nodeid = forget->forget_one.nodeid,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001126 .unique = fuse_get_unique(fiq),
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001127 .len = sizeof(ih) + sizeof(arg),
1128 };
1129
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001130 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001131 kfree(forget);
1132 if (nbytes < ih.len)
1133 return -EINVAL;
1134
1135 err = fuse_copy_one(cs, &ih, sizeof(ih));
1136 if (!err)
1137 err = fuse_copy_one(cs, &arg, sizeof(arg));
1138 fuse_copy_finish(cs);
1139
1140 if (err)
1141 return err;
1142
1143 return ih.len;
1144}
1145
Miklos Szeredifd22d622015-07-01 16:26:03 +02001146static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001147 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001148__releases(fiq->waitq.lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001149{
1150 int err;
1151 unsigned max_forgets;
1152 unsigned count;
1153 struct fuse_forget_link *head;
1154 struct fuse_batch_forget_in arg = { .count = 0 };
1155 struct fuse_in_header ih = {
1156 .opcode = FUSE_BATCH_FORGET,
Miklos Szeredif88996a2015-07-01 16:26:01 +02001157 .unique = fuse_get_unique(fiq),
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001158 .len = sizeof(ih) + sizeof(arg),
1159 };
1160
1161 if (nbytes < ih.len) {
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001162 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001163 return -EINVAL;
1164 }
1165
1166 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
Miklos Szeredif88996a2015-07-01 16:26:01 +02001167 head = dequeue_forget(fiq, max_forgets, &count);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001168 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001169
1170 arg.count = count;
1171 ih.len += count * sizeof(struct fuse_forget_one);
1172 err = fuse_copy_one(cs, &ih, sizeof(ih));
1173 if (!err)
1174 err = fuse_copy_one(cs, &arg, sizeof(arg));
1175
1176 while (head) {
1177 struct fuse_forget_link *forget = head;
1178
1179 if (!err) {
1180 err = fuse_copy_one(cs, &forget->forget_one,
1181 sizeof(forget->forget_one));
1182 }
1183 head = forget->next;
1184 kfree(forget);
1185 }
1186
1187 fuse_copy_finish(cs);
1188
1189 if (err)
1190 return err;
1191
1192 return ih.len;
1193}
1194
Miklos Szeredifd22d622015-07-01 16:26:03 +02001195static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1196 struct fuse_copy_state *cs,
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001197 size_t nbytes)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001198__releases(fiq->waitq.lock)
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001199{
Miklos Szeredif88996a2015-07-01 16:26:01 +02001200 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001201 return fuse_read_single_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001202 else
Miklos Szeredifd22d622015-07-01 16:26:03 +02001203 return fuse_read_batch_forget(fiq, cs, nbytes);
Miklos Szeredi02c048b2010-12-07 20:16:56 +01001204}
1205
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001206/*
Miklos Szeredi334f4852005-09-09 13:10:27 -07001207 * Read a single request into the userspace filesystem's buffer. This
1208 * function waits until a request is available, then removes it from
1209 * the pending list and copies request data to userspace buffer. If
Miklos Szeredif9a28422006-06-25 05:48:53 -07001210 * no reply is needed (FORGET) or request has been aborted or there
1211 * was an error during the copying then it's finished by calling
Miklos Szeredi334f4852005-09-09 13:10:27 -07001212 * request_end(). Otherwise add it to the processing list, and set
1213 * the 'sent' flag.
1214 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001215static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
Miklos Szeredic3021622010-05-25 15:06:07 +02001216 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001217{
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001218 ssize_t err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001219 struct fuse_conn *fc = fud->fc;
Miklos Szeredif88996a2015-07-01 16:26:01 +02001220 struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredic36960462015-07-01 16:26:09 +02001221 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001222 struct fuse_req *req;
1223 struct fuse_in *in;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001224 unsigned reqsize;
1225
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001226 restart:
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001227 spin_lock(&fiq->waitq.lock);
Jeff Dikee5ac1d12006-04-10 22:54:53 -07001228 err = -EAGAIN;
Miklos Szeredie16714d2015-07-01 16:26:01 +02001229 if ((file->f_flags & O_NONBLOCK) && fiq->connected &&
Miklos Szeredif88996a2015-07-01 16:26:01 +02001230 !request_pending(fiq))
Jeff Dikee5ac1d12006-04-10 22:54:53 -07001231 goto err_unlock;
1232
Miklos Szeredi52509212015-07-01 16:26:03 +02001233 err = wait_event_interruptible_exclusive_locked(fiq->waitq,
1234 !fiq->connected || request_pending(fiq));
1235 if (err)
1236 goto err_unlock;
1237
Miklos Szeredi334f4852005-09-09 13:10:27 -07001238 err = -ENODEV;
Miklos Szeredie16714d2015-07-01 16:26:01 +02001239 if (!fiq->connected)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001240 goto err_unlock;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001241
Miklos Szeredif88996a2015-07-01 16:26:01 +02001242 if (!list_empty(&fiq->interrupts)) {
1243 req = list_entry(fiq->interrupts.next, struct fuse_req,
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001244 intr_entry);
Miklos Szeredifd22d622015-07-01 16:26:03 +02001245 return fuse_read_interrupt(fiq, cs, nbytes, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001246 }
1247
Miklos Szeredif88996a2015-07-01 16:26:01 +02001248 if (forget_pending(fiq)) {
1249 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
Miklos Szeredifd22d622015-07-01 16:26:03 +02001250 return fuse_read_forget(fc, fiq, cs, nbytes);
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001251
Miklos Szeredif88996a2015-07-01 16:26:01 +02001252 if (fiq->forget_batch <= -8)
1253 fiq->forget_batch = 16;
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001254 }
1255
Miklos Szeredif88996a2015-07-01 16:26:01 +02001256 req = list_entry(fiq->pending.next, struct fuse_req, list);
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001257 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredief759252015-07-01 16:26:02 +02001258 list_del_init(&req->list);
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001259 spin_unlock(&fiq->waitq.lock);
1260
Miklos Szeredi334f4852005-09-09 13:10:27 -07001261 in = &req->in;
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001262 reqsize = in->h.len;
1263 /* If request is too large, reply with an error and restart the read */
Miklos Szeredic3021622010-05-25 15:06:07 +02001264 if (nbytes < reqsize) {
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001265 req->out.h.error = -EIO;
1266 /* SETXATTR is special, since it may contain too large data */
1267 if (in->h.opcode == FUSE_SETXATTR)
1268 req->out.h.error = -E2BIG;
1269 request_end(fc, req);
1270 goto restart;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001271 }
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001272 spin_lock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001273 list_add(&req->list, &fpq->io);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001274 spin_unlock(&fpq->lock);
Miklos Szeredic3021622010-05-25 15:06:07 +02001275 cs->req = req;
1276 err = fuse_copy_one(cs, &in->h, sizeof(in->h));
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001277 if (!err)
Miklos Szeredic3021622010-05-25 15:06:07 +02001278 err = fuse_copy_args(cs, in->numargs, in->argpages,
Miklos Szeredi1d3d7522006-01-06 00:19:40 -08001279 (struct fuse_arg *) in->args, 0);
Miklos Szeredic3021622010-05-25 15:06:07 +02001280 fuse_copy_finish(cs);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001281 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001282 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001283 if (!fpq->connected) {
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001284 err = -ENODEV;
1285 goto out_end;
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001286 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07001287 if (err) {
Miklos Szeredic9c9d7d2007-10-16 23:31:05 -07001288 req->out.h.error = -EIO;
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001289 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001290 }
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001291 if (!test_bit(FR_ISREPLY, &req->flags)) {
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001292 err = reqsize;
1293 goto out_end;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001294 }
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001295 list_move_tail(&req->list, &fpq->processing);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001296 spin_unlock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001297 set_bit(FR_SENT, &req->flags);
1298 /* matches barrier in request_wait_answer() */
1299 smp_mb__after_atomic();
1300 if (test_bit(FR_INTERRUPTED, &req->flags))
1301 queue_interrupt(fiq, req);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001302
Miklos Szeredi334f4852005-09-09 13:10:27 -07001303 return reqsize;
1304
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001305out_end:
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001306 if (!test_bit(FR_PRIVATE, &req->flags))
1307 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001308 spin_unlock(&fpq->lock);
Miklos Szeredi82cbdcd2015-07-01 16:26:05 +02001309 request_end(fc, req);
1310 return err;
1311
Miklos Szeredi334f4852005-09-09 13:10:27 -07001312 err_unlock:
Miklos Szeredi4ce60812015-07-01 16:26:02 +02001313 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001314 return err;
1315}
1316
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01001317static int fuse_dev_open(struct inode *inode, struct file *file)
1318{
1319 /*
1320 * The fuse device's file's private_data is used to hold
1321 * the fuse_conn(ection) when it is mounted, and is used to
1322 * keep track of whether the file has been mounted already.
1323 */
1324 file->private_data = NULL;
1325 return 0;
1326}
1327
Al Virofbdbacc2015-04-03 21:53:39 -04001328static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
Miklos Szeredic3021622010-05-25 15:06:07 +02001329{
1330 struct fuse_copy_state cs;
1331 struct file *file = iocb->ki_filp;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001332 struct fuse_dev *fud = fuse_get_dev(file);
1333
1334 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001335 return -EPERM;
1336
Al Virofbdbacc2015-04-03 21:53:39 -04001337 if (!iter_is_iovec(to))
1338 return -EINVAL;
Miklos Szeredic3021622010-05-25 15:06:07 +02001339
Miklos Szeredidc008092015-07-01 16:25:58 +02001340 fuse_copy_init(&cs, 1, to);
Al Virofbdbacc2015-04-03 21:53:39 -04001341
Miklos Szeredic36960462015-07-01 16:26:09 +02001342 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
Miklos Szeredic3021622010-05-25 15:06:07 +02001343}
1344
Miklos Szeredic3021622010-05-25 15:06:07 +02001345static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1346 struct pipe_inode_info *pipe,
1347 size_t len, unsigned int flags)
1348{
Al Virod82718e2016-09-17 22:56:25 -04001349 int total, ret;
Miklos Szeredic3021622010-05-25 15:06:07 +02001350 int page_nr = 0;
Miklos Szeredic3021622010-05-25 15:06:07 +02001351 struct pipe_buffer *bufs;
1352 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001353 struct fuse_dev *fud = fuse_get_dev(in);
1354
1355 if (!fud)
Miklos Szeredic3021622010-05-25 15:06:07 +02001356 return -EPERM;
1357
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001358 bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001359 if (!bufs)
1360 return -ENOMEM;
1361
Miklos Szeredidc008092015-07-01 16:25:58 +02001362 fuse_copy_init(&cs, 1, NULL);
Miklos Szeredic3021622010-05-25 15:06:07 +02001363 cs.pipebufs = bufs;
1364 cs.pipe = pipe;
Miklos Szeredic36960462015-07-01 16:26:09 +02001365 ret = fuse_dev_do_read(fud, in, &cs, len);
Miklos Szeredic3021622010-05-25 15:06:07 +02001366 if (ret < 0)
1367 goto out;
1368
Miklos Szeredic3021622010-05-25 15:06:07 +02001369 if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1370 ret = -EIO;
Al Virod82718e2016-09-17 22:56:25 -04001371 goto out;
Miklos Szeredic3021622010-05-25 15:06:07 +02001372 }
1373
Al Virod82718e2016-09-17 22:56:25 -04001374 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
Miklos Szeredi28a625c2014-01-22 19:36:57 +01001375 /*
1376 * Need to be careful about this. Having buf->ops in module
1377 * code can Oops if the buffer persists after module unload.
1378 */
Al Virod82718e2016-09-17 22:56:25 -04001379 bufs[page_nr].ops = &nosteal_pipe_buf_ops;
Miklos Szeredi84588a92017-02-16 15:08:20 +01001380 bufs[page_nr].flags = 0;
Al Virod82718e2016-09-17 22:56:25 -04001381 ret = add_to_pipe(pipe, &bufs[page_nr++]);
1382 if (unlikely(ret < 0))
1383 break;
Miklos Szeredic3021622010-05-25 15:06:07 +02001384 }
Al Virod82718e2016-09-17 22:56:25 -04001385 if (total)
1386 ret = total;
Miklos Szeredic3021622010-05-25 15:06:07 +02001387out:
1388 for (; page_nr < cs.nr_segs; page_nr++)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001389 put_page(bufs[page_nr].page);
Miklos Szeredic3021622010-05-25 15:06:07 +02001390
1391 kfree(bufs);
1392 return ret;
1393}
1394
Tejun Heo95668a62008-11-26 12:03:55 +01001395static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1396 struct fuse_copy_state *cs)
1397{
1398 struct fuse_notify_poll_wakeup_out outarg;
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001399 int err = -EINVAL;
Tejun Heo95668a62008-11-26 12:03:55 +01001400
1401 if (size != sizeof(outarg))
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001402 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001403
1404 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1405 if (err)
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001406 goto err;
Tejun Heo95668a62008-11-26 12:03:55 +01001407
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001408 fuse_copy_finish(cs);
Tejun Heo95668a62008-11-26 12:03:55 +01001409 return fuse_notify_poll_wakeup(fc, &outarg);
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001410
1411err:
1412 fuse_copy_finish(cs);
1413 return err;
Tejun Heo95668a62008-11-26 12:03:55 +01001414}
1415
John Muir3b463ae2009-05-31 11:13:57 -04001416static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1417 struct fuse_copy_state *cs)
1418{
1419 struct fuse_notify_inval_inode_out outarg;
1420 int err = -EINVAL;
1421
1422 if (size != sizeof(outarg))
1423 goto err;
1424
1425 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1426 if (err)
1427 goto err;
1428 fuse_copy_finish(cs);
1429
1430 down_read(&fc->killsb);
1431 err = -ENOENT;
Miklos Szeredib21dda42010-02-05 12:08:31 +01001432 if (fc->sb) {
1433 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1434 outarg.off, outarg.len);
1435 }
John Muir3b463ae2009-05-31 11:13:57 -04001436 up_read(&fc->killsb);
1437 return err;
1438
1439err:
1440 fuse_copy_finish(cs);
1441 return err;
1442}
1443
1444static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1445 struct fuse_copy_state *cs)
1446{
1447 struct fuse_notify_inval_entry_out outarg;
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001448 int err = -ENOMEM;
1449 char *buf;
John Muir3b463ae2009-05-31 11:13:57 -04001450 struct qstr name;
1451
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001452 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1453 if (!buf)
1454 goto err;
1455
1456 err = -EINVAL;
John Muir3b463ae2009-05-31 11:13:57 -04001457 if (size < sizeof(outarg))
1458 goto err;
1459
1460 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1461 if (err)
1462 goto err;
1463
1464 err = -ENAMETOOLONG;
1465 if (outarg.namelen > FUSE_NAME_MAX)
1466 goto err;
1467
Miklos Szeredic2183d12011-08-24 10:20:17 +02001468 err = -EINVAL;
1469 if (size != sizeof(outarg) + outarg.namelen + 1)
1470 goto err;
1471
John Muir3b463ae2009-05-31 11:13:57 -04001472 name.name = buf;
1473 name.len = outarg.namelen;
1474 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1475 if (err)
1476 goto err;
1477 fuse_copy_finish(cs);
1478 buf[outarg.namelen] = 0;
John Muir3b463ae2009-05-31 11:13:57 -04001479
1480 down_read(&fc->killsb);
1481 err = -ENOENT;
Miklos Szeredib21dda42010-02-05 12:08:31 +01001482 if (fc->sb)
John Muir451d0f52011-12-06 21:50:06 +01001483 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1484 up_read(&fc->killsb);
1485 kfree(buf);
1486 return err;
1487
1488err:
1489 kfree(buf);
1490 fuse_copy_finish(cs);
1491 return err;
1492}
1493
1494static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1495 struct fuse_copy_state *cs)
1496{
1497 struct fuse_notify_delete_out outarg;
1498 int err = -ENOMEM;
1499 char *buf;
1500 struct qstr name;
1501
1502 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1503 if (!buf)
1504 goto err;
1505
1506 err = -EINVAL;
1507 if (size < sizeof(outarg))
1508 goto err;
1509
1510 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1511 if (err)
1512 goto err;
1513
1514 err = -ENAMETOOLONG;
1515 if (outarg.namelen > FUSE_NAME_MAX)
1516 goto err;
1517
1518 err = -EINVAL;
1519 if (size != sizeof(outarg) + outarg.namelen + 1)
1520 goto err;
1521
1522 name.name = buf;
1523 name.len = outarg.namelen;
1524 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1525 if (err)
1526 goto err;
1527 fuse_copy_finish(cs);
1528 buf[outarg.namelen] = 0;
John Muir451d0f52011-12-06 21:50:06 +01001529
1530 down_read(&fc->killsb);
1531 err = -ENOENT;
1532 if (fc->sb)
1533 err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1534 outarg.child, &name);
John Muir3b463ae2009-05-31 11:13:57 -04001535 up_read(&fc->killsb);
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001536 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001537 return err;
1538
1539err:
Fang Wenqib2d82ee2009-12-30 18:37:13 +08001540 kfree(buf);
John Muir3b463ae2009-05-31 11:13:57 -04001541 fuse_copy_finish(cs);
1542 return err;
1543}
1544
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001545static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1546 struct fuse_copy_state *cs)
1547{
1548 struct fuse_notify_store_out outarg;
1549 struct inode *inode;
1550 struct address_space *mapping;
1551 u64 nodeid;
1552 int err;
1553 pgoff_t index;
1554 unsigned int offset;
1555 unsigned int num;
1556 loff_t file_size;
1557 loff_t end;
1558
1559 err = -EINVAL;
1560 if (size < sizeof(outarg))
1561 goto out_finish;
1562
1563 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1564 if (err)
1565 goto out_finish;
1566
1567 err = -EINVAL;
1568 if (size - sizeof(outarg) != outarg.size)
1569 goto out_finish;
1570
1571 nodeid = outarg.nodeid;
1572
1573 down_read(&fc->killsb);
1574
1575 err = -ENOENT;
1576 if (!fc->sb)
1577 goto out_up_killsb;
1578
1579 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1580 if (!inode)
1581 goto out_up_killsb;
1582
1583 mapping = inode->i_mapping;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001584 index = outarg.offset >> PAGE_SHIFT;
1585 offset = outarg.offset & ~PAGE_MASK;
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001586 file_size = i_size_read(inode);
1587 end = outarg.offset + outarg.size;
1588 if (end > file_size) {
1589 file_size = end;
1590 fuse_write_update_size(inode, file_size);
1591 }
1592
1593 num = outarg.size;
1594 while (num) {
1595 struct page *page;
1596 unsigned int this_num;
1597
1598 err = -ENOMEM;
1599 page = find_or_create_page(mapping, index,
1600 mapping_gfp_mask(mapping));
1601 if (!page)
1602 goto out_iput;
1603
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001604 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001605 err = fuse_copy_page(cs, &page, offset, this_num, 0);
Miklos Szeredi063ec1e2014-01-22 19:36:58 +01001606 if (!err && offset == 0 &&
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001607 (this_num == PAGE_SIZE || file_size == end))
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001608 SetPageUptodate(page);
1609 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001610 put_page(page);
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001611
1612 if (err)
1613 goto out_iput;
1614
1615 num -= this_num;
1616 offset = 0;
1617 index++;
1618 }
1619
1620 err = 0;
1621
1622out_iput:
1623 iput(inode);
1624out_up_killsb:
1625 up_read(&fc->killsb);
1626out_finish:
1627 fuse_copy_finish(cs);
1628 return err;
1629}
1630
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001631static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1632{
Mel Gormanb745bc82014-06-04 16:10:22 -07001633 release_pages(req->pages, req->num_pages, false);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001634}
1635
1636static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1637 struct fuse_notify_retrieve_out *outarg)
1638{
1639 int err;
1640 struct address_space *mapping = inode->i_mapping;
1641 struct fuse_req *req;
1642 pgoff_t index;
1643 loff_t file_size;
1644 unsigned int num;
1645 unsigned int offset;
Geert Uytterhoeven01574432010-09-30 22:06:21 +02001646 size_t total_len = 0;
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001647 int num_pages;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001648
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001649 offset = outarg->offset & ~PAGE_MASK;
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001650 file_size = i_size_read(inode);
1651
1652 num = outarg->size;
1653 if (outarg->offset > file_size)
1654 num = 0;
1655 else if (outarg->offset + num > file_size)
1656 num = file_size - outarg->offset;
1657
1658 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1659 num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1660
1661 req = fuse_get_req(fc, num_pages);
1662 if (IS_ERR(req))
1663 return PTR_ERR(req);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001664
1665 req->in.h.opcode = FUSE_NOTIFY_REPLY;
1666 req->in.h.nodeid = outarg->nodeid;
1667 req->in.numargs = 2;
1668 req->in.argpages = 1;
Maxim Patlasovb2430d72012-10-26 19:49:24 +04001669 req->page_descs[0].offset = offset;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001670 req->end = fuse_retrieve_end;
1671
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001672 index = outarg->offset >> PAGE_SHIFT;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001673
Maxim Patlasov4d53dc92012-10-26 19:48:42 +04001674 while (num && req->num_pages < num_pages) {
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001675 struct page *page;
1676 unsigned int this_num;
1677
1678 page = find_get_page(mapping, index);
1679 if (!page)
1680 break;
1681
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001682 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001683 req->pages[req->num_pages] = page;
Maxim Patlasov85f40ae2012-10-26 19:49:33 +04001684 req->page_descs[req->num_pages].length = this_num;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001685 req->num_pages++;
1686
Miklos Szeredic9e67d42012-09-04 18:45:54 +02001687 offset = 0;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001688 num -= this_num;
1689 total_len += this_num;
Miklos Szeredi48706d02011-12-13 10:36:59 +01001690 index++;
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001691 }
1692 req->misc.retrieve_in.offset = outarg->offset;
1693 req->misc.retrieve_in.size = total_len;
1694 req->in.args[0].size = sizeof(req->misc.retrieve_in);
1695 req->in.args[0].value = &req->misc.retrieve_in;
1696 req->in.args[1].size = total_len;
1697
1698 err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1699 if (err)
1700 fuse_retrieve_end(fc, req);
1701
1702 return err;
1703}
1704
1705static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1706 struct fuse_copy_state *cs)
1707{
1708 struct fuse_notify_retrieve_out outarg;
1709 struct inode *inode;
1710 int err;
1711
1712 err = -EINVAL;
1713 if (size != sizeof(outarg))
1714 goto copy_finish;
1715
1716 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1717 if (err)
1718 goto copy_finish;
1719
1720 fuse_copy_finish(cs);
1721
1722 down_read(&fc->killsb);
1723 err = -ENOENT;
1724 if (fc->sb) {
1725 u64 nodeid = outarg.nodeid;
1726
1727 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1728 if (inode) {
1729 err = fuse_retrieve(fc, inode, &outarg);
1730 iput(inode);
1731 }
1732 }
1733 up_read(&fc->killsb);
1734
1735 return err;
1736
1737copy_finish:
1738 fuse_copy_finish(cs);
1739 return err;
1740}
1741
Tejun Heo85993962008-11-26 12:03:55 +01001742static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1743 unsigned int size, struct fuse_copy_state *cs)
1744{
Miklos Szeredi0d278362015-02-26 11:45:47 +01001745 /* Don't try to move pages (yet) */
1746 cs->move_pages = 0;
1747
Tejun Heo85993962008-11-26 12:03:55 +01001748 switch (code) {
Tejun Heo95668a62008-11-26 12:03:55 +01001749 case FUSE_NOTIFY_POLL:
1750 return fuse_notify_poll(fc, size, cs);
1751
John Muir3b463ae2009-05-31 11:13:57 -04001752 case FUSE_NOTIFY_INVAL_INODE:
1753 return fuse_notify_inval_inode(fc, size, cs);
1754
1755 case FUSE_NOTIFY_INVAL_ENTRY:
1756 return fuse_notify_inval_entry(fc, size, cs);
1757
Miklos Szeredia1d75f22010-07-12 14:41:40 +02001758 case FUSE_NOTIFY_STORE:
1759 return fuse_notify_store(fc, size, cs);
1760
Miklos Szeredi2d45ba32010-07-12 14:41:40 +02001761 case FUSE_NOTIFY_RETRIEVE:
1762 return fuse_notify_retrieve(fc, size, cs);
1763
John Muir451d0f52011-12-06 21:50:06 +01001764 case FUSE_NOTIFY_DELETE:
1765 return fuse_notify_delete(fc, size, cs);
1766
Tejun Heo85993962008-11-26 12:03:55 +01001767 default:
Miklos Szeredif6d47a12009-01-26 15:00:59 +01001768 fuse_copy_finish(cs);
Tejun Heo85993962008-11-26 12:03:55 +01001769 return -EINVAL;
1770 }
1771}
1772
Miklos Szeredi334f4852005-09-09 13:10:27 -07001773/* Look up request on processing list by unique ID */
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001774static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001775{
Dong Fang05726ac2013-07-30 22:50:01 -04001776 struct fuse_req *req;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001777
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001778 list_for_each_entry(req, &fpq->processing, list) {
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001779 if (req->in.h.unique == unique || req->intr_unique == unique)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001780 return req;
1781 }
1782 return NULL;
1783}
1784
1785static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1786 unsigned nbytes)
1787{
1788 unsigned reqsize = sizeof(struct fuse_out_header);
1789
1790 if (out->h.error)
1791 return nbytes != reqsize ? -EINVAL : 0;
1792
1793 reqsize += len_args(out->numargs, out->args);
1794
1795 if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1796 return -EINVAL;
1797 else if (reqsize > nbytes) {
1798 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1799 unsigned diffsize = reqsize - nbytes;
1800 if (diffsize > lastarg->size)
1801 return -EINVAL;
1802 lastarg->size -= diffsize;
1803 }
1804 return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1805 out->page_zeroing);
1806}
1807
1808/*
1809 * Write a single reply to a request. First the header is copied from
1810 * the write buffer. The request is then searched on the processing
1811 * list by the unique ID found in the header. If found, then remove
1812 * it from the list and copy the rest of the buffer to the request.
1813 * The request is finished by calling request_end()
1814 */
Miklos Szeredic36960462015-07-01 16:26:09 +02001815static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001816 struct fuse_copy_state *cs, size_t nbytes)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001817{
1818 int err;
Miklos Szeredic36960462015-07-01 16:26:09 +02001819 struct fuse_conn *fc = fud->fc;
1820 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001821 struct fuse_req *req;
1822 struct fuse_out_header oh;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001823
Miklos Szeredi334f4852005-09-09 13:10:27 -07001824 if (nbytes < sizeof(struct fuse_out_header))
1825 return -EINVAL;
1826
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001827 err = fuse_copy_one(cs, &oh, sizeof(oh));
Miklos Szeredi334f4852005-09-09 13:10:27 -07001828 if (err)
1829 goto err_finish;
Tejun Heo85993962008-11-26 12:03:55 +01001830
Miklos Szeredi334f4852005-09-09 13:10:27 -07001831 err = -EINVAL;
Tejun Heo85993962008-11-26 12:03:55 +01001832 if (oh.len != nbytes)
1833 goto err_finish;
1834
1835 /*
1836 * Zero oh.unique indicates unsolicited notification message
1837 * and error contains notification code.
1838 */
1839 if (!oh.unique) {
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001840 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
Tejun Heo85993962008-11-26 12:03:55 +01001841 return err ? err : nbytes;
1842 }
1843
1844 err = -EINVAL;
1845 if (oh.error <= -1000 || oh.error > 0)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001846 goto err_finish;
1847
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001848 spin_lock(&fpq->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08001849 err = -ENOENT;
Miklos Szeredie96edd92015-07-01 16:26:04 +02001850 if (!fpq->connected)
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001851 goto err_unlock_pq;
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08001852
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001853 req = request_find(fpq, oh.unique);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001854 if (!req)
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001855 goto err_unlock_pq;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001856
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001857 /* Is it an interrupt reply? */
1858 if (req->intr_unique == oh.unique) {
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001859 spin_unlock(&fpq->lock);
1860
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001861 err = -EINVAL;
1862 if (nbytes != sizeof(struct fuse_out_header))
Miklos Szeredi46c34a32015-07-01 16:26:07 +02001863 goto err_finish;
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001864
1865 if (oh.error == -ENOSYS)
1866 fc->no_interrupt = 1;
1867 else if (oh.error == -EAGAIN)
Miklos Szeredif88996a2015-07-01 16:26:01 +02001868 queue_interrupt(&fc->iq, req);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001869
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001870 fuse_copy_finish(cs);
Miklos Szeredia4d27e72006-06-25 05:48:54 -07001871 return nbytes;
1872 }
1873
Miklos Szeredi33e14b42015-07-01 16:26:01 +02001874 clear_bit(FR_SENT, &req->flags);
Miklos Szeredi3a2b5b92015-07-01 16:26:04 +02001875 list_move(&req->list, &fpq->io);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001876 req->out.h = oh;
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001877 set_bit(FR_LOCKED, &req->flags);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001878 spin_unlock(&fpq->lock);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001879 cs->req = req;
Miklos Szeredice534fb2010-05-25 15:06:07 +02001880 if (!req->out.page_replace)
1881 cs->move_pages = 0;
Miklos Szeredi334f4852005-09-09 13:10:27 -07001882
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001883 err = copy_out_args(cs, &req->out, nbytes);
1884 fuse_copy_finish(cs);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001885
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001886 spin_lock(&fpq->lock);
Miklos Szeredi825d6d32015-07-01 16:25:58 +02001887 clear_bit(FR_LOCKED, &req->flags);
Miklos Szeredie96edd92015-07-01 16:26:04 +02001888 if (!fpq->connected)
Miklos Szeredi0d8e84b2015-07-01 16:25:58 +02001889 err = -ENOENT;
1890 else if (err)
Miklos Szeredi334f4852005-09-09 13:10:27 -07001891 req->out.h.error = -EIO;
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02001892 if (!test_bit(FR_PRIVATE, &req->flags))
1893 list_del_init(&req->list);
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001894 spin_unlock(&fpq->lock);
Miklos Szeredi46c34a32015-07-01 16:26:07 +02001895
Miklos Szeredi334f4852005-09-09 13:10:27 -07001896 request_end(fc, req);
1897
1898 return err ? err : nbytes;
1899
Miklos Szeredi45a91cb2015-07-01 16:26:06 +02001900 err_unlock_pq:
1901 spin_unlock(&fpq->lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001902 err_finish:
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001903 fuse_copy_finish(cs);
Miklos Szeredi334f4852005-09-09 13:10:27 -07001904 return err;
1905}
1906
Al Virofbdbacc2015-04-03 21:53:39 -04001907static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001908{
1909 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001910 struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
1911
1912 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001913 return -EPERM;
1914
Al Virofbdbacc2015-04-03 21:53:39 -04001915 if (!iter_is_iovec(from))
1916 return -EINVAL;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001917
Miklos Szeredidc008092015-07-01 16:25:58 +02001918 fuse_copy_init(&cs, 0, from);
Al Virofbdbacc2015-04-03 21:53:39 -04001919
Miklos Szeredic36960462015-07-01 16:26:09 +02001920 return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001921}
1922
1923static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1924 struct file *out, loff_t *ppos,
1925 size_t len, unsigned int flags)
1926{
1927 unsigned nbuf;
1928 unsigned idx;
1929 struct pipe_buffer *bufs;
1930 struct fuse_copy_state cs;
Miklos Szeredicc080e92015-07-01 16:26:08 +02001931 struct fuse_dev *fud;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001932 size_t rem;
1933 ssize_t ret;
1934
Miklos Szeredicc080e92015-07-01 16:26:08 +02001935 fud = fuse_get_dev(out);
1936 if (!fud)
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001937 return -EPERM;
1938
Miklos Szeredi07e77dc2010-12-07 20:16:56 +01001939 bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001940 if (!bufs)
1941 return -ENOMEM;
1942
1943 pipe_lock(pipe);
1944 nbuf = 0;
1945 rem = 0;
1946 for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1947 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1948
1949 ret = -EINVAL;
1950 if (rem < len) {
1951 pipe_unlock(pipe);
1952 goto out;
1953 }
1954
1955 rem = len;
1956 while (rem) {
1957 struct pipe_buffer *ibuf;
1958 struct pipe_buffer *obuf;
1959
1960 BUG_ON(nbuf >= pipe->buffers);
1961 BUG_ON(!pipe->nrbufs);
1962 ibuf = &pipe->bufs[pipe->curbuf];
1963 obuf = &bufs[nbuf];
1964
1965 if (rem >= ibuf->len) {
1966 *obuf = *ibuf;
1967 ibuf->ops = NULL;
1968 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1969 pipe->nrbufs--;
1970 } else {
Miklos Szeredi7bf2d1d2016-09-27 10:45:12 +02001971 pipe_buf_get(pipe, ibuf);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001972 *obuf = *ibuf;
1973 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1974 obuf->len = rem;
1975 ibuf->offset += obuf->len;
1976 ibuf->len -= obuf->len;
1977 }
1978 nbuf++;
1979 rem -= obuf->len;
1980 }
1981 pipe_unlock(pipe);
1982
Miklos Szeredidc008092015-07-01 16:25:58 +02001983 fuse_copy_init(&cs, 0, NULL);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001984 cs.pipebufs = bufs;
Al Viro6c09e942015-04-03 22:06:08 -04001985 cs.nr_segs = nbuf;
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001986 cs.pipe = pipe;
1987
Miklos Szeredice534fb2010-05-25 15:06:07 +02001988 if (flags & SPLICE_F_MOVE)
1989 cs.move_pages = 1;
1990
Miklos Szeredic36960462015-07-01 16:26:09 +02001991 ret = fuse_dev_do_write(fud, &cs, len);
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001992
Miklos Szeredia7796382016-09-27 10:45:12 +02001993 for (idx = 0; idx < nbuf; idx++)
1994 pipe_buf_release(pipe, &bufs[idx]);
1995
Miklos Szeredidd3bb142010-05-25 15:06:06 +02001996out:
1997 kfree(bufs);
1998 return ret;
1999}
2000
Miklos Szeredi334f4852005-09-09 13:10:27 -07002001static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
2002{
Miklos Szeredi334f4852005-09-09 13:10:27 -07002003 unsigned mask = POLLOUT | POLLWRNORM;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002004 struct fuse_iqueue *fiq;
Miklos Szeredicc080e92015-07-01 16:26:08 +02002005 struct fuse_dev *fud = fuse_get_dev(file);
2006
2007 if (!fud)
Miklos Szeredi7025d9a2006-04-10 22:54:50 -07002008 return POLLERR;
Miklos Szeredi334f4852005-09-09 13:10:27 -07002009
Miklos Szeredicc080e92015-07-01 16:26:08 +02002010 fiq = &fud->fc->iq;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002011 poll_wait(file, &fiq->waitq, wait);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002012
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002013 spin_lock(&fiq->waitq.lock);
Miklos Szeredie16714d2015-07-01 16:26:01 +02002014 if (!fiq->connected)
Miklos Szeredi7025d9a2006-04-10 22:54:50 -07002015 mask = POLLERR;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002016 else if (request_pending(fiq))
Miklos Szeredi7025d9a2006-04-10 22:54:50 -07002017 mask |= POLLIN | POLLRDNORM;
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002018 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002019
2020 return mask;
2021}
2022
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002023/*
2024 * Abort all requests on the given list (pending or processing)
2025 *
Miklos Szeredid7133112006-04-10 22:54:55 -07002026 * This function releases and reacquires fc->lock
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002027 */
Miklos Szeredi334f4852005-09-09 13:10:27 -07002028static void end_requests(struct fuse_conn *fc, struct list_head *head)
2029{
2030 while (!list_empty(head)) {
2031 struct fuse_req *req;
2032 req = list_entry(head->next, struct fuse_req, list);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002033 req->out.h.error = -ECONNABORTED;
Miklos Szeredi33e14b42015-07-01 16:26:01 +02002034 clear_bit(FR_SENT, &req->flags);
Miklos Szeredif377cb72015-07-01 16:26:04 +02002035 list_del_init(&req->list);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002036 request_end(fc, req);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002037 }
2038}
2039
Bryan Green357ccf22011-03-01 16:43:52 -08002040static void end_polls(struct fuse_conn *fc)
2041{
2042 struct rb_node *p;
2043
2044 p = rb_first(&fc->polled_files);
2045
2046 while (p) {
2047 struct fuse_file *ff;
2048 ff = rb_entry(p, struct fuse_file, polled_node);
2049 wake_up_interruptible_all(&ff->poll_wait);
2050
2051 p = rb_next(p);
2052 }
2053}
2054
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002055/*
2056 * Abort all requests.
2057 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002058 * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2059 * filesystem.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002060 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002061 * The same effect is usually achievable through killing the filesystem daemon
2062 * and all users of the filesystem. The exception is the combination of an
2063 * asynchronous request and the tricky deadlock (see
2064 * Documentation/filesystems/fuse.txt).
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002065 *
Miklos Szeredib716d422015-07-01 16:25:59 +02002066 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2067 * requests, they should be finished off immediately. Locked requests will be
2068 * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2069 * requests. It is possible that some request will finish before we can. This
2070 * is OK, the request will in that case be removed from the list before we touch
2071 * it.
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002072 */
2073void fuse_abort_conn(struct fuse_conn *fc)
2074{
Miklos Szeredif88996a2015-07-01 16:26:01 +02002075 struct fuse_iqueue *fiq = &fc->iq;
2076
Miklos Szeredid7133112006-04-10 22:54:55 -07002077 spin_lock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002078 if (fc->connected) {
Miklos Szeredic36960462015-07-01 16:26:09 +02002079 struct fuse_dev *fud;
Miklos Szeredib716d422015-07-01 16:25:59 +02002080 struct fuse_req *req, *next;
Miklos Szeredi41f98272015-07-01 16:25:59 +02002081 LIST_HEAD(to_end1);
2082 LIST_HEAD(to_end2);
Miklos Szeredib716d422015-07-01 16:25:59 +02002083
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002084 fc->connected = 0;
Miklos Szeredi51eb01e2006-06-25 05:48:50 -07002085 fc->blocked = 0;
Miklos Szeredi9759bd512015-01-06 10:45:35 +01002086 fuse_set_initialized(fc);
Miklos Szeredic36960462015-07-01 16:26:09 +02002087 list_for_each_entry(fud, &fc->devices, entry) {
2088 struct fuse_pqueue *fpq = &fud->pq;
2089
2090 spin_lock(&fpq->lock);
2091 fpq->connected = 0;
2092 list_for_each_entry_safe(req, next, &fpq->io, list) {
2093 req->out.h.error = -ECONNABORTED;
2094 spin_lock(&req->waitq.lock);
2095 set_bit(FR_ABORTED, &req->flags);
2096 if (!test_bit(FR_LOCKED, &req->flags)) {
2097 set_bit(FR_PRIVATE, &req->flags);
2098 list_move(&req->list, &to_end1);
2099 }
2100 spin_unlock(&req->waitq.lock);
Miklos Szeredi77cd9d42015-07-01 16:26:06 +02002101 }
Miklos Szeredic36960462015-07-01 16:26:09 +02002102 list_splice_init(&fpq->processing, &to_end2);
2103 spin_unlock(&fpq->lock);
Miklos Szeredib716d422015-07-01 16:25:59 +02002104 }
Miklos Szeredi41f98272015-07-01 16:25:59 +02002105 fc->max_background = UINT_MAX;
2106 flush_bg_queue(fc);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002107
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002108 spin_lock(&fiq->waitq.lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002109 fiq->connected = 0;
Miklos Szeredif88996a2015-07-01 16:26:01 +02002110 list_splice_init(&fiq->pending, &to_end2);
Tahsin Erdogana8a86d72017-01-12 12:04:04 -08002111 list_for_each_entry(req, &to_end2, list)
2112 clear_bit(FR_PENDING, &req->flags);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002113 while (forget_pending(fiq))
2114 kfree(dequeue_forget(fiq, 1, NULL));
Miklos Szeredi4ce60812015-07-01 16:26:02 +02002115 wake_up_all_locked(&fiq->waitq);
2116 spin_unlock(&fiq->waitq.lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002117 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szerediee314a82015-07-01 16:26:08 +02002118 end_polls(fc);
2119 wake_up_all(&fc->blocked_waitq);
2120 spin_unlock(&fc->lock);
Miklos Szeredi8c911892015-07-01 16:26:02 +02002121
Miklos Szeredi41f98272015-07-01 16:25:59 +02002122 while (!list_empty(&to_end1)) {
2123 req = list_first_entry(&to_end1, struct fuse_req, list);
Miklos Szeredib716d422015-07-01 16:25:59 +02002124 __fuse_get_request(req);
Miklos Szeredif377cb72015-07-01 16:26:04 +02002125 list_del_init(&req->list);
Miklos Szeredib716d422015-07-01 16:25:59 +02002126 request_end(fc, req);
Miklos Szeredib716d422015-07-01 16:25:59 +02002127 }
Miklos Szeredi41f98272015-07-01 16:25:59 +02002128 end_requests(fc, &to_end2);
Miklos Szerediee314a82015-07-01 16:26:08 +02002129 } else {
2130 spin_unlock(&fc->lock);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002131 }
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002132}
Tejun Heo08cbf542009-04-14 10:54:53 +09002133EXPORT_SYMBOL_GPL(fuse_abort_conn);
Miklos Szeredi69a53bf2006-01-16 22:14:41 -08002134
Tejun Heo08cbf542009-04-14 10:54:53 +09002135int fuse_dev_release(struct inode *inode, struct file *file)
Miklos Szeredi334f4852005-09-09 13:10:27 -07002136{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002137 struct fuse_dev *fud = fuse_get_dev(file);
2138
2139 if (fud) {
2140 struct fuse_conn *fc = fud->fc;
Miklos Szeredic36960462015-07-01 16:26:09 +02002141 struct fuse_pqueue *fpq = &fud->pq;
Miklos Szeredicc080e92015-07-01 16:26:08 +02002142
Miklos Szeredic36960462015-07-01 16:26:09 +02002143 WARN_ON(!list_empty(&fpq->io));
2144 end_requests(fc, &fpq->processing);
2145 /* Are we the last open device? */
2146 if (atomic_dec_and_test(&fc->dev_count)) {
2147 WARN_ON(fc->iq.fasync != NULL);
2148 fuse_abort_conn(fc);
2149 }
Miklos Szeredicc080e92015-07-01 16:26:08 +02002150 fuse_dev_free(fud);
Jeff Dike385a17b2006-04-10 22:54:52 -07002151 }
Miklos Szeredi334f4852005-09-09 13:10:27 -07002152 return 0;
2153}
Tejun Heo08cbf542009-04-14 10:54:53 +09002154EXPORT_SYMBOL_GPL(fuse_dev_release);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002155
Jeff Dike385a17b2006-04-10 22:54:52 -07002156static int fuse_dev_fasync(int fd, struct file *file, int on)
2157{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002158 struct fuse_dev *fud = fuse_get_dev(file);
2159
2160 if (!fud)
Miklos Szeredia87046d2006-04-10 22:54:56 -07002161 return -EPERM;
Jeff Dike385a17b2006-04-10 22:54:52 -07002162
2163 /* No locking - fasync_helper does its own locking */
Miklos Szeredicc080e92015-07-01 16:26:08 +02002164 return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
Jeff Dike385a17b2006-04-10 22:54:52 -07002165}
2166
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002167static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2168{
Miklos Szeredicc080e92015-07-01 16:26:08 +02002169 struct fuse_dev *fud;
2170
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002171 if (new->private_data)
2172 return -EINVAL;
2173
Miklos Szeredicc080e92015-07-01 16:26:08 +02002174 fud = fuse_dev_alloc(fc);
2175 if (!fud)
2176 return -ENOMEM;
2177
2178 new->private_data = fud;
Miklos Szeredic36960462015-07-01 16:26:09 +02002179 atomic_inc(&fc->dev_count);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002180
2181 return 0;
2182}
2183
2184static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2185 unsigned long arg)
2186{
2187 int err = -ENOTTY;
2188
2189 if (cmd == FUSE_DEV_IOC_CLONE) {
2190 int oldfd;
2191
2192 err = -EFAULT;
2193 if (!get_user(oldfd, (__u32 __user *) arg)) {
2194 struct file *old = fget(oldfd);
2195
2196 err = -EINVAL;
2197 if (old) {
Jann Horn8ed1f0e2015-08-16 20:27:01 +02002198 struct fuse_dev *fud = NULL;
2199
2200 /*
2201 * Check against file->f_op because CUSE
2202 * uses the same ioctl handler.
2203 */
2204 if (old->f_op == file->f_op &&
2205 old->f_cred->user_ns == file->f_cred->user_ns)
2206 fud = fuse_get_dev(old);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002207
Miklos Szeredicc080e92015-07-01 16:26:08 +02002208 if (fud) {
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002209 mutex_lock(&fuse_mutex);
Miklos Szeredicc080e92015-07-01 16:26:08 +02002210 err = fuse_device_clone(fud->fc, file);
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002211 mutex_unlock(&fuse_mutex);
2212 }
2213 fput(old);
2214 }
2215 }
2216 }
2217 return err;
2218}
2219
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -08002220const struct file_operations fuse_dev_operations = {
Miklos Szeredi334f4852005-09-09 13:10:27 -07002221 .owner = THIS_MODULE,
Tom Van Braeckel94e4fe22015-01-12 05:22:16 +01002222 .open = fuse_dev_open,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002223 .llseek = no_llseek,
Al Virofbdbacc2015-04-03 21:53:39 -04002224 .read_iter = fuse_dev_read,
Miklos Szeredic3021622010-05-25 15:06:07 +02002225 .splice_read = fuse_dev_splice_read,
Al Virofbdbacc2015-04-03 21:53:39 -04002226 .write_iter = fuse_dev_write,
Miklos Szeredidd3bb142010-05-25 15:06:06 +02002227 .splice_write = fuse_dev_splice_write,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002228 .poll = fuse_dev_poll,
2229 .release = fuse_dev_release,
Jeff Dike385a17b2006-04-10 22:54:52 -07002230 .fasync = fuse_dev_fasync,
Miklos Szeredi00c570f2015-07-01 16:26:08 +02002231 .unlocked_ioctl = fuse_dev_ioctl,
2232 .compat_ioctl = fuse_dev_ioctl,
Miklos Szeredi334f4852005-09-09 13:10:27 -07002233};
Tejun Heo08cbf542009-04-14 10:54:53 +09002234EXPORT_SYMBOL_GPL(fuse_dev_operations);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002235
2236static struct miscdevice fuse_miscdevice = {
2237 .minor = FUSE_MINOR,
2238 .name = "fuse",
2239 .fops = &fuse_dev_operations,
2240};
2241
2242int __init fuse_dev_init(void)
2243{
2244 int err = -ENOMEM;
2245 fuse_req_cachep = kmem_cache_create("fuse_request",
2246 sizeof(struct fuse_req),
Paul Mundt20c2df82007-07-20 10:11:58 +09002247 0, 0, NULL);
Miklos Szeredi334f4852005-09-09 13:10:27 -07002248 if (!fuse_req_cachep)
2249 goto out;
2250
2251 err = misc_register(&fuse_miscdevice);
2252 if (err)
2253 goto out_cache_clean;
2254
2255 return 0;
2256
2257 out_cache_clean:
2258 kmem_cache_destroy(fuse_req_cachep);
2259 out:
2260 return err;
2261}
2262
2263void fuse_dev_cleanup(void)
2264{
2265 misc_deregister(&fuse_miscdevice);
2266 kmem_cache_destroy(fuse_req_cachep);
2267}