blob: c3f652395a80b8ded540bc60fe235ce504e239f7 [file] [log] [blame]
Tom Tucker1d8206b92007-12-30 21:07:15 -06001/*
2 * linux/net/sunrpc/svc_xprt.c
3 *
4 * Author: Tom Tucker <tom@opengridcomputing.com>
5 */
6
7#include <linux/sched.h>
8#include <linux/errno.h>
Tom Tucker1d8206b92007-12-30 21:07:15 -06009#include <linux/freezer.h>
Jeff Layton70867212008-02-07 16:34:54 -050010#include <linux/kthread.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090011#include <linux/slab.h>
Tom Tucker1d8206b92007-12-30 21:07:15 -060012#include <net/sock.h>
Scott Mayhewc3d48792015-12-11 16:45:58 -050013#include <linux/sunrpc/addr.h>
Tom Tucker1d8206b92007-12-30 21:07:15 -060014#include <linux/sunrpc/stats.h>
15#include <linux/sunrpc/svc_xprt.h>
H Hartley Sweetendcf1a352009-04-22 20:18:19 -040016#include <linux/sunrpc/svcsock.h>
J. Bruce Fields99de8ea2010-12-08 12:45:44 -050017#include <linux/sunrpc/xprt.h>
Paul Gortmaker3a9a2312011-05-27 09:12:25 -040018#include <linux/module.h>
Scott Mayhewc3d48792015-12-11 16:45:58 -050019#include <linux/netdevice.h>
Jeff Layton860a0d92014-10-28 14:24:12 -040020#include <trace/events/sunrpc.h>
Tom Tucker1d8206b92007-12-30 21:07:15 -060021
22#define RPCDBG_FACILITY RPCDBG_SVCXPRT
23
Trond Myklebustff3ac5c2016-06-24 10:55:50 -040024static unsigned int svc_rpc_per_connection_limit __read_mostly;
25module_param(svc_rpc_per_connection_limit, uint, 0644);
26
27
Tom Tucker0f0257e2007-12-30 21:08:27 -060028static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt);
29static int svc_deferred_recv(struct svc_rqst *rqstp);
30static struct cache_deferred_req *svc_defer(struct cache_req *req);
31static void svc_age_temp_xprts(unsigned long closure);
J. Bruce Fields7710ec32011-11-25 18:44:05 -050032static void svc_delete_xprt(struct svc_xprt *xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -060033
34/* apparently the "standard" is that clients close
35 * idle connections after 5 minutes, servers after
36 * 6 minutes
37 * http://www.connectathon.org/talks96/nfstcp.pdf
38 */
39static int svc_conn_age_period = 6*60;
40
Tom Tucker1d8206b92007-12-30 21:07:15 -060041/* List of registered transport classes */
42static DEFINE_SPINLOCK(svc_xprt_class_lock);
43static LIST_HEAD(svc_xprt_class_list);
44
Tom Tucker0f0257e2007-12-30 21:08:27 -060045/* SMP locking strategy:
46 *
47 * svc_pool->sp_lock protects most of the fields of that pool.
48 * svc_serv->sv_lock protects sv_tempsocks, sv_permsocks, sv_tmpcnt.
49 * when both need to be taken (rare), svc_serv->sv_lock is first.
Jeff Layton3c519912015-01-22 08:19:32 -050050 * The "service mutex" protects svc_serv->sv_nrthread.
Tom Tucker0f0257e2007-12-30 21:08:27 -060051 * svc_sock->sk_lock protects the svc_sock->sk_deferred list
52 * and the ->sk_info_authunix cache.
53 *
54 * The XPT_BUSY bit in xprt->xpt_flags prevents a transport being
55 * enqueued multiply. During normal transport processing this bit
56 * is set by svc_xprt_enqueue and cleared by svc_xprt_received.
57 * Providers should not manipulate this bit directly.
58 *
59 * Some flags can be set to certain values at any time
60 * providing that certain rules are followed:
61 *
62 * XPT_CONN, XPT_DATA:
63 * - Can be set or cleared at any time.
64 * - After a set, svc_xprt_enqueue must be called to enqueue
65 * the transport for processing.
66 * - After a clear, the transport must be read/accepted.
67 * If this succeeds, it must be set again.
68 * XPT_CLOSE:
69 * - Can set at any time. It is never cleared.
70 * XPT_DEAD:
71 * - Can only be set while XPT_BUSY is held which ensures
72 * that no other thread will be using the transport or will
73 * try to set XPT_DEAD.
74 */
Tom Tucker1d8206b92007-12-30 21:07:15 -060075int svc_reg_xprt_class(struct svc_xprt_class *xcl)
76{
77 struct svc_xprt_class *cl;
78 int res = -EEXIST;
79
80 dprintk("svc: Adding svc transport class '%s'\n", xcl->xcl_name);
81
82 INIT_LIST_HEAD(&xcl->xcl_list);
83 spin_lock(&svc_xprt_class_lock);
84 /* Make sure there isn't already a class with the same name */
85 list_for_each_entry(cl, &svc_xprt_class_list, xcl_list) {
86 if (strcmp(xcl->xcl_name, cl->xcl_name) == 0)
87 goto out;
88 }
89 list_add_tail(&xcl->xcl_list, &svc_xprt_class_list);
90 res = 0;
91out:
92 spin_unlock(&svc_xprt_class_lock);
93 return res;
94}
95EXPORT_SYMBOL_GPL(svc_reg_xprt_class);
96
97void svc_unreg_xprt_class(struct svc_xprt_class *xcl)
98{
99 dprintk("svc: Removing svc transport class '%s'\n", xcl->xcl_name);
100 spin_lock(&svc_xprt_class_lock);
101 list_del_init(&xcl->xcl_list);
102 spin_unlock(&svc_xprt_class_lock);
103}
104EXPORT_SYMBOL_GPL(svc_unreg_xprt_class);
105
Tom Tuckerdc9a16e2007-12-30 21:08:31 -0600106/*
107 * Format the transport list for printing
108 */
109int svc_print_xprts(char *buf, int maxlen)
110{
Pavel Emelyanov8f3a6de2010-10-05 23:30:19 +0400111 struct svc_xprt_class *xcl;
Tom Tuckerdc9a16e2007-12-30 21:08:31 -0600112 char tmpstr[80];
113 int len = 0;
114 buf[0] = '\0';
115
116 spin_lock(&svc_xprt_class_lock);
Pavel Emelyanov8f3a6de2010-10-05 23:30:19 +0400117 list_for_each_entry(xcl, &svc_xprt_class_list, xcl_list) {
Tom Tuckerdc9a16e2007-12-30 21:08:31 -0600118 int slen;
Tom Tuckerdc9a16e2007-12-30 21:08:31 -0600119
120 sprintf(tmpstr, "%s %d\n", xcl->xcl_name, xcl->xcl_max_payload);
121 slen = strlen(tmpstr);
122 if (len + slen > maxlen)
123 break;
124 len += slen;
125 strcat(buf, tmpstr);
126 }
127 spin_unlock(&svc_xprt_class_lock);
128
129 return len;
130}
131
Tom Tuckere1b31572007-12-30 21:07:46 -0600132static void svc_xprt_free(struct kref *kref)
133{
134 struct svc_xprt *xprt =
135 container_of(kref, struct svc_xprt, xpt_ref);
136 struct module *owner = xprt->xpt_class->xcl_owner;
Pavel Emelyanove3bfca02010-09-27 13:58:42 +0400137 if (test_bit(XPT_CACHE_AUTH, &xprt->xpt_flags))
138 svcauth_unix_info_release(xprt);
Pavel Emelyanov4fb85182010-09-27 14:00:49 +0400139 put_net(xprt->xpt_net);
J. Bruce Fields99de8ea2010-12-08 12:45:44 -0500140 /* See comment on corresponding get in xs_setup_bc_tcp(): */
141 if (xprt->xpt_bc_xprt)
142 xprt_put(xprt->xpt_bc_xprt);
J. Bruce Fields39a9bea2016-05-17 12:38:21 -0400143 if (xprt->xpt_bc_xps)
144 xprt_switch_put(xprt->xpt_bc_xps);
Tom Tuckere1b31572007-12-30 21:07:46 -0600145 xprt->xpt_ops->xpo_free(xprt);
146 module_put(owner);
147}
148
149void svc_xprt_put(struct svc_xprt *xprt)
150{
151 kref_put(&xprt->xpt_ref, svc_xprt_free);
152}
153EXPORT_SYMBOL_GPL(svc_xprt_put);
154
Tom Tucker1d8206b92007-12-30 21:07:15 -0600155/*
156 * Called by transport drivers to initialize the transport independent
157 * portion of the transport instance.
158 */
Stanislav Kinsburskybd4620d2011-12-06 14:19:10 +0300159void svc_xprt_init(struct net *net, struct svc_xprt_class *xcl,
160 struct svc_xprt *xprt, struct svc_serv *serv)
Tom Tucker1d8206b92007-12-30 21:07:15 -0600161{
162 memset(xprt, 0, sizeof(*xprt));
163 xprt->xpt_class = xcl;
164 xprt->xpt_ops = xcl->xcl_ops;
Tom Tuckere1b31572007-12-30 21:07:46 -0600165 kref_init(&xprt->xpt_ref);
Tom Tuckerbb5cf162007-12-30 21:07:50 -0600166 xprt->xpt_server = serv;
Tom Tucker7a182082007-12-30 21:07:53 -0600167 INIT_LIST_HEAD(&xprt->xpt_list);
168 INIT_LIST_HEAD(&xprt->xpt_ready);
Tom Tucker8c7b0172007-12-30 21:08:10 -0600169 INIT_LIST_HEAD(&xprt->xpt_deferred);
J. Bruce Fieldsedc7a892010-03-22 15:37:17 -0400170 INIT_LIST_HEAD(&xprt->xpt_users);
Tom Tuckera50fea22007-12-30 21:07:59 -0600171 mutex_init(&xprt->xpt_mutex);
Tom Tuckerdef13d72007-12-30 21:08:08 -0600172 spin_lock_init(&xprt->xpt_lock);
Tom Tucker4e5caaa2007-12-30 21:08:20 -0600173 set_bit(XPT_BUSY, &xprt->xpt_flags);
Rahul Iyer4cfc7e62009-09-10 17:32:28 +0300174 rpc_init_wait_queue(&xprt->xpt_bc_pending, "xpt_bc_pending");
Stanislav Kinsburskybd4620d2011-12-06 14:19:10 +0300175 xprt->xpt_net = get_net(net);
Tom Tucker1d8206b92007-12-30 21:07:15 -0600176}
177EXPORT_SYMBOL_GPL(svc_xprt_init);
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600178
Chuck Lever5dd248f2008-06-30 18:45:37 -0400179static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl,
180 struct svc_serv *serv,
Pavel Emelyanov62832c02010-09-29 16:04:18 +0400181 struct net *net,
Chuck Lever9652ada2009-03-18 20:46:21 -0400182 const int family,
183 const unsigned short port,
184 int flags)
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600185{
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600186 struct sockaddr_in sin = {
187 .sin_family = AF_INET,
Al Viroe6f1ceb2008-03-17 22:44:53 -0700188 .sin_addr.s_addr = htonl(INADDR_ANY),
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600189 .sin_port = htons(port),
190 };
Eric Dumazetdfd56b82011-12-10 09:48:31 +0000191#if IS_ENABLED(CONFIG_IPV6)
Chuck Lever5dd248f2008-06-30 18:45:37 -0400192 struct sockaddr_in6 sin6 = {
193 .sin6_family = AF_INET6,
194 .sin6_addr = IN6ADDR_ANY_INIT,
195 .sin6_port = htons(port),
196 };
Eric Dumazetdfd56b82011-12-10 09:48:31 +0000197#endif
Chuck Lever5dd248f2008-06-30 18:45:37 -0400198 struct sockaddr *sap;
199 size_t len;
200
Chuck Lever9652ada2009-03-18 20:46:21 -0400201 switch (family) {
202 case PF_INET:
Chuck Lever5dd248f2008-06-30 18:45:37 -0400203 sap = (struct sockaddr *)&sin;
204 len = sizeof(sin);
205 break;
Eric Dumazetdfd56b82011-12-10 09:48:31 +0000206#if IS_ENABLED(CONFIG_IPV6)
Chuck Lever9652ada2009-03-18 20:46:21 -0400207 case PF_INET6:
Chuck Lever5dd248f2008-06-30 18:45:37 -0400208 sap = (struct sockaddr *)&sin6;
209 len = sizeof(sin6);
210 break;
Eric Dumazetdfd56b82011-12-10 09:48:31 +0000211#endif
Chuck Lever5dd248f2008-06-30 18:45:37 -0400212 default:
213 return ERR_PTR(-EAFNOSUPPORT);
214 }
215
Pavel Emelyanov62832c02010-09-29 16:04:18 +0400216 return xcl->xcl_ops->xpo_create(serv, net, sap, len, flags);
Chuck Lever5dd248f2008-06-30 18:45:37 -0400217}
218
J. Bruce Fields67410192012-08-17 22:12:19 -0400219/*
220 * svc_xprt_received conditionally queues the transport for processing
221 * by another thread. The caller must hold the XPT_BUSY bit and must
222 * not thereafter touch transport data.
223 *
224 * Note: XPT_DATA only gets cleared when a read-attempt finds no (or
225 * insufficient) data.
226 */
227static void svc_xprt_received(struct svc_xprt *xprt)
228{
Jeff Laytonacf06a72014-12-01 13:45:24 -0500229 if (!test_bit(XPT_BUSY, &xprt->xpt_flags)) {
230 WARN_ONCE(1, "xprt=0x%p already busy!", xprt);
Weston Andros Adamsonff1fdb92012-10-23 10:43:40 -0400231 return;
Jeff Laytonacf06a72014-12-01 13:45:24 -0500232 }
233
J. Bruce Fields67410192012-08-17 22:12:19 -0400234 /* As soon as we clear busy, the xprt could be closed and
Jeff Laytonb9e13cd2015-06-08 12:06:51 -0700235 * 'put', so we need a reference to call svc_enqueue_xprt with:
J. Bruce Fields67410192012-08-17 22:12:19 -0400236 */
237 svc_xprt_get(xprt);
Trond Myklebust09713742014-07-24 23:59:31 -0400238 smp_mb__before_atomic();
J. Bruce Fields67410192012-08-17 22:12:19 -0400239 clear_bit(XPT_BUSY, &xprt->xpt_flags);
Jeff Laytonb9e13cd2015-06-08 12:06:51 -0700240 xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
J. Bruce Fields67410192012-08-17 22:12:19 -0400241 svc_xprt_put(xprt);
242}
243
J. Bruce Fields39b55302012-08-14 15:50:34 -0400244void svc_add_new_perm_xprt(struct svc_serv *serv, struct svc_xprt *new)
245{
246 clear_bit(XPT_TEMP, &new->xpt_flags);
247 spin_lock_bh(&serv->sv_lock);
248 list_add(&new->xpt_list, &serv->sv_permsocks);
249 spin_unlock_bh(&serv->sv_lock);
250 svc_xprt_received(new);
251}
252
J. Bruce Fieldsd96b9c92016-05-18 14:50:14 -0400253int _svc_create_xprt(struct svc_serv *serv, const char *xprt_name,
Pavel Emelyanovfc5d00b2010-09-29 16:03:50 +0400254 struct net *net, const int family,
255 const unsigned short port, int flags)
Chuck Lever5dd248f2008-06-30 18:45:37 -0400256{
257 struct svc_xprt_class *xcl;
258
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600259 spin_lock(&svc_xprt_class_lock);
260 list_for_each_entry(xcl, &svc_xprt_class_list, xcl_list) {
Tom Tucker4e5caaa2007-12-30 21:08:20 -0600261 struct svc_xprt *newxprt;
NeilBrowned2849d2010-11-16 16:55:19 +1100262 unsigned short newport;
Tom Tucker4e5caaa2007-12-30 21:08:20 -0600263
264 if (strcmp(xprt_name, xcl->xcl_name))
265 continue;
266
267 if (!try_module_get(xcl->xcl_owner))
268 goto err;
269
270 spin_unlock(&svc_xprt_class_lock);
Pavel Emelyanov62832c02010-09-29 16:04:18 +0400271 newxprt = __svc_xpo_create(xcl, serv, net, family, port, flags);
Tom Tucker4e5caaa2007-12-30 21:08:20 -0600272 if (IS_ERR(newxprt)) {
273 module_put(xcl->xcl_owner);
274 return PTR_ERR(newxprt);
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600275 }
J. Bruce Fields39b55302012-08-14 15:50:34 -0400276 svc_add_new_perm_xprt(serv, newxprt);
NeilBrowned2849d2010-11-16 16:55:19 +1100277 newport = svc_xprt_local_port(newxprt);
NeilBrowned2849d2010-11-16 16:55:19 +1100278 return newport;
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600279 }
Tom Tucker4e5caaa2007-12-30 21:08:20 -0600280 err:
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600281 spin_unlock(&svc_xprt_class_lock);
Chuck Lever68717902010-01-26 14:04:13 -0500282 /* This errno is exposed to user space. Provide a reasonable
283 * perror msg for a bad transport. */
284 return -EPROTONOSUPPORT;
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600285}
J. Bruce Fieldsd96b9c92016-05-18 14:50:14 -0400286
287int svc_create_xprt(struct svc_serv *serv, const char *xprt_name,
288 struct net *net, const int family,
289 const unsigned short port, int flags)
290{
291 int err;
292
293 dprintk("svc: creating transport %s[%d]\n", xprt_name, port);
294 err = _svc_create_xprt(serv, xprt_name, net, family, port, flags);
295 if (err == -EPROTONOSUPPORT) {
296 request_module("svc%s", xprt_name);
297 err = _svc_create_xprt(serv, xprt_name, net, family, port, flags);
298 }
299 if (err)
300 dprintk("svc: transport %s not found, err %d\n",
301 xprt_name, err);
302 return err;
303}
Tom Tuckerb700cbb2007-12-30 21:07:42 -0600304EXPORT_SYMBOL_GPL(svc_create_xprt);
Tom Tucker9dbc2402007-12-30 21:08:12 -0600305
306/*
307 * Copy the local and remote xprt addresses to the rqstp structure
308 */
309void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt)
310{
Tom Tucker9dbc2402007-12-30 21:08:12 -0600311 memcpy(&rqstp->rq_addr, &xprt->xpt_remote, xprt->xpt_remotelen);
312 rqstp->rq_addrlen = xprt->xpt_remotelen;
313
314 /*
315 * Destination address in request is needed for binding the
316 * source address in RPC replies/callbacks later.
317 */
Mi Jinlong849a1cf2011-08-30 17:18:41 +0800318 memcpy(&rqstp->rq_daddr, &xprt->xpt_local, xprt->xpt_locallen);
319 rqstp->rq_daddrlen = xprt->xpt_locallen;
Tom Tucker9dbc2402007-12-30 21:08:12 -0600320}
321EXPORT_SYMBOL_GPL(svc_xprt_copy_addrs);
322
Tom Tucker0f0257e2007-12-30 21:08:27 -0600323/**
324 * svc_print_addr - Format rq_addr field for printing
325 * @rqstp: svc_rqst struct containing address to print
326 * @buf: target buffer for formatted address
327 * @len: length of target buffer
328 *
329 */
330char *svc_print_addr(struct svc_rqst *rqstp, char *buf, size_t len)
331{
332 return __svc_print_addr(svc_addr(rqstp), buf, len);
333}
334EXPORT_SYMBOL_GPL(svc_print_addr);
335
Trond Myklebustff3ac5c2016-06-24 10:55:50 -0400336static bool svc_xprt_slots_in_range(struct svc_xprt *xprt)
337{
338 unsigned int limit = svc_rpc_per_connection_limit;
339 int nrqsts = atomic_read(&xprt->xpt_nr_rqsts);
340
341 return limit == 0 || (nrqsts >= 0 && nrqsts < limit);
342}
343
344static bool svc_xprt_reserve_slot(struct svc_rqst *rqstp, struct svc_xprt *xprt)
345{
346 if (!test_bit(RQ_DATA, &rqstp->rq_flags)) {
347 if (!svc_xprt_slots_in_range(xprt))
348 return false;
349 atomic_inc(&xprt->xpt_nr_rqsts);
350 set_bit(RQ_DATA, &rqstp->rq_flags);
351 }
352 return true;
353}
354
355static void svc_xprt_release_slot(struct svc_rqst *rqstp)
356{
357 struct svc_xprt *xprt = rqstp->rq_xprt;
358 if (test_and_clear_bit(RQ_DATA, &rqstp->rq_flags)) {
359 atomic_dec(&xprt->xpt_nr_rqsts);
360 svc_xprt_enqueue(xprt);
361 }
362}
363
J. Bruce Fields9c335c02010-10-26 11:32:03 -0400364static bool svc_xprt_has_something_to_do(struct svc_xprt *xprt)
365{
366 if (xprt->xpt_flags & ((1<<XPT_CONN)|(1<<XPT_CLOSE)))
367 return true;
Trond Myklebust82ea2d72016-06-24 10:55:45 -0400368 if (xprt->xpt_flags & ((1<<XPT_DATA)|(1<<XPT_DEFERRED))) {
Trond Myklebustff3ac5c2016-06-24 10:55:50 -0400369 if (xprt->xpt_ops->xpo_has_wspace(xprt) &&
370 svc_xprt_slots_in_range(xprt))
Trond Myklebust82ea2d72016-06-24 10:55:45 -0400371 return true;
372 trace_svc_xprt_no_write_space(xprt);
373 return false;
374 }
J. Bruce Fields9c335c02010-10-26 11:32:03 -0400375 return false;
376}
377
Jeff Laytonb9e13cd2015-06-08 12:06:51 -0700378void svc_xprt_do_enqueue(struct svc_xprt *xprt)
Tom Tucker0f0257e2007-12-30 21:08:27 -0600379{
Tom Tucker0f0257e2007-12-30 21:08:27 -0600380 struct svc_pool *pool;
Jeff Layton83a712e2014-11-21 14:19:31 -0500381 struct svc_rqst *rqstp = NULL;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600382 int cpu;
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500383 bool queued = false;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600384
J. Bruce Fields9c335c02010-10-26 11:32:03 -0400385 if (!svc_xprt_has_something_to_do(xprt))
Jeff Layton83a712e2014-11-21 14:19:31 -0500386 goto out;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600387
Tom Tucker0f0257e2007-12-30 21:08:27 -0600388 /* Mark transport as busy. It will remain in this state until
389 * the provider calls svc_xprt_received. We update XPT_BUSY
390 * atomically because it also guards against trying to enqueue
391 * the transport twice.
392 */
393 if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags)) {
394 /* Don't enqueue transport while already enqueued */
395 dprintk("svc: transport %p busy, not enqueued\n", xprt);
Jeff Layton83a712e2014-11-21 14:19:31 -0500396 goto out;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600397 }
Tom Tucker0f0257e2007-12-30 21:08:27 -0600398
Trond Myklebust0c0746d2014-08-03 13:03:12 -0400399 cpu = get_cpu();
400 pool = svc_pool_for_cpu(xprt->xpt_server, cpu);
Trond Myklebust0c0746d2014-08-03 13:03:12 -0400401
Jeff Layton403c7b42014-11-21 14:19:29 -0500402 atomic_long_inc(&pool->sp_stats.packets);
Trond Myklebust0c0746d2014-08-03 13:03:12 -0400403
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500404redo_search:
405 /* find a thread for this xprt */
406 rcu_read_lock();
407 list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {
408 /* Do a lockless check first */
409 if (test_bit(RQ_BUSY, &rqstp->rq_flags))
410 continue;
411
412 /*
413 * Once the xprt has been queued, it can only be dequeued by
414 * the task that intends to service it. All we can do at that
415 * point is to try to wake this thread back up so that it can
416 * do so.
Trond Myklebust983c6842014-08-03 13:03:10 -0400417 */
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500418 if (!queued) {
419 spin_lock_bh(&rqstp->rq_lock);
420 if (test_and_set_bit(RQ_BUSY, &rqstp->rq_flags)) {
421 /* already busy, move on... */
422 spin_unlock_bh(&rqstp->rq_lock);
423 continue;
424 }
425
426 /* this one will do */
427 rqstp->rq_xprt = xprt;
428 svc_xprt_get(xprt);
429 spin_unlock_bh(&rqstp->rq_lock);
430 }
431 rcu_read_unlock();
432
Jeff Layton403c7b42014-11-21 14:19:29 -0500433 atomic_long_inc(&pool->sp_stats.threads_woken);
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500434 wake_up_process(rqstp->rq_task);
435 put_cpu();
Jeff Layton83a712e2014-11-21 14:19:31 -0500436 goto out;
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500437 }
438 rcu_read_unlock();
439
440 /*
441 * We didn't find an idle thread to use, so we need to queue the xprt.
442 * Do so and then search again. If we find one, we can't hook this one
443 * up to it directly but we can wake the thread up in the hopes that it
444 * will pick it up once it searches for a xprt to service.
445 */
446 if (!queued) {
447 queued = true;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600448 dprintk("svc: transport %p put into queue\n", xprt);
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500449 spin_lock_bh(&pool->sp_lock);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600450 list_add_tail(&xprt->xpt_ready, &pool->sp_sockets);
Greg Banks03cf6c92009-01-13 21:26:36 +1100451 pool->sp_stats.sockets_queued++;
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500452 spin_unlock_bh(&pool->sp_lock);
453 goto redo_search;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600454 }
Jeff Layton83a712e2014-11-21 14:19:31 -0500455 rqstp = NULL;
Trond Myklebust983c6842014-08-03 13:03:10 -0400456 put_cpu();
Jeff Layton83a712e2014-11-21 14:19:31 -0500457out:
458 trace_svc_xprt_do_enqueue(xprt, rqstp);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600459}
Jeff Laytonb9e13cd2015-06-08 12:06:51 -0700460EXPORT_SYMBOL_GPL(svc_xprt_do_enqueue);
Trond Myklebust09713742014-07-24 23:59:31 -0400461
462/*
463 * Queue up a transport with data pending. If there are idle nfsd
464 * processes, wake 'em up.
465 *
466 */
467void svc_xprt_enqueue(struct svc_xprt *xprt)
468{
469 if (test_bit(XPT_BUSY, &xprt->xpt_flags))
470 return;
Jeff Laytonb9e13cd2015-06-08 12:06:51 -0700471 xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
Trond Myklebust09713742014-07-24 23:59:31 -0400472}
Tom Tucker0f0257e2007-12-30 21:08:27 -0600473EXPORT_SYMBOL_GPL(svc_xprt_enqueue);
474
475/*
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500476 * Dequeue the first transport, if there is one.
Tom Tucker0f0257e2007-12-30 21:08:27 -0600477 */
478static struct svc_xprt *svc_xprt_dequeue(struct svc_pool *pool)
479{
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500480 struct svc_xprt *xprt = NULL;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600481
482 if (list_empty(&pool->sp_sockets))
Jeff Layton83a712e2014-11-21 14:19:31 -0500483 goto out;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600484
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500485 spin_lock_bh(&pool->sp_lock);
486 if (likely(!list_empty(&pool->sp_sockets))) {
487 xprt = list_first_entry(&pool->sp_sockets,
488 struct svc_xprt, xpt_ready);
489 list_del_init(&xprt->xpt_ready);
490 svc_xprt_get(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600491
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500492 dprintk("svc: transport %p dequeued, inuse=%d\n",
493 xprt, atomic_read(&xprt->xpt_ref.refcount));
494 }
495 spin_unlock_bh(&pool->sp_lock);
Jeff Layton83a712e2014-11-21 14:19:31 -0500496out:
497 trace_svc_xprt_dequeue(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600498 return xprt;
499}
500
Tom Tucker0f0257e2007-12-30 21:08:27 -0600501/**
502 * svc_reserve - change the space reserved for the reply to a request.
503 * @rqstp: The request in question
504 * @space: new max space to reserve
505 *
506 * Each request reserves some space on the output queue of the transport
507 * to make sure the reply fits. This function reduces that reserved
508 * space to be the amount of space used already, plus @space.
509 *
510 */
511void svc_reserve(struct svc_rqst *rqstp, int space)
512{
513 space += rqstp->rq_res.head[0].iov_len;
514
515 if (space < rqstp->rq_reserved) {
516 struct svc_xprt *xprt = rqstp->rq_xprt;
517 atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);
518 rqstp->rq_reserved = space;
519
520 svc_xprt_enqueue(xprt);
521 }
522}
Trond Myklebust24c37672008-12-23 16:30:12 -0500523EXPORT_SYMBOL_GPL(svc_reserve);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600524
525static void svc_xprt_release(struct svc_rqst *rqstp)
526{
527 struct svc_xprt *xprt = rqstp->rq_xprt;
528
529 rqstp->rq_xprt->xpt_ops->xpo_release_rqst(rqstp);
530
Tom Tucker2779e3a2009-01-05 11:12:52 -0600531 kfree(rqstp->rq_deferred);
532 rqstp->rq_deferred = NULL;
533
Tom Tucker0f0257e2007-12-30 21:08:27 -0600534 svc_free_res_pages(rqstp);
535 rqstp->rq_res.page_len = 0;
536 rqstp->rq_res.page_base = 0;
537
538 /* Reset response buffer and release
539 * the reservation.
540 * But first, check that enough space was reserved
541 * for the reply, otherwise we have a bug!
542 */
543 if ((rqstp->rq_res.len) > rqstp->rq_reserved)
544 printk(KERN_ERR "RPC request reserved %d but used %d\n",
545 rqstp->rq_reserved,
546 rqstp->rq_res.len);
547
548 rqstp->rq_res.head[0].iov_len = 0;
549 svc_reserve(rqstp, 0);
Trond Myklebustff3ac5c2016-06-24 10:55:50 -0400550 svc_xprt_release_slot(rqstp);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600551 rqstp->rq_xprt = NULL;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600552 svc_xprt_put(xprt);
553}
554
555/*
Jeff Laytonceff7392014-11-19 07:51:21 -0500556 * Some svc_serv's will have occasional work to do, even when a xprt is not
557 * waiting to be serviced. This function is there to "kick" a task in one of
558 * those services so that it can wake up and do that work. Note that we only
559 * bother with pool 0 as we don't need to wake up more than one thread for
560 * this purpose.
Tom Tucker0f0257e2007-12-30 21:08:27 -0600561 */
562void svc_wake_up(struct svc_serv *serv)
563{
564 struct svc_rqst *rqstp;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600565 struct svc_pool *pool;
566
Jeff Laytonceff7392014-11-19 07:51:21 -0500567 pool = &serv->sv_pools[0];
Tom Tucker0f0257e2007-12-30 21:08:27 -0600568
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500569 rcu_read_lock();
570 list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {
571 /* skip any that aren't queued */
572 if (test_bit(RQ_BUSY, &rqstp->rq_flags))
573 continue;
574 rcu_read_unlock();
Jeff Laytonceff7392014-11-19 07:51:21 -0500575 dprintk("svc: daemon %p woken up.\n", rqstp);
576 wake_up_process(rqstp->rq_task);
Jeff Layton83a712e2014-11-21 14:19:31 -0500577 trace_svc_wake_up(rqstp->rq_task->pid);
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500578 return;
579 }
580 rcu_read_unlock();
581
582 /* No free entries available */
583 set_bit(SP_TASK_PENDING, &pool->sp_flags);
584 smp_wmb();
Jeff Layton83a712e2014-11-21 14:19:31 -0500585 trace_svc_wake_up(0);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600586}
Trond Myklebust24c37672008-12-23 16:30:12 -0500587EXPORT_SYMBOL_GPL(svc_wake_up);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600588
589int svc_port_is_privileged(struct sockaddr *sin)
590{
591 switch (sin->sa_family) {
592 case AF_INET:
593 return ntohs(((struct sockaddr_in *)sin)->sin_port)
594 < PROT_SOCK;
595 case AF_INET6:
596 return ntohs(((struct sockaddr_in6 *)sin)->sin6_port)
597 < PROT_SOCK;
598 default:
599 return 0;
600 }
601}
602
603/*
Jeff Laytonc9233eb2008-10-20 11:51:57 -0400604 * Make sure that we don't have too many active connections. If we have,
605 * something must be dropped. It's not clear what will happen if we allow
606 * "too many" connections, but when dealing with network-facing software,
607 * we have to code defensively. Here we do that by imposing hard limits.
Tom Tucker0f0257e2007-12-30 21:08:27 -0600608 *
609 * There's no point in trying to do random drop here for DoS
610 * prevention. The NFS clients does 1 reconnect in 15 seconds. An
611 * attacker can easily beat that.
612 *
613 * The only somewhat efficient mechanism would be if drop old
614 * connections from the same IP first. But right now we don't even
615 * record the client IP in svc_sock.
Jeff Laytonc9233eb2008-10-20 11:51:57 -0400616 *
617 * single-threaded services that expect a lot of clients will probably
618 * need to set sv_maxconn to override the default value which is based
619 * on the number of threads
Tom Tucker0f0257e2007-12-30 21:08:27 -0600620 */
621static void svc_check_conn_limits(struct svc_serv *serv)
622{
Jeff Laytonc9233eb2008-10-20 11:51:57 -0400623 unsigned int limit = serv->sv_maxconn ? serv->sv_maxconn :
624 (serv->sv_nrthreads+3) * 20;
625
626 if (serv->sv_tmpcnt > limit) {
Tom Tucker0f0257e2007-12-30 21:08:27 -0600627 struct svc_xprt *xprt = NULL;
628 spin_lock_bh(&serv->sv_lock);
629 if (!list_empty(&serv->sv_tempsocks)) {
Joe Perchese87cc472012-05-13 21:56:26 +0000630 /* Try to help the admin */
631 net_notice_ratelimited("%s: too many open connections, consider increasing the %s\n",
632 serv->sv_name, serv->sv_maxconn ?
633 "max number of connections" :
634 "number of threads");
Tom Tucker0f0257e2007-12-30 21:08:27 -0600635 /*
636 * Always select the oldest connection. It's not fair,
637 * but so is life
638 */
639 xprt = list_entry(serv->sv_tempsocks.prev,
640 struct svc_xprt,
641 xpt_list);
642 set_bit(XPT_CLOSE, &xprt->xpt_flags);
643 svc_xprt_get(xprt);
644 }
645 spin_unlock_bh(&serv->sv_lock);
646
647 if (xprt) {
648 svc_xprt_enqueue(xprt);
649 svc_xprt_put(xprt);
650 }
651 }
652}
653
Rashika Kheriae1d83ee2014-02-09 22:33:03 +0530654static int svc_alloc_arg(struct svc_rqst *rqstp)
Tom Tucker0f0257e2007-12-30 21:08:27 -0600655{
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400656 struct svc_serv *serv = rqstp->rq_server;
657 struct xdr_buf *arg;
658 int pages;
659 int i;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600660
661 /* now allocate needed pages. If we get a failure, sleep briefly */
662 pages = (serv->sv_max_mesg + PAGE_SIZE) / PAGE_SIZE;
Weston Andros Adamsonb25cd052012-10-23 10:43:41 -0400663 WARN_ON_ONCE(pages >= RPCSVC_MAXPAGES);
664 if (pages >= RPCSVC_MAXPAGES)
665 /* use as many pages as possible */
666 pages = RPCSVC_MAXPAGES - 1;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600667 for (i = 0; i < pages ; i++)
668 while (rqstp->rq_pages[i] == NULL) {
669 struct page *p = alloc_page(GFP_KERNEL);
670 if (!p) {
Jeff Layton7b54fe62008-02-12 11:47:24 -0500671 set_current_state(TASK_INTERRUPTIBLE);
672 if (signalled() || kthread_should_stop()) {
673 set_current_state(TASK_RUNNING);
Jeff Layton70867212008-02-07 16:34:54 -0500674 return -EINTR;
Jeff Layton7b54fe62008-02-12 11:47:24 -0500675 }
676 schedule_timeout(msecs_to_jiffies(500));
Tom Tucker0f0257e2007-12-30 21:08:27 -0600677 }
678 rqstp->rq_pages[i] = p;
679 }
J. Bruce Fields2825a7f2013-08-26 16:04:46 -0400680 rqstp->rq_page_end = &rqstp->rq_pages[i];
Tom Tucker0f0257e2007-12-30 21:08:27 -0600681 rqstp->rq_pages[i++] = NULL; /* this might be seen in nfs_read_actor */
Tom Tucker0f0257e2007-12-30 21:08:27 -0600682
683 /* Make arg->head point to first page and arg->pages point to rest */
684 arg = &rqstp->rq_arg;
685 arg->head[0].iov_base = page_address(rqstp->rq_pages[0]);
686 arg->head[0].iov_len = PAGE_SIZE;
687 arg->pages = rqstp->rq_pages + 1;
688 arg->page_base = 0;
689 /* save at least one page for response */
690 arg->page_len = (pages-2)*PAGE_SIZE;
691 arg->len = (pages-1)*PAGE_SIZE;
692 arg->tail[0].iov_len = 0;
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400693 return 0;
694}
Tom Tucker0f0257e2007-12-30 21:08:27 -0600695
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500696static bool
697rqst_should_sleep(struct svc_rqst *rqstp)
698{
699 struct svc_pool *pool = rqstp->rq_pool;
700
701 /* did someone call svc_wake_up? */
702 if (test_and_clear_bit(SP_TASK_PENDING, &pool->sp_flags))
703 return false;
704
705 /* was a socket queued? */
706 if (!list_empty(&pool->sp_sockets))
707 return false;
708
709 /* are we shutting down? */
710 if (signalled() || kthread_should_stop())
711 return false;
712
713 /* are we freezing? */
714 if (freezing(current))
715 return false;
716
717 return true;
718}
719
Rashika Kheriae1d83ee2014-02-09 22:33:03 +0530720static struct svc_xprt *svc_get_next_xprt(struct svc_rqst *rqstp, long timeout)
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400721{
722 struct svc_xprt *xprt;
723 struct svc_pool *pool = rqstp->rq_pool;
Trond Myklebusta4aa8052014-08-03 13:03:11 -0400724 long time_left = 0;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600725
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500726 /* rq_xprt should be clear on entry */
727 WARN_ON_ONCE(rqstp->rq_xprt);
728
NeilBrownf16b6e82010-08-12 17:04:06 +1000729 /* Normally we will wait up to 5 seconds for any required
730 * cache information to be provided.
731 */
732 rqstp->rq_chandle.thread_wait = 5*HZ;
733
Tom Tucker0f0257e2007-12-30 21:08:27 -0600734 xprt = svc_xprt_dequeue(pool);
735 if (xprt) {
736 rqstp->rq_xprt = xprt;
NeilBrownf16b6e82010-08-12 17:04:06 +1000737
738 /* As there is a shortage of threads and this request
J. Bruce Fields6610f722010-08-26 13:19:52 -0400739 * had to be queued, don't allow the thread to wait so
NeilBrownf16b6e82010-08-12 17:04:06 +1000740 * long for cache updates.
741 */
742 rqstp->rq_chandle.thread_wait = 1*HZ;
Jeff Layton4d5db3f2014-11-19 07:51:20 -0500743 clear_bit(SP_TASK_PENDING, &pool->sp_flags);
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500744 return xprt;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600745 }
Jeff Laytonb1691bc2014-11-21 14:19:30 -0500746
747 /*
748 * We have to be able to interrupt this wait
749 * to bring down the daemons ...
750 */
751 set_current_state(TASK_INTERRUPTIBLE);
752 clear_bit(RQ_BUSY, &rqstp->rq_flags);
753 smp_mb();
754
755 if (likely(rqst_should_sleep(rqstp)))
756 time_left = schedule_timeout(timeout);
757 else
758 __set_current_state(TASK_RUNNING);
759
760 try_to_freeze();
761
762 spin_lock_bh(&rqstp->rq_lock);
763 set_bit(RQ_BUSY, &rqstp->rq_flags);
764 spin_unlock_bh(&rqstp->rq_lock);
765
766 xprt = rqstp->rq_xprt;
767 if (xprt != NULL)
768 return xprt;
769
770 if (!time_left)
771 atomic_long_inc(&pool->sp_stats.threads_timedout);
772
773 if (signalled() || kthread_should_stop())
774 return ERR_PTR(-EINTR);
775 return ERR_PTR(-EAGAIN);
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400776}
Tom Tucker0f0257e2007-12-30 21:08:27 -0600777
Rashika Kheriae1d83ee2014-02-09 22:33:03 +0530778static void svc_add_new_temp_xprt(struct svc_serv *serv, struct svc_xprt *newxpt)
J. Bruce Fields65b2e662012-08-18 15:44:33 -0400779{
780 spin_lock_bh(&serv->sv_lock);
781 set_bit(XPT_TEMP, &newxpt->xpt_flags);
782 list_add(&newxpt->xpt_list, &serv->sv_tempsocks);
783 serv->sv_tmpcnt++;
784 if (serv->sv_temptimer.function == NULL) {
785 /* setup timer to age temp transports */
786 setup_timer(&serv->sv_temptimer, svc_age_temp_xprts,
787 (unsigned long)serv);
788 mod_timer(&serv->sv_temptimer,
789 jiffies + svc_conn_age_period * HZ);
790 }
791 spin_unlock_bh(&serv->sv_lock);
792 svc_xprt_received(newxpt);
793}
794
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400795static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt)
796{
797 struct svc_serv *serv = rqstp->rq_server;
798 int len = 0;
799
J. Bruce Fields1b644b62010-02-28 16:33:31 -0500800 if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) {
801 dprintk("svc_recv: found XPT_CLOSE\n");
802 svc_delete_xprt(xprt);
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400803 /* Leave XPT_BUSY set on the dead xprt: */
Jeff Layton83a712e2014-11-21 14:19:31 -0500804 goto out;
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400805 }
806 if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) {
Tom Tucker0f0257e2007-12-30 21:08:27 -0600807 struct svc_xprt *newxpt;
J. Bruce Fields65b2e662012-08-18 15:44:33 -0400808 /*
809 * We know this module_get will succeed because the
810 * listener holds a reference too
811 */
812 __module_get(xprt->xpt_class->xcl_owner);
813 svc_check_conn_limits(xprt->xpt_server);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600814 newxpt = xprt->xpt_ops->xpo_accept(xprt);
J. Bruce Fields65b2e662012-08-18 15:44:33 -0400815 if (newxpt)
816 svc_add_new_temp_xprt(serv, newxpt);
Trond Myklebustc7891022014-05-18 14:05:22 -0400817 else
818 module_put(xprt->xpt_class->xcl_owner);
Trond Myklebustff3ac5c2016-06-24 10:55:50 -0400819 } else if (svc_xprt_reserve_slot(rqstp, xprt)) {
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400820 /* XPT_DATA|XPT_DEFERRED case: */
Tom Tucker0f0257e2007-12-30 21:08:27 -0600821 dprintk("svc: server %p, pool %u, transport %p, inuse=%d\n",
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400822 rqstp, rqstp->rq_pool->sp_id, xprt,
Tom Tucker0f0257e2007-12-30 21:08:27 -0600823 atomic_read(&xprt->xpt_ref.refcount));
824 rqstp->rq_deferred = svc_deferred_dequeue(xprt);
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400825 if (rqstp->rq_deferred)
Tom Tucker0f0257e2007-12-30 21:08:27 -0600826 len = svc_deferred_recv(rqstp);
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400827 else
Tom Tucker0f0257e2007-12-30 21:08:27 -0600828 len = xprt->xpt_ops->xpo_recvfrom(rqstp);
829 dprintk("svc: got len=%d\n", len);
J. Bruce Fieldsd10f27a2012-08-17 17:31:53 -0400830 rqstp->rq_reserved = serv->sv_max_mesg;
831 atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600832 }
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400833 /* clear XPT_BUSY: */
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400834 svc_xprt_received(xprt);
Jeff Layton83a712e2014-11-21 14:19:31 -0500835out:
836 trace_svc_handle_xprt(xprt, len);
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400837 return len;
838}
839
840/*
841 * Receive the next request on any transport. This code is carefully
842 * organised not to touch any cachelines in the shared svc_serv
843 * structure, only cachelines in the local svc_pool.
844 */
845int svc_recv(struct svc_rqst *rqstp, long timeout)
846{
847 struct svc_xprt *xprt = NULL;
848 struct svc_serv *serv = rqstp->rq_server;
849 int len, err;
850
851 dprintk("svc: server %p waiting for data (to = %ld)\n",
852 rqstp, timeout);
853
854 if (rqstp->rq_xprt)
855 printk(KERN_ERR
856 "svc_recv: service %p, transport not NULL!\n",
857 rqstp);
Trond Myklebust983c6842014-08-03 13:03:10 -0400858
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400859 err = svc_alloc_arg(rqstp);
860 if (err)
Jeff Layton860a0d92014-10-28 14:24:12 -0400861 goto out;
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400862
863 try_to_freeze();
864 cond_resched();
Jeff Layton860a0d92014-10-28 14:24:12 -0400865 err = -EINTR;
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400866 if (signalled() || kthread_should_stop())
Jeff Layton860a0d92014-10-28 14:24:12 -0400867 goto out;
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400868
869 xprt = svc_get_next_xprt(rqstp, timeout);
Jeff Layton860a0d92014-10-28 14:24:12 -0400870 if (IS_ERR(xprt)) {
871 err = PTR_ERR(xprt);
872 goto out;
873 }
J. Bruce Fields6797fa52012-08-18 15:33:51 -0400874
875 len = svc_handle_xprt(rqstp, xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600876
877 /* No data, incomplete (TCP) read, or accept() */
Jeff Layton860a0d92014-10-28 14:24:12 -0400878 err = -EAGAIN;
J. Bruce Fields9f9d2eb2012-08-17 21:35:24 -0400879 if (len <= 0)
Jeff Layton860a0d92014-10-28 14:24:12 -0400880 goto out_release;
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400881
Tom Tucker0f0257e2007-12-30 21:08:27 -0600882 clear_bit(XPT_OLD, &xprt->xpt_flags);
883
Jeff Layton4d152e22014-11-19 07:51:14 -0500884 if (xprt->xpt_ops->xpo_secure_port(rqstp))
885 set_bit(RQ_SECURE, &rqstp->rq_flags);
886 else
887 clear_bit(RQ_SECURE, &rqstp->rq_flags);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600888 rqstp->rq_chandle.defer = svc_defer;
Jeff Layton860a0d92014-10-28 14:24:12 -0400889 rqstp->rq_xid = svc_getu32(&rqstp->rq_arg.head[0]);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600890
891 if (serv->sv_stats)
892 serv->sv_stats->netcnt++;
Jeff Layton860a0d92014-10-28 14:24:12 -0400893 trace_svc_recv(rqstp, len);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600894 return len;
Jeff Layton860a0d92014-10-28 14:24:12 -0400895out_release:
J. Bruce Fieldsca7896c2010-10-25 14:12:40 -0400896 rqstp->rq_res.len = 0;
897 svc_xprt_release(rqstp);
Jeff Layton860a0d92014-10-28 14:24:12 -0400898out:
899 trace_svc_recv(rqstp, err);
900 return err;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600901}
Trond Myklebust24c37672008-12-23 16:30:12 -0500902EXPORT_SYMBOL_GPL(svc_recv);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600903
904/*
905 * Drop request
906 */
907void svc_drop(struct svc_rqst *rqstp)
908{
Trond Myklebust104f6352016-06-24 10:55:46 -0400909 trace_svc_drop(rqstp);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600910 dprintk("svc: xprt %p dropped request\n", rqstp->rq_xprt);
911 svc_xprt_release(rqstp);
912}
Trond Myklebust24c37672008-12-23 16:30:12 -0500913EXPORT_SYMBOL_GPL(svc_drop);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600914
915/*
916 * Return reply to client.
917 */
918int svc_send(struct svc_rqst *rqstp)
919{
920 struct svc_xprt *xprt;
Jeff Layton860a0d92014-10-28 14:24:12 -0400921 int len = -EFAULT;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600922 struct xdr_buf *xb;
923
924 xprt = rqstp->rq_xprt;
925 if (!xprt)
Jeff Layton860a0d92014-10-28 14:24:12 -0400926 goto out;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600927
928 /* release the receive skb before sending the reply */
929 rqstp->rq_xprt->xpt_ops->xpo_release_rqst(rqstp);
930
931 /* calculate over-all length */
932 xb = &rqstp->rq_res;
933 xb->len = xb->head[0].iov_len +
934 xb->page_len +
935 xb->tail[0].iov_len;
936
937 /* Grab mutex to serialize outgoing data. */
938 mutex_lock(&xprt->xpt_mutex);
J. Bruce Fieldsf06f00a2012-08-20 16:04:40 -0400939 if (test_bit(XPT_DEAD, &xprt->xpt_flags)
940 || test_bit(XPT_CLOSE, &xprt->xpt_flags))
Tom Tucker0f0257e2007-12-30 21:08:27 -0600941 len = -ENOTCONN;
942 else
943 len = xprt->xpt_ops->xpo_sendto(rqstp);
944 mutex_unlock(&xprt->xpt_mutex);
Rahul Iyer4cfc7e62009-09-10 17:32:28 +0300945 rpc_wake_up(&xprt->xpt_bc_pending);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600946 svc_xprt_release(rqstp);
947
948 if (len == -ECONNREFUSED || len == -ENOTCONN || len == -EAGAIN)
Jeff Layton860a0d92014-10-28 14:24:12 -0400949 len = 0;
950out:
951 trace_svc_send(rqstp, len);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600952 return len;
953}
954
955/*
956 * Timer function to close old temporary transports, using
957 * a mark-and-sweep algorithm.
958 */
959static void svc_age_temp_xprts(unsigned long closure)
960{
961 struct svc_serv *serv = (struct svc_serv *)closure;
962 struct svc_xprt *xprt;
963 struct list_head *le, *next;
Tom Tucker0f0257e2007-12-30 21:08:27 -0600964
965 dprintk("svc_age_temp_xprts\n");
966
967 if (!spin_trylock_bh(&serv->sv_lock)) {
968 /* busy, try again 1 sec later */
969 dprintk("svc_age_temp_xprts: busy\n");
970 mod_timer(&serv->sv_temptimer, jiffies + HZ);
971 return;
972 }
973
974 list_for_each_safe(le, next, &serv->sv_tempsocks) {
975 xprt = list_entry(le, struct svc_xprt, xpt_list);
976
977 /* First time through, just mark it OLD. Second time
978 * through, close it. */
979 if (!test_and_set_bit(XPT_OLD, &xprt->xpt_flags))
980 continue;
Joe Perchesf64f9e72009-11-29 16:55:45 -0800981 if (atomic_read(&xprt->xpt_ref.refcount) > 1 ||
982 test_bit(XPT_BUSY, &xprt->xpt_flags))
Tom Tucker0f0257e2007-12-30 21:08:27 -0600983 continue;
J. Bruce Fieldse75bafb2013-02-10 11:33:48 -0500984 list_del_init(le);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600985 set_bit(XPT_CLOSE, &xprt->xpt_flags);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600986 dprintk("queuing xprt %p for closing\n", xprt);
987
988 /* a thread will dequeue and close it soon */
989 svc_xprt_enqueue(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600990 }
J. Bruce Fieldse75bafb2013-02-10 11:33:48 -0500991 spin_unlock_bh(&serv->sv_lock);
Tom Tucker0f0257e2007-12-30 21:08:27 -0600992
993 mod_timer(&serv->sv_temptimer, jiffies + svc_conn_age_period * HZ);
994}
995
Scott Mayhewc3d48792015-12-11 16:45:58 -0500996/* Close temporary transports whose xpt_local matches server_addr immediately
997 * instead of waiting for them to be picked up by the timer.
998 *
999 * This is meant to be called from a notifier_block that runs when an ip
1000 * address is deleted.
1001 */
1002void svc_age_temp_xprts_now(struct svc_serv *serv, struct sockaddr *server_addr)
1003{
1004 struct svc_xprt *xprt;
1005 struct svc_sock *svsk;
1006 struct socket *sock;
1007 struct list_head *le, *next;
1008 LIST_HEAD(to_be_closed);
1009 struct linger no_linger = {
1010 .l_onoff = 1,
1011 .l_linger = 0,
1012 };
1013
1014 spin_lock_bh(&serv->sv_lock);
1015 list_for_each_safe(le, next, &serv->sv_tempsocks) {
1016 xprt = list_entry(le, struct svc_xprt, xpt_list);
1017 if (rpc_cmp_addr(server_addr, (struct sockaddr *)
1018 &xprt->xpt_local)) {
1019 dprintk("svc_age_temp_xprts_now: found %p\n", xprt);
1020 list_move(le, &to_be_closed);
1021 }
1022 }
1023 spin_unlock_bh(&serv->sv_lock);
1024
1025 while (!list_empty(&to_be_closed)) {
1026 le = to_be_closed.next;
1027 list_del_init(le);
1028 xprt = list_entry(le, struct svc_xprt, xpt_list);
1029 dprintk("svc_age_temp_xprts_now: closing %p\n", xprt);
1030 svsk = container_of(xprt, struct svc_sock, sk_xprt);
1031 sock = svsk->sk_sock;
1032 kernel_setsockopt(sock, SOL_SOCKET, SO_LINGER,
1033 (char *)&no_linger, sizeof(no_linger));
1034 svc_close_xprt(xprt);
1035 }
1036}
1037EXPORT_SYMBOL_GPL(svc_age_temp_xprts_now);
1038
J. Bruce Fieldsedc7a892010-03-22 15:37:17 -04001039static void call_xpt_users(struct svc_xprt *xprt)
1040{
1041 struct svc_xpt_user *u;
1042
1043 spin_lock(&xprt->xpt_lock);
1044 while (!list_empty(&xprt->xpt_users)) {
1045 u = list_first_entry(&xprt->xpt_users, struct svc_xpt_user, list);
1046 list_del(&u->list);
1047 u->callback(u);
1048 }
1049 spin_unlock(&xprt->xpt_lock);
1050}
1051
Tom Tucker0f0257e2007-12-30 21:08:27 -06001052/*
1053 * Remove a dead transport
1054 */
J. Bruce Fields7710ec32011-11-25 18:44:05 -05001055static void svc_delete_xprt(struct svc_xprt *xprt)
Tom Tucker0f0257e2007-12-30 21:08:27 -06001056{
1057 struct svc_serv *serv = xprt->xpt_server;
Tom Tucker22945e42009-01-05 15:21:19 -06001058 struct svc_deferred_req *dr;
1059
1060 /* Only do this once */
1061 if (test_and_set_bit(XPT_DEAD, &xprt->xpt_flags))
J. Bruce Fieldsac9303e2010-10-23 11:16:10 -04001062 BUG();
Tom Tucker0f0257e2007-12-30 21:08:27 -06001063
1064 dprintk("svc: svc_delete_xprt(%p)\n", xprt);
1065 xprt->xpt_ops->xpo_detach(xprt);
1066
1067 spin_lock_bh(&serv->sv_lock);
Jeff Layton8d65ef72014-11-17 17:02:57 -05001068 list_del_init(&xprt->xpt_list);
Weston Andros Adamson01047292012-10-23 10:43:48 -04001069 WARN_ON_ONCE(!list_empty(&xprt->xpt_ready));
Tom Tucker22945e42009-01-05 15:21:19 -06001070 if (test_bit(XPT_TEMP, &xprt->xpt_flags))
1071 serv->sv_tmpcnt--;
J. Bruce Fields788e69e2010-03-29 21:02:31 -04001072 spin_unlock_bh(&serv->sv_lock);
Tom Tucker22945e42009-01-05 15:21:19 -06001073
Neil Brownab1b18f2010-02-27 09:33:40 +11001074 while ((dr = svc_deferred_dequeue(xprt)) != NULL)
Tom Tucker22945e42009-01-05 15:21:19 -06001075 kfree(dr);
Tom Tucker22945e42009-01-05 15:21:19 -06001076
J. Bruce Fieldsedc7a892010-03-22 15:37:17 -04001077 call_xpt_users(xprt);
Tom Tucker22945e42009-01-05 15:21:19 -06001078 svc_xprt_put(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001079}
1080
1081void svc_close_xprt(struct svc_xprt *xprt)
1082{
1083 set_bit(XPT_CLOSE, &xprt->xpt_flags);
1084 if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags))
1085 /* someone else will have to effect the close */
1086 return;
J. Bruce Fieldsb1763312010-10-25 20:24:48 -04001087 /*
1088 * We expect svc_close_xprt() to work even when no threads are
1089 * running (e.g., while configuring the server before starting
1090 * any threads), so if the transport isn't busy, we delete
1091 * it ourself:
1092 */
Tom Tucker0f0257e2007-12-30 21:08:27 -06001093 svc_delete_xprt(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001094}
Tom Tuckera2178132007-12-30 21:08:35 -06001095EXPORT_SYMBOL_GPL(svc_close_xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001096
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001097static int svc_close_list(struct svc_serv *serv, struct list_head *xprt_list, struct net *net)
Tom Tucker0f0257e2007-12-30 21:08:27 -06001098{
1099 struct svc_xprt *xprt;
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001100 int ret = 0;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001101
J. Bruce Fields719f8bc2012-08-13 17:03:00 -04001102 spin_lock(&serv->sv_lock);
J. Bruce Fieldsb4f36f82011-11-29 17:00:26 -05001103 list_for_each_entry(xprt, xprt_list, xpt_list) {
Stanislav Kinsbursky7b147f12012-01-31 14:09:17 +04001104 if (xprt->xpt_net != net)
1105 continue;
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001106 ret++;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001107 set_bit(XPT_CLOSE, &xprt->xpt_flags);
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001108 svc_xprt_enqueue(xprt);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001109 }
J. Bruce Fields719f8bc2012-08-13 17:03:00 -04001110 spin_unlock(&serv->sv_lock);
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001111 return ret;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001112}
1113
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001114static struct svc_xprt *svc_dequeue_net(struct svc_serv *serv, struct net *net)
J. Bruce Fields2fefb8a2011-11-29 11:35:35 -05001115{
J. Bruce Fieldsb4f36f82011-11-29 17:00:26 -05001116 struct svc_pool *pool;
1117 struct svc_xprt *xprt;
1118 struct svc_xprt *tmp;
1119 int i;
1120
J. Bruce Fieldsb4f36f82011-11-29 17:00:26 -05001121 for (i = 0; i < serv->sv_nrpools; i++) {
1122 pool = &serv->sv_pools[i];
1123
1124 spin_lock_bh(&pool->sp_lock);
Stanislav Kinsbursky6f513362012-01-31 14:09:00 +04001125 list_for_each_entry_safe(xprt, tmp, &pool->sp_sockets, xpt_ready) {
Stanislav Kinsbursky7b147f12012-01-31 14:09:17 +04001126 if (xprt->xpt_net != net)
1127 continue;
J. Bruce Fieldsb4f36f82011-11-29 17:00:26 -05001128 list_del_init(&xprt->xpt_ready);
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001129 spin_unlock_bh(&pool->sp_lock);
1130 return xprt;
J. Bruce Fieldsb4f36f82011-11-29 17:00:26 -05001131 }
1132 spin_unlock_bh(&pool->sp_lock);
1133 }
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001134 return NULL;
Stanislav Kinsbursky6f513362012-01-31 14:09:00 +04001135}
1136
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001137static void svc_clean_up_xprts(struct svc_serv *serv, struct net *net)
Stanislav Kinsbursky6f513362012-01-31 14:09:00 +04001138{
1139 struct svc_xprt *xprt;
Stanislav Kinsbursky6f513362012-01-31 14:09:00 +04001140
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001141 while ((xprt = svc_dequeue_net(serv, net))) {
1142 set_bit(XPT_CLOSE, &xprt->xpt_flags);
J. Bruce Fields719f8bc2012-08-13 17:03:00 -04001143 svc_delete_xprt(xprt);
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001144 }
Stanislav Kinsbursky3a22bf52012-01-31 14:09:08 +04001145}
1146
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001147/*
1148 * Server threads may still be running (especially in the case where the
1149 * service is still running in other network namespaces).
1150 *
1151 * So we shut down sockets the same way we would on a running server, by
1152 * setting XPT_CLOSE, enqueuing, and letting a thread pick it up to do
1153 * the close. In the case there are no such other threads,
1154 * threads running, svc_clean_up_xprts() does a simple version of a
1155 * server's main event loop, and in the case where there are other
1156 * threads, we may need to wait a little while and then check again to
1157 * see if they're done.
1158 */
Stanislav Kinsbursky7b147f12012-01-31 14:09:17 +04001159void svc_close_net(struct svc_serv *serv, struct net *net)
Stanislav Kinsbursky3a22bf52012-01-31 14:09:08 +04001160{
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001161 int delay = 0;
Stanislav Kinsbursky6f513362012-01-31 14:09:00 +04001162
J. Bruce Fieldscc630d92013-02-10 16:08:11 -05001163 while (svc_close_list(serv, &serv->sv_permsocks, net) +
1164 svc_close_list(serv, &serv->sv_tempsocks, net)) {
1165
1166 svc_clean_up_xprts(serv, net);
1167 msleep(delay++);
1168 }
J. Bruce Fields2fefb8a2011-11-29 11:35:35 -05001169}
1170
Tom Tucker0f0257e2007-12-30 21:08:27 -06001171/*
1172 * Handle defer and revisit of requests
1173 */
1174
1175static void svc_revisit(struct cache_deferred_req *dreq, int too_many)
1176{
1177 struct svc_deferred_req *dr =
1178 container_of(dreq, struct svc_deferred_req, handle);
1179 struct svc_xprt *xprt = dr->xprt;
1180
Tom Tucker22945e42009-01-05 15:21:19 -06001181 spin_lock(&xprt->xpt_lock);
1182 set_bit(XPT_DEFERRED, &xprt->xpt_flags);
1183 if (too_many || test_bit(XPT_DEAD, &xprt->xpt_flags)) {
1184 spin_unlock(&xprt->xpt_lock);
1185 dprintk("revisit canceled\n");
Tom Tucker0f0257e2007-12-30 21:08:27 -06001186 svc_xprt_put(xprt);
Trond Myklebust104f6352016-06-24 10:55:46 -04001187 trace_svc_drop_deferred(dr);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001188 kfree(dr);
1189 return;
1190 }
1191 dprintk("revisit queued\n");
1192 dr->xprt = NULL;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001193 list_add(&dr->handle.recent, &xprt->xpt_deferred);
1194 spin_unlock(&xprt->xpt_lock);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001195 svc_xprt_enqueue(xprt);
1196 svc_xprt_put(xprt);
1197}
1198
Tom Tucker260c1d12007-12-30 21:08:29 -06001199/*
1200 * Save the request off for later processing. The request buffer looks
1201 * like this:
1202 *
1203 * <xprt-header><rpc-header><rpc-pagelist><rpc-tail>
1204 *
1205 * This code can only handle requests that consist of an xprt-header
1206 * and rpc-header.
1207 */
Tom Tucker0f0257e2007-12-30 21:08:27 -06001208static struct cache_deferred_req *svc_defer(struct cache_req *req)
1209{
1210 struct svc_rqst *rqstp = container_of(req, struct svc_rqst, rq_chandle);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001211 struct svc_deferred_req *dr;
1212
Jeff Layton30660e04b2014-11-19 07:51:16 -05001213 if (rqstp->rq_arg.page_len || !test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags))
Tom Tucker0f0257e2007-12-30 21:08:27 -06001214 return NULL; /* if more than a page, give up FIXME */
1215 if (rqstp->rq_deferred) {
1216 dr = rqstp->rq_deferred;
1217 rqstp->rq_deferred = NULL;
1218 } else {
Tom Tucker260c1d12007-12-30 21:08:29 -06001219 size_t skip;
1220 size_t size;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001221 /* FIXME maybe discard if size too large */
Tom Tucker260c1d12007-12-30 21:08:29 -06001222 size = sizeof(struct svc_deferred_req) + rqstp->rq_arg.len;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001223 dr = kmalloc(size, GFP_KERNEL);
1224 if (dr == NULL)
1225 return NULL;
1226
1227 dr->handle.owner = rqstp->rq_server;
1228 dr->prot = rqstp->rq_prot;
1229 memcpy(&dr->addr, &rqstp->rq_addr, rqstp->rq_addrlen);
1230 dr->addrlen = rqstp->rq_addrlen;
1231 dr->daddr = rqstp->rq_daddr;
1232 dr->argslen = rqstp->rq_arg.len >> 2;
Tom Tucker260c1d12007-12-30 21:08:29 -06001233 dr->xprt_hlen = rqstp->rq_xprt_hlen;
1234
1235 /* back up head to the start of the buffer and copy */
1236 skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;
1237 memcpy(dr->args, rqstp->rq_arg.head[0].iov_base - skip,
1238 dr->argslen << 2);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001239 }
1240 svc_xprt_get(rqstp->rq_xprt);
1241 dr->xprt = rqstp->rq_xprt;
Jeff Layton78b65eb2014-11-19 07:51:17 -05001242 set_bit(RQ_DROPME, &rqstp->rq_flags);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001243
1244 dr->handle.revisit = svc_revisit;
Trond Myklebust104f6352016-06-24 10:55:46 -04001245 trace_svc_defer(rqstp);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001246 return &dr->handle;
1247}
1248
1249/*
1250 * recv data from a deferred request into an active one
1251 */
1252static int svc_deferred_recv(struct svc_rqst *rqstp)
1253{
1254 struct svc_deferred_req *dr = rqstp->rq_deferred;
1255
Tom Tucker260c1d12007-12-30 21:08:29 -06001256 /* setup iov_base past transport header */
1257 rqstp->rq_arg.head[0].iov_base = dr->args + (dr->xprt_hlen>>2);
1258 /* The iov_len does not include the transport header bytes */
1259 rqstp->rq_arg.head[0].iov_len = (dr->argslen<<2) - dr->xprt_hlen;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001260 rqstp->rq_arg.page_len = 0;
Tom Tucker260c1d12007-12-30 21:08:29 -06001261 /* The rq_arg.len includes the transport header bytes */
1262 rqstp->rq_arg.len = dr->argslen<<2;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001263 rqstp->rq_prot = dr->prot;
1264 memcpy(&rqstp->rq_addr, &dr->addr, dr->addrlen);
1265 rqstp->rq_addrlen = dr->addrlen;
Tom Tucker260c1d12007-12-30 21:08:29 -06001266 /* Save off transport header len in case we get deferred again */
1267 rqstp->rq_xprt_hlen = dr->xprt_hlen;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001268 rqstp->rq_daddr = dr->daddr;
1269 rqstp->rq_respages = rqstp->rq_pages;
Tom Tucker260c1d12007-12-30 21:08:29 -06001270 return (dr->argslen<<2) - dr->xprt_hlen;
Tom Tucker0f0257e2007-12-30 21:08:27 -06001271}
1272
1273
1274static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt)
1275{
1276 struct svc_deferred_req *dr = NULL;
1277
1278 if (!test_bit(XPT_DEFERRED, &xprt->xpt_flags))
1279 return NULL;
1280 spin_lock(&xprt->xpt_lock);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001281 if (!list_empty(&xprt->xpt_deferred)) {
1282 dr = list_entry(xprt->xpt_deferred.next,
1283 struct svc_deferred_req,
1284 handle.recent);
1285 list_del_init(&dr->handle.recent);
Trond Myklebust104f6352016-06-24 10:55:46 -04001286 trace_svc_revisit_deferred(dr);
J. Bruce Fields62bac4a2010-10-25 12:50:15 -04001287 } else
1288 clear_bit(XPT_DEFERRED, &xprt->xpt_flags);
Tom Tucker0f0257e2007-12-30 21:08:27 -06001289 spin_unlock(&xprt->xpt_lock);
1290 return dr;
1291}
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001292
Chuck Lever156e6202009-03-18 20:45:58 -04001293/**
1294 * svc_find_xprt - find an RPC transport instance
1295 * @serv: pointer to svc_serv to search
1296 * @xcl_name: C string containing transport's class name
Stanislav Kinsbursky4cb54ca2012-01-20 16:50:53 +04001297 * @net: owner net pointer
Chuck Lever156e6202009-03-18 20:45:58 -04001298 * @af: Address family of transport's local address
1299 * @port: transport's IP port number
1300 *
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001301 * Return the transport instance pointer for the endpoint accepting
1302 * connections/peer traffic from the specified transport class,
1303 * address family and port.
1304 *
1305 * Specifying 0 for the address family or port is effectively a
1306 * wild-card, and will result in matching the first transport in the
1307 * service's list that has a matching class name.
1308 */
Chuck Lever156e6202009-03-18 20:45:58 -04001309struct svc_xprt *svc_find_xprt(struct svc_serv *serv, const char *xcl_name,
Stanislav Kinsbursky4cb54ca2012-01-20 16:50:53 +04001310 struct net *net, const sa_family_t af,
1311 const unsigned short port)
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001312{
1313 struct svc_xprt *xprt;
1314 struct svc_xprt *found = NULL;
1315
1316 /* Sanity check the args */
Chuck Lever156e6202009-03-18 20:45:58 -04001317 if (serv == NULL || xcl_name == NULL)
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001318 return found;
1319
1320 spin_lock_bh(&serv->sv_lock);
1321 list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {
Stanislav Kinsbursky4cb54ca2012-01-20 16:50:53 +04001322 if (xprt->xpt_net != net)
1323 continue;
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001324 if (strcmp(xprt->xpt_class->xcl_name, xcl_name))
1325 continue;
1326 if (af != AF_UNSPEC && af != xprt->xpt_local.ss_family)
1327 continue;
Chuck Lever156e6202009-03-18 20:45:58 -04001328 if (port != 0 && port != svc_xprt_local_port(xprt))
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001329 continue;
1330 found = xprt;
Tom Tuckera2178132007-12-30 21:08:35 -06001331 svc_xprt_get(xprt);
Tom Tucker7fcb98d2007-12-30 21:08:33 -06001332 break;
1333 }
1334 spin_unlock_bh(&serv->sv_lock);
1335 return found;
1336}
1337EXPORT_SYMBOL_GPL(svc_find_xprt);
Tom Tucker9571af12007-12-30 21:08:37 -06001338
Chuck Lever335c54b2009-04-23 19:32:25 -04001339static int svc_one_xprt_name(const struct svc_xprt *xprt,
1340 char *pos, int remaining)
1341{
1342 int len;
1343
1344 len = snprintf(pos, remaining, "%s %u\n",
1345 xprt->xpt_class->xcl_name,
1346 svc_xprt_local_port(xprt));
1347 if (len >= remaining)
1348 return -ENAMETOOLONG;
1349 return len;
1350}
1351
1352/**
1353 * svc_xprt_names - format a buffer with a list of transport names
1354 * @serv: pointer to an RPC service
1355 * @buf: pointer to a buffer to be filled in
1356 * @buflen: length of buffer to be filled in
1357 *
1358 * Fills in @buf with a string containing a list of transport names,
1359 * each name terminated with '\n'.
1360 *
1361 * Returns positive length of the filled-in string on success; otherwise
1362 * a negative errno value is returned if an error occurs.
Tom Tucker9571af12007-12-30 21:08:37 -06001363 */
Chuck Lever335c54b2009-04-23 19:32:25 -04001364int svc_xprt_names(struct svc_serv *serv, char *buf, const int buflen)
Tom Tucker9571af12007-12-30 21:08:37 -06001365{
1366 struct svc_xprt *xprt;
Chuck Lever335c54b2009-04-23 19:32:25 -04001367 int len, totlen;
1368 char *pos;
Tom Tucker9571af12007-12-30 21:08:37 -06001369
1370 /* Sanity check args */
1371 if (!serv)
1372 return 0;
1373
1374 spin_lock_bh(&serv->sv_lock);
Chuck Lever335c54b2009-04-23 19:32:25 -04001375
1376 pos = buf;
1377 totlen = 0;
Tom Tucker9571af12007-12-30 21:08:37 -06001378 list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {
Chuck Lever335c54b2009-04-23 19:32:25 -04001379 len = svc_one_xprt_name(xprt, pos, buflen - totlen);
1380 if (len < 0) {
1381 *buf = '\0';
1382 totlen = len;
1383 }
1384 if (len <= 0)
Tom Tucker9571af12007-12-30 21:08:37 -06001385 break;
Chuck Lever335c54b2009-04-23 19:32:25 -04001386
1387 pos += len;
Tom Tucker9571af12007-12-30 21:08:37 -06001388 totlen += len;
1389 }
Chuck Lever335c54b2009-04-23 19:32:25 -04001390
Tom Tucker9571af12007-12-30 21:08:37 -06001391 spin_unlock_bh(&serv->sv_lock);
1392 return totlen;
1393}
1394EXPORT_SYMBOL_GPL(svc_xprt_names);
Greg Banks03cf6c92009-01-13 21:26:36 +11001395
1396
1397/*----------------------------------------------------------------------------*/
1398
1399static void *svc_pool_stats_start(struct seq_file *m, loff_t *pos)
1400{
1401 unsigned int pidx = (unsigned int)*pos;
1402 struct svc_serv *serv = m->private;
1403
1404 dprintk("svc_pool_stats_start, *pidx=%u\n", pidx);
1405
Greg Banks03cf6c92009-01-13 21:26:36 +11001406 if (!pidx)
1407 return SEQ_START_TOKEN;
1408 return (pidx > serv->sv_nrpools ? NULL : &serv->sv_pools[pidx-1]);
1409}
1410
1411static void *svc_pool_stats_next(struct seq_file *m, void *p, loff_t *pos)
1412{
1413 struct svc_pool *pool = p;
1414 struct svc_serv *serv = m->private;
1415
1416 dprintk("svc_pool_stats_next, *pos=%llu\n", *pos);
1417
1418 if (p == SEQ_START_TOKEN) {
1419 pool = &serv->sv_pools[0];
1420 } else {
1421 unsigned int pidx = (pool - &serv->sv_pools[0]);
1422 if (pidx < serv->sv_nrpools-1)
1423 pool = &serv->sv_pools[pidx+1];
1424 else
1425 pool = NULL;
1426 }
1427 ++*pos;
1428 return pool;
1429}
1430
1431static void svc_pool_stats_stop(struct seq_file *m, void *p)
1432{
Greg Banks03cf6c92009-01-13 21:26:36 +11001433}
1434
1435static int svc_pool_stats_show(struct seq_file *m, void *p)
1436{
1437 struct svc_pool *pool = p;
1438
1439 if (p == SEQ_START_TOKEN) {
J. Bruce Fields78c210e2009-08-06 15:41:34 -04001440 seq_puts(m, "# pool packets-arrived sockets-enqueued threads-woken threads-timedout\n");
Greg Banks03cf6c92009-01-13 21:26:36 +11001441 return 0;
1442 }
1443
J. Bruce Fields78c210e2009-08-06 15:41:34 -04001444 seq_printf(m, "%u %lu %lu %lu %lu\n",
Greg Banks03cf6c92009-01-13 21:26:36 +11001445 pool->sp_id,
Jeff Layton403c7b42014-11-21 14:19:29 -05001446 (unsigned long)atomic_long_read(&pool->sp_stats.packets),
Greg Banks03cf6c92009-01-13 21:26:36 +11001447 pool->sp_stats.sockets_queued,
Jeff Layton403c7b42014-11-21 14:19:29 -05001448 (unsigned long)atomic_long_read(&pool->sp_stats.threads_woken),
1449 (unsigned long)atomic_long_read(&pool->sp_stats.threads_timedout));
Greg Banks03cf6c92009-01-13 21:26:36 +11001450
1451 return 0;
1452}
1453
1454static const struct seq_operations svc_pool_stats_seq_ops = {
1455 .start = svc_pool_stats_start,
1456 .next = svc_pool_stats_next,
1457 .stop = svc_pool_stats_stop,
1458 .show = svc_pool_stats_show,
1459};
1460
1461int svc_pool_stats_open(struct svc_serv *serv, struct file *file)
1462{
1463 int err;
1464
1465 err = seq_open(file, &svc_pool_stats_seq_ops);
1466 if (!err)
1467 ((struct seq_file *) file->private_data)->private = serv;
1468 return err;
1469}
1470EXPORT_SYMBOL(svc_pool_stats_open);
1471
1472/*----------------------------------------------------------------------------*/