blob: 8313960cac524dd36d220f9b55d124435400f25a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/xprt.c
3 *
4 * This is a generic RPC call interface supporting congestion avoidance,
5 * and asynchronous calls.
6 *
7 * The interface works like this:
8 *
9 * - When a process places a call, it allocates a request slot if
10 * one is available. Otherwise, it sleeps on the backlog queue
11 * (xprt_reserve).
12 * - Next, the caller puts together the RPC message, stuffs it into
Chuck Lever55aa4f52005-08-11 16:25:47 -040013 * the request struct, and calls xprt_transmit().
14 * - xprt_transmit sends the message and installs the caller on the
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -040015 * transport's wait list. At the same time, if a reply is expected,
16 * it installs a timer that is run after the packet's timeout has
17 * expired.
Linus Torvalds1da177e2005-04-16 15:20:36 -070018 * - When a packet arrives, the data_ready handler walks the list of
Chuck Lever55aa4f52005-08-11 16:25:47 -040019 * pending requests for that transport. If a matching XID is found, the
Linus Torvalds1da177e2005-04-16 15:20:36 -070020 * caller is woken up, and the timer removed.
21 * - When no reply arrives within the timeout interval, the timer is
22 * fired by the kernel and runs xprt_timer(). It either adjusts the
23 * timeout values (minor timeout) or wakes up the caller with a status
24 * of -ETIMEDOUT.
25 * - When the caller receives a notification from RPC that a reply arrived,
26 * it should release the RPC slot, and process the reply.
27 * If the call timed out, it may choose to retry the operation by
28 * adjusting the initial timeout value, and simply calling rpc_call
29 * again.
30 *
31 * Support for async RPC is done through a set of RPC-specific scheduling
32 * primitives that `transparently' work for processes as well as async
33 * tasks that rely on callbacks.
34 *
35 * Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de>
Chuck Lever55aa4f52005-08-11 16:25:47 -040036 *
37 * Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 */
39
Chuck Levera246b012005-08-11 16:25:23 -040040#include <linux/module.h>
41
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <linux/types.h>
Chuck Levera246b012005-08-11 16:25:23 -040043#include <linux/interrupt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <linux/workqueue.h>
Chuck Leverbf3fcf82006-05-25 01:40:51 -040045#include <linux/net.h>
Chuck Leverff839972010-05-07 13:34:47 -040046#include <linux/ktime.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
Chuck Levera246b012005-08-11 16:25:23 -040048#include <linux/sunrpc/clnt.h>
Chuck Lever11c556b2006-03-20 13:44:22 -050049#include <linux/sunrpc/metrics.h>
Trond Myklebustc9acb422010-03-19 15:36:22 -040050#include <linux/sunrpc/bc_xprt.h>
Trond Myklebustfda1bfe2015-02-14 17:48:49 -050051#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Jeff Layton3705ad62014-10-28 14:24:13 -040053#include <trace/events/sunrpc.h>
54
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -040055#include "sunrpc.h"
56
Linus Torvalds1da177e2005-04-16 15:20:36 -070057/*
58 * Local variables
59 */
60
Jeff Laytonf895b252014-11-17 16:58:04 -050061#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062# define RPCDBG_FACILITY RPCDBG_XPRT
63#endif
64
Linus Torvalds1da177e2005-04-16 15:20:36 -070065/*
66 * Local functions
67 */
Trond Myklebust21de0a92011-07-17 16:57:32 -040068static void xprt_init(struct rpc_xprt *xprt, struct net *net);
Linus Torvalds1da177e2005-04-16 15:20:36 -070069static void xprt_request_init(struct rpc_task *, struct rpc_xprt *);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070static void xprt_connect_status(struct rpc_task *task);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071static int __xprt_get_cong(struct rpc_xprt *, struct rpc_task *);
Neil Brown29807312015-06-15 15:55:30 +100072static void __xprt_put_cong(struct rpc_xprt *, struct rpc_rqst *);
Trond Myklebust4e0038b2012-03-01 17:01:05 -050073static void xprt_destroy(struct rpc_xprt *xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
Jiri Slaby5ba03e82007-11-22 19:40:22 +080075static DEFINE_SPINLOCK(xprt_list_lock);
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -040076static LIST_HEAD(xprt_list);
77
Chuck Lever12a80462005-08-25 16:25:51 -070078/**
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -040079 * xprt_register_transport - register a transport implementation
80 * @transport: transport to register
81 *
82 * If a transport implementation is loaded as a kernel module, it can
83 * call this interface to make itself known to the RPC client.
84 *
85 * Returns:
86 * 0: transport successfully registered
87 * -EEXIST: transport already registered
88 * -EINVAL: transport module being unloaded
89 */
90int xprt_register_transport(struct xprt_class *transport)
91{
92 struct xprt_class *t;
93 int result;
94
95 result = -EEXIST;
96 spin_lock(&xprt_list_lock);
97 list_for_each_entry(t, &xprt_list, list) {
98 /* don't register the same transport class twice */
\"Talpey, Thomas\4fa016e2007-09-10 13:47:57 -040099 if (t->ident == transport->ident)
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -0400100 goto out;
101 }
102
Denis V. Lunevc9f6cde2008-07-31 09:53:56 +0400103 list_add_tail(&transport->list, &xprt_list);
104 printk(KERN_INFO "RPC: Registered %s transport module.\n",
105 transport->name);
106 result = 0;
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -0400107
108out:
109 spin_unlock(&xprt_list_lock);
110 return result;
111}
112EXPORT_SYMBOL_GPL(xprt_register_transport);
113
114/**
115 * xprt_unregister_transport - unregister a transport implementation
Randy Dunlap65b6e422008-02-13 15:03:23 -0800116 * @transport: transport to unregister
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -0400117 *
118 * Returns:
119 * 0: transport successfully unregistered
120 * -ENOENT: transport never registered
121 */
122int xprt_unregister_transport(struct xprt_class *transport)
123{
124 struct xprt_class *t;
125 int result;
126
127 result = 0;
128 spin_lock(&xprt_list_lock);
129 list_for_each_entry(t, &xprt_list, list) {
130 if (t == transport) {
131 printk(KERN_INFO
132 "RPC: Unregistered %s transport module.\n",
133 transport->name);
134 list_del_init(&transport->list);
\"Talpey, Thomas\81c098a2007-09-10 13:46:00 -0400135 goto out;
136 }
137 }
138 result = -ENOENT;
139
140out:
141 spin_unlock(&xprt_list_lock);
142 return result;
143}
144EXPORT_SYMBOL_GPL(xprt_unregister_transport);
145
146/**
Tom Talpey441e3e22009-03-11 14:37:56 -0400147 * xprt_load_transport - load a transport implementation
148 * @transport_name: transport to load
149 *
150 * Returns:
151 * 0: transport successfully loaded
152 * -ENOENT: transport module not available
153 */
154int xprt_load_transport(const char *transport_name)
155{
156 struct xprt_class *t;
Tom Talpey441e3e22009-03-11 14:37:56 -0400157 int result;
158
159 result = 0;
160 spin_lock(&xprt_list_lock);
161 list_for_each_entry(t, &xprt_list, list) {
162 if (strcmp(t->name, transport_name) == 0) {
163 spin_unlock(&xprt_list_lock);
164 goto out;
165 }
166 }
167 spin_unlock(&xprt_list_lock);
Alex Riesenef7ffe82010-05-24 14:33:05 -0700168 result = request_module("xprt%s", transport_name);
Tom Talpey441e3e22009-03-11 14:37:56 -0400169out:
170 return result;
171}
172EXPORT_SYMBOL_GPL(xprt_load_transport);
173
174/**
Chuck Lever12a80462005-08-25 16:25:51 -0700175 * xprt_reserve_xprt - serialize write access to transports
176 * @task: task that is requesting access to the transport
Randy Dunlap177c27b2011-07-28 06:54:36 +0000177 * @xprt: pointer to the target transport
Chuck Lever12a80462005-08-25 16:25:51 -0700178 *
179 * This prevents mixing the payload of separate requests, and prevents
180 * transport connects from colliding with writes. No congestion control
181 * is provided.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 */
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400183int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184{
Chuck Lever12a80462005-08-25 16:25:51 -0700185 struct rpc_rqst *req = task->tk_rqstp;
Trond Myklebust34006ce2011-07-17 18:11:34 -0400186 int priority;
Chuck Lever12a80462005-08-25 16:25:51 -0700187
188 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
189 if (task == xprt->snd_task)
190 return 1;
Chuck Lever12a80462005-08-25 16:25:51 -0700191 goto out_sleep;
192 }
193 xprt->snd_task = task;
Trond Myklebust92551942013-09-27 11:28:40 -0400194 if (req != NULL)
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400195 req->rq_ntrans++;
j223yang@asset.uwaterloo.ca4d4a76f2011-03-10 12:40:28 -0500196
Chuck Lever12a80462005-08-25 16:25:51 -0700197 return 1;
198
199out_sleep:
Chuck Lever46121cf2007-01-31 12:14:08 -0500200 dprintk("RPC: %5u failed to lock transport %p\n",
Chuck Lever12a80462005-08-25 16:25:51 -0700201 task->tk_pid, xprt);
202 task->tk_timeout = 0;
203 task->tk_status = -EAGAIN;
Trond Myklebust34006ce2011-07-17 18:11:34 -0400204 if (req == NULL)
205 priority = RPC_PRIORITY_LOW;
206 else if (!req->rq_ntrans)
207 priority = RPC_PRIORITY_NORMAL;
Chuck Lever12a80462005-08-25 16:25:51 -0700208 else
Trond Myklebust34006ce2011-07-17 18:11:34 -0400209 priority = RPC_PRIORITY_HIGH;
210 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
Chuck Lever12a80462005-08-25 16:25:51 -0700211 return 0;
212}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400213EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
Chuck Lever12a80462005-08-25 16:25:51 -0700214
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100215static void xprt_clear_locked(struct rpc_xprt *xprt)
216{
217 xprt->snd_task = NULL;
Trond Myklebustd19751e2012-09-11 17:21:25 -0400218 if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100219 smp_mb__before_atomic();
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100220 clear_bit(XPRT_LOCKED, &xprt->state);
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100221 smp_mb__after_atomic();
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100222 } else
Trond Myklebust40a5f1b2016-05-27 10:39:50 -0400223 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100224}
225
Chuck Lever12a80462005-08-25 16:25:51 -0700226/*
227 * xprt_reserve_xprt_cong - serialize write access to transports
228 * @task: task that is requesting access to the transport
229 *
230 * Same as xprt_reserve_xprt, but Van Jacobson congestion control is
231 * integrated into the decision of whether a request is allowed to be
232 * woken up and given access to the transport.
233 */
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400234int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
Chuck Lever12a80462005-08-25 16:25:51 -0700235{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 struct rpc_rqst *req = task->tk_rqstp;
Trond Myklebust34006ce2011-07-17 18:11:34 -0400237 int priority;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
Chuck Lever2226feb2005-08-11 16:25:38 -0400239 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 if (task == xprt->snd_task)
241 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 goto out_sleep;
243 }
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400244 if (req == NULL) {
245 xprt->snd_task = task;
246 return 1;
247 }
Chuck Lever12a80462005-08-25 16:25:51 -0700248 if (__xprt_get_cong(xprt, task)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 xprt->snd_task = task;
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400250 req->rq_ntrans++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 return 1;
252 }
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100253 xprt_clear_locked(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254out_sleep:
Neil Brown29807312015-06-15 15:55:30 +1000255 if (req)
256 __xprt_put_cong(xprt, req);
Chuck Lever46121cf2007-01-31 12:14:08 -0500257 dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 task->tk_timeout = 0;
259 task->tk_status = -EAGAIN;
Trond Myklebust34006ce2011-07-17 18:11:34 -0400260 if (req == NULL)
261 priority = RPC_PRIORITY_LOW;
262 else if (!req->rq_ntrans)
263 priority = RPC_PRIORITY_NORMAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 else
Trond Myklebust34006ce2011-07-17 18:11:34 -0400265 priority = RPC_PRIORITY_HIGH;
266 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 return 0;
268}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400269EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
Chuck Lever12a80462005-08-25 16:25:51 -0700271static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272{
273 int retval;
274
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400275 spin_lock_bh(&xprt->transport_lock);
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400276 retval = xprt->ops->reserve_xprt(xprt, task);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400277 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 return retval;
279}
280
Trond Myklebust961a8282012-01-17 22:57:37 -0500281static bool __xprt_lock_write_func(struct rpc_task *task, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282{
Trond Myklebust961a8282012-01-17 22:57:37 -0500283 struct rpc_xprt *xprt = data;
Chuck Lever49e9a892005-08-25 16:25:51 -0700284 struct rpc_rqst *req;
285
Chuck Lever49e9a892005-08-25 16:25:51 -0700286 req = task->tk_rqstp;
287 xprt->snd_task = task;
Trond Myklebust92551942013-09-27 11:28:40 -0400288 if (req)
Chuck Lever49e9a892005-08-25 16:25:51 -0700289 req->rq_ntrans++;
Trond Myklebust961a8282012-01-17 22:57:37 -0500290 return true;
291}
Chuck Lever49e9a892005-08-25 16:25:51 -0700292
Trond Myklebust961a8282012-01-17 22:57:37 -0500293static void __xprt_lock_write_next(struct rpc_xprt *xprt)
294{
295 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
296 return;
297
Trond Myklebustf1dc2372016-05-27 12:59:33 -0400298 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
299 __xprt_lock_write_func, xprt))
Trond Myklebust961a8282012-01-17 22:57:37 -0500300 return;
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100301 xprt_clear_locked(xprt);
Chuck Lever49e9a892005-08-25 16:25:51 -0700302}
303
Trond Myklebust961a8282012-01-17 22:57:37 -0500304static bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data)
Chuck Lever49e9a892005-08-25 16:25:51 -0700305{
Trond Myklebust961a8282012-01-17 22:57:37 -0500306 struct rpc_xprt *xprt = data;
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400307 struct rpc_rqst *req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400309 req = task->tk_rqstp;
310 if (req == NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 xprt->snd_task = task;
Trond Myklebust961a8282012-01-17 22:57:37 -0500312 return true;
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400313 }
314 if (__xprt_get_cong(xprt, task)) {
315 xprt->snd_task = task;
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -0400316 req->rq_ntrans++;
Trond Myklebust961a8282012-01-17 22:57:37 -0500317 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 }
Trond Myklebust961a8282012-01-17 22:57:37 -0500319 return false;
320}
321
322static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
323{
324 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
325 return;
326 if (RPCXPRT_CONGESTED(xprt))
327 goto out_unlock;
Trond Myklebustf1dc2372016-05-27 12:59:33 -0400328 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
329 __xprt_lock_write_cong_func, xprt))
Trond Myklebust961a8282012-01-17 22:57:37 -0500330 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331out_unlock:
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100332 xprt_clear_locked(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333}
334
Trond Myklebust06953142015-03-23 16:10:00 -0400335static void xprt_task_clear_bytes_sent(struct rpc_task *task)
336{
337 if (task != NULL) {
338 struct rpc_rqst *req = task->tk_rqstp;
339 if (req != NULL)
340 req->rq_bytes_sent = 0;
341 }
342}
343
Chuck Lever49e9a892005-08-25 16:25:51 -0700344/**
345 * xprt_release_xprt - allow other requests to use a transport
346 * @xprt: transport with other tasks potentially waiting
347 * @task: task that is releasing access to the transport
348 *
349 * Note that "task" can be NULL. No congestion control is provided.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 */
Chuck Lever49e9a892005-08-25 16:25:51 -0700351void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352{
353 if (xprt->snd_task == task) {
Trond Myklebust06953142015-03-23 16:10:00 -0400354 xprt_task_clear_bytes_sent(task);
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100355 xprt_clear_locked(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 __xprt_lock_write_next(xprt);
357 }
358}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400359EXPORT_SYMBOL_GPL(xprt_release_xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
Chuck Lever49e9a892005-08-25 16:25:51 -0700361/**
362 * xprt_release_xprt_cong - allow other requests to use a transport
363 * @xprt: transport with other tasks potentially waiting
364 * @task: task that is releasing access to the transport
365 *
366 * Note that "task" can be NULL. Another task is awoken to use the
367 * transport if the transport's congestion window allows it.
368 */
369void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
370{
371 if (xprt->snd_task == task) {
Trond Myklebust06953142015-03-23 16:10:00 -0400372 xprt_task_clear_bytes_sent(task);
Trond Myklebust632e3bd2006-01-03 09:55:55 +0100373 xprt_clear_locked(xprt);
Chuck Lever49e9a892005-08-25 16:25:51 -0700374 __xprt_lock_write_next_cong(xprt);
375 }
376}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400377EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
Chuck Lever49e9a892005-08-25 16:25:51 -0700378
379static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380{
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400381 spin_lock_bh(&xprt->transport_lock);
Chuck Lever49e9a892005-08-25 16:25:51 -0700382 xprt->ops->release_xprt(xprt, task);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400383 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385
386/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 * Van Jacobson congestion avoidance. Check if the congestion window
388 * overflowed. Put the task to sleep if this is the case.
389 */
390static int
391__xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)
392{
393 struct rpc_rqst *req = task->tk_rqstp;
394
395 if (req->rq_cong)
396 return 1;
Chuck Lever46121cf2007-01-31 12:14:08 -0500397 dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 task->tk_pid, xprt->cong, xprt->cwnd);
399 if (RPCXPRT_CONGESTED(xprt))
400 return 0;
401 req->rq_cong = 1;
402 xprt->cong += RPC_CWNDSCALE;
403 return 1;
404}
405
406/*
407 * Adjust the congestion window, and wake up the next task
408 * that has been sleeping due to congestion
409 */
410static void
411__xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
412{
413 if (!req->rq_cong)
414 return;
415 req->rq_cong = 0;
416 xprt->cong -= RPC_CWNDSCALE;
Chuck Lever49e9a892005-08-25 16:25:51 -0700417 __xprt_lock_write_next_cong(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418}
419
Chuck Lever46c0ee82005-08-25 16:25:52 -0700420/**
Chuck Levera58dd392005-08-25 16:25:53 -0700421 * xprt_release_rqst_cong - housekeeping when request is complete
422 * @task: RPC request that recently completed
423 *
424 * Useful for transports that require congestion control.
425 */
426void xprt_release_rqst_cong(struct rpc_task *task)
427{
Trond Myklebusta4f08352013-01-08 09:10:21 -0500428 struct rpc_rqst *req = task->tk_rqstp;
429
430 __xprt_put_cong(req->rq_xprt, req);
Chuck Levera58dd392005-08-25 16:25:53 -0700431}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400432EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
Chuck Levera58dd392005-08-25 16:25:53 -0700433
434/**
Chuck Lever46c0ee82005-08-25 16:25:52 -0700435 * xprt_adjust_cwnd - adjust transport congestion window
Trond Myklebust6a24dfb2013-01-08 09:48:15 -0500436 * @xprt: pointer to xprt
Chuck Lever46c0ee82005-08-25 16:25:52 -0700437 * @task: recently completed RPC request used to adjust window
438 * @result: result code of completed RPC request
439 *
Chuck Lever4f4cf5a2014-05-28 10:34:49 -0400440 * The transport code maintains an estimate on the maximum number of out-
441 * standing RPC requests, using a smoothed version of the congestion
442 * avoidance implemented in 44BSD. This is basically the Van Jacobson
443 * congestion algorithm: If a retransmit occurs, the congestion window is
444 * halved; otherwise, it is incremented by 1/cwnd when
445 *
446 * - a reply is received and
447 * - a full number of requests are outstanding and
448 * - the congestion window hasn't been updated recently.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 */
Trond Myklebust6a24dfb2013-01-08 09:48:15 -0500450void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451{
Chuck Lever46c0ee82005-08-25 16:25:52 -0700452 struct rpc_rqst *req = task->tk_rqstp;
Chuck Lever46c0ee82005-08-25 16:25:52 -0700453 unsigned long cwnd = xprt->cwnd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 if (result >= 0 && cwnd <= xprt->cong) {
456 /* The (cwnd >> 1) term makes sure
457 * the result gets rounded properly. */
458 cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
459 if (cwnd > RPC_MAXCWND(xprt))
460 cwnd = RPC_MAXCWND(xprt);
Chuck Lever49e9a892005-08-25 16:25:51 -0700461 __xprt_lock_write_next_cong(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 } else if (result == -ETIMEDOUT) {
463 cwnd >>= 1;
464 if (cwnd < RPC_CWNDSCALE)
465 cwnd = RPC_CWNDSCALE;
466 }
Chuck Lever46121cf2007-01-31 12:14:08 -0500467 dprintk("RPC: cong %ld, cwnd was %ld, now %ld\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 xprt->cong, xprt->cwnd, cwnd);
469 xprt->cwnd = cwnd;
Chuck Lever46c0ee82005-08-25 16:25:52 -0700470 __xprt_put_cong(xprt, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400472EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473
Chuck Lever44fbac22005-08-11 16:25:44 -0400474/**
475 * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue
476 * @xprt: transport with waiting tasks
477 * @status: result code to plant in each task before waking it
478 *
479 */
480void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
481{
482 if (status < 0)
483 rpc_wake_up_status(&xprt->pending, status);
484 else
485 rpc_wake_up(&xprt->pending);
486}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400487EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
Chuck Lever44fbac22005-08-11 16:25:44 -0400488
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400489/**
490 * xprt_wait_for_buffer_space - wait for transport output buffer to clear
491 * @task: task to be put to sleep
Randy Dunlap0b80ae42008-04-26 22:59:02 -0700492 * @action: function pointer to be executed after wait
Trond Myklebusta9a6b522013-02-22 14:57:57 -0500493 *
494 * Note that we only set the timer for the case of RPC_IS_SOFT(), since
495 * we don't in general want to force a socket disconnection due to
496 * an incomplete RPC call transmission.
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400497 */
Trond Myklebustb6ddf642008-04-17 18:52:19 -0400498void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400499{
500 struct rpc_rqst *req = task->tk_rqstp;
501 struct rpc_xprt *xprt = req->rq_xprt;
502
Trond Myklebusta9a6b522013-02-22 14:57:57 -0500503 task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
Trond Myklebustb6ddf642008-04-17 18:52:19 -0400504 rpc_sleep_on(&xprt->pending, task, action);
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400505}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400506EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400507
508/**
509 * xprt_write_space - wake the task waiting for transport output buffer space
510 * @xprt: transport with waiting tasks
511 *
512 * Can be called in a soft IRQ context, so xprt_write_space never sleeps.
513 */
514void xprt_write_space(struct rpc_xprt *xprt)
515{
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400516 spin_lock_bh(&xprt->transport_lock);
517 if (xprt->snd_task) {
Chuck Lever46121cf2007-01-31 12:14:08 -0500518 dprintk("RPC: write space: waking waiting task on "
519 "xprt %p\n", xprt);
Trond Myklebustfda13932008-02-22 16:34:12 -0500520 rpc_wake_up_queued_task(&xprt->pending, xprt->snd_task);
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400521 }
522 spin_unlock_bh(&xprt->transport_lock);
523}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400524EXPORT_SYMBOL_GPL(xprt_write_space);
Chuck Leverc7b2cae2005-08-11 16:25:50 -0400525
Chuck Leverfe3aca22005-08-25 16:25:50 -0700526/**
527 * xprt_set_retrans_timeout_def - set a request's retransmit timeout
528 * @task: task whose timeout is to be set
529 *
530 * Set a request's retransmit timeout based on the transport's
531 * default timeout parameters. Used by transports that don't adjust
532 * the retransmit timeout based on round-trip time estimation.
533 */
534void xprt_set_retrans_timeout_def(struct rpc_task *task)
535{
536 task->tk_timeout = task->tk_rqstp->rq_timeout;
537}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400538EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
Chuck Leverfe3aca22005-08-25 16:25:50 -0700539
Ben Hutchings2c530402012-07-10 10:55:09 +0000540/**
Chuck Leverfe3aca22005-08-25 16:25:50 -0700541 * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout
542 * @task: task whose timeout is to be set
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800543 *
Chuck Leverfe3aca22005-08-25 16:25:50 -0700544 * Set a request's retransmit timeout using the RTT estimator.
545 */
546void xprt_set_retrans_timeout_rtt(struct rpc_task *task)
547{
548 int timer = task->tk_msg.rpc_proc->p_timer;
Trond Myklebustba7392b2007-12-20 16:03:55 -0500549 struct rpc_clnt *clnt = task->tk_client;
550 struct rpc_rtt *rtt = clnt->cl_rtt;
Chuck Leverfe3aca22005-08-25 16:25:50 -0700551 struct rpc_rqst *req = task->tk_rqstp;
Trond Myklebustba7392b2007-12-20 16:03:55 -0500552 unsigned long max_timeout = clnt->cl_timeout->to_maxval;
Chuck Leverfe3aca22005-08-25 16:25:50 -0700553
554 task->tk_timeout = rpc_calc_rto(rtt, timer);
555 task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
556 if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
557 task->tk_timeout = max_timeout;
558}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400559EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
Chuck Leverfe3aca22005-08-25 16:25:50 -0700560
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561static void xprt_reset_majortimeo(struct rpc_rqst *req)
562{
Trond Myklebustba7392b2007-12-20 16:03:55 -0500563 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564
565 req->rq_majortimeo = req->rq_timeout;
566 if (to->to_exponential)
567 req->rq_majortimeo <<= to->to_retries;
568 else
569 req->rq_majortimeo += to->to_increment * to->to_retries;
570 if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
571 req->rq_majortimeo = to->to_maxval;
572 req->rq_majortimeo += jiffies;
573}
574
Chuck Lever9903cd12005-08-11 16:25:26 -0400575/**
576 * xprt_adjust_timeout - adjust timeout values for next retransmit
577 * @req: RPC request containing parameters to use for the adjustment
578 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 */
580int xprt_adjust_timeout(struct rpc_rqst *req)
581{
582 struct rpc_xprt *xprt = req->rq_xprt;
Trond Myklebustba7392b2007-12-20 16:03:55 -0500583 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 int status = 0;
585
586 if (time_before(jiffies, req->rq_majortimeo)) {
587 if (to->to_exponential)
588 req->rq_timeout <<= 1;
589 else
590 req->rq_timeout += to->to_increment;
591 if (to->to_maxval && req->rq_timeout >= to->to_maxval)
592 req->rq_timeout = to->to_maxval;
593 req->rq_retries++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 } else {
595 req->rq_timeout = to->to_initval;
596 req->rq_retries = 0;
597 xprt_reset_majortimeo(req);
598 /* Reset the RTT counters == "slow start" */
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400599 spin_lock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400601 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 status = -ETIMEDOUT;
603 }
604
605 if (req->rq_timeout == 0) {
606 printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
607 req->rq_timeout = 5 * HZ;
608 }
609 return status;
610}
611
David Howells65f27f32006-11-22 14:55:48 +0000612static void xprt_autoclose(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613{
David Howells65f27f32006-11-22 14:55:48 +0000614 struct rpc_xprt *xprt =
615 container_of(work, struct rpc_xprt, task_cleanup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
Trond Myklebust66af1e52007-11-06 10:18:36 -0500617 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
Trond Myklebust4876cc72015-06-19 16:17:57 -0400618 xprt->ops->close(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 xprt_release_write(xprt, NULL);
Trond Myklebust79234c32015-09-18 15:53:24 -0400620 wake_up_bit(&xprt->state, XPRT_LOCKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621}
622
Chuck Lever9903cd12005-08-11 16:25:26 -0400623/**
Trond Myklebust62da3b22007-11-06 18:44:20 -0500624 * xprt_disconnect_done - mark a transport as disconnected
Chuck Lever9903cd12005-08-11 16:25:26 -0400625 * @xprt: transport to flag for disconnect
626 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 */
Trond Myklebust62da3b22007-11-06 18:44:20 -0500628void xprt_disconnect_done(struct rpc_xprt *xprt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629{
Chuck Lever46121cf2007-01-31 12:14:08 -0500630 dprintk("RPC: disconnected transport %p\n", xprt);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400631 spin_lock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 xprt_clear_connected(xprt);
Trond Myklebust2a491992009-03-11 14:38:00 -0400633 xprt_wake_pending_tasks(xprt, -EAGAIN);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400634 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635}
Trond Myklebust62da3b22007-11-06 18:44:20 -0500636EXPORT_SYMBOL_GPL(xprt_disconnect_done);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637
Trond Myklebust66af1e52007-11-06 10:18:36 -0500638/**
639 * xprt_force_disconnect - force a transport to disconnect
640 * @xprt: transport to disconnect
641 *
642 */
643void xprt_force_disconnect(struct rpc_xprt *xprt)
644{
645 /* Don't race with the test_bit() in xprt_clear_locked() */
646 spin_lock_bh(&xprt->transport_lock);
647 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
648 /* Try to schedule an autoclose RPC call */
649 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
Trond Myklebust40a5f1b2016-05-27 10:39:50 -0400650 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
Trond Myklebust2a491992009-03-11 14:38:00 -0400651 xprt_wake_pending_tasks(xprt, -EAGAIN);
Trond Myklebust66af1e52007-11-06 10:18:36 -0500652 spin_unlock_bh(&xprt->transport_lock);
653}
Trond Myklebust66af1e52007-11-06 10:18:36 -0500654
Trond Myklebust7c1d71c2008-04-17 16:52:57 -0400655/**
656 * xprt_conditional_disconnect - force a transport to disconnect
657 * @xprt: transport to disconnect
658 * @cookie: 'connection cookie'
659 *
660 * This attempts to break the connection if and only if 'cookie' matches
661 * the current transport 'connection cookie'. It ensures that we don't
662 * try to break the connection more than once when we need to retransmit
663 * a batch of RPC requests.
664 *
665 */
666void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
667{
668 /* Don't race with the test_bit() in xprt_clear_locked() */
669 spin_lock_bh(&xprt->transport_lock);
670 if (cookie != xprt->connect_cookie)
671 goto out;
672 if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
673 goto out;
674 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
675 /* Try to schedule an autoclose RPC call */
676 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
Trond Myklebust40a5f1b2016-05-27 10:39:50 -0400677 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
Trond Myklebust2a491992009-03-11 14:38:00 -0400678 xprt_wake_pending_tasks(xprt, -EAGAIN);
Trond Myklebust7c1d71c2008-04-17 16:52:57 -0400679out:
680 spin_unlock_bh(&xprt->transport_lock);
681}
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683static void
684xprt_init_autodisconnect(unsigned long data)
685{
686 struct rpc_xprt *xprt = (struct rpc_xprt *)data;
687
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400688 spin_lock(&xprt->transport_lock);
Trond Myklebustd19751e2012-09-11 17:21:25 -0400689 if (!list_empty(&xprt->recv))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690 goto out_abort;
Chuck Lever2226feb2005-08-11 16:25:38 -0400691 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 goto out_abort;
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400693 spin_unlock(&xprt->transport_lock);
Trond Myklebust40a5f1b2016-05-27 10:39:50 -0400694 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 return;
696out_abort:
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400697 spin_unlock(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698}
699
Trond Myklebust718ba5b2015-02-08 18:19:25 -0500700bool xprt_lock_connect(struct rpc_xprt *xprt,
701 struct rpc_task *task,
702 void *cookie)
703{
704 bool ret = false;
705
706 spin_lock_bh(&xprt->transport_lock);
707 if (!test_bit(XPRT_LOCKED, &xprt->state))
708 goto out;
709 if (xprt->snd_task != task)
710 goto out;
Trond Myklebust06953142015-03-23 16:10:00 -0400711 xprt_task_clear_bytes_sent(task);
Trond Myklebust718ba5b2015-02-08 18:19:25 -0500712 xprt->snd_task = cookie;
713 ret = true;
714out:
715 spin_unlock_bh(&xprt->transport_lock);
716 return ret;
717}
718
719void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie)
720{
721 spin_lock_bh(&xprt->transport_lock);
722 if (xprt->snd_task != cookie)
723 goto out;
724 if (!test_bit(XPRT_LOCKED, &xprt->state))
725 goto out;
726 xprt->snd_task =NULL;
727 xprt->ops->release_xprt(xprt, NULL);
728out:
729 spin_unlock_bh(&xprt->transport_lock);
Trond Myklebust79234c32015-09-18 15:53:24 -0400730 wake_up_bit(&xprt->state, XPRT_LOCKED);
Trond Myklebust718ba5b2015-02-08 18:19:25 -0500731}
732
Chuck Lever9903cd12005-08-11 16:25:26 -0400733/**
734 * xprt_connect - schedule a transport connect operation
735 * @task: RPC task that is requesting the connect
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 *
737 */
738void xprt_connect(struct rpc_task *task)
739{
Trond Myklebustad2368d2013-01-08 10:08:33 -0500740 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
Chuck Lever46121cf2007-01-31 12:14:08 -0500742 dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 xprt, (xprt_connected(xprt) ? "is" : "is not"));
744
Chuck Leverec739ef2006-08-22 20:06:15 -0400745 if (!xprt_bound(xprt)) {
Trond Myklebust01d37c42009-03-11 14:09:39 -0400746 task->tk_status = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 return;
748 }
749 if (!xprt_lock_write(xprt, task))
750 return;
Trond Myklebustfeb8ca32009-12-03 08:10:17 -0500751
752 if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
753 xprt->ops->close(xprt);
754
Trond Myklebust718ba5b2015-02-08 18:19:25 -0500755 if (!xprt_connected(xprt)) {
Dan Carpenter87e3c052012-02-01 10:46:20 +0300756 task->tk_rqstp->rq_bytes_sent = 0;
Trond Myklebusta8ce4a82010-04-16 16:42:12 -0400757 task->tk_timeout = task->tk_rqstp->rq_timeout;
Trond Myklebust5d008372008-02-22 16:34:17 -0500758 rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
Trond Myklebust0b9e7942010-04-16 16:41:57 -0400759
760 if (test_bit(XPRT_CLOSING, &xprt->state))
761 return;
762 if (xprt_test_and_set_connecting(xprt))
763 return;
Chuck Lever262ca072006-03-20 13:44:16 -0500764 xprt->stat.connect_start = jiffies;
Trond Myklebust1b092092013-01-08 09:26:49 -0500765 xprt->ops->connect(xprt, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 }
Trond Myklebust718ba5b2015-02-08 18:19:25 -0500767 xprt_release_write(xprt, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768}
769
Chuck Lever9903cd12005-08-11 16:25:26 -0400770static void xprt_connect_status(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771{
Trond Myklebustad2368d2013-01-08 10:08:33 -0500772 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Chuck Levercd983ef2008-06-11 17:56:13 -0400774 if (task->tk_status == 0) {
Chuck Lever262ca072006-03-20 13:44:16 -0500775 xprt->stat.connect_count++;
776 xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;
Chuck Lever46121cf2007-01-31 12:14:08 -0500777 dprintk("RPC: %5u xprt_connect_status: connection established\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 task->tk_pid);
779 return;
780 }
781
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 switch (task->tk_status) {
Trond Myklebust0fe8d042013-12-31 13:13:30 -0500783 case -ECONNREFUSED:
784 case -ECONNRESET:
785 case -ECONNABORTED:
786 case -ENETUNREACH:
787 case -EHOSTUNREACH:
Trond Myklebust2fc193c2014-07-03 00:02:57 -0400788 case -EPIPE:
Trond Myklebust2a491992009-03-11 14:38:00 -0400789 case -EAGAIN:
790 dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid);
Chuck Lever23475d62005-08-11 16:25:08 -0400791 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 case -ETIMEDOUT:
Chuck Lever46121cf2007-01-31 12:14:08 -0500793 dprintk("RPC: %5u xprt_connect_status: connect attempt timed "
794 "out\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 break;
796 default:
Chuck Lever46121cf2007-01-31 12:14:08 -0500797 dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
798 "server %s\n", task->tk_pid, -task->tk_status,
Trond Myklebust4e0038b2012-03-01 17:01:05 -0500799 xprt->servername);
Chuck Lever23475d62005-08-11 16:25:08 -0400800 task->tk_status = -EIO;
Chuck Lever23475d62005-08-11 16:25:08 -0400801 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802}
803
Chuck Lever9903cd12005-08-11 16:25:26 -0400804/**
805 * xprt_lookup_rqst - find an RPC request corresponding to an XID
806 * @xprt: transport on which the original request was transmitted
807 * @xid: RPC XID of incoming reply
808 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 */
Alexey Dobriyand8ed0292006-09-26 22:29:38 -0700810struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811{
Pavel Emelyanov8f3a6de2010-10-05 23:30:19 +0400812 struct rpc_rqst *entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
Pavel Emelyanov8f3a6de2010-10-05 23:30:19 +0400814 list_for_each_entry(entry, &xprt->recv, rq_list)
Jeff Layton3705ad62014-10-28 14:24:13 -0400815 if (entry->rq_xid == xid) {
816 trace_xprt_lookup_rqst(xprt, xid, 0);
Chuck Lever262ca072006-03-20 13:44:16 -0500817 return entry;
Jeff Layton3705ad62014-10-28 14:24:13 -0400818 }
Chuck Lever46121cf2007-01-31 12:14:08 -0500819
820 dprintk("RPC: xprt_lookup_rqst did not find xid %08x\n",
821 ntohl(xid));
Jeff Layton3705ad62014-10-28 14:24:13 -0400822 trace_xprt_lookup_rqst(xprt, xid, -ENOENT);
Chuck Lever262ca072006-03-20 13:44:16 -0500823 xprt->stat.bad_xids++;
824 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400826EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827
Chuck Leverbbc72ce2010-05-07 13:34:27 -0400828static void xprt_update_rtt(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829{
Chuck Lever1570c1e2005-08-25 16:25:52 -0700830 struct rpc_rqst *req = task->tk_rqstp;
831 struct rpc_rtt *rtt = task->tk_client->cl_rtt;
Eric Dumazet95c96172012-04-15 05:58:06 +0000832 unsigned int timer = task->tk_msg.rpc_proc->p_timer;
Trond Myklebustd60dbb22010-05-13 12:51:49 -0400833 long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834
Chuck Lever1570c1e2005-08-25 16:25:52 -0700835 if (timer) {
836 if (req->rq_ntrans == 1)
Chuck Leverff839972010-05-07 13:34:47 -0400837 rpc_update_rtt(rtt, timer, m);
Chuck Lever1570c1e2005-08-25 16:25:52 -0700838 rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 }
Chuck Lever1570c1e2005-08-25 16:25:52 -0700840}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841
Chuck Lever1570c1e2005-08-25 16:25:52 -0700842/**
843 * xprt_complete_rqst - called when reply processing is complete
844 * @task: RPC request that recently completed
845 * @copied: actual number of bytes received from the transport
846 *
847 * Caller holds transport lock.
848 */
849void xprt_complete_rqst(struct rpc_task *task, int copied)
850{
851 struct rpc_rqst *req = task->tk_rqstp;
Trond Myklebustfda13932008-02-22 16:34:12 -0500852 struct rpc_xprt *xprt = req->rq_xprt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853
Chuck Lever1570c1e2005-08-25 16:25:52 -0700854 dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
855 task->tk_pid, ntohl(req->rq_xid), copied);
Jeff Layton3705ad62014-10-28 14:24:13 -0400856 trace_xprt_complete_rqst(xprt, req->rq_xid, copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857
Trond Myklebustfda13932008-02-22 16:34:12 -0500858 xprt->stat.recvs++;
Trond Myklebustd60dbb22010-05-13 12:51:49 -0400859 req->rq_rtt = ktime_sub(ktime_get(), req->rq_xtime);
Chuck Leverbbc72ce2010-05-07 13:34:27 -0400860 if (xprt->ops->timer != NULL)
861 xprt_update_rtt(task);
Chuck Leveref759a22006-03-20 13:44:17 -0500862
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 list_del_init(&req->rq_list);
Trond Myklebust1e799b62008-03-21 16:19:41 -0400864 req->rq_private_buf.len = copied;
Ricardo Labiagadd2b63d2009-04-01 09:23:28 -0400865 /* Ensure all writes are done before we update */
866 /* req->rq_reply_bytes_recvd */
Trond Myklebust43ac3f22006-03-20 13:44:51 -0500867 smp_wmb();
Ricardo Labiagadd2b63d2009-04-01 09:23:28 -0400868 req->rq_reply_bytes_recvd = copied;
Trond Myklebustfda13932008-02-22 16:34:12 -0500869 rpc_wake_up_queued_task(&xprt->pending, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400871EXPORT_SYMBOL_GPL(xprt_complete_rqst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872
Chuck Lever46c0ee82005-08-25 16:25:52 -0700873static void xprt_timer(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874{
Chuck Lever46c0ee82005-08-25 16:25:52 -0700875 struct rpc_rqst *req = task->tk_rqstp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 struct rpc_xprt *xprt = req->rq_xprt;
877
Trond Myklebust5d008372008-02-22 16:34:17 -0500878 if (task->tk_status != -ETIMEDOUT)
879 return;
Chuck Lever46121cf2007-01-31 12:14:08 -0500880 dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
Chuck Lever46c0ee82005-08-25 16:25:52 -0700881
Trond Myklebust5d008372008-02-22 16:34:17 -0500882 spin_lock_bh(&xprt->transport_lock);
Ricardo Labiagadd2b63d2009-04-01 09:23:28 -0400883 if (!req->rq_reply_bytes_recvd) {
Chuck Lever46c0ee82005-08-25 16:25:52 -0700884 if (xprt->ops->timer)
Trond Myklebust6a24dfb2013-01-08 09:48:15 -0500885 xprt->ops->timer(xprt, task);
Trond Myklebust5d008372008-02-22 16:34:17 -0500886 } else
887 task->tk_status = 0;
888 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889}
890
Rahul Iyer4cfc7e62009-09-10 17:32:28 +0300891static inline int xprt_has_timer(struct rpc_xprt *xprt)
892{
893 return xprt->idle_timeout != 0;
894}
895
Chuck Lever9903cd12005-08-11 16:25:26 -0400896/**
897 * xprt_prepare_transmit - reserve the transport before sending a request
898 * @task: RPC task about to send a request
899 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 */
Trond Myklebust90051ea2013-09-25 12:17:18 -0400901bool xprt_prepare_transmit(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902{
903 struct rpc_rqst *req = task->tk_rqstp;
904 struct rpc_xprt *xprt = req->rq_xprt;
Trond Myklebust90051ea2013-09-25 12:17:18 -0400905 bool ret = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906
Chuck Lever46121cf2007-01-31 12:14:08 -0500907 dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400909 spin_lock_bh(&xprt->transport_lock);
Trond Myklebust8a19a0b2013-09-24 12:00:27 -0400910 if (!req->rq_bytes_sent) {
911 if (req->rq_reply_bytes_recvd) {
912 task->tk_status = req->rq_reply_bytes_recvd;
913 goto out_unlock;
914 }
915 if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT)
916 && xprt_connected(xprt)
917 && req->rq_connect_cookie == xprt->connect_cookie) {
918 xprt->ops->set_retrans_timeout(task);
919 rpc_sleep_on(&xprt->pending, task, xprt_timer);
920 goto out_unlock;
921 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 }
Trond Myklebust90051ea2013-09-25 12:17:18 -0400923 if (!xprt->ops->reserve_xprt(xprt, task)) {
924 task->tk_status = -EAGAIN;
925 goto out_unlock;
926 }
927 ret = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928out_unlock:
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400929 spin_unlock_bh(&xprt->transport_lock);
Trond Myklebust90051ea2013-09-25 12:17:18 -0400930 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931}
932
Trond Myklebuste0ab53d2006-07-27 17:22:50 -0400933void xprt_end_transmit(struct rpc_task *task)
Trond Myklebust5e5ce5b2005-10-18 14:20:11 -0700934{
Rahul Iyer343952f2009-04-01 09:23:17 -0400935 xprt_release_write(task->tk_rqstp->rq_xprt, task);
Trond Myklebust5e5ce5b2005-10-18 14:20:11 -0700936}
937
Chuck Lever9903cd12005-08-11 16:25:26 -0400938/**
939 * xprt_transmit - send an RPC request on a transport
940 * @task: controlling RPC task
941 *
942 * We have to copy the iovec because sendmsg fiddles with its contents.
943 */
944void xprt_transmit(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 struct rpc_rqst *req = task->tk_rqstp;
947 struct rpc_xprt *xprt = req->rq_xprt;
Andy Adamson15a45202012-02-14 16:19:18 -0500948 int status, numreqs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949
Chuck Lever46121cf2007-01-31 12:14:08 -0500950 dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951
Ricardo Labiagadd2b63d2009-04-01 09:23:28 -0400952 if (!req->rq_reply_bytes_recvd) {
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -0400953 if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
954 /*
955 * Add to the list only if we're expecting a reply
956 */
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400957 spin_lock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 /* Update the softirq receive buffer */
959 memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
960 sizeof(req->rq_private_buf));
961 /* Add request to the receive list */
962 list_add_tail(&req->rq_list, &xprt->recv);
Chuck Lever4a0f8c02005-08-11 16:25:32 -0400963 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 xprt_reset_majortimeo(req);
Trond Myklebust0f9dc2b2005-06-22 17:16:28 +0000965 /* Turn off autodisconnect */
966 del_singleshot_timer_sync(&xprt->timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 }
968 } else if (!req->rq_bytes_sent)
969 return;
970
Chuck Leverff839972010-05-07 13:34:47 -0400971 req->rq_xtime = ktime_get();
Chuck Levera246b012005-08-11 16:25:23 -0400972 status = xprt->ops->send_request(task);
Jeff Layton3705ad62014-10-28 14:24:13 -0400973 trace_xprt_transmit(xprt, req->rq_xid, status);
Trond Myklebustc8485e42009-03-11 14:37:59 -0400974 if (status != 0) {
975 task->tk_status = status;
Chuck Leverfe3aca22005-08-25 16:25:50 -0700976 return;
977 }
Chuck Lever4a068252015-05-11 14:02:25 -0400978 xprt_inject_disconnect(xprt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Trond Myklebustc8485e42009-03-11 14:37:59 -0400980 dprintk("RPC: %5u xmit complete\n", task->tk_pid);
Bryan Schumaker468f8612011-04-18 15:57:32 -0400981 task->tk_flags |= RPC_TASK_SENT;
Trond Myklebustc8485e42009-03-11 14:37:59 -0400982 spin_lock_bh(&xprt->transport_lock);
983
984 xprt->ops->set_retrans_timeout(task);
985
Andy Adamson15a45202012-02-14 16:19:18 -0500986 numreqs = atomic_read(&xprt->num_reqs);
987 if (numreqs > xprt->stat.max_slots)
988 xprt->stat.max_slots = numreqs;
Trond Myklebustc8485e42009-03-11 14:37:59 -0400989 xprt->stat.sends++;
990 xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
991 xprt->stat.bklog_u += xprt->backlog.qlen;
Andy Adamson15a45202012-02-14 16:19:18 -0500992 xprt->stat.sending_u += xprt->sending.qlen;
993 xprt->stat.pending_u += xprt->pending.qlen;
Trond Myklebustc8485e42009-03-11 14:37:59 -0400994
995 /* Don't race with disconnect */
996 if (!xprt_connected(xprt))
997 task->tk_status = -ENOTCONN;
Trond Myklebust0a660522013-09-25 11:31:54 -0400998 else {
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -0400999 /*
1000 * Sleep on the pending queue since
1001 * we're expecting a reply.
1002 */
Trond Myklebust0a660522013-09-25 11:31:54 -04001003 if (!req->rq_reply_bytes_recvd && rpc_reply_expected(task))
1004 rpc_sleep_on(&xprt->pending, task, xprt_timer);
1005 req->rq_connect_cookie = xprt->connect_cookie;
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001006 }
Trond Myklebustc8485e42009-03-11 14:37:59 -04001007 spin_unlock_bh(&xprt->transport_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008}
1009
Trond Myklebustba60eb22013-04-14 10:49:37 -04001010static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)
1011{
1012 set_bit(XPRT_CONGESTED, &xprt->state);
1013 rpc_sleep_on(&xprt->backlog, task, NULL);
1014}
1015
1016static void xprt_wake_up_backlog(struct rpc_xprt *xprt)
1017{
1018 if (rpc_wake_up_next(&xprt->backlog) == NULL)
1019 clear_bit(XPRT_CONGESTED, &xprt->state);
1020}
1021
1022static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)
1023{
1024 bool ret = false;
1025
1026 if (!test_bit(XPRT_CONGESTED, &xprt->state))
1027 goto out;
1028 spin_lock(&xprt->reserve_lock);
1029 if (test_bit(XPRT_CONGESTED, &xprt->state)) {
1030 rpc_sleep_on(&xprt->backlog, task, NULL);
1031 ret = true;
1032 }
1033 spin_unlock(&xprt->reserve_lock);
1034out:
1035 return ret;
1036}
1037
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001038static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt, gfp_t gfp_flags)
1039{
1040 struct rpc_rqst *req = ERR_PTR(-EAGAIN);
1041
1042 if (!atomic_add_unless(&xprt->num_reqs, 1, xprt->max_reqs))
1043 goto out;
1044 req = kzalloc(sizeof(struct rpc_rqst), gfp_flags);
1045 if (req != NULL)
1046 goto out;
1047 atomic_dec(&xprt->num_reqs);
1048 req = ERR_PTR(-ENOMEM);
1049out:
1050 return req;
1051}
1052
1053static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1054{
1055 if (atomic_add_unless(&xprt->num_reqs, -1, xprt->min_reqs)) {
1056 kfree(req);
1057 return true;
1058 }
1059 return false;
1060}
1061
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001062void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063{
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001064 struct rpc_rqst *req;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001066 spin_lock(&xprt->reserve_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 if (!list_empty(&xprt->free)) {
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001068 req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
1069 list_del(&req->rq_list);
1070 goto out_init_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071 }
Jeff Layton6b343092012-05-16 13:30:35 -04001072 req = xprt_dynamic_alloc_slot(xprt, GFP_NOWAIT|__GFP_NOWARN);
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001073 if (!IS_ERR(req))
1074 goto out_init_req;
1075 switch (PTR_ERR(req)) {
1076 case -ENOMEM:
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001077 dprintk("RPC: dynamic allocation of request slot "
1078 "failed! Retrying\n");
Trond Myklebust1afeaf52012-05-19 12:12:53 -04001079 task->tk_status = -ENOMEM;
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001080 break;
1081 case -EAGAIN:
Trond Myklebustba60eb22013-04-14 10:49:37 -04001082 xprt_add_backlog(xprt, task);
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001083 dprintk("RPC: waiting for request slot\n");
Trond Myklebust1afeaf52012-05-19 12:12:53 -04001084 default:
1085 task->tk_status = -EAGAIN;
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001086 }
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001087 spin_unlock(&xprt->reserve_lock);
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001088 return;
1089out_init_req:
1090 task->tk_status = 0;
1091 task->tk_rqstp = req;
1092 xprt_request_init(task, xprt);
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001093 spin_unlock(&xprt->reserve_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094}
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001095EXPORT_SYMBOL_GPL(xprt_alloc_slot);
1096
1097void xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1098{
1099 /* Note: grabbing the xprt_lock_write() ensures that we throttle
1100 * new slot allocation if the transport is congested (i.e. when
1101 * reconnecting a stream transport or when out of socket write
1102 * buffer space).
1103 */
1104 if (xprt_lock_write(xprt, task)) {
1105 xprt_alloc_slot(xprt, task);
1106 xprt_release_write(xprt, task);
1107 }
1108}
1109EXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Trond Myklebustee5ebe82010-04-16 16:37:01 -04001111static void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1112{
Trond Myklebustee5ebe82010-04-16 16:37:01 -04001113 spin_lock(&xprt->reserve_lock);
Trond Myklebustc25573b2011-12-01 14:16:17 -05001114 if (!xprt_dynamic_free_slot(xprt, req)) {
1115 memset(req, 0, sizeof(*req)); /* mark unused */
1116 list_add(&req->rq_list, &xprt->free);
1117 }
Trond Myklebustba60eb22013-04-14 10:49:37 -04001118 xprt_wake_up_backlog(xprt);
Trond Myklebustee5ebe82010-04-16 16:37:01 -04001119 spin_unlock(&xprt->reserve_lock);
1120}
1121
Trond Myklebust21de0a92011-07-17 16:57:32 -04001122static void xprt_free_all_slots(struct rpc_xprt *xprt)
1123{
1124 struct rpc_rqst *req;
1125 while (!list_empty(&xprt->free)) {
1126 req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list);
1127 list_del(&req->rq_list);
1128 kfree(req);
1129 }
1130}
1131
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001132struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
1133 unsigned int num_prealloc,
1134 unsigned int max_alloc)
Pavel Emelyanovbd1722d2010-09-29 16:02:43 +04001135{
1136 struct rpc_xprt *xprt;
Trond Myklebust21de0a92011-07-17 16:57:32 -04001137 struct rpc_rqst *req;
1138 int i;
Pavel Emelyanovbd1722d2010-09-29 16:02:43 +04001139
1140 xprt = kzalloc(size, GFP_KERNEL);
1141 if (xprt == NULL)
1142 goto out;
1143
Trond Myklebust21de0a92011-07-17 16:57:32 -04001144 xprt_init(xprt, net);
1145
1146 for (i = 0; i < num_prealloc; i++) {
1147 req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
1148 if (!req)
wangweidong83131642013-10-15 11:44:30 +08001149 goto out_free;
Trond Myklebust21de0a92011-07-17 16:57:32 -04001150 list_add(&req->rq_list, &xprt->free);
1151 }
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001152 if (max_alloc > num_prealloc)
1153 xprt->max_reqs = max_alloc;
1154 else
1155 xprt->max_reqs = num_prealloc;
1156 xprt->min_reqs = num_prealloc;
1157 atomic_set(&xprt->num_reqs, num_prealloc);
Pavel Emelyanovbd1722d2010-09-29 16:02:43 +04001158
1159 return xprt;
1160
1161out_free:
Trond Myklebust21de0a92011-07-17 16:57:32 -04001162 xprt_free(xprt);
Pavel Emelyanovbd1722d2010-09-29 16:02:43 +04001163out:
1164 return NULL;
1165}
1166EXPORT_SYMBOL_GPL(xprt_alloc);
1167
Pavel Emelyanove204e622010-09-29 16:03:13 +04001168void xprt_free(struct rpc_xprt *xprt)
1169{
Pavel Emelyanov37aa2132010-09-29 16:05:43 +04001170 put_net(xprt->xprt_net);
Trond Myklebust21de0a92011-07-17 16:57:32 -04001171 xprt_free_all_slots(xprt);
Trond Myklebustfda1bfe2015-02-14 17:48:49 -05001172 kfree_rcu(xprt, rcu);
Pavel Emelyanove204e622010-09-29 16:03:13 +04001173}
1174EXPORT_SYMBOL_GPL(xprt_free);
1175
Chuck Lever9903cd12005-08-11 16:25:26 -04001176/**
1177 * xprt_reserve - allocate an RPC request slot
1178 * @task: RPC task requesting a slot allocation
1179 *
Trond Myklebustba60eb22013-04-14 10:49:37 -04001180 * If the transport is marked as being congested, or if no more
1181 * slots are available, place the task on the transport's
Chuck Lever9903cd12005-08-11 16:25:26 -04001182 * backlog queue.
1183 */
1184void xprt_reserve(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185{
Trond Myklebustfb43d172016-01-30 16:39:26 -05001186 struct rpc_xprt *xprt = task->tk_xprt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -04001188 task->tk_status = 0;
1189 if (task->tk_rqstp != NULL)
1190 return;
1191
Trond Myklebust43cedbf0e2011-07-17 16:01:03 -04001192 task->tk_timeout = 0;
1193 task->tk_status = -EAGAIN;
Trond Myklebustba60eb22013-04-14 10:49:37 -04001194 if (!xprt_throttle_congested(xprt, task))
1195 xprt->ops->alloc_slot(xprt, task);
Trond Myklebustba60eb22013-04-14 10:49:37 -04001196}
1197
1198/**
1199 * xprt_retry_reserve - allocate an RPC request slot
1200 * @task: RPC task requesting a slot allocation
1201 *
1202 * If no more slots are available, place the task on the transport's
1203 * backlog queue.
1204 * Note that the only difference with xprt_reserve is that we now
1205 * ignore the value of the XPRT_CONGESTED flag.
1206 */
1207void xprt_retry_reserve(struct rpc_task *task)
1208{
Trond Myklebustfb43d172016-01-30 16:39:26 -05001209 struct rpc_xprt *xprt = task->tk_xprt;
Trond Myklebustba60eb22013-04-14 10:49:37 -04001210
1211 task->tk_status = 0;
1212 if (task->tk_rqstp != NULL)
1213 return;
1214
1215 task->tk_timeout = 0;
1216 task->tk_status = -EAGAIN;
Trond Myklebustf39c1bf2012-09-07 11:08:50 -04001217 xprt->ops->alloc_slot(xprt, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218}
1219
Alexey Dobriyand8ed0292006-09-26 22:29:38 -07001220static inline __be32 xprt_alloc_xid(struct rpc_xprt *xprt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221{
Eric Dumazet0eae88f2010-04-20 19:06:52 -07001222 return (__force __be32)xprt->xid++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223}
1224
1225static inline void xprt_init_xid(struct rpc_xprt *xprt)
1226{
Aruna-Hewapathirane63862b52014-01-11 07:15:59 -05001227 xprt->xid = prandom_u32();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228}
1229
Chuck Lever9903cd12005-08-11 16:25:26 -04001230static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231{
1232 struct rpc_rqst *req = task->tk_rqstp;
1233
Trond Myklebustd9ba1312011-07-17 18:11:30 -04001234 INIT_LIST_HEAD(&req->rq_list);
Trond Myklebustba7392b2007-12-20 16:03:55 -05001235 req->rq_timeout = task->tk_client->cl_timeout->to_initval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236 req->rq_task = task;
1237 req->rq_xprt = xprt;
Chuck Lever02107142006-01-03 09:55:49 +01001238 req->rq_buffer = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239 req->rq_xid = xprt_alloc_xid(xprt);
Trond Myklebust0a660522013-09-25 11:31:54 -04001240 req->rq_connect_cookie = xprt->connect_cookie - 1;
Trond Myklebust92551942013-09-27 11:28:40 -04001241 req->rq_bytes_sent = 0;
1242 req->rq_snd_buf.len = 0;
1243 req->rq_snd_buf.buflen = 0;
1244 req->rq_rcv_buf.len = 0;
1245 req->rq_rcv_buf.buflen = 0;
J. Bruce Fieldsead5e1c2005-10-13 16:54:43 -04001246 req->rq_release_snd_buf = NULL;
Trond Myklebustda458282006-08-31 15:44:52 -04001247 xprt_reset_majortimeo(req);
Chuck Lever46121cf2007-01-31 12:14:08 -05001248 dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 req, ntohl(req->rq_xid));
1250}
1251
Chuck Lever9903cd12005-08-11 16:25:26 -04001252/**
1253 * xprt_release - release an RPC request slot
1254 * @task: task which is finished with the slot
1255 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 */
Chuck Lever9903cd12005-08-11 16:25:26 -04001257void xprt_release(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258{
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001259 struct rpc_xprt *xprt;
Trond Myklebust87ed5002013-01-07 14:30:46 -05001260 struct rpc_rqst *req = task->tk_rqstp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261
Trond Myklebust87ed5002013-01-07 14:30:46 -05001262 if (req == NULL) {
1263 if (task->tk_client) {
Trond Myklebustfb43d172016-01-30 16:39:26 -05001264 xprt = task->tk_xprt;
Trond Myklebust87ed5002013-01-07 14:30:46 -05001265 if (xprt->snd_task == task)
1266 xprt_release_write(xprt, task);
Trond Myklebust87ed5002013-01-07 14:30:46 -05001267 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 return;
Trond Myklebust87ed5002013-01-07 14:30:46 -05001269 }
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001270
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001271 xprt = req->rq_xprt;
Weston Andros Adamson0a702192012-02-17 13:15:24 -05001272 if (task->tk_ops->rpc_count_stats != NULL)
1273 task->tk_ops->rpc_count_stats(task, task->tk_calldata);
1274 else if (task->tk_client)
1275 rpc_count_iostats(task, task->tk_client->cl_metrics);
Chuck Lever4a0f8c02005-08-11 16:25:32 -04001276 spin_lock_bh(&xprt->transport_lock);
Chuck Lever49e9a892005-08-25 16:25:51 -07001277 xprt->ops->release_xprt(xprt, task);
Chuck Levera58dd392005-08-25 16:25:53 -07001278 if (xprt->ops->release_request)
1279 xprt->ops->release_request(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 if (!list_empty(&req->rq_list))
1281 list_del(&req->rq_list);
1282 xprt->last_used = jiffies;
Rahul Iyer4cfc7e62009-09-10 17:32:28 +03001283 if (list_empty(&xprt->recv) && xprt_has_timer(xprt))
Chuck Levera246b012005-08-11 16:25:23 -04001284 mod_timer(&xprt->timer,
Chuck Lever03bf4b72005-08-25 16:25:55 -07001285 xprt->last_used + xprt->idle_timeout);
Chuck Lever4a0f8c02005-08-11 16:25:32 -04001286 spin_unlock_bh(&xprt->transport_lock);
Trond Myklebustee5ebe82010-04-16 16:37:01 -04001287 if (req->rq_buffer)
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001288 xprt->ops->buf_free(req->rq_buffer);
Chuck Lever4a068252015-05-11 14:02:25 -04001289 xprt_inject_disconnect(xprt);
Trond Myklebusta17c2152010-07-31 14:29:08 -04001290 if (req->rq_cred != NULL)
1291 put_rpccred(req->rq_cred);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 task->tk_rqstp = NULL;
J. Bruce Fieldsead5e1c2005-10-13 16:54:43 -04001293 if (req->rq_release_snd_buf)
1294 req->rq_release_snd_buf(req);
Ricardo Labiaga55ae1aa2009-04-01 09:23:03 -04001295
Chuck Lever46121cf2007-01-31 12:14:08 -05001296 dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
Trond Myklebustee5ebe82010-04-16 16:37:01 -04001297 if (likely(!bc_prealloc(req)))
1298 xprt_free_slot(xprt, req);
1299 else
Trond Myklebustc9acb422010-03-19 15:36:22 -04001300 xprt_free_bc_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301}
1302
Trond Myklebust21de0a92011-07-17 16:57:32 -04001303static void xprt_init(struct rpc_xprt *xprt, struct net *net)
Chuck Leverc2866762006-08-22 20:06:20 -04001304{
Trond Myklebust30c51162015-02-24 20:31:39 -05001305 kref_init(&xprt->kref);
Chuck Leverc2866762006-08-22 20:06:20 -04001306
1307 spin_lock_init(&xprt->transport_lock);
1308 spin_lock_init(&xprt->reserve_lock);
1309
1310 INIT_LIST_HEAD(&xprt->free);
1311 INIT_LIST_HEAD(&xprt->recv);
Trond Myklebust9e00abc2011-07-13 19:20:49 -04001312#if defined(CONFIG_SUNRPC_BACKCHANNEL)
Ricardo Labiagaf9acac12009-04-01 09:22:59 -04001313 spin_lock_init(&xprt->bc_pa_lock);
1314 INIT_LIST_HEAD(&xprt->bc_pa_list);
Trond Myklebust9e00abc2011-07-13 19:20:49 -04001315#endif /* CONFIG_SUNRPC_BACKCHANNEL */
Trond Myklebust80b14d52015-02-14 20:31:59 -05001316 INIT_LIST_HEAD(&xprt->xprt_switch);
Ricardo Labiagaf9acac12009-04-01 09:22:59 -04001317
Chuck Leverc2866762006-08-22 20:06:20 -04001318 xprt->last_used = jiffies;
1319 xprt->cwnd = RPC_INITCWND;
Chuck Levera5090502007-03-29 16:48:04 -04001320 xprt->bind_index = 0;
Chuck Leverc2866762006-08-22 20:06:20 -04001321
1322 rpc_init_wait_queue(&xprt->binding, "xprt_binding");
1323 rpc_init_wait_queue(&xprt->pending, "xprt_pending");
Trond Myklebust34006ce2011-07-17 18:11:34 -04001324 rpc_init_priority_wait_queue(&xprt->sending, "xprt_sending");
Chuck Leverc2866762006-08-22 20:06:20 -04001325 rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");
1326
Chuck Leverc2866762006-08-22 20:06:20 -04001327 xprt_init_xid(xprt);
1328
Trond Myklebust21de0a92011-07-17 16:57:32 -04001329 xprt->xprt_net = get_net(net);
Trond Myklebust8d9266f2011-07-17 16:01:09 -04001330}
1331
1332/**
1333 * xprt_create_transport - create an RPC transport
1334 * @args: rpc transport creation arguments
1335 *
1336 */
1337struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
1338{
1339 struct rpc_xprt *xprt;
1340 struct xprt_class *t;
1341
1342 spin_lock(&xprt_list_lock);
1343 list_for_each_entry(t, &xprt_list, list) {
1344 if (t->ident == args->ident) {
1345 spin_unlock(&xprt_list_lock);
1346 goto found;
1347 }
1348 }
1349 spin_unlock(&xprt_list_lock);
Chuck Lever3c45ddf2014-07-16 15:38:32 -04001350 dprintk("RPC: transport (%d) not supported\n", args->ident);
Trond Myklebust8d9266f2011-07-17 16:01:09 -04001351 return ERR_PTR(-EIO);
1352
1353found:
1354 xprt = t->setup(args);
1355 if (IS_ERR(xprt)) {
1356 dprintk("RPC: xprt_create_transport: failed, %ld\n",
1357 -PTR_ERR(xprt));
Trond Myklebust21de0a92011-07-17 16:57:32 -04001358 goto out;
Trond Myklebust8d9266f2011-07-17 16:01:09 -04001359 }
J. Bruce Fields33d90ac2013-04-11 15:06:36 -04001360 if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
1361 xprt->idle_timeout = 0;
Trond Myklebust21de0a92011-07-17 16:57:32 -04001362 INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
1363 if (xprt_has_timer(xprt))
1364 setup_timer(&xprt->timer, xprt_init_autodisconnect,
1365 (unsigned long)xprt);
1366 else
1367 init_timer(&xprt->timer);
Trond Myklebust4e0038b2012-03-01 17:01:05 -05001368
1369 if (strlen(args->servername) > RPC_MAXNETNAMELEN) {
1370 xprt_destroy(xprt);
1371 return ERR_PTR(-EINVAL);
1372 }
1373 xprt->servername = kstrdup(args->servername, GFP_KERNEL);
1374 if (xprt->servername == NULL) {
1375 xprt_destroy(xprt);
1376 return ERR_PTR(-ENOMEM);
1377 }
1378
Jeff Layton3f940092015-03-31 12:03:28 -04001379 rpc_xprt_debugfs_register(xprt);
Jeff Layton388f0c72014-11-26 14:44:44 -05001380
Chuck Lever46121cf2007-01-31 12:14:08 -05001381 dprintk("RPC: created transport %p with %u slots\n", xprt,
Chuck Leverc2866762006-08-22 20:06:20 -04001382 xprt->max_reqs);
Trond Myklebust21de0a92011-07-17 16:57:32 -04001383out:
Chuck Leverc2866762006-08-22 20:06:20 -04001384 return xprt;
1385}
1386
Chuck Lever9903cd12005-08-11 16:25:26 -04001387/**
1388 * xprt_destroy - destroy an RPC transport, killing off all requests.
Trond Myklebusta8de2402011-03-15 19:56:30 -04001389 * @xprt: transport to destroy
Chuck Lever9903cd12005-08-11 16:25:26 -04001390 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 */
Trond Myklebusta8de2402011-03-15 19:56:30 -04001392static void xprt_destroy(struct rpc_xprt *xprt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393{
Chuck Lever46121cf2007-01-31 12:14:08 -05001394 dprintk("RPC: destroying transport %p\n", xprt);
Trond Myklebust79234c32015-09-18 15:53:24 -04001395
1396 /* Exclude transport connect/disconnect handlers */
1397 wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
1398
Trond Myklebust0065db32006-01-03 09:55:56 +01001399 del_timer_sync(&xprt->timer);
Chuck Leverc8541ec2006-10-17 14:44:27 -04001400
Jeff Layton388f0c72014-11-26 14:44:44 -05001401 rpc_xprt_debugfs_unregister(xprt);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001402 rpc_destroy_wait_queue(&xprt->binding);
1403 rpc_destroy_wait_queue(&xprt->pending);
1404 rpc_destroy_wait_queue(&xprt->sending);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001405 rpc_destroy_wait_queue(&xprt->backlog);
J. Bruce Fieldsc3ae62ae2010-08-03 17:22:20 -04001406 cancel_work_sync(&xprt->task_cleanup);
Trond Myklebust4e0038b2012-03-01 17:01:05 -05001407 kfree(xprt->servername);
Chuck Leverc8541ec2006-10-17 14:44:27 -04001408 /*
1409 * Tear down transport state and free the rpc_xprt
1410 */
Chuck Levera246b012005-08-11 16:25:23 -04001411 xprt->ops->destroy(xprt);
Trond Myklebust6b6ca862006-09-05 12:55:57 -04001412}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413
Trond Myklebust30c51162015-02-24 20:31:39 -05001414static void xprt_destroy_kref(struct kref *kref)
1415{
1416 xprt_destroy(container_of(kref, struct rpc_xprt, kref));
1417}
1418
1419/**
1420 * xprt_get - return a reference to an RPC transport.
1421 * @xprt: pointer to the transport
1422 *
1423 */
1424struct rpc_xprt *xprt_get(struct rpc_xprt *xprt)
1425{
1426 if (xprt != NULL && kref_get_unless_zero(&xprt->kref))
1427 return xprt;
1428 return NULL;
1429}
1430EXPORT_SYMBOL_GPL(xprt_get);
1431
Trond Myklebust6b6ca862006-09-05 12:55:57 -04001432/**
1433 * xprt_put - release a reference to an RPC transport.
1434 * @xprt: pointer to the transport
1435 *
1436 */
1437void xprt_put(struct rpc_xprt *xprt)
1438{
Trond Myklebust30c51162015-02-24 20:31:39 -05001439 if (xprt != NULL)
1440 kref_put(&xprt->kref, xprt_destroy_kref);
Trond Myklebust6b6ca862006-09-05 12:55:57 -04001441}
Chuck Lever5d252f92016-01-07 14:50:10 -05001442EXPORT_SYMBOL_GPL(xprt_put);