blob: b4133bd13915317a96d94de9dcf2e5d48551870e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080020#include <linux/mutex.h>
Jeff Laytond3103102011-12-01 22:44:39 +010021#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Benny Halevy69518672009-09-10 12:25:04 +030025#include "sunrpc.h"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#ifdef RPC_DEBUG
28#define RPCDBG_FACILITY RPCDBG_SCHED
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#endif
30
Trond Myklebust82b0a4c2012-01-20 14:52:23 -050031#define CREATE_TRACE_POINTS
32#include <trace/events/sunrpc.h>
33
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
35 * RPC slabs and memory pools
36 */
37#define RPC_BUFFER_MAXSIZE (2048)
38#define RPC_BUFFER_POOLSIZE (8)
39#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080040static struct kmem_cache *rpc_task_slabp __read_mostly;
41static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070042static mempool_t *rpc_task_mempool __read_mostly;
43static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
David Howells65f27f32006-11-22 14:55:48 +000045static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080046static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040047static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * RPC tasks sit here while waiting for conditions to improve.
51 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040052static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 * rpciod-related stuff
56 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050057struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
59/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 * Disable the timer for a given RPC task. Should be called with
61 * queue->lock and bh_disabled in order to avoid races within
62 * rpc_run_timer().
63 */
Trond Myklebust5d008372008-02-22 16:34:17 -050064static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050065__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040067 if (task->tk_timeout == 0)
68 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050069 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040071 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050072 if (list_empty(&queue->timer_list.list))
73 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040074}
75
76static void
77rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
78{
79 queue->timer_list.expires = expires;
80 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070081}
82
83/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 * Set up a timer for the current task.
85 */
Trond Myklebust5d008372008-02-22 16:34:17 -050086static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050087__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088{
89 if (!task->tk_timeout)
90 return;
91
Chuck Lever46121cf2007-01-31 12:14:08 -050092 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 task->tk_pid, task->tk_timeout * 1000 / HZ);
94
Trond Myklebusteb276c02008-02-22 17:27:59 -050095 task->u.tk_wait.expires = jiffies + task->tk_timeout;
96 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
97 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
98 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099}
100
Trond Myklebustc05eecf2012-11-30 23:59:29 -0500101static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
102{
103 queue->priority = priority;
104}
105
106static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
107{
108 queue->owner = pid;
109 queue->nr = RPC_BATCH_COUNT;
110}
111
112static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
113{
114 rpc_set_waitqueue_priority(queue, queue->maxpriority);
115 rpc_set_waitqueue_owner(queue, 0);
116}
117
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118/*
119 * Add new request to a priority queue.
120 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400121static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
122 struct rpc_task *task,
123 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124{
125 struct list_head *q;
126 struct rpc_task *t;
127
128 INIT_LIST_HEAD(&task->u.tk_wait.links);
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400129 if (unlikely(queue_priority > queue->maxpriority))
Trond Myklebustc05eecf2012-11-30 23:59:29 -0500130 queue_priority = queue->maxpriority;
131 if (queue_priority > queue->priority)
132 rpc_set_waitqueue_priority(queue, queue_priority);
133 q = &queue->tasks[queue_priority];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400135 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
137 return;
138 }
139 }
140 list_add_tail(&task->u.tk_wait.list, q);
141}
142
143/*
144 * Add new request to wait queue.
145 *
146 * Swapper tasks always get inserted at the head of the queue.
147 * This should avoid many nasty memory deadlocks and hopefully
148 * improve overall performance.
149 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
150 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400151static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
152 struct rpc_task *task,
153 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154{
Weston Andros Adamson2bd4eef2012-10-23 10:43:47 -0400155 WARN_ON_ONCE(RPC_IS_QUEUED(task));
156 if (RPC_IS_QUEUED(task))
157 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158
159 if (RPC_IS_PRIORITY(queue))
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400160 __rpc_add_wait_queue_priority(queue, task, queue_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 else if (RPC_IS_SWAPPER(task))
162 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
163 else
164 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500165 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500166 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 rpc_set_queued(task);
168
Chuck Lever46121cf2007-01-31 12:14:08 -0500169 dprintk("RPC: %5u added to queue %p \"%s\"\n",
170 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172
173/*
174 * Remove request from a priority queue.
175 */
176static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
177{
178 struct rpc_task *t;
179
180 if (!list_empty(&task->u.tk_wait.links)) {
181 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
182 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
183 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
184 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185}
186
187/*
188 * Remove request from queue.
189 * Note: must be called with spin lock held.
190 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500191static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500193 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 if (RPC_IS_PRIORITY(queue))
195 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400196 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500197 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500198 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
199 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Trond Myklebust3ff75762007-07-14 15:40:00 -0400202static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203{
204 int i;
205
206 spin_lock_init(&queue->lock);
207 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
208 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400209 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400211 queue->qlen = 0;
212 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
213 INIT_LIST_HEAD(&queue->timer_list.list);
Trond Myklebust2f09c242012-02-08 22:01:15 -0500214 rpc_assign_waitqueue_name(queue, qname);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215}
216
217void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
218{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400219 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220}
Alexandros Batsakis689cf5c2009-12-14 21:27:56 -0800221EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
223void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
224{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400225 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400227EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500229void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
230{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400231 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500232}
233EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
234
Matthew Wilcox150030b2007-12-06 16:24:39 -0500235static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100236{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500237 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100238 return -ERESTARTSYS;
Jeff Laytond3103102011-12-01 22:44:39 +0100239 freezable_schedule();
Trond Myklebust44c28872006-01-03 09:55:06 +0100240 return 0;
241}
242
Trond Myklebustc44fe702007-06-16 14:17:01 -0400243#ifdef RPC_DEBUG
244static void rpc_task_set_debuginfo(struct rpc_task *task)
245{
246 static atomic_t rpc_pid;
247
Trond Myklebustc44fe702007-06-16 14:17:01 -0400248 task->tk_pid = atomic_inc_return(&rpc_pid);
249}
250#else
251static inline void rpc_task_set_debuginfo(struct rpc_task *task)
252{
253}
254#endif
255
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500256static void rpc_set_active(struct rpc_task *task)
257{
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500258 trace_rpc_task_begin(task->tk_client, task, NULL);
259
Trond Myklebustc44fe702007-06-16 14:17:01 -0400260 rpc_task_set_debuginfo(task);
Trond Myklebust58f96122010-07-31 14:29:08 -0400261 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500262}
263
Trond Myklebust44c28872006-01-03 09:55:06 +0100264/*
265 * Mark an RPC call as having completed by clearing the 'active' bit
Trond Myklebustbf294b42011-02-21 11:05:41 -0800266 * and then waking up all tasks that were sleeping.
Trond Myklebust44c28872006-01-03 09:55:06 +0100267 */
Trond Myklebustbf294b42011-02-21 11:05:41 -0800268static int rpc_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100269{
Trond Myklebustbf294b42011-02-21 11:05:41 -0800270 void *m = &task->tk_runstate;
271 wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE);
272 struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE);
273 unsigned long flags;
274 int ret;
275
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500276 trace_rpc_task_complete(task->tk_client, task, NULL);
277
Trond Myklebustbf294b42011-02-21 11:05:41 -0800278 spin_lock_irqsave(&wq->lock, flags);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500279 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800280 ret = atomic_dec_and_test(&task->tk_count);
281 if (waitqueue_active(wq))
282 __wake_up_locked_key(wq, TASK_NORMAL, &k);
283 spin_unlock_irqrestore(&wq->lock, flags);
284 return ret;
Trond Myklebust44c28872006-01-03 09:55:06 +0100285}
286
287/*
288 * Allow callers to wait for completion of an RPC call
Trond Myklebustbf294b42011-02-21 11:05:41 -0800289 *
290 * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit()
291 * to enforce taking of the wq->lock and hence avoid races with
292 * rpc_complete_task().
Trond Myklebust44c28872006-01-03 09:55:06 +0100293 */
294int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
295{
296 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500297 action = rpc_wait_bit_killable;
Trond Myklebustbf294b42011-02-21 11:05:41 -0800298 return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500299 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100300}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400301EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100302
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303/*
304 * Make an RPC task runnable.
305 *
Jeff Layton506026c2012-07-23 15:51:55 -0400306 * Note: If the task is ASYNC, and is being made runnable after sitting on an
307 * rpc_wait_queue, this must be called with the queue spinlock held to protect
308 * the wait queue operation.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 */
310static void rpc_make_runnable(struct rpc_task *task)
311{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 rpc_clear_queued(task);
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100313 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 return;
315 if (RPC_IS_ASYNC(task)) {
David Howells65f27f32006-11-22 14:55:48 +0000316 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Fred Isamancee6a532011-02-11 15:42:35 +0000317 queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000319 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320}
321
322/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 * Prepare for sleeping on a wait queue.
324 * By always appending tasks to the list we ensure FIFO behavior.
325 * NB: An RPC task will only receive interrupt-driven events as long
326 * as it's on a wait queue.
327 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400328static void __rpc_sleep_on_priority(struct rpc_wait_queue *q,
329 struct rpc_task *task,
330 rpc_action action,
331 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
Chuck Lever46121cf2007-01-31 12:14:08 -0500333 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
334 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500336 trace_rpc_task_sleep(task->tk_client, task, q);
337
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400338 __rpc_add_wait_queue(q, task, queue_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
Weston Andros Adamsonf50ad422012-10-23 10:43:46 -0400340 WARN_ON_ONCE(task->tk_callback != NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500342 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343}
344
345void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500346 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347{
Trond Myklebust58f96122010-07-31 14:29:08 -0400348 /* We shouldn't ever put an inactive task to sleep */
Weston Andros Adamsone454a7a2012-10-23 10:43:32 -0400349 WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));
350 if (!RPC_IS_ACTIVATED(task)) {
351 task->tk_status = -EIO;
352 rpc_put_task_async(task);
353 return;
354 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500355
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 /*
357 * Protect the queue operations.
358 */
359 spin_lock_bh(&q->lock);
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400360 __rpc_sleep_on_priority(q, task, action, task->tk_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 spin_unlock_bh(&q->lock);
362}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400363EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400365void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task,
366 rpc_action action, int priority)
367{
368 /* We shouldn't ever put an inactive task to sleep */
Weston Andros Adamsone454a7a2012-10-23 10:43:32 -0400369 WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));
370 if (!RPC_IS_ACTIVATED(task)) {
371 task->tk_status = -EIO;
372 rpc_put_task_async(task);
373 return;
374 }
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400375
376 /*
377 * Protect the queue operations.
378 */
379 spin_lock_bh(&q->lock);
380 __rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW);
381 spin_unlock_bh(&q->lock);
382}
Trond Myklebust1e1093c2012-11-01 16:44:05 -0400383EXPORT_SYMBOL_GPL(rpc_sleep_on_priority);
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400384
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385/**
386 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500387 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 * @task: task to be woken up
389 *
390 * Caller must hold queue->lock, and have cleared the task queued flag.
391 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500392static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393{
Chuck Lever46121cf2007-01-31 12:14:08 -0500394 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
395 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 /* Has the task been executed yet? If not, we cannot wake it up! */
398 if (!RPC_IS_ACTIVATED(task)) {
399 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
400 return;
401 }
402
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500403 trace_rpc_task_wakeup(task->tk_client, task, queue);
404
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500405 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
407 rpc_make_runnable(task);
408
Chuck Lever46121cf2007-01-31 12:14:08 -0500409 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410}
411
412/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500413 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500415static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800417 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
418 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419}
420
421/*
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800422 * Tests whether rpc queue is empty
423 */
424int rpc_queue_empty(struct rpc_wait_queue *queue)
425{
426 int res;
427
428 spin_lock_bh(&queue->lock);
429 res = queue->qlen;
430 spin_unlock_bh(&queue->lock);
Eric Dumazeta02cec22010-09-22 20:43:57 +0000431 return res == 0;
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800432}
433EXPORT_SYMBOL_GPL(rpc_queue_empty);
434
435/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500436 * Wake up a task on a specific queue
437 */
438void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
439{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800440 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500441 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800442 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500443}
444EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
445
446/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 * Wake up the next task on a priority queue.
448 */
Trond Myklebust961a8282012-01-17 22:57:37 -0500449static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450{
451 struct list_head *q;
452 struct rpc_task *task;
453
454 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400455 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 */
457 q = &queue->tasks[queue->priority];
458 if (!list_empty(q)) {
459 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400460 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 if (--queue->nr)
462 goto out;
463 list_move_tail(&task->u.tk_wait.list, q);
464 }
465 /*
466 * Check if we need to switch queues.
467 */
Trond Myklebustc05eecf2012-11-30 23:59:29 -0500468 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 }
470
471 /*
472 * Service the next queue.
473 */
474 do {
475 if (q == &queue->tasks[0])
476 q = &queue->tasks[queue->maxpriority];
477 else
478 q = q - 1;
479 if (!list_empty(q)) {
480 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
481 goto new_queue;
482 }
483 } while (q != &queue->tasks[queue->priority]);
484
485 rpc_reset_waitqueue_priority(queue);
486 return NULL;
487
488new_queue:
489 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400490new_owner:
491 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 return task;
494}
495
Trond Myklebust961a8282012-01-17 22:57:37 -0500496static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)
497{
498 if (RPC_IS_PRIORITY(queue))
499 return __rpc_find_next_queued_priority(queue);
500 if (!list_empty(&queue->tasks[0]))
501 return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);
502 return NULL;
503}
504
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505/*
Trond Myklebust961a8282012-01-17 22:57:37 -0500506 * Wake up the first task on the wait queue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 */
Trond Myklebust961a8282012-01-17 22:57:37 -0500508struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,
509 bool (*func)(struct rpc_task *, void *), void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
511 struct rpc_task *task = NULL;
512
Trond Myklebust961a8282012-01-17 22:57:37 -0500513 dprintk("RPC: wake_up_first(%p \"%s\")\n",
Chuck Lever46121cf2007-01-31 12:14:08 -0500514 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800515 spin_lock_bh(&queue->lock);
Trond Myklebust961a8282012-01-17 22:57:37 -0500516 task = __rpc_find_next_queued(queue);
517 if (task != NULL) {
518 if (func(task, data))
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500519 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust961a8282012-01-17 22:57:37 -0500520 else
521 task = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800523 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
525 return task;
526}
Trond Myklebust961a8282012-01-17 22:57:37 -0500527EXPORT_SYMBOL_GPL(rpc_wake_up_first);
528
529static bool rpc_wake_up_next_func(struct rpc_task *task, void *data)
530{
531 return true;
532}
533
534/*
535 * Wake up the next task on the wait queue.
536*/
537struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)
538{
539 return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);
540}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400541EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542
543/**
544 * rpc_wake_up - wake up all rpc_tasks
545 * @queue: rpc_wait_queue on which the tasks are sleeping
546 *
547 * Grabs queue->lock
548 */
549void rpc_wake_up(struct rpc_wait_queue *queue)
550{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800552
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800553 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 head = &queue->tasks[queue->maxpriority];
555 for (;;) {
Trond Myklebust540a0f72012-03-19 13:39:35 -0400556 while (!list_empty(head)) {
557 struct rpc_task *task;
558 task = list_first_entry(head,
559 struct rpc_task,
560 u.tk_wait.list);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500561 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust540a0f72012-03-19 13:39:35 -0400562 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 if (head == &queue->tasks[0])
564 break;
565 head--;
566 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800567 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400569EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570
571/**
572 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
573 * @queue: rpc_wait_queue on which the tasks are sleeping
574 * @status: status value to set
575 *
576 * Grabs queue->lock
577 */
578void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
579{
580 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800582 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 head = &queue->tasks[queue->maxpriority];
584 for (;;) {
Trond Myklebust540a0f72012-03-19 13:39:35 -0400585 while (!list_empty(head)) {
586 struct rpc_task *task;
587 task = list_first_entry(head,
588 struct rpc_task,
589 u.tk_wait.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500591 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 }
593 if (head == &queue->tasks[0])
594 break;
595 head--;
596 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800597 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400599EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400601static void __rpc_queue_timer_fn(unsigned long ptr)
602{
603 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
604 struct rpc_task *task, *n;
605 unsigned long expires, now, timeo;
606
607 spin_lock(&queue->lock);
608 expires = now = jiffies;
609 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
610 timeo = task->u.tk_wait.expires;
611 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400612 dprintk("RPC: %5u timeout\n", task->tk_pid);
613 task->tk_status = -ETIMEDOUT;
614 rpc_wake_up_task_queue_locked(queue, task);
615 continue;
616 }
617 if (expires == now || time_after(expires, timeo))
618 expires = timeo;
619 }
620 if (!list_empty(&queue->timer_list.list))
621 rpc_set_queue_timer(queue, expires);
622 spin_unlock(&queue->lock);
623}
624
Trond Myklebust80147932006-08-31 18:24:08 -0400625static void __rpc_atrun(struct rpc_task *task)
626{
Trond Myklebust5d008372008-02-22 16:34:17 -0500627 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400628}
629
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630/*
631 * Run a task at a later time
632 */
Trond Myklebust80147932006-08-31 18:24:08 -0400633void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
635 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500636 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400638EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100641 * Helper to call task->tk_ops->rpc_call_prepare
642 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400643void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100644{
645 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
646}
647
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500648static void
649rpc_init_task_statistics(struct rpc_task *task)
650{
651 /* Initialize retry counters */
652 task->tk_garb_retry = 2;
653 task->tk_cred_retry = 2;
654 task->tk_rebind_retry = 2;
655
656 /* starting timestamp */
657 task->tk_start = ktime_get();
658}
659
660static void
661rpc_reset_task_statistics(struct rpc_task *task)
662{
663 task->tk_timeouts = 0;
664 task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT);
665
666 rpc_init_task_statistics(task);
667}
668
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100669/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100670 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000671 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100672void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000673{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100674 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100675 if (task->tk_ops->rpc_call_done != NULL) {
676 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000677 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100678 WARN_ON(RPC_ASSASSINATED(task));
679 /* Always release the RPC slot and buffer memory */
680 xprt_release(task);
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500681 rpc_reset_task_statistics(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000682 }
683 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000684}
Trond Myklebustd9b6cd92010-07-31 14:29:08 -0400685
686void rpc_exit(struct rpc_task *task, int status)
687{
688 task->tk_status = status;
689 task->tk_action = rpc_exit_task;
690 if (RPC_IS_QUEUED(task))
691 rpc_wake_up_queued_task(task->tk_waitqueue, task);
692}
693EXPORT_SYMBOL_GPL(rpc_exit);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000694
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400695void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
696{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400697 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400698 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400699}
700
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000701/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 * This is the RPC `scheduler' (or rather, the finite state machine).
703 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800704static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400706 struct rpc_wait_queue *queue;
707 int task_is_async = RPC_IS_ASYNC(task);
708 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
Chuck Lever46121cf2007-01-31 12:14:08 -0500710 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
711 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712
Weston Andros Adamson2bd4eef2012-10-23 10:43:47 -0400713 WARN_ON_ONCE(RPC_IS_QUEUED(task));
714 if (RPC_IS_QUEUED(task))
715 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000717 for (;;) {
Trond Myklebustb55c5982011-07-06 19:58:23 -0400718 void (*do_action)(struct rpc_task *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719
720 /*
Trond Myklebustb55c5982011-07-06 19:58:23 -0400721 * Execute any pending callback first.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 */
Trond Myklebustb55c5982011-07-06 19:58:23 -0400723 do_action = task->tk_callback;
724 task->tk_callback = NULL;
725 if (do_action == NULL) {
Trond Myklebuste020c682011-03-15 19:56:30 -0400726 /*
727 * Perform the next FSM step.
Trond Myklebustb55c5982011-07-06 19:58:23 -0400728 * tk_action may be NULL if the task has been killed.
729 * In particular, note that rpc_killall_tasks may
730 * do this at any time, so beware when dereferencing.
Trond Myklebuste020c682011-03-15 19:56:30 -0400731 */
Trond Myklebustb55c5982011-07-06 19:58:23 -0400732 do_action = task->tk_action;
733 if (do_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 }
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500736 trace_rpc_task_run_action(task->tk_client, task, task->tk_action);
Trond Myklebustb55c5982011-07-06 19:58:23 -0400737 do_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
739 /*
740 * Lockless check for whether task is sleeping or not.
741 */
742 if (!RPC_IS_QUEUED(task))
743 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400744 /*
745 * The queue->lock protects against races with
746 * rpc_make_runnable().
747 *
748 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
749 * rpc_task, rpc_make_runnable() can assign it to a
750 * different workqueue. We therefore cannot assume that the
751 * rpc_task pointer may still be dereferenced.
752 */
753 queue = task->tk_waitqueue;
754 spin_lock_bh(&queue->lock);
755 if (!RPC_IS_QUEUED(task)) {
756 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 continue;
758 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400759 rpc_clear_running(task);
760 spin_unlock_bh(&queue->lock);
761 if (task_is_async)
762 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
764 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500765 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000766 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500767 RPC_TASK_QUEUED, rpc_wait_bit_killable,
768 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000769 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 /*
771 * When a sync task receives a signal, it exits with
772 * -ERESTARTSYS. In order to catch any callbacks that
773 * clean up after sleeping on some queue, we don't
774 * break the loop here, but go around once more.
775 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500776 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000777 task->tk_flags |= RPC_TASK_KILLED;
778 rpc_exit(task, -ERESTARTSYS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 }
780 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500781 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 }
783
Chuck Lever46121cf2007-01-31 12:14:08 -0500784 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
785 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 /* Release all resources associated with the task */
787 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788}
789
790/*
791 * User-visible entry point to the scheduler.
792 *
793 * This may be called recursively if e.g. an async NFS task updates
794 * the attributes and finds that dirty pages must be flushed.
795 * NOTE: Upon exit of this function the task is guaranteed to be
796 * released. In particular note that tk_release() will have
797 * been called, so your task memory may have been freed.
798 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800799void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800{
Trond Myklebust44c28872006-01-03 09:55:06 +0100801 rpc_set_active(task);
Trond Myklebustd6a1ed02010-07-31 14:29:08 -0400802 rpc_make_runnable(task);
803 if (!RPC_IS_ASYNC(task))
804 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805}
806
David Howells65f27f32006-11-22 14:55:48 +0000807static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808{
Jeff Layton5cf02d02012-07-23 13:58:51 -0400809 current->flags |= PF_FSTRANS;
David Howells65f27f32006-11-22 14:55:48 +0000810 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Jeff Layton5cf02d02012-07-23 13:58:51 -0400811 current->flags &= ~PF_FSTRANS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812}
813
Chuck Lever02107142006-01-03 09:55:49 +0100814/**
815 * rpc_malloc - allocate an RPC buffer
816 * @task: RPC task that will use this buffer
817 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400819 * To prevent rpciod from hanging, this allocator never sleeps,
820 * returning NULL if the request cannot be serviced immediately.
821 * The caller can arrange to sleep in a way that is safe for rpciod.
822 *
823 * Most requests are 'small' (under 2KiB) and can be serviced from a
824 * mempool, ensuring that NFS reads and writes can always proceed,
825 * and that there is good locality of reference for these buffers.
826 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400828 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400830void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400832 struct rpc_buffer *buf;
Mel Gormana564b8f2012-07-31 16:45:12 -0700833 gfp_t gfp = GFP_NOWAIT;
834
835 if (RPC_IS_SWAPPER(task))
836 gfp |= __GFP_MEMALLOC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400838 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400839 if (size <= RPC_BUFFER_MAXSIZE)
840 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400842 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200843
844 if (!buf)
845 return NULL;
846
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400847 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200848 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400849 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400850 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400852EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853
Chuck Lever02107142006-01-03 09:55:49 +0100854/**
855 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400856 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100857 *
858 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400859void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400861 size_t size;
862 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100863
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400864 if (!buffer)
865 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400866
867 buf = container_of(buffer, struct rpc_buffer, data);
868 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400869
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200870 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400871 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400872
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400873 if (size <= RPC_BUFFER_MAXSIZE)
874 mempool_free(buf, rpc_buffer_mempool);
875 else
876 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400878EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879
880/*
881 * Creation and deletion of RPC task structures
882 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400883static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884{
885 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100886 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400887 task->tk_flags = task_setup_data->flags;
888 task->tk_ops = task_setup_data->callback_ops;
889 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400890 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891
Trond Myklebust3ff75762007-07-14 15:40:00 -0400892 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
893 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894
895 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500896 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897
Trond Myklebust84115e12007-07-14 15:39:59 -0400898 if (task->tk_ops->rpc_call_prepare != NULL)
899 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100900
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500901 rpc_init_task_statistics(task);
Chuck Leveref759a22006-03-20 13:44:17 -0500902
Chuck Lever46121cf2007-01-31 12:14:08 -0500903 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700904 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905}
906
907static struct rpc_task *
908rpc_alloc_task(void)
909{
Mel Gormana564b8f2012-07-31 16:45:12 -0700910 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911}
912
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400914 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400916struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400918 struct rpc_task *task = setup_data->task;
919 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400921 if (task == NULL) {
922 task = rpc_alloc_task();
Trond Myklebust19445b92010-04-16 16:41:10 -0400923 if (task == NULL) {
924 rpc_release_calldata(setup_data->callback_ops,
925 setup_data->callback_data);
926 return ERR_PTR(-ENOMEM);
927 }
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400928 flags = RPC_TASK_DYNAMIC;
929 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930
Trond Myklebust84115e12007-07-14 15:39:59 -0400931 rpc_init_task(task, setup_data);
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400932 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500933 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935}
936
Trond Myklebustc6567ed2013-01-04 12:23:21 -0500937/*
938 * rpc_free_task - release rpc task and perform cleanups
939 *
940 * Note that we free up the rpc_task _after_ rpc_release_calldata()
941 * in order to work around a workqueue dependency issue.
942 *
943 * Tejun Heo states:
944 * "Workqueue currently considers two work items to be the same if they're
945 * on the same address and won't execute them concurrently - ie. it
946 * makes a work item which is queued again while being executed wait
947 * for the previous execution to complete.
948 *
949 * If a work function frees the work item, and then waits for an event
950 * which should be performed by another work item and *that* work item
951 * recycles the freed work item, it can create a false dependency loop.
952 * There really is no reliable way to detect this short of verifying
953 * every memory free."
954 *
955 */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500956static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957{
Trond Myklebustc6567ed2013-01-04 12:23:21 -0500958 unsigned short tk_flags = task->tk_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959
Trond Myklebustc6567ed2013-01-04 12:23:21 -0500960 rpc_release_calldata(task->tk_ops, task->tk_calldata);
961
962 if (tk_flags & RPC_TASK_DYNAMIC) {
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800963 dprintk("RPC: %5u freeing task\n", task->tk_pid);
964 mempool_free(task, rpc_task_mempool);
965 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500966}
967
968static void rpc_async_release(struct work_struct *work)
969{
970 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
971}
972
Trond Myklebustbf294b42011-02-21 11:05:41 -0800973static void rpc_release_resources_task(struct rpc_task *task)
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500974{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500975 if (task->tk_rqstp)
976 xprt_release(task);
OGAWA Hirofumia271c5a2011-03-27 17:48:57 +0200977 if (task->tk_msg.rpc_cred) {
Trond Myklebusta17c2152010-07-31 14:29:08 -0400978 put_rpccred(task->tk_msg.rpc_cred);
OGAWA Hirofumia271c5a2011-03-27 17:48:57 +0200979 task->tk_msg.rpc_cred = NULL;
980 }
Trond Myklebust58f96122010-07-31 14:29:08 -0400981 rpc_task_release_client(task);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800982}
983
984static void rpc_final_put_task(struct rpc_task *task,
985 struct workqueue_struct *q)
986{
987 if (q != NULL) {
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500988 INIT_WORK(&task->u.tk_work, rpc_async_release);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800989 queue_work(q, &task->u.tk_work);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500990 } else
991 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500992}
Trond Myklebustbf294b42011-02-21 11:05:41 -0800993
994static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q)
995{
996 if (atomic_dec_and_test(&task->tk_count)) {
997 rpc_release_resources_task(task);
998 rpc_final_put_task(task, q);
999 }
1000}
1001
1002void rpc_put_task(struct rpc_task *task)
1003{
1004 rpc_do_put_task(task, NULL);
1005}
Trond Myklebuste8914c62007-07-14 15:39:59 -04001006EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -05001007
Trond Myklebustbf294b42011-02-21 11:05:41 -08001008void rpc_put_task_async(struct rpc_task *task)
1009{
1010 rpc_do_put_task(task, task->tk_workqueue);
1011}
1012EXPORT_SYMBOL_GPL(rpc_put_task_async);
1013
Trond Myklebustbde8f002007-01-24 11:54:53 -08001014static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -05001015{
Chuck Lever46121cf2007-01-31 12:14:08 -05001016 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017
Weston Andros Adamson0a0c2a52012-10-23 10:43:49 -04001018 WARN_ON_ONCE(RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019
Trond Myklebustbf294b42011-02-21 11:05:41 -08001020 rpc_release_resources_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -05001021
Trond Myklebustbf294b42011-02-21 11:05:41 -08001022 /*
1023 * Note: at this point we have been removed from rpc_clnt->cl_tasks,
1024 * so it should be safe to use task->tk_count as a test for whether
1025 * or not any other processes still hold references to our rpc_task.
1026 */
1027 if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) {
1028 /* Wake up anyone who may be waiting for task completion */
1029 if (!rpc_complete_task(task))
1030 return;
1031 } else {
1032 if (!atomic_dec_and_test(&task->tk_count))
1033 return;
1034 }
1035 rpc_final_put_task(task, task->tk_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036}
1037
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001038int rpciod_up(void)
1039{
1040 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
1041}
1042
1043void rpciod_down(void)
1044{
1045 module_put(THIS_MODULE);
1046}
1047
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001049 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001051static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052{
1053 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 /*
1056 * Create the rpciod thread and wait for it to start.
1057 */
Trond Myklebustab418d72007-06-14 17:08:36 -04001058 dprintk("RPC: creating workqueue rpciod\n");
Trond Myklebust9b96ce72012-09-28 20:24:16 -04001059 wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001061 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062}
1063
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001064static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001066 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -04001067
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001068 if (rpciod_workqueue == NULL)
1069 return;
Trond Myklebustab418d72007-06-14 17:08:36 -04001070 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001072 wq = rpciod_workqueue;
1073 rpciod_workqueue = NULL;
1074 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075}
1076
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077void
1078rpc_destroy_mempool(void)
1079{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001080 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 if (rpc_buffer_mempool)
1082 mempool_destroy(rpc_buffer_mempool);
1083 if (rpc_task_mempool)
1084 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001085 if (rpc_task_slabp)
1086 kmem_cache_destroy(rpc_task_slabp);
1087 if (rpc_buffer_slabp)
1088 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001089 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090}
1091
1092int
1093rpc_init_mempool(void)
1094{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001095 /*
1096 * The following is not strictly a mempool initialisation,
1097 * but there is no harm in doing it here
1098 */
1099 rpc_init_wait_queue(&delay_queue, "delayq");
1100 if (!rpciod_start())
1101 goto err_nomem;
1102
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1104 sizeof(struct rpc_task),
1105 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001106 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 if (!rpc_task_slabp)
1108 goto err_nomem;
1109 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1110 RPC_BUFFER_MAXSIZE,
1111 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001112 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 if (!rpc_buffer_slabp)
1114 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001115 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1116 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 if (!rpc_task_mempool)
1118 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001119 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1120 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 if (!rpc_buffer_mempool)
1122 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 return 0;
1124err_nomem:
1125 rpc_destroy_mempool();
1126 return -ENOMEM;
1127}