blob: 8f459abe97cff614525775560ec355f9d4c69246 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080020#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24#ifdef RPC_DEBUG
25#define RPCDBG_FACILITY RPCDBG_SCHED
26#define RPC_TASK_MAGIC_ID 0xf00baa
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#endif
28
29/*
30 * RPC slabs and memory pools
31 */
32#define RPC_BUFFER_MAXSIZE (2048)
33#define RPC_BUFFER_POOLSIZE (8)
34#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080035static struct kmem_cache *rpc_task_slabp __read_mostly;
36static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070037static mempool_t *rpc_task_mempool __read_mostly;
38static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
David Howells65f27f32006-11-22 14:55:48 +000040static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080041static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040042static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
44/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 * RPC tasks sit here while waiting for conditions to improve.
46 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040047static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * rpciod-related stuff
51 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050052struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 * Disable the timer for a given RPC task. Should be called with
56 * queue->lock and bh_disabled in order to avoid races within
57 * rpc_run_timer().
58 */
Trond Myklebust5d008372008-02-22 16:34:17 -050059static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050060__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070061{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040062 if (task->tk_timeout == 0)
63 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050064 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040066 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050067 if (list_empty(&queue->timer_list.list))
68 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040069}
70
71static void
72rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
73{
74 queue->timer_list.expires = expires;
75 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070076}
77
78/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 * Set up a timer for the current task.
80 */
Trond Myklebust5d008372008-02-22 16:34:17 -050081static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050082__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070083{
84 if (!task->tk_timeout)
85 return;
86
Chuck Lever46121cf2007-01-31 12:14:08 -050087 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 task->tk_pid, task->tk_timeout * 1000 / HZ);
89
Trond Myklebusteb276c02008-02-22 17:27:59 -050090 task->u.tk_wait.expires = jiffies + task->tk_timeout;
91 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
92 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
93 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070094}
95
96/*
97 * Add new request to a priority queue.
98 */
99static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
100{
101 struct list_head *q;
102 struct rpc_task *t;
103
104 INIT_LIST_HEAD(&task->u.tk_wait.links);
105 q = &queue->tasks[task->tk_priority];
106 if (unlikely(task->tk_priority > queue->maxpriority))
107 q = &queue->tasks[queue->maxpriority];
108 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400109 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
111 return;
112 }
113 }
114 list_add_tail(&task->u.tk_wait.list, q);
115}
116
117/*
118 * Add new request to wait queue.
119 *
120 * Swapper tasks always get inserted at the head of the queue.
121 * This should avoid many nasty memory deadlocks and hopefully
122 * improve overall performance.
123 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
124 */
125static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
126{
127 BUG_ON (RPC_IS_QUEUED(task));
128
129 if (RPC_IS_PRIORITY(queue))
130 __rpc_add_wait_queue_priority(queue, task);
131 else if (RPC_IS_SWAPPER(task))
132 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
133 else
134 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500135 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500136 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 rpc_set_queued(task);
138
Chuck Lever46121cf2007-01-31 12:14:08 -0500139 dprintk("RPC: %5u added to queue %p \"%s\"\n",
140 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141}
142
143/*
144 * Remove request from a priority queue.
145 */
146static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
147{
148 struct rpc_task *t;
149
150 if (!list_empty(&task->u.tk_wait.links)) {
151 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
152 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
153 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
154 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155}
156
157/*
158 * Remove request from queue.
159 * Note: must be called with spin lock held.
160 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500161static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500163 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 if (RPC_IS_PRIORITY(queue))
165 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400166 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500167 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500168 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
169 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170}
171
172static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
173{
174 queue->priority = priority;
175 queue->count = 1 << (priority * 2);
176}
177
Trond Myklebust3ff75762007-07-14 15:40:00 -0400178static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400180 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 queue->nr = RPC_BATCH_COUNT;
182}
183
184static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
185{
186 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400187 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188}
189
Trond Myklebust3ff75762007-07-14 15:40:00 -0400190static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191{
192 int i;
193
194 spin_lock_init(&queue->lock);
195 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
196 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400197 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400199 queue->qlen = 0;
200 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
201 INIT_LIST_HEAD(&queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202#ifdef RPC_DEBUG
203 queue->name = qname;
204#endif
205}
206
207void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
208{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400209 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210}
211
212void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
213{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400214 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400216EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500218void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
219{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400220 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500221}
222EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
223
Matthew Wilcox150030b2007-12-06 16:24:39 -0500224static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100225{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500226 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100227 return -ERESTARTSYS;
228 schedule();
229 return 0;
230}
231
Trond Myklebustc44fe702007-06-16 14:17:01 -0400232#ifdef RPC_DEBUG
233static void rpc_task_set_debuginfo(struct rpc_task *task)
234{
235 static atomic_t rpc_pid;
236
237 task->tk_magic = RPC_TASK_MAGIC_ID;
238 task->tk_pid = atomic_inc_return(&rpc_pid);
239}
240#else
241static inline void rpc_task_set_debuginfo(struct rpc_task *task)
242{
243}
244#endif
245
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500246static void rpc_set_active(struct rpc_task *task)
247{
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400248 struct rpc_clnt *clnt;
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500249 if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
250 return;
Trond Myklebustc44fe702007-06-16 14:17:01 -0400251 rpc_task_set_debuginfo(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500252 /* Add to global list of all tasks */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400253 clnt = task->tk_client;
254 if (clnt != NULL) {
255 spin_lock(&clnt->cl_lock);
256 list_add_tail(&task->tk_task, &clnt->cl_tasks);
257 spin_unlock(&clnt->cl_lock);
258 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500259}
260
Trond Myklebust44c28872006-01-03 09:55:06 +0100261/*
262 * Mark an RPC call as having completed by clearing the 'active' bit
263 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500264static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100265{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500266 smp_mb__before_clear_bit();
267 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
268 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100269 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
270}
271
272/*
273 * Allow callers to wait for completion of an RPC call
274 */
275int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
276{
277 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500278 action = rpc_wait_bit_killable;
Trond Myklebust44c28872006-01-03 09:55:06 +0100279 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500280 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100281}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400282EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100283
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284/*
285 * Make an RPC task runnable.
286 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800287 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 * the spinlock held to protect the wait queue operation.
289 */
290static void rpc_make_runnable(struct rpc_task *task)
291{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 rpc_clear_queued(task);
Christophe Saoutcc4dc592006-11-05 18:42:48 +0100293 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 return;
295 if (RPC_IS_ASYNC(task)) {
296 int status;
297
David Howells65f27f32006-11-22 14:55:48 +0000298 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500299 status = queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 if (status < 0) {
301 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
302 task->tk_status = status;
303 return;
304 }
305 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000306 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307}
308
309/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 * Prepare for sleeping on a wait queue.
311 * By always appending tasks to the list we ensure FIFO behavior.
312 * NB: An RPC task will only receive interrupt-driven events as long
313 * as it's on a wait queue.
314 */
315static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500316 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317{
Chuck Lever46121cf2007-01-31 12:14:08 -0500318 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
319 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320
321 if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
322 printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
323 return;
324 }
325
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 __rpc_add_wait_queue(q, task);
327
328 BUG_ON(task->tk_callback != NULL);
329 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500330 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
333void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500334 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500336 /* Mark the task as being activated if so needed */
337 rpc_set_active(task);
338
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 /*
340 * Protect the queue operations.
341 */
342 spin_lock_bh(&q->lock);
Trond Myklebust5d008372008-02-22 16:34:17 -0500343 __rpc_sleep_on(q, task, action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 spin_unlock_bh(&q->lock);
345}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400346EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347
348/**
349 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500350 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 * @task: task to be woken up
352 *
353 * Caller must hold queue->lock, and have cleared the task queued flag.
354 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500355static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356{
Chuck Lever46121cf2007-01-31 12:14:08 -0500357 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
358 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359
360#ifdef RPC_DEBUG
361 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
362#endif
363 /* Has the task been executed yet? If not, we cannot wake it up! */
364 if (!RPC_IS_ACTIVATED(task)) {
365 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
366 return;
367 }
368
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500369 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
371 rpc_make_runnable(task);
372
Chuck Lever46121cf2007-01-31 12:14:08 -0500373 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374}
375
376/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500377 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500379static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800381 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
382 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
385/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500386 * Wake up a task on a specific queue
387 */
388void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
389{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800390 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500391 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800392 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500393}
394EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
395
396/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 * Wake up the specified task
398 */
Trond Myklebustfda13932008-02-22 16:34:12 -0500399static void rpc_wake_up_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500401 rpc_wake_up_queued_task(task->tk_waitqueue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402}
403
404/*
405 * Wake up the next task on a priority queue.
406 */
407static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
408{
409 struct list_head *q;
410 struct rpc_task *task;
411
412 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400413 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 */
415 q = &queue->tasks[queue->priority];
416 if (!list_empty(q)) {
417 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400418 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 if (--queue->nr)
420 goto out;
421 list_move_tail(&task->u.tk_wait.list, q);
422 }
423 /*
424 * Check if we need to switch queues.
425 */
426 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400427 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 }
429
430 /*
431 * Service the next queue.
432 */
433 do {
434 if (q == &queue->tasks[0])
435 q = &queue->tasks[queue->maxpriority];
436 else
437 q = q - 1;
438 if (!list_empty(q)) {
439 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
440 goto new_queue;
441 }
442 } while (q != &queue->tasks[queue->priority]);
443
444 rpc_reset_waitqueue_priority(queue);
445 return NULL;
446
447new_queue:
448 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400449new_owner:
450 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451out:
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500452 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 return task;
454}
455
456/*
457 * Wake up the next task on the wait queue.
458 */
459struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
460{
461 struct rpc_task *task = NULL;
462
Chuck Lever46121cf2007-01-31 12:14:08 -0500463 dprintk("RPC: wake_up_next(%p \"%s\")\n",
464 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800465 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 if (RPC_IS_PRIORITY(queue))
467 task = __rpc_wake_up_next_priority(queue);
468 else {
469 task_for_first(task, &queue->tasks[0])
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500470 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800472 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473
474 return task;
475}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400476EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
478/**
479 * rpc_wake_up - wake up all rpc_tasks
480 * @queue: rpc_wait_queue on which the tasks are sleeping
481 *
482 * Grabs queue->lock
483 */
484void rpc_wake_up(struct rpc_wait_queue *queue)
485{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800486 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800488
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800489 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 head = &queue->tasks[queue->maxpriority];
491 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800492 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500493 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 if (head == &queue->tasks[0])
495 break;
496 head--;
497 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800498 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400500EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
502/**
503 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
504 * @queue: rpc_wait_queue on which the tasks are sleeping
505 * @status: status value to set
506 *
507 * Grabs queue->lock
508 */
509void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
510{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800511 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800514 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 head = &queue->tasks[queue->maxpriority];
516 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800517 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500519 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 }
521 if (head == &queue->tasks[0])
522 break;
523 head--;
524 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800525 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400527EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400529static void __rpc_queue_timer_fn(unsigned long ptr)
530{
531 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
532 struct rpc_task *task, *n;
533 unsigned long expires, now, timeo;
534
535 spin_lock(&queue->lock);
536 expires = now = jiffies;
537 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
538 timeo = task->u.tk_wait.expires;
539 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400540 dprintk("RPC: %5u timeout\n", task->tk_pid);
541 task->tk_status = -ETIMEDOUT;
542 rpc_wake_up_task_queue_locked(queue, task);
543 continue;
544 }
545 if (expires == now || time_after(expires, timeo))
546 expires = timeo;
547 }
548 if (!list_empty(&queue->timer_list.list))
549 rpc_set_queue_timer(queue, expires);
550 spin_unlock(&queue->lock);
551}
552
Trond Myklebust80147932006-08-31 18:24:08 -0400553static void __rpc_atrun(struct rpc_task *task)
554{
Trond Myklebust5d008372008-02-22 16:34:17 -0500555 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400556}
557
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558/*
559 * Run a task at a later time
560 */
Trond Myklebust80147932006-08-31 18:24:08 -0400561void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562{
563 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500564 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400566EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100569 * Helper to call task->tk_ops->rpc_call_prepare
570 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400571void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100572{
573 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
574}
575
576/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100577 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000578 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100579void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000580{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100581 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100582 if (task->tk_ops->rpc_call_done != NULL) {
583 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000584 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100585 WARN_ON(RPC_ASSASSINATED(task));
586 /* Always release the RPC slot and buffer memory */
587 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000588 }
589 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000590}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400591EXPORT_SYMBOL_GPL(rpc_exit_task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000592
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400593void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
594{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400595 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400596 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400597}
598
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000599/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 * This is the RPC `scheduler' (or rather, the finite state machine).
601 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800602static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400604 struct rpc_wait_queue *queue;
605 int task_is_async = RPC_IS_ASYNC(task);
606 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Chuck Lever46121cf2007-01-31 12:14:08 -0500608 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
609 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
611 BUG_ON(RPC_IS_QUEUED(task));
612
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000613 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
615 /*
616 * Execute any pending callback.
617 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400618 if (task->tk_callback) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800620
621 /*
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400622 * We set tk_callback to NULL before calling it,
623 * in case it sets the tk_callback field itself:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400625 save_callback = task->tk_callback;
626 task->tk_callback = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 }
629
630 /*
631 * Perform the next FSM step.
632 * tk_action may be NULL when the task has been killed
633 * by someone else.
634 */
635 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100636 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100638 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 }
640
641 /*
642 * Lockless check for whether task is sleeping or not.
643 */
644 if (!RPC_IS_QUEUED(task))
645 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400646 /*
647 * The queue->lock protects against races with
648 * rpc_make_runnable().
649 *
650 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
651 * rpc_task, rpc_make_runnable() can assign it to a
652 * different workqueue. We therefore cannot assume that the
653 * rpc_task pointer may still be dereferenced.
654 */
655 queue = task->tk_waitqueue;
656 spin_lock_bh(&queue->lock);
657 if (!RPC_IS_QUEUED(task)) {
658 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 continue;
660 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400661 rpc_clear_running(task);
662 spin_unlock_bh(&queue->lock);
663 if (task_is_async)
664 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665
666 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500667 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000668 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500669 RPC_TASK_QUEUED, rpc_wait_bit_killable,
670 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000671 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 /*
673 * When a sync task receives a signal, it exits with
674 * -ERESTARTSYS. In order to catch any callbacks that
675 * clean up after sleeping on some queue, we don't
676 * break the loop here, but go around once more.
677 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500678 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000679 task->tk_flags |= RPC_TASK_KILLED;
680 rpc_exit(task, -ERESTARTSYS);
681 rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 }
683 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500684 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 }
686
Chuck Lever46121cf2007-01-31 12:14:08 -0500687 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
688 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 /* Release all resources associated with the task */
690 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691}
692
693/*
694 * User-visible entry point to the scheduler.
695 *
696 * This may be called recursively if e.g. an async NFS task updates
697 * the attributes and finds that dirty pages must be flushed.
698 * NOTE: Upon exit of this function the task is guaranteed to be
699 * released. In particular note that tk_release() will have
700 * been called, so your task memory may have been freed.
701 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800702void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703{
Trond Myklebust44c28872006-01-03 09:55:06 +0100704 rpc_set_active(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 rpc_set_running(task);
Trond Myklebust2efef832007-02-03 13:38:41 -0800706 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707}
708
David Howells65f27f32006-11-22 14:55:48 +0000709static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710{
David Howells65f27f32006-11-22 14:55:48 +0000711 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712}
713
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400714struct rpc_buffer {
715 size_t len;
716 char data[];
717};
718
Chuck Lever02107142006-01-03 09:55:49 +0100719/**
720 * rpc_malloc - allocate an RPC buffer
721 * @task: RPC task that will use this buffer
722 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400724 * To prevent rpciod from hanging, this allocator never sleeps,
725 * returning NULL if the request cannot be serviced immediately.
726 * The caller can arrange to sleep in a way that is safe for rpciod.
727 *
728 * Most requests are 'small' (under 2KiB) and can be serviced from a
729 * mempool, ensuring that NFS reads and writes can always proceed,
730 * and that there is good locality of reference for these buffers.
731 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400733 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400735void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400737 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400738 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400740 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400741 if (size <= RPC_BUFFER_MAXSIZE)
742 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400744 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200745
746 if (!buf)
747 return NULL;
748
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400749 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200750 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400751 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400752 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400754EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755
Chuck Lever02107142006-01-03 09:55:49 +0100756/**
757 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400758 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100759 *
760 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400761void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400763 size_t size;
764 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100765
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400766 if (!buffer)
767 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400768
769 buf = container_of(buffer, struct rpc_buffer, data);
770 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400771
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200772 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400773 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400774
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400775 if (size <= RPC_BUFFER_MAXSIZE)
776 mempool_free(buf, rpc_buffer_mempool);
777 else
778 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400780EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781
782/*
783 * Creation and deletion of RPC task structures
784 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400785static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786{
787 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100788 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400789 task->tk_flags = task_setup_data->flags;
790 task->tk_ops = task_setup_data->callback_ops;
791 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400792 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
794 /* Initialize retry counters */
795 task->tk_garb_retry = 2;
796 task->tk_cred_retry = 2;
797
Trond Myklebust3ff75762007-07-14 15:40:00 -0400798 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
799 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
801 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500802 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803
Trond Myklebust84115e12007-07-14 15:39:59 -0400804 task->tk_client = task_setup_data->rpc_client;
805 if (task->tk_client != NULL) {
806 kref_get(&task->tk_client->cl_kref);
807 if (task->tk_client->cl_softrtry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 task->tk_flags |= RPC_TASK_SOFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 }
810
Trond Myklebust84115e12007-07-14 15:39:59 -0400811 if (task->tk_ops->rpc_call_prepare != NULL)
812 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100813
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400814 if (task_setup_data->rpc_message != NULL) {
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400815 task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
816 task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
817 task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400818 /* Bind the user cred */
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400819 rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400820 if (task->tk_action == NULL)
821 rpc_call_start(task);
822 }
823
Chuck Leveref759a22006-03-20 13:44:17 -0500824 /* starting timestamp */
825 task->tk_start = jiffies;
826
Chuck Lever46121cf2007-01-31 12:14:08 -0500827 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700828 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829}
830
831static struct rpc_task *
832rpc_alloc_task(void)
833{
834 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
835}
836
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400838 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400840struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400842 struct rpc_task *task = setup_data->task;
843 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400845 if (task == NULL) {
846 task = rpc_alloc_task();
847 if (task == NULL)
848 goto out;
849 flags = RPC_TASK_DYNAMIC;
850 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851
Trond Myklebust84115e12007-07-14 15:39:59 -0400852 rpc_init_task(task, setup_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400854 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500855 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856out:
857 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858}
859
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500860static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100862 const struct rpc_call_ops *tk_ops = task->tk_ops;
863 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800865 if (task->tk_flags & RPC_TASK_DYNAMIC) {
866 dprintk("RPC: %5u freeing task\n", task->tk_pid);
867 mempool_free(task, rpc_task_mempool);
868 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500869 rpc_release_calldata(tk_ops, calldata);
870}
871
872static void rpc_async_release(struct work_struct *work)
873{
874 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
875}
876
877void rpc_put_task(struct rpc_task *task)
878{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500879 if (!atomic_dec_and_test(&task->tk_count))
880 return;
881 /* Release resources */
882 if (task->tk_rqstp)
883 xprt_release(task);
884 if (task->tk_msg.rpc_cred)
885 rpcauth_unbindcred(task);
886 if (task->tk_client) {
887 rpc_release_client(task->tk_client);
888 task->tk_client = NULL;
889 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500890 if (task->tk_workqueue != NULL) {
891 INIT_WORK(&task->u.tk_work, rpc_async_release);
892 queue_work(task->tk_workqueue, &task->u.tk_work);
893 } else
894 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500895}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400896EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500897
Trond Myklebustbde8f002007-01-24 11:54:53 -0800898static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500899{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900#ifdef RPC_DEBUG
901 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
902#endif
Chuck Lever46121cf2007-01-31 12:14:08 -0500903 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904
Trond Myklebust6529eba2007-06-14 16:40:14 -0400905 if (!list_empty(&task->tk_task)) {
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400906 struct rpc_clnt *clnt = task->tk_client;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400907 /* Remove from client task list */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400908 spin_lock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400909 list_del(&task->tk_task);
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400910 spin_unlock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400911 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914#ifdef RPC_DEBUG
915 task->tk_magic = 0;
916#endif
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500917 /* Wake up anyone who is waiting for task completion */
918 rpc_mark_complete_task(task);
919
920 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
922
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923/*
924 * Kill all tasks for the given client.
925 * XXX: kill their descendants as well?
926 */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400927void rpc_killall_tasks(struct rpc_clnt *clnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
929 struct rpc_task *rovr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400932 if (list_empty(&clnt->cl_tasks))
933 return;
934 dprintk("RPC: killing all tasks for client %p\n", clnt);
935 /*
936 * Spin lock all_tasks to prevent changes...
937 */
938 spin_lock(&clnt->cl_lock);
939 list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 if (! RPC_IS_ACTIVATED(rovr))
941 continue;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400942 if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 rovr->tk_flags |= RPC_TASK_KILLED;
944 rpc_exit(rovr, -EIO);
945 rpc_wake_up_task(rovr);
946 }
947 }
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400948 spin_unlock(&clnt->cl_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400950EXPORT_SYMBOL_GPL(rpc_killall_tasks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400952int rpciod_up(void)
953{
954 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
955}
956
957void rpciod_down(void)
958{
959 module_put(THIS_MODULE);
960}
961
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400963 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400965static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966{
967 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 /*
970 * Create the rpciod thread and wait for it to start.
971 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400972 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400975 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976}
977
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400978static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400980 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -0400981
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400982 if (rpciod_workqueue == NULL)
983 return;
Trond Myklebustab418d72007-06-14 17:08:36 -0400984 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400986 wq = rpciod_workqueue;
987 rpciod_workqueue = NULL;
988 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989}
990
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991void
992rpc_destroy_mempool(void)
993{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400994 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 if (rpc_buffer_mempool)
996 mempool_destroy(rpc_buffer_mempool);
997 if (rpc_task_mempool)
998 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -0700999 if (rpc_task_slabp)
1000 kmem_cache_destroy(rpc_task_slabp);
1001 if (rpc_buffer_slabp)
1002 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001003 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004}
1005
1006int
1007rpc_init_mempool(void)
1008{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001009 /*
1010 * The following is not strictly a mempool initialisation,
1011 * but there is no harm in doing it here
1012 */
1013 rpc_init_wait_queue(&delay_queue, "delayq");
1014 if (!rpciod_start())
1015 goto err_nomem;
1016
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1018 sizeof(struct rpc_task),
1019 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001020 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 if (!rpc_task_slabp)
1022 goto err_nomem;
1023 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1024 RPC_BUFFER_MAXSIZE,
1025 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001026 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 if (!rpc_buffer_slabp)
1028 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001029 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1030 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 if (!rpc_task_mempool)
1032 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001033 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1034 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 if (!rpc_buffer_mempool)
1036 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 return 0;
1038err_nomem:
1039 rpc_destroy_mempool();
1040 return -ENOMEM;
1041}