blob: b9061bcf6fc155cce2eaa2e582b0c7168967f344 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080021#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#ifdef RPC_DEBUG
26#define RPCDBG_FACILITY RPCDBG_SCHED
27#define RPC_TASK_MAGIC_ID 0xf00baa
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#endif
29
30/*
31 * RPC slabs and memory pools
32 */
33#define RPC_BUFFER_MAXSIZE (2048)
34#define RPC_BUFFER_POOLSIZE (8)
35#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080036static struct kmem_cache *rpc_task_slabp __read_mostly;
37static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070038static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41static void __rpc_default_timer(struct rpc_task *task);
David Howells65f27f32006-11-22 14:55:48 +000042static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080043static void rpc_release_task(struct rpc_task *task);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * RPC tasks sit here while waiting for conditions to improve.
47 */
48static RPC_WAITQ(delay_queue, "delayq");
49
50/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 * rpciod-related stuff
52 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050053struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * Disable the timer for a given RPC task. Should be called with
57 * queue->lock and bh_disabled in order to avoid races within
58 * rpc_run_timer().
59 */
60static inline void
61__rpc_disable_timer(struct rpc_task *task)
62{
Chuck Lever46121cf2007-01-31 12:14:08 -050063 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 task->tk_timeout_fn = NULL;
65 task->tk_timeout = 0;
66}
67
68/*
69 * Run a timeout function.
70 * We use the callback in order to allow __rpc_wake_up_task()
71 * and friends to disable the timer synchronously on SMP systems
72 * without calling del_timer_sync(). The latter could cause a
73 * deadlock if called while we're holding spinlocks...
74 */
75static void rpc_run_timer(struct rpc_task *task)
76{
77 void (*callback)(struct rpc_task *);
78
79 callback = task->tk_timeout_fn;
80 task->tk_timeout_fn = NULL;
81 if (callback && RPC_IS_QUEUED(task)) {
Chuck Lever46121cf2007-01-31 12:14:08 -050082 dprintk("RPC: %5u running timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 callback(task);
84 }
85 smp_mb__before_clear_bit();
86 clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate);
87 smp_mb__after_clear_bit();
88}
89
90/*
91 * Set up a timer for the current task.
92 */
93static inline void
94__rpc_add_timer(struct rpc_task *task, rpc_action timer)
95{
96 if (!task->tk_timeout)
97 return;
98
Chuck Lever46121cf2007-01-31 12:14:08 -050099 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 task->tk_pid, task->tk_timeout * 1000 / HZ);
101
102 if (timer)
103 task->tk_timeout_fn = timer;
104 else
105 task->tk_timeout_fn = __rpc_default_timer;
106 set_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate);
107 mod_timer(&task->tk_timer, jiffies + task->tk_timeout);
108}
109
110/*
111 * Delete any timer for the current task. Because we use del_timer_sync(),
112 * this function should never be called while holding queue->lock.
113 */
114static void
115rpc_delete_timer(struct rpc_task *task)
116{
117 if (RPC_IS_QUEUED(task))
118 return;
119 if (test_and_clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate)) {
120 del_singleshot_timer_sync(&task->tk_timer);
Chuck Lever46121cf2007-01-31 12:14:08 -0500121 dprintk("RPC: %5u deleting timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 }
123}
124
125/*
126 * Add new request to a priority queue.
127 */
128static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
129{
130 struct list_head *q;
131 struct rpc_task *t;
132
133 INIT_LIST_HEAD(&task->u.tk_wait.links);
134 q = &queue->tasks[task->tk_priority];
135 if (unlikely(task->tk_priority > queue->maxpriority))
136 q = &queue->tasks[queue->maxpriority];
137 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400138 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
140 return;
141 }
142 }
143 list_add_tail(&task->u.tk_wait.list, q);
144}
145
146/*
147 * Add new request to wait queue.
148 *
149 * Swapper tasks always get inserted at the head of the queue.
150 * This should avoid many nasty memory deadlocks and hopefully
151 * improve overall performance.
152 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
153 */
154static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
155{
156 BUG_ON (RPC_IS_QUEUED(task));
157
158 if (RPC_IS_PRIORITY(queue))
159 __rpc_add_wait_queue_priority(queue, task);
160 else if (RPC_IS_SWAPPER(task))
161 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
162 else
163 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
164 task->u.tk_wait.rpc_waitq = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500165 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 rpc_set_queued(task);
167
Chuck Lever46121cf2007-01-31 12:14:08 -0500168 dprintk("RPC: %5u added to queue %p \"%s\"\n",
169 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170}
171
172/*
173 * Remove request from a priority queue.
174 */
175static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
176{
177 struct rpc_task *t;
178
179 if (!list_empty(&task->u.tk_wait.links)) {
180 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
181 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
182 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
183 }
184 list_del(&task->u.tk_wait.list);
185}
186
187/*
188 * Remove request from queue.
189 * Note: must be called with spin lock held.
190 */
191static void __rpc_remove_wait_queue(struct rpc_task *task)
192{
193 struct rpc_wait_queue *queue;
194 queue = task->u.tk_wait.rpc_waitq;
195
196 if (RPC_IS_PRIORITY(queue))
197 __rpc_remove_wait_queue_priority(task);
198 else
199 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500200 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500201 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
202 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
205static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
206{
207 queue->priority = priority;
208 queue->count = 1 << (priority * 2);
209}
210
Trond Myklebust3ff75762007-07-14 15:40:00 -0400211static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400213 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 queue->nr = RPC_BATCH_COUNT;
215}
216
217static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
218{
219 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400220 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221}
222
Trond Myklebust3ff75762007-07-14 15:40:00 -0400223static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224{
225 int i;
226
227 spin_lock_init(&queue->lock);
228 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
229 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400230 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 rpc_reset_waitqueue_priority(queue);
232#ifdef RPC_DEBUG
233 queue->name = qname;
234#endif
235}
236
237void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
238{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400239 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240}
241
242void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
243{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400244 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400246EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247
Trond Myklebust44c28872006-01-03 09:55:06 +0100248static int rpc_wait_bit_interruptible(void *word)
249{
250 if (signal_pending(current))
251 return -ERESTARTSYS;
252 schedule();
253 return 0;
254}
255
Trond Myklebustc44fe702007-06-16 14:17:01 -0400256#ifdef RPC_DEBUG
257static void rpc_task_set_debuginfo(struct rpc_task *task)
258{
259 static atomic_t rpc_pid;
260
261 task->tk_magic = RPC_TASK_MAGIC_ID;
262 task->tk_pid = atomic_inc_return(&rpc_pid);
263}
264#else
265static inline void rpc_task_set_debuginfo(struct rpc_task *task)
266{
267}
268#endif
269
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500270static void rpc_set_active(struct rpc_task *task)
271{
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400272 struct rpc_clnt *clnt;
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500273 if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
274 return;
Trond Myklebustc44fe702007-06-16 14:17:01 -0400275 rpc_task_set_debuginfo(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500276 /* Add to global list of all tasks */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400277 clnt = task->tk_client;
278 if (clnt != NULL) {
279 spin_lock(&clnt->cl_lock);
280 list_add_tail(&task->tk_task, &clnt->cl_tasks);
281 spin_unlock(&clnt->cl_lock);
282 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500283}
284
Trond Myklebust44c28872006-01-03 09:55:06 +0100285/*
286 * Mark an RPC call as having completed by clearing the 'active' bit
287 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500288static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100289{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500290 smp_mb__before_clear_bit();
291 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
292 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100293 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
294}
295
296/*
297 * Allow callers to wait for completion of an RPC call
298 */
299int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
300{
301 if (action == NULL)
302 action = rpc_wait_bit_interruptible;
303 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
304 action, TASK_INTERRUPTIBLE);
305}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400306EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100307
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308/*
309 * Make an RPC task runnable.
310 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800311 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 * the spinlock held to protect the wait queue operation.
313 */
314static void rpc_make_runnable(struct rpc_task *task)
315{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 BUG_ON(task->tk_timeout_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 rpc_clear_queued(task);
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100318 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 return;
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100320 /* We might have raced */
321 if (RPC_IS_QUEUED(task)) {
322 rpc_clear_running(task);
323 return;
324 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 if (RPC_IS_ASYNC(task)) {
326 int status;
327
David Howells65f27f32006-11-22 14:55:48 +0000328 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 status = queue_work(task->tk_workqueue, &task->u.tk_work);
330 if (status < 0) {
331 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
332 task->tk_status = status;
333 return;
334 }
335 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000336 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
339/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 * Prepare for sleeping on a wait queue.
341 * By always appending tasks to the list we ensure FIFO behavior.
342 * NB: An RPC task will only receive interrupt-driven events as long
343 * as it's on a wait queue.
344 */
345static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
346 rpc_action action, rpc_action timer)
347{
Chuck Lever46121cf2007-01-31 12:14:08 -0500348 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
349 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
351 if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
352 printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
353 return;
354 }
355
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 __rpc_add_wait_queue(q, task);
357
358 BUG_ON(task->tk_callback != NULL);
359 task->tk_callback = action;
360 __rpc_add_timer(task, timer);
361}
362
363void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
364 rpc_action action, rpc_action timer)
365{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500366 /* Mark the task as being activated if so needed */
367 rpc_set_active(task);
368
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 /*
370 * Protect the queue operations.
371 */
372 spin_lock_bh(&q->lock);
373 __rpc_sleep_on(q, task, action, timer);
374 spin_unlock_bh(&q->lock);
375}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400376EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
378/**
379 * __rpc_do_wake_up_task - wake up a single rpc_task
380 * @task: task to be woken up
381 *
382 * Caller must hold queue->lock, and have cleared the task queued flag.
383 */
384static void __rpc_do_wake_up_task(struct rpc_task *task)
385{
Chuck Lever46121cf2007-01-31 12:14:08 -0500386 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
387 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
389#ifdef RPC_DEBUG
390 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
391#endif
392 /* Has the task been executed yet? If not, we cannot wake it up! */
393 if (!RPC_IS_ACTIVATED(task)) {
394 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
395 return;
396 }
397
398 __rpc_disable_timer(task);
399 __rpc_remove_wait_queue(task);
400
401 rpc_make_runnable(task);
402
Chuck Lever46121cf2007-01-31 12:14:08 -0500403 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404}
405
406/*
407 * Wake up the specified task
408 */
409static void __rpc_wake_up_task(struct rpc_task *task)
410{
411 if (rpc_start_wakeup(task)) {
412 if (RPC_IS_QUEUED(task))
413 __rpc_do_wake_up_task(task);
414 rpc_finish_wakeup(task);
415 }
416}
417
418/*
419 * Default timeout handler if none specified by user
420 */
421static void
422__rpc_default_timer(struct rpc_task *task)
423{
Chuck Lever46121cf2007-01-31 12:14:08 -0500424 dprintk("RPC: %5u timeout (default timer)\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 task->tk_status = -ETIMEDOUT;
426 rpc_wake_up_task(task);
427}
428
429/*
430 * Wake up the specified task
431 */
432void rpc_wake_up_task(struct rpc_task *task)
433{
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500434 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 if (rpc_start_wakeup(task)) {
436 if (RPC_IS_QUEUED(task)) {
437 struct rpc_wait_queue *queue = task->u.tk_wait.rpc_waitq;
438
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500439 /* Note: we're already in a bh-safe context */
440 spin_lock(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 __rpc_do_wake_up_task(task);
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500442 spin_unlock(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 }
444 rpc_finish_wakeup(task);
445 }
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500446 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400448EXPORT_SYMBOL_GPL(rpc_wake_up_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
450/*
451 * Wake up the next task on a priority queue.
452 */
453static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
454{
455 struct list_head *q;
456 struct rpc_task *task;
457
458 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400459 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 */
461 q = &queue->tasks[queue->priority];
462 if (!list_empty(q)) {
463 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400464 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 if (--queue->nr)
466 goto out;
467 list_move_tail(&task->u.tk_wait.list, q);
468 }
469 /*
470 * Check if we need to switch queues.
471 */
472 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400473 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 }
475
476 /*
477 * Service the next queue.
478 */
479 do {
480 if (q == &queue->tasks[0])
481 q = &queue->tasks[queue->maxpriority];
482 else
483 q = q - 1;
484 if (!list_empty(q)) {
485 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
486 goto new_queue;
487 }
488 } while (q != &queue->tasks[queue->priority]);
489
490 rpc_reset_waitqueue_priority(queue);
491 return NULL;
492
493new_queue:
494 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400495new_owner:
496 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497out:
498 __rpc_wake_up_task(task);
499 return task;
500}
501
502/*
503 * Wake up the next task on the wait queue.
504 */
505struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
506{
507 struct rpc_task *task = NULL;
508
Chuck Lever46121cf2007-01-31 12:14:08 -0500509 dprintk("RPC: wake_up_next(%p \"%s\")\n",
510 queue, rpc_qname(queue));
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500511 rcu_read_lock_bh();
512 spin_lock(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 if (RPC_IS_PRIORITY(queue))
514 task = __rpc_wake_up_next_priority(queue);
515 else {
516 task_for_first(task, &queue->tasks[0])
517 __rpc_wake_up_task(task);
518 }
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500519 spin_unlock(&queue->lock);
520 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521
522 return task;
523}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400524EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
526/**
527 * rpc_wake_up - wake up all rpc_tasks
528 * @queue: rpc_wait_queue on which the tasks are sleeping
529 *
530 * Grabs queue->lock
531 */
532void rpc_wake_up(struct rpc_wait_queue *queue)
533{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800534 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800536
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500537 rcu_read_lock_bh();
538 spin_lock(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 head = &queue->tasks[queue->maxpriority];
540 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800541 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 __rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 if (head == &queue->tasks[0])
544 break;
545 head--;
546 }
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500547 spin_unlock(&queue->lock);
548 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400550EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551
552/**
553 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
554 * @queue: rpc_wait_queue on which the tasks are sleeping
555 * @status: status value to set
556 *
557 * Grabs queue->lock
558 */
559void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
560{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800561 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500564 rcu_read_lock_bh();
565 spin_lock(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 head = &queue->tasks[queue->maxpriority];
567 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800568 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 task->tk_status = status;
570 __rpc_wake_up_task(task);
571 }
572 if (head == &queue->tasks[0])
573 break;
574 head--;
575 }
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500576 spin_unlock(&queue->lock);
577 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400579EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580
Trond Myklebust80147932006-08-31 18:24:08 -0400581static void __rpc_atrun(struct rpc_task *task)
582{
583 rpc_wake_up_task(task);
584}
585
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586/*
587 * Run a task at a later time
588 */
Trond Myklebust80147932006-08-31 18:24:08 -0400589void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590{
591 task->tk_timeout = delay;
592 rpc_sleep_on(&delay_queue, task, NULL, __rpc_atrun);
593}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400594EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100597 * Helper to call task->tk_ops->rpc_call_prepare
598 */
599static void rpc_prepare_task(struct rpc_task *task)
600{
Trond Myklebust6d5fcb52006-10-18 16:01:06 -0400601 lock_kernel();
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100602 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
Trond Myklebust6d5fcb52006-10-18 16:01:06 -0400603 unlock_kernel();
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100604}
605
606/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100607 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000608 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100609void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000610{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100611 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100612 if (task->tk_ops->rpc_call_done != NULL) {
Trond Myklebust6d5fcb52006-10-18 16:01:06 -0400613 lock_kernel();
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100614 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebust6d5fcb52006-10-18 16:01:06 -0400615 unlock_kernel();
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000616 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100617 WARN_ON(RPC_ASSASSINATED(task));
618 /* Always release the RPC slot and buffer memory */
619 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000620 }
621 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000622}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400623EXPORT_SYMBOL_GPL(rpc_exit_task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000624
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400625void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
626{
627 if (ops->rpc_release != NULL) {
628 lock_kernel();
629 ops->rpc_release(calldata);
630 unlock_kernel();
631 }
632}
633
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000634/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 * This is the RPC `scheduler' (or rather, the finite state machine).
636 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800637static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638{
639 int status = 0;
640
Chuck Lever46121cf2007-01-31 12:14:08 -0500641 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
642 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643
644 BUG_ON(RPC_IS_QUEUED(task));
645
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000646 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 /*
648 * Garbage collection of pending timers...
649 */
650 rpc_delete_timer(task);
651
652 /*
653 * Execute any pending callback.
654 */
655 if (RPC_DO_CALLBACK(task)) {
656 /* Define a callback save pointer */
657 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800658
659 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 * If a callback exists, save it, reset it,
661 * call it.
662 * The save is needed to stop from resetting
663 * another callback set within the callback handler
664 * - Dave
665 */
666 save_callback=task->tk_callback;
667 task->tk_callback=NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 }
670
671 /*
672 * Perform the next FSM step.
673 * tk_action may be NULL when the task has been killed
674 * by someone else.
675 */
676 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100677 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100679 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 }
681
682 /*
683 * Lockless check for whether task is sleeping or not.
684 */
685 if (!RPC_IS_QUEUED(task))
686 continue;
687 rpc_clear_running(task);
688 if (RPC_IS_ASYNC(task)) {
689 /* Careful! we may have raced... */
690 if (RPC_IS_QUEUED(task))
Trond Myklebust2efef832007-02-03 13:38:41 -0800691 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 if (rpc_test_and_set_running(task))
Trond Myklebust2efef832007-02-03 13:38:41 -0800693 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 continue;
695 }
696
697 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500698 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000699 /* Note: Caller should be using rpc_clnt_sigmask() */
700 status = out_of_line_wait_on_bit(&task->tk_runstate,
701 RPC_TASK_QUEUED, rpc_wait_bit_interruptible,
702 TASK_INTERRUPTIBLE);
703 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 /*
705 * When a sync task receives a signal, it exits with
706 * -ERESTARTSYS. In order to catch any callbacks that
707 * clean up after sleeping on some queue, we don't
708 * break the loop here, but go around once more.
709 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500710 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000711 task->tk_flags |= RPC_TASK_KILLED;
712 rpc_exit(task, -ERESTARTSYS);
713 rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714 }
715 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500716 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 }
718
Chuck Lever46121cf2007-01-31 12:14:08 -0500719 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
720 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 /* Release all resources associated with the task */
722 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723}
724
725/*
726 * User-visible entry point to the scheduler.
727 *
728 * This may be called recursively if e.g. an async NFS task updates
729 * the attributes and finds that dirty pages must be flushed.
730 * NOTE: Upon exit of this function the task is guaranteed to be
731 * released. In particular note that tk_release() will have
732 * been called, so your task memory may have been freed.
733 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800734void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735{
Trond Myklebust44c28872006-01-03 09:55:06 +0100736 rpc_set_active(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 rpc_set_running(task);
Trond Myklebust2efef832007-02-03 13:38:41 -0800738 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400740EXPORT_SYMBOL_GPL(rpc_execute);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
David Howells65f27f32006-11-22 14:55:48 +0000742static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743{
David Howells65f27f32006-11-22 14:55:48 +0000744 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400747struct rpc_buffer {
748 size_t len;
749 char data[];
750};
751
Chuck Lever02107142006-01-03 09:55:49 +0100752/**
753 * rpc_malloc - allocate an RPC buffer
754 * @task: RPC task that will use this buffer
755 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400757 * To prevent rpciod from hanging, this allocator never sleeps,
758 * returning NULL if the request cannot be serviced immediately.
759 * The caller can arrange to sleep in a way that is safe for rpciod.
760 *
761 * Most requests are 'small' (under 2KiB) and can be serviced from a
762 * mempool, ensuring that NFS reads and writes can always proceed,
763 * and that there is good locality of reference for these buffers.
764 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400766 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400768void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400770 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400771 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400773 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400774 if (size <= RPC_BUFFER_MAXSIZE)
775 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400777 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200778
779 if (!buf)
780 return NULL;
781
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400782 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200783 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400784 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400785 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400787EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788
Chuck Lever02107142006-01-03 09:55:49 +0100789/**
790 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400791 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100792 *
793 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400794void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400796 size_t size;
797 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100798
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400799 if (!buffer)
800 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400801
802 buf = container_of(buffer, struct rpc_buffer, data);
803 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400804
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200805 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400806 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400807
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400808 if (size <= RPC_BUFFER_MAXSIZE)
809 mempool_free(buf, rpc_buffer_mempool);
810 else
811 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400813EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814
815/*
816 * Creation and deletion of RPC task structures
817 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400818void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819{
820 memset(task, 0, sizeof(*task));
Pavel Emelyanovb24b8a22008-01-23 21:20:07 -0800821 setup_timer(&task->tk_timer, (void (*)(unsigned long))rpc_run_timer,
822 (unsigned long)task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100823 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400824 task->tk_flags = task_setup_data->flags;
825 task->tk_ops = task_setup_data->callback_ops;
826 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400827 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828
829 /* Initialize retry counters */
830 task->tk_garb_retry = 2;
831 task->tk_cred_retry = 2;
832
Trond Myklebust3ff75762007-07-14 15:40:00 -0400833 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
834 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
836 /* Initialize workqueue for async tasks */
837 task->tk_workqueue = rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838
Trond Myklebust84115e12007-07-14 15:39:59 -0400839 task->tk_client = task_setup_data->rpc_client;
840 if (task->tk_client != NULL) {
841 kref_get(&task->tk_client->cl_kref);
842 if (task->tk_client->cl_softrtry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 task->tk_flags |= RPC_TASK_SOFT;
Trond Myklebust84115e12007-07-14 15:39:59 -0400844 if (!task->tk_client->cl_intr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 task->tk_flags |= RPC_TASK_NOINTR;
846 }
847
Trond Myklebust84115e12007-07-14 15:39:59 -0400848 if (task->tk_ops->rpc_call_prepare != NULL)
849 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100850
Chuck Leveref759a22006-03-20 13:44:17 -0500851 /* starting timestamp */
852 task->tk_start = jiffies;
853
Chuck Lever46121cf2007-01-31 12:14:08 -0500854 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700855 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400857EXPORT_SYMBOL_GPL(rpc_init_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858
859static struct rpc_task *
860rpc_alloc_task(void)
861{
862 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
863}
864
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500865static void rpc_free_task(struct rcu_head *rcu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500867 struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu);
Chuck Lever46121cf2007-01-31 12:14:08 -0500868 dprintk("RPC: %5u freeing task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 mempool_free(task, rpc_task_mempool);
870}
871
872/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400873 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400875struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876{
877 struct rpc_task *task;
878
879 task = rpc_alloc_task();
880 if (!task)
Trond Myklebust90c57552007-06-09 19:49:36 -0400881 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882
Trond Myklebust84115e12007-07-14 15:39:59 -0400883 rpc_init_task(task, setup_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884
Chuck Lever46121cf2007-01-31 12:14:08 -0500885 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 task->tk_flags |= RPC_TASK_DYNAMIC;
887out:
888 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889}
890
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500891
892void rpc_put_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100894 const struct rpc_call_ops *tk_ops = task->tk_ops;
895 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500897 if (!atomic_dec_and_test(&task->tk_count))
898 return;
899 /* Release resources */
900 if (task->tk_rqstp)
901 xprt_release(task);
902 if (task->tk_msg.rpc_cred)
903 rpcauth_unbindcred(task);
904 if (task->tk_client) {
905 rpc_release_client(task->tk_client);
906 task->tk_client = NULL;
907 }
908 if (task->tk_flags & RPC_TASK_DYNAMIC)
Trond Myklebust8aca67f2006-11-13 16:23:44 -0500909 call_rcu_bh(&task->u.tk_rcu, rpc_free_task);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400910 rpc_release_calldata(tk_ops, calldata);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500911}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400912EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500913
Trond Myklebustbde8f002007-01-24 11:54:53 -0800914static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500915{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916#ifdef RPC_DEBUG
917 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
918#endif
Chuck Lever46121cf2007-01-31 12:14:08 -0500919 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Trond Myklebust6529eba2007-06-14 16:40:14 -0400921 if (!list_empty(&task->tk_task)) {
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400922 struct rpc_clnt *clnt = task->tk_client;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400923 /* Remove from client task list */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400924 spin_lock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400925 list_del(&task->tk_task);
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400926 spin_unlock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400927 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 /* Synchronously delete any running timer */
931 rpc_delete_timer(task);
932
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933#ifdef RPC_DEBUG
934 task->tk_magic = 0;
935#endif
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500936 /* Wake up anyone who is waiting for task completion */
937 rpc_mark_complete_task(task);
938
939 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940}
941
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942/*
943 * Kill all tasks for the given client.
944 * XXX: kill their descendants as well?
945 */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400946void rpc_killall_tasks(struct rpc_clnt *clnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947{
948 struct rpc_task *rovr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400951 if (list_empty(&clnt->cl_tasks))
952 return;
953 dprintk("RPC: killing all tasks for client %p\n", clnt);
954 /*
955 * Spin lock all_tasks to prevent changes...
956 */
957 spin_lock(&clnt->cl_lock);
958 list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 if (! RPC_IS_ACTIVATED(rovr))
960 continue;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400961 if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 rovr->tk_flags |= RPC_TASK_KILLED;
963 rpc_exit(rovr, -EIO);
964 rpc_wake_up_task(rovr);
965 }
966 }
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400967 spin_unlock(&clnt->cl_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400969EXPORT_SYMBOL_GPL(rpc_killall_tasks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400971int rpciod_up(void)
972{
973 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
974}
975
976void rpciod_down(void)
977{
978 module_put(THIS_MODULE);
979}
980
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400982 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400984static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985{
986 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 /*
989 * Create the rpciod thread and wait for it to start.
990 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400991 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400994 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995}
996
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400997static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400999 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -04001000
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001001 if (rpciod_workqueue == NULL)
1002 return;
Trond Myklebustab418d72007-06-14 17:08:36 -04001003 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001005 wq = rpciod_workqueue;
1006 rpciod_workqueue = NULL;
1007 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008}
1009
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010void
1011rpc_destroy_mempool(void)
1012{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001013 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 if (rpc_buffer_mempool)
1015 mempool_destroy(rpc_buffer_mempool);
1016 if (rpc_task_mempool)
1017 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001018 if (rpc_task_slabp)
1019 kmem_cache_destroy(rpc_task_slabp);
1020 if (rpc_buffer_slabp)
1021 kmem_cache_destroy(rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022}
1023
1024int
1025rpc_init_mempool(void)
1026{
1027 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1028 sizeof(struct rpc_task),
1029 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001030 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 if (!rpc_task_slabp)
1032 goto err_nomem;
1033 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1034 RPC_BUFFER_MAXSIZE,
1035 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001036 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 if (!rpc_buffer_slabp)
1038 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001039 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1040 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 if (!rpc_task_mempool)
1042 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001043 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1044 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 if (!rpc_buffer_mempool)
1046 goto err_nomem;
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001047 if (!rpciod_start())
1048 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 return 0;
1050err_nomem:
1051 rpc_destroy_mempool();
1052 return -ENOMEM;
1053}