blob: 385f427bedad5e8e8ca51088cb51963da4f2019c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080021#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#ifdef RPC_DEBUG
26#define RPCDBG_FACILITY RPCDBG_SCHED
27#define RPC_TASK_MAGIC_ID 0xf00baa
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#endif
29
30/*
31 * RPC slabs and memory pools
32 */
33#define RPC_BUFFER_MAXSIZE (2048)
34#define RPC_BUFFER_POOLSIZE (8)
35#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080036static struct kmem_cache *rpc_task_slabp __read_mostly;
37static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070038static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
David Howells65f27f32006-11-22 14:55:48 +000041static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080042static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040043static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * RPC tasks sit here while waiting for conditions to improve.
47 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040048static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
50/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 * rpciod-related stuff
52 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050053struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * Disable the timer for a given RPC task. Should be called with
57 * queue->lock and bh_disabled in order to avoid races within
58 * rpc_run_timer().
59 */
Trond Myklebust5d008372008-02-22 16:34:17 -050060static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050061__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040063 if (task->tk_timeout == 0)
64 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050065 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040067 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050068 if (list_empty(&queue->timer_list.list))
69 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040070}
71
72static void
73rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
74{
75 queue->timer_list.expires = expires;
76 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070077}
78
79/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 * Set up a timer for the current task.
81 */
Trond Myklebust5d008372008-02-22 16:34:17 -050082static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050083__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084{
85 if (!task->tk_timeout)
86 return;
87
Chuck Lever46121cf2007-01-31 12:14:08 -050088 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 task->tk_pid, task->tk_timeout * 1000 / HZ);
90
Trond Myklebusteb276c02008-02-22 17:27:59 -050091 task->u.tk_wait.expires = jiffies + task->tk_timeout;
92 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
93 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
94 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
96
97/*
98 * Add new request to a priority queue.
99 */
100static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
101{
102 struct list_head *q;
103 struct rpc_task *t;
104
105 INIT_LIST_HEAD(&task->u.tk_wait.links);
106 q = &queue->tasks[task->tk_priority];
107 if (unlikely(task->tk_priority > queue->maxpriority))
108 q = &queue->tasks[queue->maxpriority];
109 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400110 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
112 return;
113 }
114 }
115 list_add_tail(&task->u.tk_wait.list, q);
116}
117
118/*
119 * Add new request to wait queue.
120 *
121 * Swapper tasks always get inserted at the head of the queue.
122 * This should avoid many nasty memory deadlocks and hopefully
123 * improve overall performance.
124 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
125 */
126static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
127{
128 BUG_ON (RPC_IS_QUEUED(task));
129
130 if (RPC_IS_PRIORITY(queue))
131 __rpc_add_wait_queue_priority(queue, task);
132 else if (RPC_IS_SWAPPER(task))
133 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
134 else
135 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500136 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500137 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 rpc_set_queued(task);
139
Chuck Lever46121cf2007-01-31 12:14:08 -0500140 dprintk("RPC: %5u added to queue %p \"%s\"\n",
141 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142}
143
144/*
145 * Remove request from a priority queue.
146 */
147static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
148{
149 struct rpc_task *t;
150
151 if (!list_empty(&task->u.tk_wait.links)) {
152 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
153 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
154 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
155 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156}
157
158/*
159 * Remove request from queue.
160 * Note: must be called with spin lock held.
161 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500162static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500164 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 if (RPC_IS_PRIORITY(queue))
166 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400167 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500168 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500169 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
170 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172
173static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
174{
175 queue->priority = priority;
176 queue->count = 1 << (priority * 2);
177}
178
Trond Myklebust3ff75762007-07-14 15:40:00 -0400179static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400181 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 queue->nr = RPC_BATCH_COUNT;
183}
184
185static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
186{
187 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400188 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189}
190
Trond Myklebust3ff75762007-07-14 15:40:00 -0400191static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
193 int i;
194
195 spin_lock_init(&queue->lock);
196 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
197 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400198 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400200 queue->qlen = 0;
201 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
202 INIT_LIST_HEAD(&queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203#ifdef RPC_DEBUG
204 queue->name = qname;
205#endif
206}
207
208void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
209{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400210 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
213void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
214{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400215 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400217EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500219void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
220{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400221 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500222}
223EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
224
Matthew Wilcox150030b2007-12-06 16:24:39 -0500225static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100226{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500227 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100228 return -ERESTARTSYS;
229 schedule();
230 return 0;
231}
232
Trond Myklebustc44fe702007-06-16 14:17:01 -0400233#ifdef RPC_DEBUG
234static void rpc_task_set_debuginfo(struct rpc_task *task)
235{
236 static atomic_t rpc_pid;
237
238 task->tk_magic = RPC_TASK_MAGIC_ID;
239 task->tk_pid = atomic_inc_return(&rpc_pid);
240}
241#else
242static inline void rpc_task_set_debuginfo(struct rpc_task *task)
243{
244}
245#endif
246
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500247static void rpc_set_active(struct rpc_task *task)
248{
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400249 struct rpc_clnt *clnt;
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500250 if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
251 return;
Trond Myklebustc44fe702007-06-16 14:17:01 -0400252 rpc_task_set_debuginfo(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500253 /* Add to global list of all tasks */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400254 clnt = task->tk_client;
255 if (clnt != NULL) {
256 spin_lock(&clnt->cl_lock);
257 list_add_tail(&task->tk_task, &clnt->cl_tasks);
258 spin_unlock(&clnt->cl_lock);
259 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500260}
261
Trond Myklebust44c28872006-01-03 09:55:06 +0100262/*
263 * Mark an RPC call as having completed by clearing the 'active' bit
264 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500265static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100266{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500267 smp_mb__before_clear_bit();
268 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
269 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100270 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
271}
272
273/*
274 * Allow callers to wait for completion of an RPC call
275 */
276int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
277{
278 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500279 action = rpc_wait_bit_killable;
Trond Myklebust44c28872006-01-03 09:55:06 +0100280 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500281 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100282}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400283EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100284
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285/*
286 * Make an RPC task runnable.
287 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800288 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 * the spinlock held to protect the wait queue operation.
290 */
291static void rpc_make_runnable(struct rpc_task *task)
292{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 rpc_clear_queued(task);
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100294 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 return;
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100296 /* We might have raced */
297 if (RPC_IS_QUEUED(task)) {
298 rpc_clear_running(task);
299 return;
300 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 if (RPC_IS_ASYNC(task)) {
302 int status;
303
David Howells65f27f32006-11-22 14:55:48 +0000304 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500305 status = queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 if (status < 0) {
307 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
308 task->tk_status = status;
309 return;
310 }
311 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000312 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313}
314
315/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 * Prepare for sleeping on a wait queue.
317 * By always appending tasks to the list we ensure FIFO behavior.
318 * NB: An RPC task will only receive interrupt-driven events as long
319 * as it's on a wait queue.
320 */
321static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500322 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323{
Chuck Lever46121cf2007-01-31 12:14:08 -0500324 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
325 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326
327 if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
328 printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
329 return;
330 }
331
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 __rpc_add_wait_queue(q, task);
333
334 BUG_ON(task->tk_callback != NULL);
335 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500336 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
339void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500340 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500342 /* Mark the task as being activated if so needed */
343 rpc_set_active(task);
344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 /*
346 * Protect the queue operations.
347 */
348 spin_lock_bh(&q->lock);
Trond Myklebust5d008372008-02-22 16:34:17 -0500349 __rpc_sleep_on(q, task, action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 spin_unlock_bh(&q->lock);
351}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400352EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
354/**
355 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500356 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 * @task: task to be woken up
358 *
359 * Caller must hold queue->lock, and have cleared the task queued flag.
360 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500361static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362{
Chuck Lever46121cf2007-01-31 12:14:08 -0500363 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
364 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
366#ifdef RPC_DEBUG
367 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
368#endif
369 /* Has the task been executed yet? If not, we cannot wake it up! */
370 if (!RPC_IS_ACTIVATED(task)) {
371 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
372 return;
373 }
374
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500375 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376
377 rpc_make_runnable(task);
378
Chuck Lever46121cf2007-01-31 12:14:08 -0500379 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380}
381
382/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500383 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500385static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800387 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
388 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
391/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500392 * Wake up a task on a specific queue
393 */
394void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
395{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800396 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500397 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800398 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500399}
400EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
401
402/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 * Wake up the specified task
404 */
Trond Myklebustfda13932008-02-22 16:34:12 -0500405static void rpc_wake_up_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406{
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500407 rpc_wake_up_queued_task(task->tk_waitqueue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408}
409
410/*
411 * Wake up the next task on a priority queue.
412 */
413static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
414{
415 struct list_head *q;
416 struct rpc_task *task;
417
418 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400419 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 */
421 q = &queue->tasks[queue->priority];
422 if (!list_empty(q)) {
423 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400424 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 if (--queue->nr)
426 goto out;
427 list_move_tail(&task->u.tk_wait.list, q);
428 }
429 /*
430 * Check if we need to switch queues.
431 */
432 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400433 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 }
435
436 /*
437 * Service the next queue.
438 */
439 do {
440 if (q == &queue->tasks[0])
441 q = &queue->tasks[queue->maxpriority];
442 else
443 q = q - 1;
444 if (!list_empty(q)) {
445 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
446 goto new_queue;
447 }
448 } while (q != &queue->tasks[queue->priority]);
449
450 rpc_reset_waitqueue_priority(queue);
451 return NULL;
452
453new_queue:
454 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400455new_owner:
456 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457out:
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500458 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 return task;
460}
461
462/*
463 * Wake up the next task on the wait queue.
464 */
465struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
466{
467 struct rpc_task *task = NULL;
468
Chuck Lever46121cf2007-01-31 12:14:08 -0500469 dprintk("RPC: wake_up_next(%p \"%s\")\n",
470 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800471 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 if (RPC_IS_PRIORITY(queue))
473 task = __rpc_wake_up_next_priority(queue);
474 else {
475 task_for_first(task, &queue->tasks[0])
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500476 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800478 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479
480 return task;
481}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400482EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483
484/**
485 * rpc_wake_up - wake up all rpc_tasks
486 * @queue: rpc_wait_queue on which the tasks are sleeping
487 *
488 * Grabs queue->lock
489 */
490void rpc_wake_up(struct rpc_wait_queue *queue)
491{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800492 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800494
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800495 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 head = &queue->tasks[queue->maxpriority];
497 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800498 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500499 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 if (head == &queue->tasks[0])
501 break;
502 head--;
503 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800504 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400506EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
508/**
509 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
510 * @queue: rpc_wait_queue on which the tasks are sleeping
511 * @status: status value to set
512 *
513 * Grabs queue->lock
514 */
515void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
516{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800517 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800520 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 head = &queue->tasks[queue->maxpriority];
522 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800523 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500525 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 }
527 if (head == &queue->tasks[0])
528 break;
529 head--;
530 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800531 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400533EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400535static void __rpc_queue_timer_fn(unsigned long ptr)
536{
537 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
538 struct rpc_task *task, *n;
539 unsigned long expires, now, timeo;
540
541 spin_lock(&queue->lock);
542 expires = now = jiffies;
543 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
544 timeo = task->u.tk_wait.expires;
545 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400546 dprintk("RPC: %5u timeout\n", task->tk_pid);
547 task->tk_status = -ETIMEDOUT;
548 rpc_wake_up_task_queue_locked(queue, task);
549 continue;
550 }
551 if (expires == now || time_after(expires, timeo))
552 expires = timeo;
553 }
554 if (!list_empty(&queue->timer_list.list))
555 rpc_set_queue_timer(queue, expires);
556 spin_unlock(&queue->lock);
557}
558
Trond Myklebust80147932006-08-31 18:24:08 -0400559static void __rpc_atrun(struct rpc_task *task)
560{
Trond Myklebust5d008372008-02-22 16:34:17 -0500561 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400562}
563
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564/*
565 * Run a task at a later time
566 */
Trond Myklebust80147932006-08-31 18:24:08 -0400567void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568{
569 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500570 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400572EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100575 * Helper to call task->tk_ops->rpc_call_prepare
576 */
577static void rpc_prepare_task(struct rpc_task *task)
578{
579 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
580}
581
582/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100583 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000584 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100585void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000586{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100587 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100588 if (task->tk_ops->rpc_call_done != NULL) {
589 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000590 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100591 WARN_ON(RPC_ASSASSINATED(task));
592 /* Always release the RPC slot and buffer memory */
593 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000594 }
595 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000596}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400597EXPORT_SYMBOL_GPL(rpc_exit_task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000598
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400599void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
600{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400601 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400602 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400603}
604
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000605/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 * This is the RPC `scheduler' (or rather, the finite state machine).
607 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800608static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609{
610 int status = 0;
611
Chuck Lever46121cf2007-01-31 12:14:08 -0500612 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
613 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
615 BUG_ON(RPC_IS_QUEUED(task));
616
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000617 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
619 /*
620 * Execute any pending callback.
621 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400622 if (task->tk_callback) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800624
625 /*
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400626 * We set tk_callback to NULL before calling it,
627 * in case it sets the tk_callback field itself:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400629 save_callback = task->tk_callback;
630 task->tk_callback = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 }
633
634 /*
635 * Perform the next FSM step.
636 * tk_action may be NULL when the task has been killed
637 * by someone else.
638 */
639 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100640 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100642 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 }
644
645 /*
646 * Lockless check for whether task is sleeping or not.
647 */
648 if (!RPC_IS_QUEUED(task))
649 continue;
650 rpc_clear_running(task);
651 if (RPC_IS_ASYNC(task)) {
652 /* Careful! we may have raced... */
653 if (RPC_IS_QUEUED(task))
Trond Myklebust2efef832007-02-03 13:38:41 -0800654 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 if (rpc_test_and_set_running(task))
Trond Myklebust2efef832007-02-03 13:38:41 -0800656 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 continue;
658 }
659
660 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500661 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000662 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500663 RPC_TASK_QUEUED, rpc_wait_bit_killable,
664 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000665 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 /*
667 * When a sync task receives a signal, it exits with
668 * -ERESTARTSYS. In order to catch any callbacks that
669 * clean up after sleeping on some queue, we don't
670 * break the loop here, but go around once more.
671 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500672 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000673 task->tk_flags |= RPC_TASK_KILLED;
674 rpc_exit(task, -ERESTARTSYS);
675 rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 }
677 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500678 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 }
680
Chuck Lever46121cf2007-01-31 12:14:08 -0500681 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
682 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 /* Release all resources associated with the task */
684 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685}
686
687/*
688 * User-visible entry point to the scheduler.
689 *
690 * This may be called recursively if e.g. an async NFS task updates
691 * the attributes and finds that dirty pages must be flushed.
692 * NOTE: Upon exit of this function the task is guaranteed to be
693 * released. In particular note that tk_release() will have
694 * been called, so your task memory may have been freed.
695 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800696void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
Trond Myklebust44c28872006-01-03 09:55:06 +0100698 rpc_set_active(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 rpc_set_running(task);
Trond Myklebust2efef832007-02-03 13:38:41 -0800700 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701}
702
David Howells65f27f32006-11-22 14:55:48 +0000703static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704{
David Howells65f27f32006-11-22 14:55:48 +0000705 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706}
707
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400708struct rpc_buffer {
709 size_t len;
710 char data[];
711};
712
Chuck Lever02107142006-01-03 09:55:49 +0100713/**
714 * rpc_malloc - allocate an RPC buffer
715 * @task: RPC task that will use this buffer
716 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400718 * To prevent rpciod from hanging, this allocator never sleeps,
719 * returning NULL if the request cannot be serviced immediately.
720 * The caller can arrange to sleep in a way that is safe for rpciod.
721 *
722 * Most requests are 'small' (under 2KiB) and can be serviced from a
723 * mempool, ensuring that NFS reads and writes can always proceed,
724 * and that there is good locality of reference for these buffers.
725 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400727 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400729void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400731 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400732 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400734 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400735 if (size <= RPC_BUFFER_MAXSIZE)
736 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400738 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200739
740 if (!buf)
741 return NULL;
742
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400743 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200744 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400745 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400746 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400748EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749
Chuck Lever02107142006-01-03 09:55:49 +0100750/**
751 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400752 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100753 *
754 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400755void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400757 size_t size;
758 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100759
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400760 if (!buffer)
761 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400762
763 buf = container_of(buffer, struct rpc_buffer, data);
764 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400765
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200766 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400767 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400768
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400769 if (size <= RPC_BUFFER_MAXSIZE)
770 mempool_free(buf, rpc_buffer_mempool);
771 else
772 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400774EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
776/*
777 * Creation and deletion of RPC task structures
778 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400779static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780{
781 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100782 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400783 task->tk_flags = task_setup_data->flags;
784 task->tk_ops = task_setup_data->callback_ops;
785 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400786 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787
788 /* Initialize retry counters */
789 task->tk_garb_retry = 2;
790 task->tk_cred_retry = 2;
791
Trond Myklebust3ff75762007-07-14 15:40:00 -0400792 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
793 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794
795 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500796 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
Trond Myklebust84115e12007-07-14 15:39:59 -0400798 task->tk_client = task_setup_data->rpc_client;
799 if (task->tk_client != NULL) {
800 kref_get(&task->tk_client->cl_kref);
801 if (task->tk_client->cl_softrtry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 task->tk_flags |= RPC_TASK_SOFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 }
804
Trond Myklebust84115e12007-07-14 15:39:59 -0400805 if (task->tk_ops->rpc_call_prepare != NULL)
806 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100807
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400808 if (task_setup_data->rpc_message != NULL) {
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400809 task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
810 task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
811 task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400812 /* Bind the user cred */
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400813 rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400814 if (task->tk_action == NULL)
815 rpc_call_start(task);
816 }
817
Chuck Leveref759a22006-03-20 13:44:17 -0500818 /* starting timestamp */
819 task->tk_start = jiffies;
820
Chuck Lever46121cf2007-01-31 12:14:08 -0500821 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700822 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823}
824
825static struct rpc_task *
826rpc_alloc_task(void)
827{
828 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
829}
830
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400832 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400834struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400836 struct rpc_task *task = setup_data->task;
837 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400839 if (task == NULL) {
840 task = rpc_alloc_task();
841 if (task == NULL)
842 goto out;
843 flags = RPC_TASK_DYNAMIC;
844 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845
Trond Myklebust84115e12007-07-14 15:39:59 -0400846 rpc_init_task(task, setup_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400848 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500849 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850out:
851 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852}
853
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500854static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100856 const struct rpc_call_ops *tk_ops = task->tk_ops;
857 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800859 if (task->tk_flags & RPC_TASK_DYNAMIC) {
860 dprintk("RPC: %5u freeing task\n", task->tk_pid);
861 mempool_free(task, rpc_task_mempool);
862 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500863 rpc_release_calldata(tk_ops, calldata);
864}
865
866static void rpc_async_release(struct work_struct *work)
867{
868 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
869}
870
871void rpc_put_task(struct rpc_task *task)
872{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500873 if (!atomic_dec_and_test(&task->tk_count))
874 return;
875 /* Release resources */
876 if (task->tk_rqstp)
877 xprt_release(task);
878 if (task->tk_msg.rpc_cred)
879 rpcauth_unbindcred(task);
880 if (task->tk_client) {
881 rpc_release_client(task->tk_client);
882 task->tk_client = NULL;
883 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500884 if (task->tk_workqueue != NULL) {
885 INIT_WORK(&task->u.tk_work, rpc_async_release);
886 queue_work(task->tk_workqueue, &task->u.tk_work);
887 } else
888 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500889}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400890EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500891
Trond Myklebustbde8f002007-01-24 11:54:53 -0800892static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500893{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894#ifdef RPC_DEBUG
895 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
896#endif
Chuck Lever46121cf2007-01-31 12:14:08 -0500897 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898
Trond Myklebust6529eba2007-06-14 16:40:14 -0400899 if (!list_empty(&task->tk_task)) {
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400900 struct rpc_clnt *clnt = task->tk_client;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400901 /* Remove from client task list */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400902 spin_lock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400903 list_del(&task->tk_task);
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400904 spin_unlock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400905 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908#ifdef RPC_DEBUG
909 task->tk_magic = 0;
910#endif
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500911 /* Wake up anyone who is waiting for task completion */
912 rpc_mark_complete_task(task);
913
914 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915}
916
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917/*
918 * Kill all tasks for the given client.
919 * XXX: kill their descendants as well?
920 */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400921void rpc_killall_tasks(struct rpc_clnt *clnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922{
923 struct rpc_task *rovr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400926 if (list_empty(&clnt->cl_tasks))
927 return;
928 dprintk("RPC: killing all tasks for client %p\n", clnt);
929 /*
930 * Spin lock all_tasks to prevent changes...
931 */
932 spin_lock(&clnt->cl_lock);
933 list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 if (! RPC_IS_ACTIVATED(rovr))
935 continue;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400936 if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 rovr->tk_flags |= RPC_TASK_KILLED;
938 rpc_exit(rovr, -EIO);
939 rpc_wake_up_task(rovr);
940 }
941 }
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400942 spin_unlock(&clnt->cl_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400944EXPORT_SYMBOL_GPL(rpc_killall_tasks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400946int rpciod_up(void)
947{
948 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
949}
950
951void rpciod_down(void)
952{
953 module_put(THIS_MODULE);
954}
955
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400957 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400959static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960{
961 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 /*
964 * Create the rpciod thread and wait for it to start.
965 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400966 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400969 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970}
971
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400972static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400974 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -0400975
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400976 if (rpciod_workqueue == NULL)
977 return;
Trond Myklebustab418d72007-06-14 17:08:36 -0400978 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400980 wq = rpciod_workqueue;
981 rpciod_workqueue = NULL;
982 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983}
984
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985void
986rpc_destroy_mempool(void)
987{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400988 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 if (rpc_buffer_mempool)
990 mempool_destroy(rpc_buffer_mempool);
991 if (rpc_task_mempool)
992 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -0700993 if (rpc_task_slabp)
994 kmem_cache_destroy(rpc_task_slabp);
995 if (rpc_buffer_slabp)
996 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500997 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998}
999
1000int
1001rpc_init_mempool(void)
1002{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001003 /*
1004 * The following is not strictly a mempool initialisation,
1005 * but there is no harm in doing it here
1006 */
1007 rpc_init_wait_queue(&delay_queue, "delayq");
1008 if (!rpciod_start())
1009 goto err_nomem;
1010
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1012 sizeof(struct rpc_task),
1013 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001014 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 if (!rpc_task_slabp)
1016 goto err_nomem;
1017 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1018 RPC_BUFFER_MAXSIZE,
1019 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001020 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 if (!rpc_buffer_slabp)
1022 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001023 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1024 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 if (!rpc_task_mempool)
1026 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001027 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1028 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 if (!rpc_buffer_mempool)
1030 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 return 0;
1032err_nomem:
1033 rpc_destroy_mempool();
1034 return -ENOMEM;
1035}