blob: 1102ce1251f7b61ad7d6a70a7df8826ce2a420c2 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080021#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#ifdef RPC_DEBUG
26#define RPCDBG_FACILITY RPCDBG_SCHED
27#define RPC_TASK_MAGIC_ID 0xf00baa
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#endif
29
30/*
31 * RPC slabs and memory pools
32 */
33#define RPC_BUFFER_MAXSIZE (2048)
34#define RPC_BUFFER_POOLSIZE (8)
35#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080036static struct kmem_cache *rpc_task_slabp __read_mostly;
37static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070038static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
David Howells65f27f32006-11-22 14:55:48 +000041static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080042static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040043static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * RPC tasks sit here while waiting for conditions to improve.
47 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040048static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
50/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 * rpciod-related stuff
52 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050053struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * Disable the timer for a given RPC task. Should be called with
57 * queue->lock and bh_disabled in order to avoid races within
58 * rpc_run_timer().
59 */
Trond Myklebust5d008372008-02-22 16:34:17 -050060static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050061__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040063 if (task->tk_timeout == 0)
64 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050065 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040067 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050068 if (list_empty(&queue->timer_list.list))
69 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040070}
71
72static void
73rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
74{
75 queue->timer_list.expires = expires;
76 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070077}
78
79/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 * Set up a timer for the current task.
81 */
Trond Myklebust5d008372008-02-22 16:34:17 -050082static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050083__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084{
85 if (!task->tk_timeout)
86 return;
87
Chuck Lever46121cf2007-01-31 12:14:08 -050088 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 task->tk_pid, task->tk_timeout * 1000 / HZ);
90
Trond Myklebusteb276c02008-02-22 17:27:59 -050091 task->u.tk_wait.expires = jiffies + task->tk_timeout;
92 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
93 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
94 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
96
97/*
98 * Add new request to a priority queue.
99 */
100static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
101{
102 struct list_head *q;
103 struct rpc_task *t;
104
105 INIT_LIST_HEAD(&task->u.tk_wait.links);
106 q = &queue->tasks[task->tk_priority];
107 if (unlikely(task->tk_priority > queue->maxpriority))
108 q = &queue->tasks[queue->maxpriority];
109 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400110 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
112 return;
113 }
114 }
115 list_add_tail(&task->u.tk_wait.list, q);
116}
117
118/*
119 * Add new request to wait queue.
120 *
121 * Swapper tasks always get inserted at the head of the queue.
122 * This should avoid many nasty memory deadlocks and hopefully
123 * improve overall performance.
124 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
125 */
126static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
127{
128 BUG_ON (RPC_IS_QUEUED(task));
129
130 if (RPC_IS_PRIORITY(queue))
131 __rpc_add_wait_queue_priority(queue, task);
132 else if (RPC_IS_SWAPPER(task))
133 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
134 else
135 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500136 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500137 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 rpc_set_queued(task);
139
Chuck Lever46121cf2007-01-31 12:14:08 -0500140 dprintk("RPC: %5u added to queue %p \"%s\"\n",
141 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142}
143
144/*
145 * Remove request from a priority queue.
146 */
147static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
148{
149 struct rpc_task *t;
150
151 if (!list_empty(&task->u.tk_wait.links)) {
152 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
153 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
154 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
155 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156}
157
158/*
159 * Remove request from queue.
160 * Note: must be called with spin lock held.
161 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500162static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500164 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 if (RPC_IS_PRIORITY(queue))
166 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400167 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500168 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500169 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
170 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172
173static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
174{
175 queue->priority = priority;
176 queue->count = 1 << (priority * 2);
177}
178
Trond Myklebust3ff75762007-07-14 15:40:00 -0400179static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400181 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 queue->nr = RPC_BATCH_COUNT;
183}
184
185static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
186{
187 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400188 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189}
190
Trond Myklebust3ff75762007-07-14 15:40:00 -0400191static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
193 int i;
194
195 spin_lock_init(&queue->lock);
196 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
197 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400198 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400200 queue->qlen = 0;
201 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
202 INIT_LIST_HEAD(&queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203#ifdef RPC_DEBUG
204 queue->name = qname;
205#endif
206}
207
208void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
209{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400210 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
213void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
214{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400215 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400217EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500219void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
220{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400221 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500222}
223EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
224
Matthew Wilcox150030b2007-12-06 16:24:39 -0500225static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100226{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500227 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100228 return -ERESTARTSYS;
229 schedule();
230 return 0;
231}
232
Trond Myklebustc44fe702007-06-16 14:17:01 -0400233#ifdef RPC_DEBUG
234static void rpc_task_set_debuginfo(struct rpc_task *task)
235{
236 static atomic_t rpc_pid;
237
238 task->tk_magic = RPC_TASK_MAGIC_ID;
239 task->tk_pid = atomic_inc_return(&rpc_pid);
240}
241#else
242static inline void rpc_task_set_debuginfo(struct rpc_task *task)
243{
244}
245#endif
246
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500247static void rpc_set_active(struct rpc_task *task)
248{
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400249 struct rpc_clnt *clnt;
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500250 if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
251 return;
Trond Myklebustc44fe702007-06-16 14:17:01 -0400252 rpc_task_set_debuginfo(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500253 /* Add to global list of all tasks */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400254 clnt = task->tk_client;
255 if (clnt != NULL) {
256 spin_lock(&clnt->cl_lock);
257 list_add_tail(&task->tk_task, &clnt->cl_tasks);
258 spin_unlock(&clnt->cl_lock);
259 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500260}
261
Trond Myklebust44c28872006-01-03 09:55:06 +0100262/*
263 * Mark an RPC call as having completed by clearing the 'active' bit
264 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500265static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100266{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500267 smp_mb__before_clear_bit();
268 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
269 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100270 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
271}
272
273/*
274 * Allow callers to wait for completion of an RPC call
275 */
276int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
277{
278 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500279 action = rpc_wait_bit_killable;
Trond Myklebust44c28872006-01-03 09:55:06 +0100280 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500281 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100282}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400283EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100284
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285/*
286 * Make an RPC task runnable.
287 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800288 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 * the spinlock held to protect the wait queue operation.
290 */
291static void rpc_make_runnable(struct rpc_task *task)
292{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 rpc_clear_queued(task);
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100294 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 return;
296 if (RPC_IS_ASYNC(task)) {
297 int status;
298
David Howells65f27f32006-11-22 14:55:48 +0000299 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500300 status = queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 if (status < 0) {
302 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
303 task->tk_status = status;
304 return;
305 }
306 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000307 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308}
309
310/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 * Prepare for sleeping on a wait queue.
312 * By always appending tasks to the list we ensure FIFO behavior.
313 * NB: An RPC task will only receive interrupt-driven events as long
314 * as it's on a wait queue.
315 */
316static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500317 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318{
Chuck Lever46121cf2007-01-31 12:14:08 -0500319 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
320 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
322 if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
323 printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
324 return;
325 }
326
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 __rpc_add_wait_queue(q, task);
328
329 BUG_ON(task->tk_callback != NULL);
330 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500331 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332}
333
334void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500335 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500337 /* Mark the task as being activated if so needed */
338 rpc_set_active(task);
339
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 /*
341 * Protect the queue operations.
342 */
343 spin_lock_bh(&q->lock);
Trond Myklebust5d008372008-02-22 16:34:17 -0500344 __rpc_sleep_on(q, task, action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 spin_unlock_bh(&q->lock);
346}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400347EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348
349/**
350 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500351 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 * @task: task to be woken up
353 *
354 * Caller must hold queue->lock, and have cleared the task queued flag.
355 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500356static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357{
Chuck Lever46121cf2007-01-31 12:14:08 -0500358 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
359 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
361#ifdef RPC_DEBUG
362 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
363#endif
364 /* Has the task been executed yet? If not, we cannot wake it up! */
365 if (!RPC_IS_ACTIVATED(task)) {
366 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
367 return;
368 }
369
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500370 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371
372 rpc_make_runnable(task);
373
Chuck Lever46121cf2007-01-31 12:14:08 -0500374 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375}
376
377/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500378 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500380static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800382 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
383 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385
386/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500387 * Wake up a task on a specific queue
388 */
389void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
390{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800391 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500392 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800393 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500394}
395EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
396
397/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 * Wake up the specified task
399 */
Trond Myklebustfda13932008-02-22 16:34:12 -0500400static void rpc_wake_up_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401{
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500402 rpc_wake_up_queued_task(task->tk_waitqueue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403}
404
405/*
406 * Wake up the next task on a priority queue.
407 */
408static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
409{
410 struct list_head *q;
411 struct rpc_task *task;
412
413 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400414 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 */
416 q = &queue->tasks[queue->priority];
417 if (!list_empty(q)) {
418 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400419 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 if (--queue->nr)
421 goto out;
422 list_move_tail(&task->u.tk_wait.list, q);
423 }
424 /*
425 * Check if we need to switch queues.
426 */
427 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400428 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 }
430
431 /*
432 * Service the next queue.
433 */
434 do {
435 if (q == &queue->tasks[0])
436 q = &queue->tasks[queue->maxpriority];
437 else
438 q = q - 1;
439 if (!list_empty(q)) {
440 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
441 goto new_queue;
442 }
443 } while (q != &queue->tasks[queue->priority]);
444
445 rpc_reset_waitqueue_priority(queue);
446 return NULL;
447
448new_queue:
449 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400450new_owner:
451 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452out:
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500453 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 return task;
455}
456
457/*
458 * Wake up the next task on the wait queue.
459 */
460struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
461{
462 struct rpc_task *task = NULL;
463
Chuck Lever46121cf2007-01-31 12:14:08 -0500464 dprintk("RPC: wake_up_next(%p \"%s\")\n",
465 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800466 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 if (RPC_IS_PRIORITY(queue))
468 task = __rpc_wake_up_next_priority(queue);
469 else {
470 task_for_first(task, &queue->tasks[0])
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500471 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800473 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
475 return task;
476}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400477EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478
479/**
480 * rpc_wake_up - wake up all rpc_tasks
481 * @queue: rpc_wait_queue on which the tasks are sleeping
482 *
483 * Grabs queue->lock
484 */
485void rpc_wake_up(struct rpc_wait_queue *queue)
486{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800487 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800489
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800490 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 head = &queue->tasks[queue->maxpriority];
492 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800493 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500494 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 if (head == &queue->tasks[0])
496 break;
497 head--;
498 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800499 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400501EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
503/**
504 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
505 * @queue: rpc_wait_queue on which the tasks are sleeping
506 * @status: status value to set
507 *
508 * Grabs queue->lock
509 */
510void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
511{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800512 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800515 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 head = &queue->tasks[queue->maxpriority];
517 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800518 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500520 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 }
522 if (head == &queue->tasks[0])
523 break;
524 head--;
525 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800526 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400528EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400530static void __rpc_queue_timer_fn(unsigned long ptr)
531{
532 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
533 struct rpc_task *task, *n;
534 unsigned long expires, now, timeo;
535
536 spin_lock(&queue->lock);
537 expires = now = jiffies;
538 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
539 timeo = task->u.tk_wait.expires;
540 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400541 dprintk("RPC: %5u timeout\n", task->tk_pid);
542 task->tk_status = -ETIMEDOUT;
543 rpc_wake_up_task_queue_locked(queue, task);
544 continue;
545 }
546 if (expires == now || time_after(expires, timeo))
547 expires = timeo;
548 }
549 if (!list_empty(&queue->timer_list.list))
550 rpc_set_queue_timer(queue, expires);
551 spin_unlock(&queue->lock);
552}
553
Trond Myklebust80147932006-08-31 18:24:08 -0400554static void __rpc_atrun(struct rpc_task *task)
555{
Trond Myklebust5d008372008-02-22 16:34:17 -0500556 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400557}
558
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559/*
560 * Run a task at a later time
561 */
Trond Myklebust80147932006-08-31 18:24:08 -0400562void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563{
564 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500565 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400567EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100570 * Helper to call task->tk_ops->rpc_call_prepare
571 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400572void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100573{
574 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
575}
576
577/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100578 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000579 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100580void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000581{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100582 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100583 if (task->tk_ops->rpc_call_done != NULL) {
584 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000585 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100586 WARN_ON(RPC_ASSASSINATED(task));
587 /* Always release the RPC slot and buffer memory */
588 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000589 }
590 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000591}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400592EXPORT_SYMBOL_GPL(rpc_exit_task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000593
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400594void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
595{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400596 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400597 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400598}
599
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000600/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 * This is the RPC `scheduler' (or rather, the finite state machine).
602 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800603static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400605 struct rpc_wait_queue *queue;
606 int task_is_async = RPC_IS_ASYNC(task);
607 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608
Chuck Lever46121cf2007-01-31 12:14:08 -0500609 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
610 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611
612 BUG_ON(RPC_IS_QUEUED(task));
613
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000614 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
616 /*
617 * Execute any pending callback.
618 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400619 if (task->tk_callback) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800621
622 /*
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400623 * We set tk_callback to NULL before calling it,
624 * in case it sets the tk_callback field itself:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400626 save_callback = task->tk_callback;
627 task->tk_callback = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630
631 /*
632 * Perform the next FSM step.
633 * tk_action may be NULL when the task has been killed
634 * by someone else.
635 */
636 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100637 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100639 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 }
641
642 /*
643 * Lockless check for whether task is sleeping or not.
644 */
645 if (!RPC_IS_QUEUED(task))
646 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400647 /*
648 * The queue->lock protects against races with
649 * rpc_make_runnable().
650 *
651 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
652 * rpc_task, rpc_make_runnable() can assign it to a
653 * different workqueue. We therefore cannot assume that the
654 * rpc_task pointer may still be dereferenced.
655 */
656 queue = task->tk_waitqueue;
657 spin_lock_bh(&queue->lock);
658 if (!RPC_IS_QUEUED(task)) {
659 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 continue;
661 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400662 rpc_clear_running(task);
663 spin_unlock_bh(&queue->lock);
664 if (task_is_async)
665 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
667 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500668 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000669 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500670 RPC_TASK_QUEUED, rpc_wait_bit_killable,
671 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000672 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 /*
674 * When a sync task receives a signal, it exits with
675 * -ERESTARTSYS. In order to catch any callbacks that
676 * clean up after sleeping on some queue, we don't
677 * break the loop here, but go around once more.
678 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500679 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000680 task->tk_flags |= RPC_TASK_KILLED;
681 rpc_exit(task, -ERESTARTSYS);
682 rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 }
684 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500685 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 }
687
Chuck Lever46121cf2007-01-31 12:14:08 -0500688 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
689 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690 /* Release all resources associated with the task */
691 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692}
693
694/*
695 * User-visible entry point to the scheduler.
696 *
697 * This may be called recursively if e.g. an async NFS task updates
698 * the attributes and finds that dirty pages must be flushed.
699 * NOTE: Upon exit of this function the task is guaranteed to be
700 * released. In particular note that tk_release() will have
701 * been called, so your task memory may have been freed.
702 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800703void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704{
Trond Myklebust44c28872006-01-03 09:55:06 +0100705 rpc_set_active(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 rpc_set_running(task);
Trond Myklebust2efef832007-02-03 13:38:41 -0800707 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708}
709
David Howells65f27f32006-11-22 14:55:48 +0000710static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711{
David Howells65f27f32006-11-22 14:55:48 +0000712 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713}
714
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400715struct rpc_buffer {
716 size_t len;
717 char data[];
718};
719
Chuck Lever02107142006-01-03 09:55:49 +0100720/**
721 * rpc_malloc - allocate an RPC buffer
722 * @task: RPC task that will use this buffer
723 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400725 * To prevent rpciod from hanging, this allocator never sleeps,
726 * returning NULL if the request cannot be serviced immediately.
727 * The caller can arrange to sleep in a way that is safe for rpciod.
728 *
729 * Most requests are 'small' (under 2KiB) and can be serviced from a
730 * mempool, ensuring that NFS reads and writes can always proceed,
731 * and that there is good locality of reference for these buffers.
732 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400734 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400736void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400738 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400739 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400741 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400742 if (size <= RPC_BUFFER_MAXSIZE)
743 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400745 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200746
747 if (!buf)
748 return NULL;
749
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400750 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200751 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400752 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400753 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400755EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
Chuck Lever02107142006-01-03 09:55:49 +0100757/**
758 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400759 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100760 *
761 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400762void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400764 size_t size;
765 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100766
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400767 if (!buffer)
768 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400769
770 buf = container_of(buffer, struct rpc_buffer, data);
771 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400772
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200773 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400774 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400775
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400776 if (size <= RPC_BUFFER_MAXSIZE)
777 mempool_free(buf, rpc_buffer_mempool);
778 else
779 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400781EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782
783/*
784 * Creation and deletion of RPC task structures
785 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400786static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787{
788 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100789 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400790 task->tk_flags = task_setup_data->flags;
791 task->tk_ops = task_setup_data->callback_ops;
792 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400793 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794
795 /* Initialize retry counters */
796 task->tk_garb_retry = 2;
797 task->tk_cred_retry = 2;
798
Trond Myklebust3ff75762007-07-14 15:40:00 -0400799 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
800 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
802 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500803 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
Trond Myklebust84115e12007-07-14 15:39:59 -0400805 task->tk_client = task_setup_data->rpc_client;
806 if (task->tk_client != NULL) {
807 kref_get(&task->tk_client->cl_kref);
808 if (task->tk_client->cl_softrtry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 task->tk_flags |= RPC_TASK_SOFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 }
811
Trond Myklebust84115e12007-07-14 15:39:59 -0400812 if (task->tk_ops->rpc_call_prepare != NULL)
813 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100814
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400815 if (task_setup_data->rpc_message != NULL) {
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400816 task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
817 task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
818 task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400819 /* Bind the user cred */
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400820 rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400821 if (task->tk_action == NULL)
822 rpc_call_start(task);
823 }
824
Chuck Leveref759a22006-03-20 13:44:17 -0500825 /* starting timestamp */
826 task->tk_start = jiffies;
827
Chuck Lever46121cf2007-01-31 12:14:08 -0500828 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700829 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830}
831
832static struct rpc_task *
833rpc_alloc_task(void)
834{
835 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
836}
837
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400839 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400841struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400843 struct rpc_task *task = setup_data->task;
844 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400846 if (task == NULL) {
847 task = rpc_alloc_task();
848 if (task == NULL)
849 goto out;
850 flags = RPC_TASK_DYNAMIC;
851 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852
Trond Myklebust84115e12007-07-14 15:39:59 -0400853 rpc_init_task(task, setup_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400855 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500856 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857out:
858 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859}
860
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500861static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100863 const struct rpc_call_ops *tk_ops = task->tk_ops;
864 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800866 if (task->tk_flags & RPC_TASK_DYNAMIC) {
867 dprintk("RPC: %5u freeing task\n", task->tk_pid);
868 mempool_free(task, rpc_task_mempool);
869 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500870 rpc_release_calldata(tk_ops, calldata);
871}
872
873static void rpc_async_release(struct work_struct *work)
874{
875 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
876}
877
878void rpc_put_task(struct rpc_task *task)
879{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500880 if (!atomic_dec_and_test(&task->tk_count))
881 return;
882 /* Release resources */
883 if (task->tk_rqstp)
884 xprt_release(task);
885 if (task->tk_msg.rpc_cred)
886 rpcauth_unbindcred(task);
887 if (task->tk_client) {
888 rpc_release_client(task->tk_client);
889 task->tk_client = NULL;
890 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500891 if (task->tk_workqueue != NULL) {
892 INIT_WORK(&task->u.tk_work, rpc_async_release);
893 queue_work(task->tk_workqueue, &task->u.tk_work);
894 } else
895 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500896}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400897EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500898
Trond Myklebustbde8f002007-01-24 11:54:53 -0800899static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500900{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901#ifdef RPC_DEBUG
902 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
903#endif
Chuck Lever46121cf2007-01-31 12:14:08 -0500904 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Trond Myklebust6529eba2007-06-14 16:40:14 -0400906 if (!list_empty(&task->tk_task)) {
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400907 struct rpc_clnt *clnt = task->tk_client;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400908 /* Remove from client task list */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400909 spin_lock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400910 list_del(&task->tk_task);
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400911 spin_unlock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400912 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915#ifdef RPC_DEBUG
916 task->tk_magic = 0;
917#endif
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500918 /* Wake up anyone who is waiting for task completion */
919 rpc_mark_complete_task(task);
920
921 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922}
923
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924/*
925 * Kill all tasks for the given client.
926 * XXX: kill their descendants as well?
927 */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400928void rpc_killall_tasks(struct rpc_clnt *clnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
930 struct rpc_task *rovr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400933 if (list_empty(&clnt->cl_tasks))
934 return;
935 dprintk("RPC: killing all tasks for client %p\n", clnt);
936 /*
937 * Spin lock all_tasks to prevent changes...
938 */
939 spin_lock(&clnt->cl_lock);
940 list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 if (! RPC_IS_ACTIVATED(rovr))
942 continue;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400943 if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 rovr->tk_flags |= RPC_TASK_KILLED;
945 rpc_exit(rovr, -EIO);
946 rpc_wake_up_task(rovr);
947 }
948 }
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400949 spin_unlock(&clnt->cl_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400951EXPORT_SYMBOL_GPL(rpc_killall_tasks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400953int rpciod_up(void)
954{
955 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
956}
957
958void rpciod_down(void)
959{
960 module_put(THIS_MODULE);
961}
962
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400964 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400966static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967{
968 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 /*
971 * Create the rpciod thread and wait for it to start.
972 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400973 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400976 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977}
978
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400979static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400981 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -0400982
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400983 if (rpciod_workqueue == NULL)
984 return;
Trond Myklebustab418d72007-06-14 17:08:36 -0400985 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400987 wq = rpciod_workqueue;
988 rpciod_workqueue = NULL;
989 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990}
991
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992void
993rpc_destroy_mempool(void)
994{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400995 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 if (rpc_buffer_mempool)
997 mempool_destroy(rpc_buffer_mempool);
998 if (rpc_task_mempool)
999 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001000 if (rpc_task_slabp)
1001 kmem_cache_destroy(rpc_task_slabp);
1002 if (rpc_buffer_slabp)
1003 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001004 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005}
1006
1007int
1008rpc_init_mempool(void)
1009{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001010 /*
1011 * The following is not strictly a mempool initialisation,
1012 * but there is no harm in doing it here
1013 */
1014 rpc_init_wait_queue(&delay_queue, "delayq");
1015 if (!rpciod_start())
1016 goto err_nomem;
1017
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1019 sizeof(struct rpc_task),
1020 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001021 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 if (!rpc_task_slabp)
1023 goto err_nomem;
1024 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1025 RPC_BUFFER_MAXSIZE,
1026 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001027 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028 if (!rpc_buffer_slabp)
1029 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001030 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1031 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 if (!rpc_task_mempool)
1033 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001034 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1035 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 if (!rpc_buffer_mempool)
1037 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038 return 0;
1039err_nomem:
1040 rpc_destroy_mempool();
1041 return -ENOMEM;
1042}