blob: aae6907fd546916317e69615dc1cb5e8040259c4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080020#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Benny Halevy69518672009-09-10 12:25:04 +030024#include "sunrpc.h"
25
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#ifdef RPC_DEBUG
27#define RPCDBG_FACILITY RPCDBG_SCHED
28#define RPC_TASK_MAGIC_ID 0xf00baa
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#endif
30
31/*
32 * RPC slabs and memory pools
33 */
34#define RPC_BUFFER_MAXSIZE (2048)
35#define RPC_BUFFER_POOLSIZE (8)
36#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080037static struct kmem_cache *rpc_task_slabp __read_mostly;
38static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070039static mempool_t *rpc_task_mempool __read_mostly;
40static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
David Howells65f27f32006-11-22 14:55:48 +000042static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080043static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040044static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
46/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 * RPC tasks sit here while waiting for conditions to improve.
48 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040049static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070052 * rpciod-related stuff
53 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050054struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
56/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070057 * Disable the timer for a given RPC task. Should be called with
58 * queue->lock and bh_disabled in order to avoid races within
59 * rpc_run_timer().
60 */
Trond Myklebust5d008372008-02-22 16:34:17 -050061static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050062__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070063{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040064 if (task->tk_timeout == 0)
65 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050066 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040068 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050069 if (list_empty(&queue->timer_list.list))
70 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040071}
72
73static void
74rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
75{
76 queue->timer_list.expires = expires;
77 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078}
79
80/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 * Set up a timer for the current task.
82 */
Trond Myklebust5d008372008-02-22 16:34:17 -050083static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050084__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085{
86 if (!task->tk_timeout)
87 return;
88
Chuck Lever46121cf2007-01-31 12:14:08 -050089 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 task->tk_pid, task->tk_timeout * 1000 / HZ);
91
Trond Myklebusteb276c02008-02-22 17:27:59 -050092 task->u.tk_wait.expires = jiffies + task->tk_timeout;
93 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
94 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
95 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070096}
97
98/*
99 * Add new request to a priority queue.
100 */
101static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
102{
103 struct list_head *q;
104 struct rpc_task *t;
105
106 INIT_LIST_HEAD(&task->u.tk_wait.links);
107 q = &queue->tasks[task->tk_priority];
108 if (unlikely(task->tk_priority > queue->maxpriority))
109 q = &queue->tasks[queue->maxpriority];
110 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400111 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
113 return;
114 }
115 }
116 list_add_tail(&task->u.tk_wait.list, q);
117}
118
119/*
120 * Add new request to wait queue.
121 *
122 * Swapper tasks always get inserted at the head of the queue.
123 * This should avoid many nasty memory deadlocks and hopefully
124 * improve overall performance.
125 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
126 */
127static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
128{
129 BUG_ON (RPC_IS_QUEUED(task));
130
131 if (RPC_IS_PRIORITY(queue))
132 __rpc_add_wait_queue_priority(queue, task);
133 else if (RPC_IS_SWAPPER(task))
134 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
135 else
136 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500137 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500138 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 rpc_set_queued(task);
140
Chuck Lever46121cf2007-01-31 12:14:08 -0500141 dprintk("RPC: %5u added to queue %p \"%s\"\n",
142 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
145/*
146 * Remove request from a priority queue.
147 */
148static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
149{
150 struct rpc_task *t;
151
152 if (!list_empty(&task->u.tk_wait.links)) {
153 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
154 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
155 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
156 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157}
158
159/*
160 * Remove request from queue.
161 * Note: must be called with spin lock held.
162 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500163static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500165 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 if (RPC_IS_PRIORITY(queue))
167 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400168 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500169 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500170 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
171 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172}
173
174static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
175{
176 queue->priority = priority;
177 queue->count = 1 << (priority * 2);
178}
179
Trond Myklebust3ff75762007-07-14 15:40:00 -0400180static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400182 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 queue->nr = RPC_BATCH_COUNT;
184}
185
186static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
187{
188 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400189 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190}
191
Trond Myklebust3ff75762007-07-14 15:40:00 -0400192static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193{
194 int i;
195
196 spin_lock_init(&queue->lock);
197 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
198 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400199 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400201 queue->qlen = 0;
202 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
203 INIT_LIST_HEAD(&queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204#ifdef RPC_DEBUG
205 queue->name = qname;
206#endif
207}
208
209void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
210{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400211 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212}
Alexandros Batsakis689cf5c2009-12-14 21:27:56 -0800213EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
215void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
216{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400217 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400219EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500221void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
222{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400223 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500224}
225EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
226
Matthew Wilcox150030b2007-12-06 16:24:39 -0500227static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100228{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500229 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100230 return -ERESTARTSYS;
231 schedule();
232 return 0;
233}
234
Trond Myklebustc44fe702007-06-16 14:17:01 -0400235#ifdef RPC_DEBUG
236static void rpc_task_set_debuginfo(struct rpc_task *task)
237{
238 static atomic_t rpc_pid;
239
240 task->tk_magic = RPC_TASK_MAGIC_ID;
241 task->tk_pid = atomic_inc_return(&rpc_pid);
242}
243#else
244static inline void rpc_task_set_debuginfo(struct rpc_task *task)
245{
246}
247#endif
248
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500249static void rpc_set_active(struct rpc_task *task)
250{
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400251 struct rpc_clnt *clnt;
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500252 if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
253 return;
Trond Myklebustc44fe702007-06-16 14:17:01 -0400254 rpc_task_set_debuginfo(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500255 /* Add to global list of all tasks */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400256 clnt = task->tk_client;
257 if (clnt != NULL) {
258 spin_lock(&clnt->cl_lock);
259 list_add_tail(&task->tk_task, &clnt->cl_tasks);
260 spin_unlock(&clnt->cl_lock);
261 }
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500262}
263
Trond Myklebust44c28872006-01-03 09:55:06 +0100264/*
265 * Mark an RPC call as having completed by clearing the 'active' bit
266 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500267static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100268{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500269 smp_mb__before_clear_bit();
270 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
271 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100272 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
273}
274
275/*
276 * Allow callers to wait for completion of an RPC call
277 */
278int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
279{
280 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500281 action = rpc_wait_bit_killable;
Trond Myklebust44c28872006-01-03 09:55:06 +0100282 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500283 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100284}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400285EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287/*
288 * Make an RPC task runnable.
289 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800290 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 * the spinlock held to protect the wait queue operation.
292 */
293static void rpc_make_runnable(struct rpc_task *task)
294{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 rpc_clear_queued(task);
Christophe Saoutcc4dc592006-11-05 18:42:48 +0100296 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 return;
298 if (RPC_IS_ASYNC(task)) {
299 int status;
300
David Howells65f27f32006-11-22 14:55:48 +0000301 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500302 status = queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 if (status < 0) {
304 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
305 task->tk_status = status;
306 return;
307 }
308 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000309 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310}
311
312/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 * Prepare for sleeping on a wait queue.
314 * By always appending tasks to the list we ensure FIFO behavior.
315 * NB: An RPC task will only receive interrupt-driven events as long
316 * as it's on a wait queue.
317 */
318static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500319 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320{
Chuck Lever46121cf2007-01-31 12:14:08 -0500321 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
322 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
324 if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
325 printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
326 return;
327 }
328
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 __rpc_add_wait_queue(q, task);
330
331 BUG_ON(task->tk_callback != NULL);
332 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500333 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500337 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500339 /* Mark the task as being activated if so needed */
340 rpc_set_active(task);
341
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 /*
343 * Protect the queue operations.
344 */
345 spin_lock_bh(&q->lock);
Trond Myklebust5d008372008-02-22 16:34:17 -0500346 __rpc_sleep_on(q, task, action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 spin_unlock_bh(&q->lock);
348}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400349EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
351/**
352 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500353 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 * @task: task to be woken up
355 *
356 * Caller must hold queue->lock, and have cleared the task queued flag.
357 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500358static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359{
Chuck Lever46121cf2007-01-31 12:14:08 -0500360 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
361 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
363#ifdef RPC_DEBUG
364 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
365#endif
366 /* Has the task been executed yet? If not, we cannot wake it up! */
367 if (!RPC_IS_ACTIVATED(task)) {
368 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
369 return;
370 }
371
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500372 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373
374 rpc_make_runnable(task);
375
Chuck Lever46121cf2007-01-31 12:14:08 -0500376 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377}
378
379/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500380 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500382static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800384 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
385 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
388/*
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800389 * Tests whether rpc queue is empty
390 */
391int rpc_queue_empty(struct rpc_wait_queue *queue)
392{
393 int res;
394
395 spin_lock_bh(&queue->lock);
396 res = queue->qlen;
397 spin_unlock_bh(&queue->lock);
398 return (res == 0);
399}
400EXPORT_SYMBOL_GPL(rpc_queue_empty);
401
402/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500403 * Wake up a task on a specific queue
404 */
405void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
406{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800407 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500408 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800409 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500410}
411EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
412
413/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 * Wake up the specified task
415 */
Trond Myklebustfda13932008-02-22 16:34:12 -0500416static void rpc_wake_up_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417{
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500418 rpc_wake_up_queued_task(task->tk_waitqueue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419}
420
421/*
422 * Wake up the next task on a priority queue.
423 */
424static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
425{
426 struct list_head *q;
427 struct rpc_task *task;
428
429 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400430 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 */
432 q = &queue->tasks[queue->priority];
433 if (!list_empty(q)) {
434 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400435 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 if (--queue->nr)
437 goto out;
438 list_move_tail(&task->u.tk_wait.list, q);
439 }
440 /*
441 * Check if we need to switch queues.
442 */
443 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400444 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 }
446
447 /*
448 * Service the next queue.
449 */
450 do {
451 if (q == &queue->tasks[0])
452 q = &queue->tasks[queue->maxpriority];
453 else
454 q = q - 1;
455 if (!list_empty(q)) {
456 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
457 goto new_queue;
458 }
459 } while (q != &queue->tasks[queue->priority]);
460
461 rpc_reset_waitqueue_priority(queue);
462 return NULL;
463
464new_queue:
465 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400466new_owner:
467 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468out:
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500469 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 return task;
471}
472
473/*
474 * Wake up the next task on the wait queue.
475 */
476struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
477{
478 struct rpc_task *task = NULL;
479
Chuck Lever46121cf2007-01-31 12:14:08 -0500480 dprintk("RPC: wake_up_next(%p \"%s\")\n",
481 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800482 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 if (RPC_IS_PRIORITY(queue))
484 task = __rpc_wake_up_next_priority(queue);
485 else {
486 task_for_first(task, &queue->tasks[0])
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500487 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800489 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
491 return task;
492}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400493EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494
495/**
496 * rpc_wake_up - wake up all rpc_tasks
497 * @queue: rpc_wait_queue on which the tasks are sleeping
498 *
499 * Grabs queue->lock
500 */
501void rpc_wake_up(struct rpc_wait_queue *queue)
502{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800503 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800505
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800506 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 head = &queue->tasks[queue->maxpriority];
508 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800509 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500510 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 if (head == &queue->tasks[0])
512 break;
513 head--;
514 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800515 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400517EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
519/**
520 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
521 * @queue: rpc_wait_queue on which the tasks are sleeping
522 * @status: status value to set
523 *
524 * Grabs queue->lock
525 */
526void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
527{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800528 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800531 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 head = &queue->tasks[queue->maxpriority];
533 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800534 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500536 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 }
538 if (head == &queue->tasks[0])
539 break;
540 head--;
541 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800542 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400544EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400546static void __rpc_queue_timer_fn(unsigned long ptr)
547{
548 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
549 struct rpc_task *task, *n;
550 unsigned long expires, now, timeo;
551
552 spin_lock(&queue->lock);
553 expires = now = jiffies;
554 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
555 timeo = task->u.tk_wait.expires;
556 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400557 dprintk("RPC: %5u timeout\n", task->tk_pid);
558 task->tk_status = -ETIMEDOUT;
559 rpc_wake_up_task_queue_locked(queue, task);
560 continue;
561 }
562 if (expires == now || time_after(expires, timeo))
563 expires = timeo;
564 }
565 if (!list_empty(&queue->timer_list.list))
566 rpc_set_queue_timer(queue, expires);
567 spin_unlock(&queue->lock);
568}
569
Trond Myklebust80147932006-08-31 18:24:08 -0400570static void __rpc_atrun(struct rpc_task *task)
571{
Trond Myklebust5d008372008-02-22 16:34:17 -0500572 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400573}
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575/*
576 * Run a task at a later time
577 */
Trond Myklebust80147932006-08-31 18:24:08 -0400578void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579{
580 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500581 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400583EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100586 * Helper to call task->tk_ops->rpc_call_prepare
587 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400588void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100589{
590 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
591}
592
593/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100594 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000595 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100596void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000597{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100598 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100599 if (task->tk_ops->rpc_call_done != NULL) {
600 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000601 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100602 WARN_ON(RPC_ASSASSINATED(task));
603 /* Always release the RPC slot and buffer memory */
604 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000605 }
606 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000607}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400608EXPORT_SYMBOL_GPL(rpc_exit_task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000609
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400610void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
611{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400612 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400613 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400614}
615
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000616/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 * This is the RPC `scheduler' (or rather, the finite state machine).
618 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800619static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400621 struct rpc_wait_queue *queue;
622 int task_is_async = RPC_IS_ASYNC(task);
623 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
Chuck Lever46121cf2007-01-31 12:14:08 -0500625 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
626 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
628 BUG_ON(RPC_IS_QUEUED(task));
629
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000630 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
632 /*
633 * Execute any pending callback.
634 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400635 if (task->tk_callback) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800637
638 /*
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400639 * We set tk_callback to NULL before calling it,
640 * in case it sets the tk_callback field itself:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400642 save_callback = task->tk_callback;
643 task->tk_callback = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 }
646
647 /*
648 * Perform the next FSM step.
649 * tk_action may be NULL when the task has been killed
650 * by someone else.
651 */
652 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100653 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100655 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 }
657
658 /*
659 * Lockless check for whether task is sleeping or not.
660 */
661 if (!RPC_IS_QUEUED(task))
662 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400663 /*
664 * The queue->lock protects against races with
665 * rpc_make_runnable().
666 *
667 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
668 * rpc_task, rpc_make_runnable() can assign it to a
669 * different workqueue. We therefore cannot assume that the
670 * rpc_task pointer may still be dereferenced.
671 */
672 queue = task->tk_waitqueue;
673 spin_lock_bh(&queue->lock);
674 if (!RPC_IS_QUEUED(task)) {
675 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 continue;
677 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400678 rpc_clear_running(task);
679 spin_unlock_bh(&queue->lock);
680 if (task_is_async)
681 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
683 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500684 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000685 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500686 RPC_TASK_QUEUED, rpc_wait_bit_killable,
687 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000688 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 /*
690 * When a sync task receives a signal, it exits with
691 * -ERESTARTSYS. In order to catch any callbacks that
692 * clean up after sleeping on some queue, we don't
693 * break the loop here, but go around once more.
694 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500695 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000696 task->tk_flags |= RPC_TASK_KILLED;
697 rpc_exit(task, -ERESTARTSYS);
698 rpc_wake_up_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 }
700 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500701 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 }
703
Chuck Lever46121cf2007-01-31 12:14:08 -0500704 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
705 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 /* Release all resources associated with the task */
707 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708}
709
710/*
711 * User-visible entry point to the scheduler.
712 *
713 * This may be called recursively if e.g. an async NFS task updates
714 * the attributes and finds that dirty pages must be flushed.
715 * NOTE: Upon exit of this function the task is guaranteed to be
716 * released. In particular note that tk_release() will have
717 * been called, so your task memory may have been freed.
718 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800719void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720{
Trond Myklebust44c28872006-01-03 09:55:06 +0100721 rpc_set_active(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 rpc_set_running(task);
Trond Myklebust2efef832007-02-03 13:38:41 -0800723 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724}
725
David Howells65f27f32006-11-22 14:55:48 +0000726static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727{
David Howells65f27f32006-11-22 14:55:48 +0000728 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729}
730
Chuck Lever02107142006-01-03 09:55:49 +0100731/**
732 * rpc_malloc - allocate an RPC buffer
733 * @task: RPC task that will use this buffer
734 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400736 * To prevent rpciod from hanging, this allocator never sleeps,
737 * returning NULL if the request cannot be serviced immediately.
738 * The caller can arrange to sleep in a way that is safe for rpciod.
739 *
740 * Most requests are 'small' (under 2KiB) and can be serviced from a
741 * mempool, ensuring that NFS reads and writes can always proceed,
742 * and that there is good locality of reference for these buffers.
743 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400745 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400747void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400749 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400750 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400752 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400753 if (size <= RPC_BUFFER_MAXSIZE)
754 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400756 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200757
758 if (!buf)
759 return NULL;
760
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400761 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200762 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400763 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400764 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400766EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767
Chuck Lever02107142006-01-03 09:55:49 +0100768/**
769 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400770 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100771 *
772 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400773void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400775 size_t size;
776 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100777
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400778 if (!buffer)
779 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400780
781 buf = container_of(buffer, struct rpc_buffer, data);
782 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400783
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200784 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400785 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400786
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400787 if (size <= RPC_BUFFER_MAXSIZE)
788 mempool_free(buf, rpc_buffer_mempool);
789 else
790 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400792EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
794/*
795 * Creation and deletion of RPC task structures
796 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400797static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798{
799 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100800 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400801 task->tk_flags = task_setup_data->flags;
802 task->tk_ops = task_setup_data->callback_ops;
803 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400804 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
806 /* Initialize retry counters */
807 task->tk_garb_retry = 2;
808 task->tk_cred_retry = 2;
809
Trond Myklebust3ff75762007-07-14 15:40:00 -0400810 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
811 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
813 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500814 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
Trond Myklebust84115e12007-07-14 15:39:59 -0400816 task->tk_client = task_setup_data->rpc_client;
817 if (task->tk_client != NULL) {
818 kref_get(&task->tk_client->cl_kref);
819 if (task->tk_client->cl_softrtry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 task->tk_flags |= RPC_TASK_SOFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 }
822
Trond Myklebust84115e12007-07-14 15:39:59 -0400823 if (task->tk_ops->rpc_call_prepare != NULL)
824 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100825
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400826 if (task_setup_data->rpc_message != NULL) {
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400827 task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
828 task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
829 task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400830 /* Bind the user cred */
Trond Myklebust4ccda2c2008-03-12 16:20:55 -0400831 rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
Trond Myklebustb3ef8b32007-10-25 18:32:34 -0400832 if (task->tk_action == NULL)
833 rpc_call_start(task);
834 }
835
Chuck Leveref759a22006-03-20 13:44:17 -0500836 /* starting timestamp */
837 task->tk_start = jiffies;
838
Chuck Lever46121cf2007-01-31 12:14:08 -0500839 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700840 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841}
842
843static struct rpc_task *
844rpc_alloc_task(void)
845{
846 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
847}
848
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400850 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400852struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400854 struct rpc_task *task = setup_data->task;
855 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400857 if (task == NULL) {
858 task = rpc_alloc_task();
859 if (task == NULL)
860 goto out;
861 flags = RPC_TASK_DYNAMIC;
862 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863
Trond Myklebust84115e12007-07-14 15:39:59 -0400864 rpc_init_task(task, setup_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400866 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500867 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868out:
869 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870}
871
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500872static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100874 const struct rpc_call_ops *tk_ops = task->tk_ops;
875 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800877 if (task->tk_flags & RPC_TASK_DYNAMIC) {
878 dprintk("RPC: %5u freeing task\n", task->tk_pid);
879 mempool_free(task, rpc_task_mempool);
880 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500881 rpc_release_calldata(tk_ops, calldata);
882}
883
884static void rpc_async_release(struct work_struct *work)
885{
886 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
887}
888
889void rpc_put_task(struct rpc_task *task)
890{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500891 if (!atomic_dec_and_test(&task->tk_count))
892 return;
893 /* Release resources */
894 if (task->tk_rqstp)
895 xprt_release(task);
896 if (task->tk_msg.rpc_cred)
897 rpcauth_unbindcred(task);
898 if (task->tk_client) {
899 rpc_release_client(task->tk_client);
900 task->tk_client = NULL;
901 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500902 if (task->tk_workqueue != NULL) {
903 INIT_WORK(&task->u.tk_work, rpc_async_release);
904 queue_work(task->tk_workqueue, &task->u.tk_work);
905 } else
906 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500907}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400908EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500909
Trond Myklebustbde8f002007-01-24 11:54:53 -0800910static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500911{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912#ifdef RPC_DEBUG
913 BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
914#endif
Chuck Lever46121cf2007-01-31 12:14:08 -0500915 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916
Trond Myklebust6529eba2007-06-14 16:40:14 -0400917 if (!list_empty(&task->tk_task)) {
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400918 struct rpc_clnt *clnt = task->tk_client;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400919 /* Remove from client task list */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400920 spin_lock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400921 list_del(&task->tk_task);
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400922 spin_unlock(&clnt->cl_lock);
Trond Myklebust6529eba2007-06-14 16:40:14 -0400923 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926#ifdef RPC_DEBUG
927 task->tk_magic = 0;
928#endif
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500929 /* Wake up anyone who is waiting for task completion */
930 rpc_mark_complete_task(task);
931
932 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933}
934
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935/*
936 * Kill all tasks for the given client.
937 * XXX: kill their descendants as well?
938 */
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400939void rpc_killall_tasks(struct rpc_clnt *clnt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940{
941 struct rpc_task *rovr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400944 if (list_empty(&clnt->cl_tasks))
945 return;
946 dprintk("RPC: killing all tasks for client %p\n", clnt);
947 /*
948 * Spin lock all_tasks to prevent changes...
949 */
950 spin_lock(&clnt->cl_lock);
951 list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 if (! RPC_IS_ACTIVATED(rovr))
953 continue;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400954 if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 rovr->tk_flags |= RPC_TASK_KILLED;
956 rpc_exit(rovr, -EIO);
957 rpc_wake_up_task(rovr);
958 }
959 }
Trond Myklebust4bef61f2007-06-16 14:17:01 -0400960 spin_unlock(&clnt->cl_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400962EXPORT_SYMBOL_GPL(rpc_killall_tasks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400964int rpciod_up(void)
965{
966 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
967}
968
969void rpciod_down(void)
970{
971 module_put(THIS_MODULE);
972}
973
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400975 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400977static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978{
979 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 /*
982 * Create the rpciod thread and wait for it to start.
983 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400984 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400987 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988}
989
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400990static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400992 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -0400993
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400994 if (rpciod_workqueue == NULL)
995 return;
Trond Myklebustab418d72007-06-14 17:08:36 -0400996 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400998 wq = rpciod_workqueue;
999 rpciod_workqueue = NULL;
1000 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001}
1002
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003void
1004rpc_destroy_mempool(void)
1005{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001006 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 if (rpc_buffer_mempool)
1008 mempool_destroy(rpc_buffer_mempool);
1009 if (rpc_task_mempool)
1010 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001011 if (rpc_task_slabp)
1012 kmem_cache_destroy(rpc_task_slabp);
1013 if (rpc_buffer_slabp)
1014 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001015 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016}
1017
1018int
1019rpc_init_mempool(void)
1020{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001021 /*
1022 * The following is not strictly a mempool initialisation,
1023 * but there is no harm in doing it here
1024 */
1025 rpc_init_wait_queue(&delay_queue, "delayq");
1026 if (!rpciod_start())
1027 goto err_nomem;
1028
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1030 sizeof(struct rpc_task),
1031 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001032 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 if (!rpc_task_slabp)
1034 goto err_nomem;
1035 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1036 RPC_BUFFER_MAXSIZE,
1037 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001038 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 if (!rpc_buffer_slabp)
1040 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001041 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1042 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 if (!rpc_task_mempool)
1044 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001045 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1046 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 if (!rpc_buffer_mempool)
1048 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 return 0;
1050err_nomem:
1051 rpc_destroy_mempool();
1052 return -ENOMEM;
1053}