blob: cace6049e4a5a1d3184829231aa776d7689ad8d0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080020#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Benny Halevy69518672009-09-10 12:25:04 +030024#include "sunrpc.h"
25
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#ifdef RPC_DEBUG
27#define RPCDBG_FACILITY RPCDBG_SCHED
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#endif
29
30/*
31 * RPC slabs and memory pools
32 */
33#define RPC_BUFFER_MAXSIZE (2048)
34#define RPC_BUFFER_POOLSIZE (8)
35#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080036static struct kmem_cache *rpc_task_slabp __read_mostly;
37static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070038static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
David Howells65f27f32006-11-22 14:55:48 +000041static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080042static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040043static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * RPC tasks sit here while waiting for conditions to improve.
47 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040048static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
50/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 * rpciod-related stuff
52 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050053struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 * Disable the timer for a given RPC task. Should be called with
57 * queue->lock and bh_disabled in order to avoid races within
58 * rpc_run_timer().
59 */
Trond Myklebust5d008372008-02-22 16:34:17 -050060static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050061__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040063 if (task->tk_timeout == 0)
64 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050065 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040067 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050068 if (list_empty(&queue->timer_list.list))
69 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040070}
71
72static void
73rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
74{
75 queue->timer_list.expires = expires;
76 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070077}
78
79/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 * Set up a timer for the current task.
81 */
Trond Myklebust5d008372008-02-22 16:34:17 -050082static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050083__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084{
85 if (!task->tk_timeout)
86 return;
87
Chuck Lever46121cf2007-01-31 12:14:08 -050088 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 task->tk_pid, task->tk_timeout * 1000 / HZ);
90
Trond Myklebusteb276c02008-02-22 17:27:59 -050091 task->u.tk_wait.expires = jiffies + task->tk_timeout;
92 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
93 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
94 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
96
97/*
98 * Add new request to a priority queue.
99 */
100static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
101{
102 struct list_head *q;
103 struct rpc_task *t;
104
105 INIT_LIST_HEAD(&task->u.tk_wait.links);
106 q = &queue->tasks[task->tk_priority];
107 if (unlikely(task->tk_priority > queue->maxpriority))
108 q = &queue->tasks[queue->maxpriority];
109 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400110 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
112 return;
113 }
114 }
115 list_add_tail(&task->u.tk_wait.list, q);
116}
117
118/*
119 * Add new request to wait queue.
120 *
121 * Swapper tasks always get inserted at the head of the queue.
122 * This should avoid many nasty memory deadlocks and hopefully
123 * improve overall performance.
124 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
125 */
126static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
127{
128 BUG_ON (RPC_IS_QUEUED(task));
129
130 if (RPC_IS_PRIORITY(queue))
131 __rpc_add_wait_queue_priority(queue, task);
132 else if (RPC_IS_SWAPPER(task))
133 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
134 else
135 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500136 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500137 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 rpc_set_queued(task);
139
Chuck Lever46121cf2007-01-31 12:14:08 -0500140 dprintk("RPC: %5u added to queue %p \"%s\"\n",
141 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142}
143
144/*
145 * Remove request from a priority queue.
146 */
147static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
148{
149 struct rpc_task *t;
150
151 if (!list_empty(&task->u.tk_wait.links)) {
152 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
153 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
154 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
155 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156}
157
158/*
159 * Remove request from queue.
160 * Note: must be called with spin lock held.
161 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500162static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500164 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 if (RPC_IS_PRIORITY(queue))
166 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400167 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500168 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500169 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
170 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172
173static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
174{
175 queue->priority = priority;
176 queue->count = 1 << (priority * 2);
177}
178
Trond Myklebust3ff75762007-07-14 15:40:00 -0400179static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400181 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 queue->nr = RPC_BATCH_COUNT;
183}
184
185static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
186{
187 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400188 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189}
190
Trond Myklebust3ff75762007-07-14 15:40:00 -0400191static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
193 int i;
194
195 spin_lock_init(&queue->lock);
196 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
197 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400198 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400200 queue->qlen = 0;
201 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
202 INIT_LIST_HEAD(&queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203#ifdef RPC_DEBUG
204 queue->name = qname;
205#endif
206}
207
208void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
209{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400210 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
Alexandros Batsakis689cf5c2009-12-14 21:27:56 -0800212EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
214void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
215{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400216 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400218EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500220void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
221{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400222 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500223}
224EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
225
Matthew Wilcox150030b2007-12-06 16:24:39 -0500226static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100227{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500228 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100229 return -ERESTARTSYS;
230 schedule();
231 return 0;
232}
233
Trond Myklebustc44fe702007-06-16 14:17:01 -0400234#ifdef RPC_DEBUG
235static void rpc_task_set_debuginfo(struct rpc_task *task)
236{
237 static atomic_t rpc_pid;
238
Trond Myklebustc44fe702007-06-16 14:17:01 -0400239 task->tk_pid = atomic_inc_return(&rpc_pid);
240}
241#else
242static inline void rpc_task_set_debuginfo(struct rpc_task *task)
243{
244}
245#endif
246
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500247static void rpc_set_active(struct rpc_task *task)
248{
Trond Myklebustc44fe702007-06-16 14:17:01 -0400249 rpc_task_set_debuginfo(task);
Trond Myklebust58f96122010-07-31 14:29:08 -0400250 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500251}
252
Trond Myklebust44c28872006-01-03 09:55:06 +0100253/*
254 * Mark an RPC call as having completed by clearing the 'active' bit
255 */
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500256static void rpc_mark_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100257{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500258 smp_mb__before_clear_bit();
259 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
260 smp_mb__after_clear_bit();
Trond Myklebust44c28872006-01-03 09:55:06 +0100261 wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE);
262}
263
264/*
265 * Allow callers to wait for completion of an RPC call
266 */
267int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
268{
269 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500270 action = rpc_wait_bit_killable;
Trond Myklebust44c28872006-01-03 09:55:06 +0100271 return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500272 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100273}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400274EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276/*
277 * Make an RPC task runnable.
278 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800279 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 * the spinlock held to protect the wait queue operation.
281 */
282static void rpc_make_runnable(struct rpc_task *task)
283{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 rpc_clear_queued(task);
Christophe Saoutcc4dc59e2006-11-05 18:42:48 +0100285 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 return;
287 if (RPC_IS_ASYNC(task)) {
288 int status;
289
David Howells65f27f32006-11-22 14:55:48 +0000290 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500291 status = queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 if (status < 0) {
293 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
294 task->tk_status = status;
295 return;
296 }
297 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000298 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299}
300
301/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 * Prepare for sleeping on a wait queue.
303 * By always appending tasks to the list we ensure FIFO behavior.
304 * NB: An RPC task will only receive interrupt-driven events as long
305 * as it's on a wait queue.
306 */
307static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500308 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309{
Chuck Lever46121cf2007-01-31 12:14:08 -0500310 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
311 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 __rpc_add_wait_queue(q, task);
314
315 BUG_ON(task->tk_callback != NULL);
316 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500317 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318}
319
320void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500321 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322{
Trond Myklebust58f96122010-07-31 14:29:08 -0400323 /* We shouldn't ever put an inactive task to sleep */
324 BUG_ON(!RPC_IS_ACTIVATED(task));
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500325
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 /*
327 * Protect the queue operations.
328 */
329 spin_lock_bh(&q->lock);
Trond Myklebust5d008372008-02-22 16:34:17 -0500330 __rpc_sleep_on(q, task, action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 spin_unlock_bh(&q->lock);
332}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400333EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334
335/**
336 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500337 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 * @task: task to be woken up
339 *
340 * Caller must hold queue->lock, and have cleared the task queued flag.
341 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500342static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
Chuck Lever46121cf2007-01-31 12:14:08 -0500344 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
345 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 /* Has the task been executed yet? If not, we cannot wake it up! */
348 if (!RPC_IS_ACTIVATED(task)) {
349 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
350 return;
351 }
352
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500353 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354
355 rpc_make_runnable(task);
356
Chuck Lever46121cf2007-01-31 12:14:08 -0500357 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358}
359
360/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500361 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500363static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800365 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
366 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367}
368
369/*
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800370 * Tests whether rpc queue is empty
371 */
372int rpc_queue_empty(struct rpc_wait_queue *queue)
373{
374 int res;
375
376 spin_lock_bh(&queue->lock);
377 res = queue->qlen;
378 spin_unlock_bh(&queue->lock);
379 return (res == 0);
380}
381EXPORT_SYMBOL_GPL(rpc_queue_empty);
382
383/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500384 * Wake up a task on a specific queue
385 */
386void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
387{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800388 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500389 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800390 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500391}
392EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
393
394/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 * Wake up the next task on a priority queue.
396 */
397static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
398{
399 struct list_head *q;
400 struct rpc_task *task;
401
402 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400403 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 */
405 q = &queue->tasks[queue->priority];
406 if (!list_empty(q)) {
407 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400408 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 if (--queue->nr)
410 goto out;
411 list_move_tail(&task->u.tk_wait.list, q);
412 }
413 /*
414 * Check if we need to switch queues.
415 */
416 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400417 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 }
419
420 /*
421 * Service the next queue.
422 */
423 do {
424 if (q == &queue->tasks[0])
425 q = &queue->tasks[queue->maxpriority];
426 else
427 q = q - 1;
428 if (!list_empty(q)) {
429 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
430 goto new_queue;
431 }
432 } while (q != &queue->tasks[queue->priority]);
433
434 rpc_reset_waitqueue_priority(queue);
435 return NULL;
436
437new_queue:
438 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400439new_owner:
440 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441out:
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500442 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 return task;
444}
445
446/*
447 * Wake up the next task on the wait queue.
448 */
449struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
450{
451 struct rpc_task *task = NULL;
452
Chuck Lever46121cf2007-01-31 12:14:08 -0500453 dprintk("RPC: wake_up_next(%p \"%s\")\n",
454 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800455 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 if (RPC_IS_PRIORITY(queue))
457 task = __rpc_wake_up_next_priority(queue);
458 else {
459 task_for_first(task, &queue->tasks[0])
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500460 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800462 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
464 return task;
465}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400466EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
468/**
469 * rpc_wake_up - wake up all rpc_tasks
470 * @queue: rpc_wait_queue on which the tasks are sleeping
471 *
472 * Grabs queue->lock
473 */
474void rpc_wake_up(struct rpc_wait_queue *queue)
475{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800476 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800478
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800479 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 head = &queue->tasks[queue->maxpriority];
481 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800482 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500483 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 if (head == &queue->tasks[0])
485 break;
486 head--;
487 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800488 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400490EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
492/**
493 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
494 * @queue: rpc_wait_queue on which the tasks are sleeping
495 * @status: status value to set
496 *
497 * Grabs queue->lock
498 */
499void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
500{
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800501 struct rpc_task *task, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800504 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 head = &queue->tasks[queue->maxpriority];
506 for (;;) {
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800507 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500509 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 }
511 if (head == &queue->tasks[0])
512 break;
513 head--;
514 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800515 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400517EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400519static void __rpc_queue_timer_fn(unsigned long ptr)
520{
521 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
522 struct rpc_task *task, *n;
523 unsigned long expires, now, timeo;
524
525 spin_lock(&queue->lock);
526 expires = now = jiffies;
527 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
528 timeo = task->u.tk_wait.expires;
529 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400530 dprintk("RPC: %5u timeout\n", task->tk_pid);
531 task->tk_status = -ETIMEDOUT;
532 rpc_wake_up_task_queue_locked(queue, task);
533 continue;
534 }
535 if (expires == now || time_after(expires, timeo))
536 expires = timeo;
537 }
538 if (!list_empty(&queue->timer_list.list))
539 rpc_set_queue_timer(queue, expires);
540 spin_unlock(&queue->lock);
541}
542
Trond Myklebust80147932006-08-31 18:24:08 -0400543static void __rpc_atrun(struct rpc_task *task)
544{
Trond Myklebust5d008372008-02-22 16:34:17 -0500545 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400546}
547
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548/*
549 * Run a task at a later time
550 */
Trond Myklebust80147932006-08-31 18:24:08 -0400551void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
553 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500554 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400556EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100559 * Helper to call task->tk_ops->rpc_call_prepare
560 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400561void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100562{
563 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
564}
565
566/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100567 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000568 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100569void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000570{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100571 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100572 if (task->tk_ops->rpc_call_done != NULL) {
573 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000574 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100575 WARN_ON(RPC_ASSASSINATED(task));
576 /* Always release the RPC slot and buffer memory */
577 xprt_release(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000578 }
579 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000580}
Trond Myklebustd9b6cd92010-07-31 14:29:08 -0400581
582void rpc_exit(struct rpc_task *task, int status)
583{
584 task->tk_status = status;
585 task->tk_action = rpc_exit_task;
586 if (RPC_IS_QUEUED(task))
587 rpc_wake_up_queued_task(task->tk_waitqueue, task);
588}
589EXPORT_SYMBOL_GPL(rpc_exit);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000590
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400591void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
592{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400593 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400594 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400595}
596
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000597/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 * This is the RPC `scheduler' (or rather, the finite state machine).
599 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800600static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400602 struct rpc_wait_queue *queue;
603 int task_is_async = RPC_IS_ASYNC(task);
604 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Chuck Lever46121cf2007-01-31 12:14:08 -0500606 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
607 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608
609 BUG_ON(RPC_IS_QUEUED(task));
610
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000611 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612
613 /*
614 * Execute any pending callback.
615 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400616 if (task->tk_callback) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 void (*save_callback)(struct rpc_task *);
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800618
619 /*
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400620 * We set tk_callback to NULL before calling it,
621 * in case it sets the tk_callback field itself:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 */
\\\"J. Bruce Fields\\\a486aed2008-06-09 16:51:35 -0400623 save_callback = task->tk_callback;
624 task->tk_callback = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 save_callback(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 }
627
628 /*
629 * Perform the next FSM step.
630 * tk_action may be NULL when the task has been killed
631 * by someone else.
632 */
633 if (!RPC_IS_QUEUED(task)) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100634 if (task->tk_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 break;
Trond Myklebustabbcf282006-01-03 09:55:03 +0100636 task->tk_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 }
638
639 /*
640 * Lockless check for whether task is sleeping or not.
641 */
642 if (!RPC_IS_QUEUED(task))
643 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400644 /*
645 * The queue->lock protects against races with
646 * rpc_make_runnable().
647 *
648 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
649 * rpc_task, rpc_make_runnable() can assign it to a
650 * different workqueue. We therefore cannot assume that the
651 * rpc_task pointer may still be dereferenced.
652 */
653 queue = task->tk_waitqueue;
654 spin_lock_bh(&queue->lock);
655 if (!RPC_IS_QUEUED(task)) {
656 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 continue;
658 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400659 rpc_clear_running(task);
660 spin_unlock_bh(&queue->lock);
661 if (task_is_async)
662 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663
664 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500665 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000666 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500667 RPC_TASK_QUEUED, rpc_wait_bit_killable,
668 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000669 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670 /*
671 * When a sync task receives a signal, it exits with
672 * -ERESTARTSYS. In order to catch any callbacks that
673 * clean up after sleeping on some queue, we don't
674 * break the loop here, but go around once more.
675 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500676 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000677 task->tk_flags |= RPC_TASK_KILLED;
678 rpc_exit(task, -ERESTARTSYS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 }
680 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500681 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 }
683
Chuck Lever46121cf2007-01-31 12:14:08 -0500684 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
685 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 /* Release all resources associated with the task */
687 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688}
689
690/*
691 * User-visible entry point to the scheduler.
692 *
693 * This may be called recursively if e.g. an async NFS task updates
694 * the attributes and finds that dirty pages must be flushed.
695 * NOTE: Upon exit of this function the task is guaranteed to be
696 * released. In particular note that tk_release() will have
697 * been called, so your task memory may have been freed.
698 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800699void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700{
Trond Myklebust44c28872006-01-03 09:55:06 +0100701 rpc_set_active(task);
Trond Myklebustd6a1ed02010-07-31 14:29:08 -0400702 rpc_make_runnable(task);
703 if (!RPC_IS_ASYNC(task))
704 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705}
706
David Howells65f27f32006-11-22 14:55:48 +0000707static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708{
David Howells65f27f32006-11-22 14:55:48 +0000709 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710}
711
Chuck Lever02107142006-01-03 09:55:49 +0100712/**
713 * rpc_malloc - allocate an RPC buffer
714 * @task: RPC task that will use this buffer
715 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400717 * To prevent rpciod from hanging, this allocator never sleeps,
718 * returning NULL if the request cannot be serviced immediately.
719 * The caller can arrange to sleep in a way that is safe for rpciod.
720 *
721 * Most requests are 'small' (under 2KiB) and can be serviced from a
722 * mempool, ensuring that NFS reads and writes can always proceed,
723 * and that there is good locality of reference for these buffers.
724 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400726 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400728void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400730 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400731 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400733 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400734 if (size <= RPC_BUFFER_MAXSIZE)
735 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400737 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200738
739 if (!buf)
740 return NULL;
741
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400742 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200743 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400744 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400745 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400747EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748
Chuck Lever02107142006-01-03 09:55:49 +0100749/**
750 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400751 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100752 *
753 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400754void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400756 size_t size;
757 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100758
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400759 if (!buffer)
760 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400761
762 buf = container_of(buffer, struct rpc_buffer, data);
763 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400764
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200765 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400766 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400767
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400768 if (size <= RPC_BUFFER_MAXSIZE)
769 mempool_free(buf, rpc_buffer_mempool);
770 else
771 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400773EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774
775/*
776 * Creation and deletion of RPC task structures
777 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400778static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779{
780 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100781 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400782 task->tk_flags = task_setup_data->flags;
783 task->tk_ops = task_setup_data->callback_ops;
784 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400785 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786
787 /* Initialize retry counters */
788 task->tk_garb_retry = 2;
789 task->tk_cred_retry = 2;
790
Trond Myklebust3ff75762007-07-14 15:40:00 -0400791 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
792 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
794 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500795 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796
Trond Myklebust84115e12007-07-14 15:39:59 -0400797 if (task->tk_ops->rpc_call_prepare != NULL)
798 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100799
Chuck Leveref759a22006-03-20 13:44:17 -0500800 /* starting timestamp */
Chuck Leverff839972010-05-07 13:34:47 -0400801 task->tk_start = ktime_get();
Chuck Leveref759a22006-03-20 13:44:17 -0500802
Chuck Lever46121cf2007-01-31 12:14:08 -0500803 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700804 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805}
806
807static struct rpc_task *
808rpc_alloc_task(void)
809{
810 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
811}
812
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400814 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400816struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400818 struct rpc_task *task = setup_data->task;
819 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400821 if (task == NULL) {
822 task = rpc_alloc_task();
Trond Myklebust19445b92010-04-16 16:41:10 -0400823 if (task == NULL) {
824 rpc_release_calldata(setup_data->callback_ops,
825 setup_data->callback_data);
826 return ERR_PTR(-ENOMEM);
827 }
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400828 flags = RPC_TASK_DYNAMIC;
829 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830
Trond Myklebust84115e12007-07-14 15:39:59 -0400831 rpc_init_task(task, setup_data);
Trond Myklebust19445b92010-04-16 16:41:10 -0400832 if (task->tk_status < 0) {
833 int err = task->tk_status;
834 rpc_put_task(task);
835 return ERR_PTR(err);
836 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400838 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500839 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841}
842
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500843static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100845 const struct rpc_call_ops *tk_ops = task->tk_ops;
846 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800848 if (task->tk_flags & RPC_TASK_DYNAMIC) {
849 dprintk("RPC: %5u freeing task\n", task->tk_pid);
850 mempool_free(task, rpc_task_mempool);
851 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500852 rpc_release_calldata(tk_ops, calldata);
853}
854
855static void rpc_async_release(struct work_struct *work)
856{
857 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
858}
859
860void rpc_put_task(struct rpc_task *task)
861{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500862 if (!atomic_dec_and_test(&task->tk_count))
863 return;
864 /* Release resources */
865 if (task->tk_rqstp)
866 xprt_release(task);
867 if (task->tk_msg.rpc_cred)
Trond Myklebusta17c2152010-07-31 14:29:08 -0400868 put_rpccred(task->tk_msg.rpc_cred);
Trond Myklebust58f96122010-07-31 14:29:08 -0400869 rpc_task_release_client(task);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500870 if (task->tk_workqueue != NULL) {
871 INIT_WORK(&task->u.tk_work, rpc_async_release);
872 queue_work(task->tk_workqueue, &task->u.tk_work);
873 } else
874 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500875}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400876EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500877
Trond Myklebustbde8f002007-01-24 11:54:53 -0800878static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500879{
Chuck Lever46121cf2007-01-31 12:14:08 -0500880 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500884 /* Wake up anyone who is waiting for task completion */
885 rpc_mark_complete_task(task);
886
887 rpc_put_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888}
889
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400890int rpciod_up(void)
891{
892 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
893}
894
895void rpciod_down(void)
896{
897 module_put(THIS_MODULE);
898}
899
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400901 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400903static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904{
905 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907 /*
908 * Create the rpciod thread and wait for it to start.
909 */
Trond Myklebustab418d72007-06-14 17:08:36 -0400910 dprintk("RPC: creating workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 wq = create_workqueue("rpciod");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400913 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914}
915
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400916static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400918 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -0400919
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400920 if (rpciod_workqueue == NULL)
921 return;
Trond Myklebustab418d72007-06-14 17:08:36 -0400922 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400924 wq = rpciod_workqueue;
925 rpciod_workqueue = NULL;
926 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927}
928
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929void
930rpc_destroy_mempool(void)
931{
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400932 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 if (rpc_buffer_mempool)
934 mempool_destroy(rpc_buffer_mempool);
935 if (rpc_task_mempool)
936 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -0700937 if (rpc_task_slabp)
938 kmem_cache_destroy(rpc_task_slabp);
939 if (rpc_buffer_slabp)
940 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500941 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942}
943
944int
945rpc_init_mempool(void)
946{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500947 /*
948 * The following is not strictly a mempool initialisation,
949 * but there is no harm in doing it here
950 */
951 rpc_init_wait_queue(&delay_queue, "delayq");
952 if (!rpciod_start())
953 goto err_nomem;
954
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 rpc_task_slabp = kmem_cache_create("rpc_tasks",
956 sizeof(struct rpc_task),
957 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +0900958 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 if (!rpc_task_slabp)
960 goto err_nomem;
961 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
962 RPC_BUFFER_MAXSIZE,
963 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +0900964 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 if (!rpc_buffer_slabp)
966 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -0800967 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
968 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 if (!rpc_task_mempool)
970 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -0800971 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
972 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 if (!rpc_buffer_mempool)
974 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 return 0;
976err_nomem:
977 rpc_destroy_mempool();
978 return -ENOMEM;
979}