blob: 994cfea2bad66f814432c2fcbb1227d0a321d34f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/net/sunrpc/sched.c
3 *
4 * Scheduling for synchronous and asynchronous RPC requests.
5 *
6 * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de>
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -08007 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * TCP NFS related read + write fixes
9 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
10 */
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/spinlock.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080020#include <linux/mutex.h>
Jeff Laytond3103102011-12-01 22:44:39 +010021#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/sunrpc/clnt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Benny Halevy69518672009-09-10 12:25:04 +030025#include "sunrpc.h"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#ifdef RPC_DEBUG
28#define RPCDBG_FACILITY RPCDBG_SCHED
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#endif
30
Trond Myklebust82b0a4c2012-01-20 14:52:23 -050031#define CREATE_TRACE_POINTS
32#include <trace/events/sunrpc.h>
33
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
35 * RPC slabs and memory pools
36 */
37#define RPC_BUFFER_MAXSIZE (2048)
38#define RPC_BUFFER_POOLSIZE (8)
39#define RPC_TASK_POOLSIZE (8)
Christoph Lametere18b8902006-12-06 20:33:20 -080040static struct kmem_cache *rpc_task_slabp __read_mostly;
41static struct kmem_cache *rpc_buffer_slabp __read_mostly;
Eric Dumazetba899662005-08-26 12:05:31 -070042static mempool_t *rpc_task_mempool __read_mostly;
43static mempool_t *rpc_buffer_mempool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
David Howells65f27f32006-11-22 14:55:48 +000045static void rpc_async_schedule(struct work_struct *);
Trond Myklebustbde8f002007-01-24 11:54:53 -080046static void rpc_release_task(struct rpc_task *task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040047static void __rpc_queue_timer_fn(unsigned long ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * RPC tasks sit here while waiting for conditions to improve.
51 */
Trond Myklebusta4a87492007-07-18 13:24:19 -040052static struct rpc_wait_queue delay_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
54/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 * rpciod-related stuff
56 */
Trond Myklebust24c5d9d2006-03-20 13:44:08 -050057struct workqueue_struct *rpciod_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070058
59/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 * Disable the timer for a given RPC task. Should be called with
61 * queue->lock and bh_disabled in order to avoid races within
62 * rpc_run_timer().
63 */
Trond Myklebust5d008372008-02-22 16:34:17 -050064static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050065__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066{
Trond Myklebust36df9aa2007-07-18 16:18:52 -040067 if (task->tk_timeout == 0)
68 return;
Chuck Lever46121cf2007-01-31 12:14:08 -050069 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 task->tk_timeout = 0;
Trond Myklebust36df9aa2007-07-18 16:18:52 -040071 list_del(&task->u.tk_wait.timer_list);
Trond Myklebusteb276c02008-02-22 17:27:59 -050072 if (list_empty(&queue->timer_list.list))
73 del_timer(&queue->timer_list.timer);
Trond Myklebust36df9aa2007-07-18 16:18:52 -040074}
75
76static void
77rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
78{
79 queue->timer_list.expires = expires;
80 mod_timer(&queue->timer_list.timer, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -070081}
82
83/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 * Set up a timer for the current task.
85 */
Trond Myklebust5d008372008-02-22 16:34:17 -050086static void
Trond Myklebusteb276c02008-02-22 17:27:59 -050087__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088{
89 if (!task->tk_timeout)
90 return;
91
Chuck Lever46121cf2007-01-31 12:14:08 -050092 dprintk("RPC: %5u setting alarm for %lu ms\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 task->tk_pid, task->tk_timeout * 1000 / HZ);
94
Trond Myklebusteb276c02008-02-22 17:27:59 -050095 task->u.tk_wait.expires = jiffies + task->tk_timeout;
96 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
97 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
98 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099}
100
101/*
102 * Add new request to a priority queue.
103 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400104static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
105 struct rpc_task *task,
106 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107{
108 struct list_head *q;
109 struct rpc_task *t;
110
111 INIT_LIST_HEAD(&task->u.tk_wait.links);
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400112 q = &queue->tasks[queue_priority];
113 if (unlikely(queue_priority > queue->maxpriority))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 q = &queue->tasks[queue->maxpriority];
115 list_for_each_entry(t, q, u.tk_wait.list) {
Trond Myklebust3ff75762007-07-14 15:40:00 -0400116 if (t->tk_owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
118 return;
119 }
120 }
121 list_add_tail(&task->u.tk_wait.list, q);
122}
123
124/*
125 * Add new request to wait queue.
126 *
127 * Swapper tasks always get inserted at the head of the queue.
128 * This should avoid many nasty memory deadlocks and hopefully
129 * improve overall performance.
130 * Everyone else gets appended to the queue to ensure proper FIFO behavior.
131 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400132static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
133 struct rpc_task *task,
134 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135{
136 BUG_ON (RPC_IS_QUEUED(task));
137
138 if (RPC_IS_PRIORITY(queue))
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400139 __rpc_add_wait_queue_priority(queue, task, queue_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 else if (RPC_IS_SWAPPER(task))
141 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
142 else
143 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500144 task->tk_waitqueue = queue;
Chuck Levere19b63d2006-03-20 13:44:15 -0500145 queue->qlen++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 rpc_set_queued(task);
147
Chuck Lever46121cf2007-01-31 12:14:08 -0500148 dprintk("RPC: %5u added to queue %p \"%s\"\n",
149 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150}
151
152/*
153 * Remove request from a priority queue.
154 */
155static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
156{
157 struct rpc_task *t;
158
159 if (!list_empty(&task->u.tk_wait.links)) {
160 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
161 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
162 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
163 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164}
165
166/*
167 * Remove request from queue.
168 * Note: must be called with spin lock held.
169 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500170static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171{
Trond Myklebusteb276c02008-02-22 17:27:59 -0500172 __rpc_disable_timer(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 if (RPC_IS_PRIORITY(queue))
174 __rpc_remove_wait_queue_priority(task);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400175 list_del(&task->u.tk_wait.list);
Chuck Levere19b63d2006-03-20 13:44:15 -0500176 queue->qlen--;
Chuck Lever46121cf2007-01-31 12:14:08 -0500177 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
178 task->tk_pid, queue, rpc_qname(queue));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179}
180
181static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
182{
183 queue->priority = priority;
184 queue->count = 1 << (priority * 2);
185}
186
Trond Myklebust3ff75762007-07-14 15:40:00 -0400187static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400189 queue->owner = pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 queue->nr = RPC_BATCH_COUNT;
191}
192
193static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
194{
195 rpc_set_waitqueue_priority(queue, queue->maxpriority);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400196 rpc_set_waitqueue_owner(queue, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197}
198
Trond Myklebust3ff75762007-07-14 15:40:00 -0400199static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200{
201 int i;
202
203 spin_lock_init(&queue->lock);
204 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
205 INIT_LIST_HEAD(&queue->tasks[i]);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400206 queue->maxpriority = nr_queues - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 rpc_reset_waitqueue_priority(queue);
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400208 queue->qlen = 0;
209 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
210 INIT_LIST_HEAD(&queue->timer_list.list);
Trond Myklebust2f09c242012-02-08 22:01:15 -0500211 rpc_assign_waitqueue_name(queue, qname);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212}
213
214void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
215{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400216 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217}
Alexandros Batsakis689cf5c2009-12-14 21:27:56 -0800218EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
220void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
221{
Trond Myklebust3ff75762007-07-14 15:40:00 -0400222 __rpc_init_priority_wait_queue(queue, qname, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400224EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500226void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
227{
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400228 del_timer_sync(&queue->timer_list.timer);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -0500229}
230EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
231
Matthew Wilcox150030b2007-12-06 16:24:39 -0500232static int rpc_wait_bit_killable(void *word)
Trond Myklebust44c28872006-01-03 09:55:06 +0100233{
Matthew Wilcox150030b2007-12-06 16:24:39 -0500234 if (fatal_signal_pending(current))
Trond Myklebust44c28872006-01-03 09:55:06 +0100235 return -ERESTARTSYS;
Jeff Laytond3103102011-12-01 22:44:39 +0100236 freezable_schedule();
Trond Myklebust44c28872006-01-03 09:55:06 +0100237 return 0;
238}
239
Trond Myklebustc44fe702007-06-16 14:17:01 -0400240#ifdef RPC_DEBUG
241static void rpc_task_set_debuginfo(struct rpc_task *task)
242{
243 static atomic_t rpc_pid;
244
Trond Myklebustc44fe702007-06-16 14:17:01 -0400245 task->tk_pid = atomic_inc_return(&rpc_pid);
246}
247#else
248static inline void rpc_task_set_debuginfo(struct rpc_task *task)
249{
250}
251#endif
252
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500253static void rpc_set_active(struct rpc_task *task)
254{
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500255 trace_rpc_task_begin(task->tk_client, task, NULL);
256
Trond Myklebustc44fe702007-06-16 14:17:01 -0400257 rpc_task_set_debuginfo(task);
Trond Myklebust58f96122010-07-31 14:29:08 -0400258 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500259}
260
Trond Myklebust44c28872006-01-03 09:55:06 +0100261/*
262 * Mark an RPC call as having completed by clearing the 'active' bit
Trond Myklebustbf294b42011-02-21 11:05:41 -0800263 * and then waking up all tasks that were sleeping.
Trond Myklebust44c28872006-01-03 09:55:06 +0100264 */
Trond Myklebustbf294b42011-02-21 11:05:41 -0800265static int rpc_complete_task(struct rpc_task *task)
Trond Myklebust44c28872006-01-03 09:55:06 +0100266{
Trond Myklebustbf294b42011-02-21 11:05:41 -0800267 void *m = &task->tk_runstate;
268 wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE);
269 struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE);
270 unsigned long flags;
271 int ret;
272
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500273 trace_rpc_task_complete(task->tk_client, task, NULL);
274
Trond Myklebustbf294b42011-02-21 11:05:41 -0800275 spin_lock_irqsave(&wq->lock, flags);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500276 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800277 ret = atomic_dec_and_test(&task->tk_count);
278 if (waitqueue_active(wq))
279 __wake_up_locked_key(wq, TASK_NORMAL, &k);
280 spin_unlock_irqrestore(&wq->lock, flags);
281 return ret;
Trond Myklebust44c28872006-01-03 09:55:06 +0100282}
283
284/*
285 * Allow callers to wait for completion of an RPC call
Trond Myklebustbf294b42011-02-21 11:05:41 -0800286 *
287 * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit()
288 * to enforce taking of the wq->lock and hence avoid races with
289 * rpc_complete_task().
Trond Myklebust44c28872006-01-03 09:55:06 +0100290 */
291int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
292{
293 if (action == NULL)
Matthew Wilcox150030b2007-12-06 16:24:39 -0500294 action = rpc_wait_bit_killable;
Trond Myklebustbf294b42011-02-21 11:05:41 -0800295 return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500296 action, TASK_KILLABLE);
Trond Myklebust44c28872006-01-03 09:55:06 +0100297}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400298EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
Trond Myklebust44c28872006-01-03 09:55:06 +0100299
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300/*
301 * Make an RPC task runnable.
302 *
YOSHIFUJI Hideakicca51722007-02-09 15:38:13 -0800303 * Note: If the task is ASYNC, this must be called with
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 * the spinlock held to protect the wait queue operation.
305 */
306static void rpc_make_runnable(struct rpc_task *task)
307{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 rpc_clear_queued(task);
Christophe Saoutcc4dc592006-11-05 18:42:48 +0100309 if (rpc_test_and_set_running(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 return;
311 if (RPC_IS_ASYNC(task)) {
David Howells65f27f32006-11-22 14:55:48 +0000312 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
Fred Isamancee6a532011-02-11 15:42:35 +0000313 queue_work(rpciod_workqueue, &task->u.tk_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 } else
Trond Myklebust96651ab2005-06-22 17:16:21 +0000315 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316}
317
318/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 * Prepare for sleeping on a wait queue.
320 * By always appending tasks to the list we ensure FIFO behavior.
321 * NB: An RPC task will only receive interrupt-driven events as long
322 * as it's on a wait queue.
323 */
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400324static void __rpc_sleep_on_priority(struct rpc_wait_queue *q,
325 struct rpc_task *task,
326 rpc_action action,
327 unsigned char queue_priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328{
Chuck Lever46121cf2007-01-31 12:14:08 -0500329 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
330 task->tk_pid, rpc_qname(q), jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500332 trace_rpc_task_sleep(task->tk_client, task, q);
333
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400334 __rpc_add_wait_queue(q, task, queue_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335
336 BUG_ON(task->tk_callback != NULL);
337 task->tk_callback = action;
Trond Myklebusteb276c02008-02-22 17:27:59 -0500338 __rpc_add_timer(q, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339}
340
341void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
Trond Myklebust5d008372008-02-22 16:34:17 -0500342 rpc_action action)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
Trond Myklebust58f96122010-07-31 14:29:08 -0400344 /* We shouldn't ever put an inactive task to sleep */
345 BUG_ON(!RPC_IS_ACTIVATED(task));
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 /*
348 * Protect the queue operations.
349 */
350 spin_lock_bh(&q->lock);
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400351 __rpc_sleep_on_priority(q, task, action, task->tk_priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 spin_unlock_bh(&q->lock);
353}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400354EXPORT_SYMBOL_GPL(rpc_sleep_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355
Trond Myklebust3b27bad2011-07-17 18:11:34 -0400356void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task,
357 rpc_action action, int priority)
358{
359 /* We shouldn't ever put an inactive task to sleep */
360 BUG_ON(!RPC_IS_ACTIVATED(task));
361
362 /*
363 * Protect the queue operations.
364 */
365 spin_lock_bh(&q->lock);
366 __rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW);
367 spin_unlock_bh(&q->lock);
368}
369
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370/**
371 * __rpc_do_wake_up_task - wake up a single rpc_task
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500372 * @queue: wait queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * @task: task to be woken up
374 *
375 * Caller must hold queue->lock, and have cleared the task queued flag.
376 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500377static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378{
Chuck Lever46121cf2007-01-31 12:14:08 -0500379 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
380 task->tk_pid, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 /* Has the task been executed yet? If not, we cannot wake it up! */
383 if (!RPC_IS_ACTIVATED(task)) {
384 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
385 return;
386 }
387
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500388 trace_rpc_task_wakeup(task->tk_client, task, queue);
389
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500390 __rpc_remove_wait_queue(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
392 rpc_make_runnable(task);
393
Chuck Lever46121cf2007-01-31 12:14:08 -0500394 dprintk("RPC: __rpc_wake_up_task done\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395}
396
397/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500398 * Wake up a queued task while the queue lock is being held
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 */
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500400static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401{
Trond Myklebustf5fb7b062008-02-25 21:40:50 -0800402 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
403 __rpc_do_wake_up_task(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404}
405
406/*
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800407 * Tests whether rpc queue is empty
408 */
409int rpc_queue_empty(struct rpc_wait_queue *queue)
410{
411 int res;
412
413 spin_lock_bh(&queue->lock);
414 res = queue->qlen;
415 spin_unlock_bh(&queue->lock);
Eric Dumazeta02cec22010-09-22 20:43:57 +0000416 return res == 0;
Alexandros Batsakis48f18612009-12-14 21:27:53 -0800417}
418EXPORT_SYMBOL_GPL(rpc_queue_empty);
419
420/*
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500421 * Wake up a task on a specific queue
422 */
423void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
424{
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800425 spin_lock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500426 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800427 spin_unlock_bh(&queue->lock);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500428}
429EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
430
431/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 * Wake up the next task on a priority queue.
433 */
Trond Myklebust961a8282012-01-17 22:57:37 -0500434static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435{
436 struct list_head *q;
437 struct rpc_task *task;
438
439 /*
Trond Myklebust3ff75762007-07-14 15:40:00 -0400440 * Service a batch of tasks from a single owner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 */
442 q = &queue->tasks[queue->priority];
443 if (!list_empty(q)) {
444 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
Trond Myklebust3ff75762007-07-14 15:40:00 -0400445 if (queue->owner == task->tk_owner) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 if (--queue->nr)
447 goto out;
448 list_move_tail(&task->u.tk_wait.list, q);
449 }
450 /*
451 * Check if we need to switch queues.
452 */
453 if (--queue->count)
Trond Myklebust3ff75762007-07-14 15:40:00 -0400454 goto new_owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 }
456
457 /*
458 * Service the next queue.
459 */
460 do {
461 if (q == &queue->tasks[0])
462 q = &queue->tasks[queue->maxpriority];
463 else
464 q = q - 1;
465 if (!list_empty(q)) {
466 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
467 goto new_queue;
468 }
469 } while (q != &queue->tasks[queue->priority]);
470
471 rpc_reset_waitqueue_priority(queue);
472 return NULL;
473
474new_queue:
475 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
Trond Myklebust3ff75762007-07-14 15:40:00 -0400476new_owner:
477 rpc_set_waitqueue_owner(queue, task->tk_owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 return task;
480}
481
Trond Myklebust961a8282012-01-17 22:57:37 -0500482static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)
483{
484 if (RPC_IS_PRIORITY(queue))
485 return __rpc_find_next_queued_priority(queue);
486 if (!list_empty(&queue->tasks[0]))
487 return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);
488 return NULL;
489}
490
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491/*
Trond Myklebust961a8282012-01-17 22:57:37 -0500492 * Wake up the first task on the wait queue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 */
Trond Myklebust961a8282012-01-17 22:57:37 -0500494struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,
495 bool (*func)(struct rpc_task *, void *), void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496{
497 struct rpc_task *task = NULL;
498
Trond Myklebust961a8282012-01-17 22:57:37 -0500499 dprintk("RPC: wake_up_first(%p \"%s\")\n",
Chuck Lever46121cf2007-01-31 12:14:08 -0500500 queue, rpc_qname(queue));
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800501 spin_lock_bh(&queue->lock);
Trond Myklebust961a8282012-01-17 22:57:37 -0500502 task = __rpc_find_next_queued(queue);
503 if (task != NULL) {
504 if (func(task, data))
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500505 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust961a8282012-01-17 22:57:37 -0500506 else
507 task = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800509 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510
511 return task;
512}
Trond Myklebust961a8282012-01-17 22:57:37 -0500513EXPORT_SYMBOL_GPL(rpc_wake_up_first);
514
515static bool rpc_wake_up_next_func(struct rpc_task *task, void *data)
516{
517 return true;
518}
519
520/*
521 * Wake up the next task on the wait queue.
522*/
523struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)
524{
525 return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);
526}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400527EXPORT_SYMBOL_GPL(rpc_wake_up_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528
529/**
530 * rpc_wake_up - wake up all rpc_tasks
531 * @queue: rpc_wait_queue on which the tasks are sleeping
532 *
533 * Grabs queue->lock
534 */
535void rpc_wake_up(struct rpc_wait_queue *queue)
536{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 struct list_head *head;
Trond Myklebuste6d83d52006-03-13 21:20:48 -0800538
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800539 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 head = &queue->tasks[queue->maxpriority];
541 for (;;) {
Trond Myklebust540a0f72012-03-19 13:39:35 -0400542 while (!list_empty(head)) {
543 struct rpc_task *task;
544 task = list_first_entry(head,
545 struct rpc_task,
546 u.tk_wait.list);
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500547 rpc_wake_up_task_queue_locked(queue, task);
Trond Myklebust540a0f72012-03-19 13:39:35 -0400548 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 if (head == &queue->tasks[0])
550 break;
551 head--;
552 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800553 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400555EXPORT_SYMBOL_GPL(rpc_wake_up);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
557/**
558 * rpc_wake_up_status - wake up all rpc_tasks and set their status value.
559 * @queue: rpc_wait_queue on which the tasks are sleeping
560 * @status: status value to set
561 *
562 * Grabs queue->lock
563 */
564void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
565{
566 struct list_head *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800568 spin_lock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 head = &queue->tasks[queue->maxpriority];
570 for (;;) {
Trond Myklebust540a0f72012-03-19 13:39:35 -0400571 while (!list_empty(head)) {
572 struct rpc_task *task;
573 task = list_first_entry(head,
574 struct rpc_task,
575 u.tk_wait.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 task->tk_status = status;
Trond Myklebust96ef13b2008-02-22 15:46:41 -0500577 rpc_wake_up_task_queue_locked(queue, task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 }
579 if (head == &queue->tasks[0])
580 break;
581 head--;
582 }
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800583 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400585EXPORT_SYMBOL_GPL(rpc_wake_up_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400587static void __rpc_queue_timer_fn(unsigned long ptr)
588{
589 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
590 struct rpc_task *task, *n;
591 unsigned long expires, now, timeo;
592
593 spin_lock(&queue->lock);
594 expires = now = jiffies;
595 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
596 timeo = task->u.tk_wait.expires;
597 if (time_after_eq(now, timeo)) {
Trond Myklebust36df9aa2007-07-18 16:18:52 -0400598 dprintk("RPC: %5u timeout\n", task->tk_pid);
599 task->tk_status = -ETIMEDOUT;
600 rpc_wake_up_task_queue_locked(queue, task);
601 continue;
602 }
603 if (expires == now || time_after(expires, timeo))
604 expires = timeo;
605 }
606 if (!list_empty(&queue->timer_list.list))
607 rpc_set_queue_timer(queue, expires);
608 spin_unlock(&queue->lock);
609}
610
Trond Myklebust80147932006-08-31 18:24:08 -0400611static void __rpc_atrun(struct rpc_task *task)
612{
Trond Myklebust5d008372008-02-22 16:34:17 -0500613 task->tk_status = 0;
Trond Myklebust80147932006-08-31 18:24:08 -0400614}
615
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616/*
617 * Run a task at a later time
618 */
Trond Myklebust80147932006-08-31 18:24:08 -0400619void rpc_delay(struct rpc_task *task, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620{
621 task->tk_timeout = delay;
Trond Myklebust5d008372008-02-22 16:34:17 -0500622 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400624EXPORT_SYMBOL_GPL(rpc_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626/*
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100627 * Helper to call task->tk_ops->rpc_call_prepare
628 */
Andy Adamsonaae20062009-04-01 09:22:40 -0400629void rpc_prepare_task(struct rpc_task *task)
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100630{
631 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
632}
633
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500634static void
635rpc_init_task_statistics(struct rpc_task *task)
636{
637 /* Initialize retry counters */
638 task->tk_garb_retry = 2;
639 task->tk_cred_retry = 2;
640 task->tk_rebind_retry = 2;
641
642 /* starting timestamp */
643 task->tk_start = ktime_get();
644}
645
646static void
647rpc_reset_task_statistics(struct rpc_task *task)
648{
649 task->tk_timeouts = 0;
650 task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT);
651
652 rpc_init_task_statistics(task);
653}
654
Trond Myklebust4ce70ad2006-01-03 09:55:05 +0100655/*
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100656 * Helper that calls task->tk_ops->rpc_call_done if it exists
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000657 */
Trond Myklebustabbcf282006-01-03 09:55:03 +0100658void rpc_exit_task(struct rpc_task *task)
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000659{
Trond Myklebustabbcf282006-01-03 09:55:03 +0100660 task->tk_action = NULL;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100661 if (task->tk_ops->rpc_call_done != NULL) {
662 task->tk_ops->rpc_call_done(task, task->tk_calldata);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000663 if (task->tk_action != NULL) {
Trond Myklebustabbcf282006-01-03 09:55:03 +0100664 WARN_ON(RPC_ASSASSINATED(task));
665 /* Always release the RPC slot and buffer memory */
666 xprt_release(task);
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500667 rpc_reset_task_statistics(task);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000668 }
669 }
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000670}
Trond Myklebustd9b6cd92010-07-31 14:29:08 -0400671
672void rpc_exit(struct rpc_task *task, int status)
673{
674 task->tk_status = status;
675 task->tk_action = rpc_exit_task;
676 if (RPC_IS_QUEUED(task))
677 rpc_wake_up_queued_task(task->tk_waitqueue, task);
678}
679EXPORT_SYMBOL_GPL(rpc_exit);
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000680
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400681void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
682{
Trond Myklebusta86dc492008-06-11 13:37:09 -0400683 if (ops->rpc_release != NULL)
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400684 ops->rpc_release(calldata);
Trond Myklebustbbd5a1f2006-10-18 16:01:05 -0400685}
686
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000687/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 * This is the RPC `scheduler' (or rather, the finite state machine).
689 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800690static void __rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691{
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400692 struct rpc_wait_queue *queue;
693 int task_is_async = RPC_IS_ASYNC(task);
694 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695
Chuck Lever46121cf2007-01-31 12:14:08 -0500696 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
697 task->tk_pid, task->tk_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
699 BUG_ON(RPC_IS_QUEUED(task));
700
Trond Myklebustd05fdb02005-06-22 17:16:19 +0000701 for (;;) {
Trond Myklebustb55c5982011-07-06 19:58:23 -0400702 void (*do_action)(struct rpc_task *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703
704 /*
Trond Myklebustb55c5982011-07-06 19:58:23 -0400705 * Execute any pending callback first.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 */
Trond Myklebustb55c5982011-07-06 19:58:23 -0400707 do_action = task->tk_callback;
708 task->tk_callback = NULL;
709 if (do_action == NULL) {
Trond Myklebuste020c682011-03-15 19:56:30 -0400710 /*
711 * Perform the next FSM step.
Trond Myklebustb55c5982011-07-06 19:58:23 -0400712 * tk_action may be NULL if the task has been killed.
713 * In particular, note that rpc_killall_tasks may
714 * do this at any time, so beware when dereferencing.
Trond Myklebuste020c682011-03-15 19:56:30 -0400715 */
Trond Myklebustb55c5982011-07-06 19:58:23 -0400716 do_action = task->tk_action;
717 if (do_action == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 }
Trond Myklebust82b0a4c2012-01-20 14:52:23 -0500720 trace_rpc_task_run_action(task->tk_client, task, task->tk_action);
Trond Myklebustb55c5982011-07-06 19:58:23 -0400721 do_action(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722
723 /*
724 * Lockless check for whether task is sleeping or not.
725 */
726 if (!RPC_IS_QUEUED(task))
727 continue;
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400728 /*
729 * The queue->lock protects against races with
730 * rpc_make_runnable().
731 *
732 * Note that once we clear RPC_TASK_RUNNING on an asynchronous
733 * rpc_task, rpc_make_runnable() can assign it to a
734 * different workqueue. We therefore cannot assume that the
735 * rpc_task pointer may still be dereferenced.
736 */
737 queue = task->tk_waitqueue;
738 spin_lock_bh(&queue->lock);
739 if (!RPC_IS_QUEUED(task)) {
740 spin_unlock_bh(&queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 continue;
742 }
Trond Myklebusteb9b55a2009-03-10 20:33:16 -0400743 rpc_clear_running(task);
744 spin_unlock_bh(&queue->lock);
745 if (task_is_async)
746 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747
748 /* sync task: sleep here */
Chuck Lever46121cf2007-01-31 12:14:08 -0500749 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000750 status = out_of_line_wait_on_bit(&task->tk_runstate,
Matthew Wilcox150030b2007-12-06 16:24:39 -0500751 RPC_TASK_QUEUED, rpc_wait_bit_killable,
752 TASK_KILLABLE);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000753 if (status == -ERESTARTSYS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 /*
755 * When a sync task receives a signal, it exits with
756 * -ERESTARTSYS. In order to catch any callbacks that
757 * clean up after sleeping on some queue, we don't
758 * break the loop here, but go around once more.
759 */
Chuck Lever46121cf2007-01-31 12:14:08 -0500760 dprintk("RPC: %5u got signal\n", task->tk_pid);
Trond Myklebust96651ab2005-06-22 17:16:21 +0000761 task->tk_flags |= RPC_TASK_KILLED;
762 rpc_exit(task, -ERESTARTSYS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 }
764 rpc_set_running(task);
Chuck Lever46121cf2007-01-31 12:14:08 -0500765 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 }
767
Chuck Lever46121cf2007-01-31 12:14:08 -0500768 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
769 task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 /* Release all resources associated with the task */
771 rpc_release_task(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772}
773
774/*
775 * User-visible entry point to the scheduler.
776 *
777 * This may be called recursively if e.g. an async NFS task updates
778 * the attributes and finds that dirty pages must be flushed.
779 * NOTE: Upon exit of this function the task is guaranteed to be
780 * released. In particular note that tk_release() will have
781 * been called, so your task memory may have been freed.
782 */
Trond Myklebust2efef832007-02-03 13:38:41 -0800783void rpc_execute(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784{
Trond Myklebust44c28872006-01-03 09:55:06 +0100785 rpc_set_active(task);
Trond Myklebustd6a1ed02010-07-31 14:29:08 -0400786 rpc_make_runnable(task);
787 if (!RPC_IS_ASYNC(task))
788 __rpc_execute(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789}
790
David Howells65f27f32006-11-22 14:55:48 +0000791static void rpc_async_schedule(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792{
David Howells65f27f32006-11-22 14:55:48 +0000793 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794}
795
Chuck Lever02107142006-01-03 09:55:49 +0100796/**
797 * rpc_malloc - allocate an RPC buffer
798 * @task: RPC task that will use this buffer
799 * @size: requested byte size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 *
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400801 * To prevent rpciod from hanging, this allocator never sleeps,
802 * returning NULL if the request cannot be serviced immediately.
803 * The caller can arrange to sleep in a way that is safe for rpciod.
804 *
805 * Most requests are 'small' (under 2KiB) and can be serviced from a
806 * mempool, ensuring that NFS reads and writes can always proceed,
807 * and that there is good locality of reference for these buffers.
808 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 * In order to avoid memory starvation triggering more writebacks of
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400810 * NFS requests, we avoid using GFP_KERNEL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400812void *rpc_malloc(struct rpc_task *task, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400814 struct rpc_buffer *buf;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400815 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400817 size += sizeof(struct rpc_buffer);
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400818 if (size <= RPC_BUFFER_MAXSIZE)
819 buf = mempool_alloc(rpc_buffer_mempool, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 else
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400821 buf = kmalloc(size, gfp);
Peter Zijlstraddce40d2007-05-09 08:30:11 +0200822
823 if (!buf)
824 return NULL;
825
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400826 buf->len = size;
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200827 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400828 task->tk_pid, size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400829 return &buf->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400831EXPORT_SYMBOL_GPL(rpc_malloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832
Chuck Lever02107142006-01-03 09:55:49 +0100833/**
834 * rpc_free - free buffer allocated via rpc_malloc
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400835 * @buffer: buffer to free
Chuck Lever02107142006-01-03 09:55:49 +0100836 *
837 */
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400838void rpc_free(void *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839{
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400840 size_t size;
841 struct rpc_buffer *buf;
Chuck Lever02107142006-01-03 09:55:49 +0100842
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400843 if (!buffer)
844 return;
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400845
846 buf = container_of(buffer, struct rpc_buffer, data);
847 size = buf->len;
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400848
Geert Uytterhoeven215d0672007-05-08 11:37:26 +0200849 dprintk("RPC: freeing buffer of size %zu at %p\n",
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400850 size, buf);
Chuck Leveraa3d1fa2007-05-08 18:23:28 -0400851
Chuck Leverc5a4dd82007-03-29 16:47:58 -0400852 if (size <= RPC_BUFFER_MAXSIZE)
853 mempool_free(buf, rpc_buffer_mempool);
854 else
855 kfree(buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856}
\"Talpey, Thomas\12444802007-09-10 13:45:36 -0400857EXPORT_SYMBOL_GPL(rpc_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858
859/*
860 * Creation and deletion of RPC task structures
861 */
Trond Myklebust47fe0642007-10-25 18:42:55 -0400862static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863{
864 memset(task, 0, sizeof(*task));
Trond Myklebust44c28872006-01-03 09:55:06 +0100865 atomic_set(&task->tk_count, 1);
Trond Myklebust84115e12007-07-14 15:39:59 -0400866 task->tk_flags = task_setup_data->flags;
867 task->tk_ops = task_setup_data->callback_ops;
868 task->tk_calldata = task_setup_data->callback_data;
Trond Myklebust6529eba2007-06-14 16:40:14 -0400869 INIT_LIST_HEAD(&task->tk_task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
Trond Myklebust3ff75762007-07-14 15:40:00 -0400871 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
872 task->tk_owner = current->tgid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873
874 /* Initialize workqueue for async tasks */
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500875 task->tk_workqueue = task_setup_data->workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876
Trond Myklebust84115e12007-07-14 15:39:59 -0400877 if (task->tk_ops->rpc_call_prepare != NULL)
878 task->tk_action = rpc_prepare_task;
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100879
Trond Myklebust7fdcf132011-12-01 14:00:15 -0500880 rpc_init_task_statistics(task);
Chuck Leveref759a22006-03-20 13:44:17 -0500881
Chuck Lever46121cf2007-01-31 12:14:08 -0500882 dprintk("RPC: new task initialized, procpid %u\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700883 task_pid_nr(current));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884}
885
886static struct rpc_task *
887rpc_alloc_task(void)
888{
889 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
890}
891
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892/*
Trond Myklebust90c57552007-06-09 19:49:36 -0400893 * Create a new task for the specified client.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 */
Trond Myklebust84115e12007-07-14 15:39:59 -0400895struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896{
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400897 struct rpc_task *task = setup_data->task;
898 unsigned short flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400900 if (task == NULL) {
901 task = rpc_alloc_task();
Trond Myklebust19445b92010-04-16 16:41:10 -0400902 if (task == NULL) {
903 rpc_release_calldata(setup_data->callback_ops,
904 setup_data->callback_data);
905 return ERR_PTR(-ENOMEM);
906 }
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400907 flags = RPC_TASK_DYNAMIC;
908 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909
Trond Myklebust84115e12007-07-14 15:39:59 -0400910 rpc_init_task(task, setup_data);
Trond Myklebuste8f5d772007-10-25 18:42:53 -0400911 task->tk_flags |= flags;
Chuck Lever46121cf2007-01-31 12:14:08 -0500912 dprintk("RPC: allocated task %p\n", task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 return task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914}
915
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500916static void rpc_free_task(struct rpc_task *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917{
Trond Myklebust963d8fe2006-01-03 09:55:04 +0100918 const struct rpc_call_ops *tk_ops = task->tk_ops;
919 void *calldata = task->tk_calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Trond Myklebust5e4424a2008-02-25 21:53:49 -0800921 if (task->tk_flags & RPC_TASK_DYNAMIC) {
922 dprintk("RPC: %5u freeing task\n", task->tk_pid);
923 mempool_free(task, rpc_task_mempool);
924 }
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500925 rpc_release_calldata(tk_ops, calldata);
926}
927
928static void rpc_async_release(struct work_struct *work)
929{
930 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
931}
932
Trond Myklebustbf294b42011-02-21 11:05:41 -0800933static void rpc_release_resources_task(struct rpc_task *task)
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500934{
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500935 if (task->tk_rqstp)
936 xprt_release(task);
OGAWA Hirofumia271c5a2011-03-27 17:48:57 +0200937 if (task->tk_msg.rpc_cred) {
Trond Myklebusta17c2152010-07-31 14:29:08 -0400938 put_rpccred(task->tk_msg.rpc_cred);
OGAWA Hirofumia271c5a2011-03-27 17:48:57 +0200939 task->tk_msg.rpc_cred = NULL;
940 }
Trond Myklebust58f96122010-07-31 14:29:08 -0400941 rpc_task_release_client(task);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800942}
943
944static void rpc_final_put_task(struct rpc_task *task,
945 struct workqueue_struct *q)
946{
947 if (q != NULL) {
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500948 INIT_WORK(&task->u.tk_work, rpc_async_release);
Trond Myklebustbf294b42011-02-21 11:05:41 -0800949 queue_work(q, &task->u.tk_work);
Trond Myklebust32bfb5c2008-02-19 20:04:21 -0500950 } else
951 rpc_free_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500952}
Trond Myklebustbf294b42011-02-21 11:05:41 -0800953
954static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q)
955{
956 if (atomic_dec_and_test(&task->tk_count)) {
957 rpc_release_resources_task(task);
958 rpc_final_put_task(task, q);
959 }
960}
961
962void rpc_put_task(struct rpc_task *task)
963{
964 rpc_do_put_task(task, NULL);
965}
Trond Myklebuste8914c62007-07-14 15:39:59 -0400966EXPORT_SYMBOL_GPL(rpc_put_task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500967
Trond Myklebustbf294b42011-02-21 11:05:41 -0800968void rpc_put_task_async(struct rpc_task *task)
969{
970 rpc_do_put_task(task, task->tk_workqueue);
971}
972EXPORT_SYMBOL_GPL(rpc_put_task_async);
973
Trond Myklebustbde8f002007-01-24 11:54:53 -0800974static void rpc_release_task(struct rpc_task *task)
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500975{
Chuck Lever46121cf2007-01-31 12:14:08 -0500976 dprintk("RPC: %5u release task\n", task->tk_pid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 BUG_ON (RPC_IS_QUEUED(task));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Trond Myklebustbf294b42011-02-21 11:05:41 -0800980 rpc_release_resources_task(task);
Trond Myklebuste6b3c4d2006-11-11 22:18:03 -0500981
Trond Myklebustbf294b42011-02-21 11:05:41 -0800982 /*
983 * Note: at this point we have been removed from rpc_clnt->cl_tasks,
984 * so it should be safe to use task->tk_count as a test for whether
985 * or not any other processes still hold references to our rpc_task.
986 */
987 if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) {
988 /* Wake up anyone who may be waiting for task completion */
989 if (!rpc_complete_task(task))
990 return;
991 } else {
992 if (!atomic_dec_and_test(&task->tk_count))
993 return;
994 }
995 rpc_final_put_task(task, task->tk_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996}
997
Trond Myklebustb247bbf2007-07-19 16:32:20 -0400998int rpciod_up(void)
999{
1000 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
1001}
1002
1003void rpciod_down(void)
1004{
1005 module_put(THIS_MODULE);
1006}
1007
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008/*
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001009 * Start up the rpciod workqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 */
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001011static int rpciod_start(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012{
1013 struct workqueue_struct *wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 /*
1016 * Create the rpciod thread and wait for it to start.
1017 */
Trond Myklebustab418d72007-06-14 17:08:36 -04001018 dprintk("RPC: creating workqueue rpciod\n");
Tejun Heoada609e2011-01-25 14:35:54 +01001019 wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 rpciod_workqueue = wq;
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001021 return rpciod_workqueue != NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022}
1023
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001024static void rpciod_stop(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001026 struct workqueue_struct *wq = NULL;
Trond Myklebustab418d72007-06-14 17:08:36 -04001027
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001028 if (rpciod_workqueue == NULL)
1029 return;
Trond Myklebustab418d72007-06-14 17:08:36 -04001030 dprintk("RPC: destroying workqueue rpciod\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001032 wq = rpciod_workqueue;
1033 rpciod_workqueue = NULL;
1034 destroy_workqueue(wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035}
1036
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037void
1038rpc_destroy_mempool(void)
1039{
Trond Myklebustb247bbf2007-07-19 16:32:20 -04001040 rpciod_stop();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 if (rpc_buffer_mempool)
1042 mempool_destroy(rpc_buffer_mempool);
1043 if (rpc_task_mempool)
1044 mempool_destroy(rpc_task_mempool);
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001045 if (rpc_task_slabp)
1046 kmem_cache_destroy(rpc_task_slabp);
1047 if (rpc_buffer_slabp)
1048 kmem_cache_destroy(rpc_buffer_slabp);
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001049 rpc_destroy_wait_queue(&delay_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050}
1051
1052int
1053rpc_init_mempool(void)
1054{
Trond Myklebustf6a1cc82008-02-22 17:06:55 -05001055 /*
1056 * The following is not strictly a mempool initialisation,
1057 * but there is no harm in doing it here
1058 */
1059 rpc_init_wait_queue(&delay_queue, "delayq");
1060 if (!rpciod_start())
1061 goto err_nomem;
1062
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1064 sizeof(struct rpc_task),
1065 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001066 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 if (!rpc_task_slabp)
1068 goto err_nomem;
1069 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1070 RPC_BUFFER_MAXSIZE,
1071 0, SLAB_HWCACHE_ALIGN,
Paul Mundt20c2df82007-07-20 10:11:58 +09001072 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073 if (!rpc_buffer_slabp)
1074 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001075 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1076 rpc_task_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 if (!rpc_task_mempool)
1078 goto err_nomem;
Matthew Dobson93d23412006-03-26 01:37:50 -08001079 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1080 rpc_buffer_slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 if (!rpc_buffer_mempool)
1082 goto err_nomem;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 return 0;
1084err_nomem:
1085 rpc_destroy_mempool();
1086 return -ENOMEM;
1087}