blob: 6e4f6c50a120dfcb78eb5f26dc4301936589cd0f [file] [log] [blame]
Chris Mason8b712842008-06-11 16:50:36 -04001/*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
19#include <linux/kthread.h>
20#include <linux/list.h>
21#include <linux/spinlock.h>
Chris Masonb51912c2009-02-04 09:23:24 -050022#include <linux/freezer.h>
Chris Mason8b712842008-06-11 16:50:36 -040023#include "async-thread.h"
24
Chris Mason4a69a412008-11-06 22:03:00 -050025#define WORK_QUEUED_BIT 0
26#define WORK_DONE_BIT 1
27#define WORK_ORDER_DONE_BIT 2
Chris Masond313d7a2009-04-20 15:50:09 -040028#define WORK_HIGH_PRIO_BIT 3
Chris Mason4a69a412008-11-06 22:03:00 -050029
Chris Mason8b712842008-06-11 16:50:36 -040030/*
31 * container for the kthread task pointer and the list of pending work
32 * One of these is allocated per thread.
33 */
34struct btrfs_worker_thread {
Chris Mason35d8ba62008-06-11 20:21:24 -040035 /* pool we belong to */
36 struct btrfs_workers *workers;
37
Chris Mason8b712842008-06-11 16:50:36 -040038 /* list of struct btrfs_work that are waiting for service */
39 struct list_head pending;
Chris Masond313d7a2009-04-20 15:50:09 -040040 struct list_head prio_pending;
Chris Mason8b712842008-06-11 16:50:36 -040041
42 /* list of worker threads from struct btrfs_workers */
43 struct list_head worker_list;
44
45 /* kthread */
46 struct task_struct *task;
47
48 /* number of things on the pending list */
49 atomic_t num_pending;
Chris Mason53863232008-08-15 15:34:18 -040050
Chris Mason4854ddd2008-08-15 15:34:17 -040051 unsigned long sequence;
Chris Mason8b712842008-06-11 16:50:36 -040052
53 /* protects the pending list. */
54 spinlock_t lock;
55
56 /* set to non-zero when this thread is already awake and kicking */
57 int working;
Chris Mason35d8ba62008-06-11 20:21:24 -040058
59 /* are we currently idle */
60 int idle;
Chris Mason8b712842008-06-11 16:50:36 -040061};
62
63/*
Chris Mason35d8ba62008-06-11 20:21:24 -040064 * helper function to move a thread onto the idle list after it
65 * has finished some requests.
66 */
67static void check_idle_worker(struct btrfs_worker_thread *worker)
68{
69 if (!worker->idle && atomic_read(&worker->num_pending) <
70 worker->workers->idle_thresh / 2) {
71 unsigned long flags;
72 spin_lock_irqsave(&worker->workers->lock, flags);
73 worker->idle = 1;
74 list_move(&worker->worker_list, &worker->workers->idle_list);
75 spin_unlock_irqrestore(&worker->workers->lock, flags);
76 }
77}
78
79/*
80 * helper function to move a thread off the idle list after new
81 * pending work is added.
82 */
83static void check_busy_worker(struct btrfs_worker_thread *worker)
84{
85 if (worker->idle && atomic_read(&worker->num_pending) >=
86 worker->workers->idle_thresh) {
87 unsigned long flags;
88 spin_lock_irqsave(&worker->workers->lock, flags);
89 worker->idle = 0;
90 list_move_tail(&worker->worker_list,
91 &worker->workers->worker_list);
92 spin_unlock_irqrestore(&worker->workers->lock, flags);
93 }
94}
95
Chris Mason4a69a412008-11-06 22:03:00 -050096static noinline int run_ordered_completions(struct btrfs_workers *workers,
97 struct btrfs_work *work)
98{
99 unsigned long flags;
100
101 if (!workers->ordered)
102 return 0;
103
104 set_bit(WORK_DONE_BIT, &work->flags);
105
106 spin_lock_irqsave(&workers->lock, flags);
107
Chris Masond313d7a2009-04-20 15:50:09 -0400108 while (1) {
109 if (!list_empty(&workers->prio_order_list)) {
110 work = list_entry(workers->prio_order_list.next,
111 struct btrfs_work, order_list);
112 } else if (!list_empty(&workers->order_list)) {
113 work = list_entry(workers->order_list.next,
114 struct btrfs_work, order_list);
115 } else {
116 break;
117 }
Chris Mason4a69a412008-11-06 22:03:00 -0500118 if (!test_bit(WORK_DONE_BIT, &work->flags))
119 break;
120
121 /* we are going to call the ordered done function, but
122 * we leave the work item on the list as a barrier so
123 * that later work items that are done don't have their
124 * functions called before this one returns
125 */
126 if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags))
127 break;
128
129 spin_unlock_irqrestore(&workers->lock, flags);
130
131 work->ordered_func(work);
132
133 /* now take the lock again and call the freeing code */
134 spin_lock_irqsave(&workers->lock, flags);
135 list_del(&work->order_list);
136 work->ordered_free(work);
137 }
138
139 spin_unlock_irqrestore(&workers->lock, flags);
140 return 0;
141}
142
Chris Mason35d8ba62008-06-11 20:21:24 -0400143/*
Chris Mason8b712842008-06-11 16:50:36 -0400144 * main loop for servicing work items
145 */
146static int worker_loop(void *arg)
147{
148 struct btrfs_worker_thread *worker = arg;
149 struct list_head *cur;
150 struct btrfs_work *work;
151 do {
152 spin_lock_irq(&worker->lock);
Chris Masonb51912c2009-02-04 09:23:24 -0500153again_locked:
Chris Masond313d7a2009-04-20 15:50:09 -0400154 while (1) {
155 if (!list_empty(&worker->prio_pending))
156 cur = worker->prio_pending.next;
157 else if (!list_empty(&worker->pending))
158 cur = worker->pending.next;
159 else
160 break;
161
Chris Mason8b712842008-06-11 16:50:36 -0400162 work = list_entry(cur, struct btrfs_work, list);
163 list_del(&work->list);
Chris Mason4a69a412008-11-06 22:03:00 -0500164 clear_bit(WORK_QUEUED_BIT, &work->flags);
Chris Mason8b712842008-06-11 16:50:36 -0400165
166 work->worker = worker;
167 spin_unlock_irq(&worker->lock);
168
169 work->func(work);
170
171 atomic_dec(&worker->num_pending);
Chris Mason4a69a412008-11-06 22:03:00 -0500172 /*
173 * unless this is an ordered work queue,
174 * 'work' was probably freed by func above.
175 */
176 run_ordered_completions(worker->workers, work);
177
Chris Mason8b712842008-06-11 16:50:36 -0400178 spin_lock_irq(&worker->lock);
Chris Mason35d8ba62008-06-11 20:21:24 -0400179 check_idle_worker(worker);
Chris Mason8b712842008-06-11 16:50:36 -0400180 }
Chris Mason8b712842008-06-11 16:50:36 -0400181 if (freezing(current)) {
Chris Masonb51912c2009-02-04 09:23:24 -0500182 worker->working = 0;
183 spin_unlock_irq(&worker->lock);
Chris Mason8b712842008-06-11 16:50:36 -0400184 refrigerator();
185 } else {
Chris Mason8b712842008-06-11 16:50:36 -0400186 spin_unlock_irq(&worker->lock);
Chris Masonb51912c2009-02-04 09:23:24 -0500187 if (!kthread_should_stop()) {
188 cpu_relax();
189 /*
190 * we've dropped the lock, did someone else
191 * jump_in?
192 */
193 smp_mb();
Chris Masond313d7a2009-04-20 15:50:09 -0400194 if (!list_empty(&worker->pending) ||
195 !list_empty(&worker->prio_pending))
Chris Masonb51912c2009-02-04 09:23:24 -0500196 continue;
197
198 /*
199 * this short schedule allows more work to
200 * come in without the queue functions
201 * needing to go through wake_up_process()
202 *
203 * worker->working is still 1, so nobody
204 * is going to try and wake us up
205 */
206 schedule_timeout(1);
207 smp_mb();
Chris Masond313d7a2009-04-20 15:50:09 -0400208 if (!list_empty(&worker->pending) ||
209 !list_empty(&worker->prio_pending))
Chris Masonb51912c2009-02-04 09:23:24 -0500210 continue;
211
Amit Gudb5555f72009-04-02 17:01:27 -0400212 if (kthread_should_stop())
213 break;
214
Chris Masonb51912c2009-02-04 09:23:24 -0500215 /* still no more work?, sleep for real */
216 spin_lock_irq(&worker->lock);
217 set_current_state(TASK_INTERRUPTIBLE);
Chris Masond313d7a2009-04-20 15:50:09 -0400218 if (!list_empty(&worker->pending) ||
219 !list_empty(&worker->prio_pending))
Chris Masonb51912c2009-02-04 09:23:24 -0500220 goto again_locked;
221
222 /*
223 * this makes sure we get a wakeup when someone
224 * adds something new to the queue
225 */
226 worker->working = 0;
227 spin_unlock_irq(&worker->lock);
228
Amit Gudb5555f72009-04-02 17:01:27 -0400229 if (!kthread_should_stop())
230 schedule();
Chris Masonb51912c2009-02-04 09:23:24 -0500231 }
Chris Mason8b712842008-06-11 16:50:36 -0400232 __set_current_state(TASK_RUNNING);
233 }
234 } while (!kthread_should_stop());
235 return 0;
236}
237
238/*
239 * this will wait for all the worker threads to shutdown
240 */
241int btrfs_stop_workers(struct btrfs_workers *workers)
242{
243 struct list_head *cur;
244 struct btrfs_worker_thread *worker;
245
Chris Mason35d8ba62008-06-11 20:21:24 -0400246 list_splice_init(&workers->idle_list, &workers->worker_list);
Chris Masond3977122009-01-05 21:25:51 -0500247 while (!list_empty(&workers->worker_list)) {
Chris Mason8b712842008-06-11 16:50:36 -0400248 cur = workers->worker_list.next;
249 worker = list_entry(cur, struct btrfs_worker_thread,
250 worker_list);
251 kthread_stop(worker->task);
252 list_del(&worker->worker_list);
253 kfree(worker);
254 }
255 return 0;
256}
257
258/*
259 * simple init on struct btrfs_workers
260 */
Chris Mason5443be42008-08-15 15:34:16 -0400261void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max)
Chris Mason8b712842008-06-11 16:50:36 -0400262{
263 workers->num_workers = 0;
264 INIT_LIST_HEAD(&workers->worker_list);
Chris Mason35d8ba62008-06-11 20:21:24 -0400265 INIT_LIST_HEAD(&workers->idle_list);
Chris Mason4a69a412008-11-06 22:03:00 -0500266 INIT_LIST_HEAD(&workers->order_list);
Chris Masond313d7a2009-04-20 15:50:09 -0400267 INIT_LIST_HEAD(&workers->prio_order_list);
Chris Mason8b712842008-06-11 16:50:36 -0400268 spin_lock_init(&workers->lock);
269 workers->max_workers = max;
Chris Mason61b49442008-07-31 15:42:53 -0400270 workers->idle_thresh = 32;
Chris Mason5443be42008-08-15 15:34:16 -0400271 workers->name = name;
Chris Mason4a69a412008-11-06 22:03:00 -0500272 workers->ordered = 0;
Chris Mason8b712842008-06-11 16:50:36 -0400273}
274
275/*
276 * starts new worker threads. This does not enforce the max worker
277 * count in case you need to temporarily go past it.
278 */
279int btrfs_start_workers(struct btrfs_workers *workers, int num_workers)
280{
281 struct btrfs_worker_thread *worker;
282 int ret = 0;
283 int i;
284
285 for (i = 0; i < num_workers; i++) {
286 worker = kzalloc(sizeof(*worker), GFP_NOFS);
287 if (!worker) {
288 ret = -ENOMEM;
289 goto fail;
290 }
291
292 INIT_LIST_HEAD(&worker->pending);
Chris Masond313d7a2009-04-20 15:50:09 -0400293 INIT_LIST_HEAD(&worker->prio_pending);
Chris Mason8b712842008-06-11 16:50:36 -0400294 INIT_LIST_HEAD(&worker->worker_list);
295 spin_lock_init(&worker->lock);
296 atomic_set(&worker->num_pending, 0);
Shin Hongfd0fb032009-06-10 20:11:29 -0400297 worker->workers = workers;
Chris Mason5443be42008-08-15 15:34:16 -0400298 worker->task = kthread_run(worker_loop, worker,
299 "btrfs-%s-%d", workers->name,
300 workers->num_workers + i);
Chris Mason8b712842008-06-11 16:50:36 -0400301 if (IS_ERR(worker->task)) {
302 ret = PTR_ERR(worker->task);
Jiri Slaby9b627e92009-07-02 13:50:58 -0400303 kfree(worker);
Chris Mason8b712842008-06-11 16:50:36 -0400304 goto fail;
305 }
306
307 spin_lock_irq(&workers->lock);
Chris Mason35d8ba62008-06-11 20:21:24 -0400308 list_add_tail(&worker->worker_list, &workers->idle_list);
Chris Mason4854ddd2008-08-15 15:34:17 -0400309 worker->idle = 1;
Chris Mason8b712842008-06-11 16:50:36 -0400310 workers->num_workers++;
311 spin_unlock_irq(&workers->lock);
312 }
313 return 0;
314fail:
315 btrfs_stop_workers(workers);
316 return ret;
317}
318
319/*
320 * run through the list and find a worker thread that doesn't have a lot
321 * to do right now. This can return null if we aren't yet at the thread
322 * count limit and all of the threads are busy.
323 */
324static struct btrfs_worker_thread *next_worker(struct btrfs_workers *workers)
325{
326 struct btrfs_worker_thread *worker;
327 struct list_head *next;
Chris Mason8b712842008-06-11 16:50:36 -0400328 int enforce_min = workers->num_workers < workers->max_workers;
329
Chris Mason8b712842008-06-11 16:50:36 -0400330 /*
Chris Mason35d8ba62008-06-11 20:21:24 -0400331 * if we find an idle thread, don't move it to the end of the
332 * idle list. This improves the chance that the next submission
333 * will reuse the same thread, and maybe catch it while it is still
334 * working
Chris Mason8b712842008-06-11 16:50:36 -0400335 */
Chris Mason35d8ba62008-06-11 20:21:24 -0400336 if (!list_empty(&workers->idle_list)) {
337 next = workers->idle_list.next;
Chris Mason8b712842008-06-11 16:50:36 -0400338 worker = list_entry(next, struct btrfs_worker_thread,
339 worker_list);
Chris Mason35d8ba62008-06-11 20:21:24 -0400340 return worker;
Chris Mason8b712842008-06-11 16:50:36 -0400341 }
Chris Mason35d8ba62008-06-11 20:21:24 -0400342 if (enforce_min || list_empty(&workers->worker_list))
343 return NULL;
344
Chris Mason8b712842008-06-11 16:50:36 -0400345 /*
Chris Mason35d8ba62008-06-11 20:21:24 -0400346 * if we pick a busy task, move the task to the end of the list.
Chris Masond352ac62008-09-29 15:18:18 -0400347 * hopefully this will keep things somewhat evenly balanced.
348 * Do the move in batches based on the sequence number. This groups
349 * requests submitted at roughly the same time onto the same worker.
Chris Mason8b712842008-06-11 16:50:36 -0400350 */
Chris Mason35d8ba62008-06-11 20:21:24 -0400351 next = workers->worker_list.next;
352 worker = list_entry(next, struct btrfs_worker_thread, worker_list);
Chris Mason4854ddd2008-08-15 15:34:17 -0400353 atomic_inc(&worker->num_pending);
354 worker->sequence++;
Chris Masond352ac62008-09-29 15:18:18 -0400355
Chris Mason53863232008-08-15 15:34:18 -0400356 if (worker->sequence % workers->idle_thresh == 0)
Chris Mason4854ddd2008-08-15 15:34:17 -0400357 list_move_tail(next, &workers->worker_list);
Chris Mason8b712842008-06-11 16:50:36 -0400358 return worker;
359}
360
Chris Masond352ac62008-09-29 15:18:18 -0400361/*
362 * selects a worker thread to take the next job. This will either find
363 * an idle worker, start a new worker up to the max count, or just return
364 * one of the existing busy workers.
365 */
Chris Mason8b712842008-06-11 16:50:36 -0400366static struct btrfs_worker_thread *find_worker(struct btrfs_workers *workers)
367{
368 struct btrfs_worker_thread *worker;
369 unsigned long flags;
370
371again:
372 spin_lock_irqsave(&workers->lock, flags);
373 worker = next_worker(workers);
374 spin_unlock_irqrestore(&workers->lock, flags);
375
376 if (!worker) {
377 spin_lock_irqsave(&workers->lock, flags);
378 if (workers->num_workers >= workers->max_workers) {
Chris Mason35d8ba62008-06-11 20:21:24 -0400379 struct list_head *fallback = NULL;
Chris Mason8b712842008-06-11 16:50:36 -0400380 /*
381 * we have failed to find any workers, just
382 * return the force one
383 */
Chris Mason35d8ba62008-06-11 20:21:24 -0400384 if (!list_empty(&workers->worker_list))
385 fallback = workers->worker_list.next;
386 if (!list_empty(&workers->idle_list))
387 fallback = workers->idle_list.next;
388 BUG_ON(!fallback);
389 worker = list_entry(fallback,
Chris Mason8b712842008-06-11 16:50:36 -0400390 struct btrfs_worker_thread, worker_list);
391 spin_unlock_irqrestore(&workers->lock, flags);
392 } else {
393 spin_unlock_irqrestore(&workers->lock, flags);
394 /* we're below the limit, start another worker */
395 btrfs_start_workers(workers, 1);
396 goto again;
397 }
398 }
399 return worker;
400}
401
402/*
403 * btrfs_requeue_work just puts the work item back on the tail of the list
404 * it was taken from. It is intended for use with long running work functions
405 * that make some progress and want to give the cpu up for others.
406 */
407int btrfs_requeue_work(struct btrfs_work *work)
408{
409 struct btrfs_worker_thread *worker = work->worker;
410 unsigned long flags;
Chris Masona6837052009-02-04 09:19:41 -0500411 int wake = 0;
Chris Mason8b712842008-06-11 16:50:36 -0400412
Chris Mason4a69a412008-11-06 22:03:00 -0500413 if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
Chris Mason8b712842008-06-11 16:50:36 -0400414 goto out;
415
416 spin_lock_irqsave(&worker->lock, flags);
Chris Masond313d7a2009-04-20 15:50:09 -0400417 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags))
418 list_add_tail(&work->list, &worker->prio_pending);
419 else
420 list_add_tail(&work->list, &worker->pending);
Chris Masonb51912c2009-02-04 09:23:24 -0500421 atomic_inc(&worker->num_pending);
Chris Mason75ccf472008-09-30 19:24:06 -0400422
423 /* by definition we're busy, take ourselves off the idle
424 * list
425 */
426 if (worker->idle) {
427 spin_lock_irqsave(&worker->workers->lock, flags);
428 worker->idle = 0;
429 list_move_tail(&worker->worker_list,
430 &worker->workers->worker_list);
431 spin_unlock_irqrestore(&worker->workers->lock, flags);
432 }
Chris Masona6837052009-02-04 09:19:41 -0500433 if (!worker->working) {
434 wake = 1;
435 worker->working = 1;
436 }
Chris Mason75ccf472008-09-30 19:24:06 -0400437
Chris Mason8b712842008-06-11 16:50:36 -0400438 spin_unlock_irqrestore(&worker->lock, flags);
Chris Masona6837052009-02-04 09:19:41 -0500439 if (wake)
440 wake_up_process(worker->task);
Chris Mason8b712842008-06-11 16:50:36 -0400441out:
Chris Masona6837052009-02-04 09:19:41 -0500442
Chris Mason8b712842008-06-11 16:50:36 -0400443 return 0;
444}
445
Chris Masond313d7a2009-04-20 15:50:09 -0400446void btrfs_set_work_high_prio(struct btrfs_work *work)
447{
448 set_bit(WORK_HIGH_PRIO_BIT, &work->flags);
449}
450
Chris Mason8b712842008-06-11 16:50:36 -0400451/*
452 * places a struct btrfs_work into the pending queue of one of the kthreads
453 */
454int btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work)
455{
456 struct btrfs_worker_thread *worker;
457 unsigned long flags;
458 int wake = 0;
459
460 /* don't requeue something already on a list */
Chris Mason4a69a412008-11-06 22:03:00 -0500461 if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
Chris Mason8b712842008-06-11 16:50:36 -0400462 goto out;
463
464 worker = find_worker(workers);
Chris Mason4a69a412008-11-06 22:03:00 -0500465 if (workers->ordered) {
466 spin_lock_irqsave(&workers->lock, flags);
Chris Masond313d7a2009-04-20 15:50:09 -0400467 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) {
468 list_add_tail(&work->order_list,
469 &workers->prio_order_list);
470 } else {
471 list_add_tail(&work->order_list, &workers->order_list);
472 }
Chris Mason4a69a412008-11-06 22:03:00 -0500473 spin_unlock_irqrestore(&workers->lock, flags);
474 } else {
475 INIT_LIST_HEAD(&work->order_list);
476 }
Chris Mason8b712842008-06-11 16:50:36 -0400477
478 spin_lock_irqsave(&worker->lock, flags);
Chris Masona6837052009-02-04 09:19:41 -0500479
Chris Masond313d7a2009-04-20 15:50:09 -0400480 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags))
481 list_add_tail(&work->list, &worker->prio_pending);
482 else
483 list_add_tail(&work->list, &worker->pending);
Chris Mason8b712842008-06-11 16:50:36 -0400484 atomic_inc(&worker->num_pending);
Chris Mason35d8ba62008-06-11 20:21:24 -0400485 check_busy_worker(worker);
Chris Mason8b712842008-06-11 16:50:36 -0400486
487 /*
488 * avoid calling into wake_up_process if this thread has already
489 * been kicked
490 */
491 if (!worker->working)
492 wake = 1;
493 worker->working = 1;
494
495 spin_unlock_irqrestore(&worker->lock, flags);
496
497 if (wake)
498 wake_up_process(worker->task);
499out:
500 return 0;
501}