blob: 0cc20b35c1c4d99bd424dc720e5764c9f80ac7ba [file] [log] [blame]
Chris Mason8b712842008-06-11 16:50:36 -04001/*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
19#include <linux/kthread.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Chris Mason8b712842008-06-11 16:50:36 -040021#include <linux/list.h>
22#include <linux/spinlock.h>
Chris Masonb51912c2009-02-04 09:23:24 -050023#include <linux/freezer.h>
Chris Mason8b712842008-06-11 16:50:36 -040024#include "async-thread.h"
25
Chris Mason4a69a412008-11-06 22:03:00 -050026#define WORK_QUEUED_BIT 0
27#define WORK_DONE_BIT 1
28#define WORK_ORDER_DONE_BIT 2
Chris Masond313d7a2009-04-20 15:50:09 -040029#define WORK_HIGH_PRIO_BIT 3
Chris Mason4a69a412008-11-06 22:03:00 -050030
Chris Mason8b712842008-06-11 16:50:36 -040031/*
32 * container for the kthread task pointer and the list of pending work
33 * One of these is allocated per thread.
34 */
35struct btrfs_worker_thread {
Chris Mason35d8ba62008-06-11 20:21:24 -040036 /* pool we belong to */
37 struct btrfs_workers *workers;
38
Chris Mason8b712842008-06-11 16:50:36 -040039 /* list of struct btrfs_work that are waiting for service */
40 struct list_head pending;
Chris Masond313d7a2009-04-20 15:50:09 -040041 struct list_head prio_pending;
Chris Mason8b712842008-06-11 16:50:36 -040042
43 /* list of worker threads from struct btrfs_workers */
44 struct list_head worker_list;
45
46 /* kthread */
47 struct task_struct *task;
48
49 /* number of things on the pending list */
50 atomic_t num_pending;
Chris Mason53863232008-08-15 15:34:18 -040051
Chris Mason90428462009-08-04 16:56:34 -040052 /* reference counter for this struct */
53 atomic_t refs;
54
Chris Mason4854ddd2008-08-15 15:34:17 -040055 unsigned long sequence;
Chris Mason8b712842008-06-11 16:50:36 -040056
57 /* protects the pending list. */
58 spinlock_t lock;
59
60 /* set to non-zero when this thread is already awake and kicking */
61 int working;
Chris Mason35d8ba62008-06-11 20:21:24 -040062
63 /* are we currently idle */
64 int idle;
Chris Mason8b712842008-06-11 16:50:36 -040065};
66
Josef Bacik0dc3b842011-11-18 14:37:27 -050067static int __btrfs_start_workers(struct btrfs_workers *workers);
68
Chris Mason8b712842008-06-11 16:50:36 -040069/*
Chris Mason61d92c32009-10-02 19:11:56 -040070 * btrfs_start_workers uses kthread_run, which can block waiting for memory
71 * for a very long time. It will actually throttle on page writeback,
72 * and so it may not make progress until after our btrfs worker threads
73 * process all of the pending work structs in their queue
74 *
75 * This means we can't use btrfs_start_workers from inside a btrfs worker
76 * thread that is used as part of cleaning dirty memory, which pretty much
77 * involves all of the worker threads.
78 *
79 * Instead we have a helper queue who never has more than one thread
80 * where we scheduler thread start operations. This worker_start struct
81 * is used to contain the work and hold a pointer to the queue that needs
82 * another worker.
83 */
84struct worker_start {
85 struct btrfs_work work;
86 struct btrfs_workers *queue;
87};
88
89static void start_new_worker_func(struct btrfs_work *work)
90{
91 struct worker_start *start;
92 start = container_of(work, struct worker_start, work);
Josef Bacik0dc3b842011-11-18 14:37:27 -050093 __btrfs_start_workers(start->queue);
Chris Mason61d92c32009-10-02 19:11:56 -040094 kfree(start);
95}
96
Chris Mason61d92c32009-10-02 19:11:56 -040097/*
Chris Mason35d8ba62008-06-11 20:21:24 -040098 * helper function to move a thread onto the idle list after it
99 * has finished some requests.
100 */
101static void check_idle_worker(struct btrfs_worker_thread *worker)
102{
103 if (!worker->idle && atomic_read(&worker->num_pending) <
104 worker->workers->idle_thresh / 2) {
105 unsigned long flags;
106 spin_lock_irqsave(&worker->workers->lock, flags);
107 worker->idle = 1;
Chris Mason3e99d8e2009-09-15 19:57:42 -0400108
109 /* the list may be empty if the worker is just starting */
110 if (!list_empty(&worker->worker_list)) {
111 list_move(&worker->worker_list,
112 &worker->workers->idle_list);
113 }
Chris Mason35d8ba62008-06-11 20:21:24 -0400114 spin_unlock_irqrestore(&worker->workers->lock, flags);
115 }
116}
117
118/*
119 * helper function to move a thread off the idle list after new
120 * pending work is added.
121 */
122static void check_busy_worker(struct btrfs_worker_thread *worker)
123{
124 if (worker->idle && atomic_read(&worker->num_pending) >=
125 worker->workers->idle_thresh) {
126 unsigned long flags;
127 spin_lock_irqsave(&worker->workers->lock, flags);
128 worker->idle = 0;
Chris Mason3e99d8e2009-09-15 19:57:42 -0400129
130 if (!list_empty(&worker->worker_list)) {
131 list_move_tail(&worker->worker_list,
132 &worker->workers->worker_list);
133 }
Chris Mason35d8ba62008-06-11 20:21:24 -0400134 spin_unlock_irqrestore(&worker->workers->lock, flags);
135 }
136}
137
Chris Mason90428462009-08-04 16:56:34 -0400138static void check_pending_worker_creates(struct btrfs_worker_thread *worker)
139{
140 struct btrfs_workers *workers = worker->workers;
Josef Bacik0dc3b842011-11-18 14:37:27 -0500141 struct worker_start *start;
Chris Mason90428462009-08-04 16:56:34 -0400142 unsigned long flags;
143
144 rmb();
145 if (!workers->atomic_start_pending)
146 return;
147
Josef Bacik0dc3b842011-11-18 14:37:27 -0500148 start = kzalloc(sizeof(*start), GFP_NOFS);
149 if (!start)
150 return;
151
152 start->work.func = start_new_worker_func;
153 start->queue = workers;
154
Chris Mason90428462009-08-04 16:56:34 -0400155 spin_lock_irqsave(&workers->lock, flags);
156 if (!workers->atomic_start_pending)
157 goto out;
158
159 workers->atomic_start_pending = 0;
Chris Mason61d92c32009-10-02 19:11:56 -0400160 if (workers->num_workers + workers->num_workers_starting >=
161 workers->max_workers)
Chris Mason90428462009-08-04 16:56:34 -0400162 goto out;
163
Chris Mason61d92c32009-10-02 19:11:56 -0400164 workers->num_workers_starting += 1;
Chris Mason90428462009-08-04 16:56:34 -0400165 spin_unlock_irqrestore(&workers->lock, flags);
Josef Bacik0dc3b842011-11-18 14:37:27 -0500166 btrfs_queue_worker(workers->atomic_worker_start, &start->work);
Chris Mason90428462009-08-04 16:56:34 -0400167 return;
168
169out:
Josef Bacik0dc3b842011-11-18 14:37:27 -0500170 kfree(start);
Chris Mason90428462009-08-04 16:56:34 -0400171 spin_unlock_irqrestore(&workers->lock, flags);
172}
173
Chris Mason4a69a412008-11-06 22:03:00 -0500174static noinline int run_ordered_completions(struct btrfs_workers *workers,
175 struct btrfs_work *work)
176{
Chris Mason4a69a412008-11-06 22:03:00 -0500177 if (!workers->ordered)
178 return 0;
179
180 set_bit(WORK_DONE_BIT, &work->flags);
181
Chris Mason4e3f9c52009-08-05 16:36:45 -0400182 spin_lock(&workers->order_lock);
Chris Mason4a69a412008-11-06 22:03:00 -0500183
Chris Masond313d7a2009-04-20 15:50:09 -0400184 while (1) {
185 if (!list_empty(&workers->prio_order_list)) {
186 work = list_entry(workers->prio_order_list.next,
187 struct btrfs_work, order_list);
188 } else if (!list_empty(&workers->order_list)) {
189 work = list_entry(workers->order_list.next,
190 struct btrfs_work, order_list);
191 } else {
192 break;
193 }
Chris Mason4a69a412008-11-06 22:03:00 -0500194 if (!test_bit(WORK_DONE_BIT, &work->flags))
195 break;
196
197 /* we are going to call the ordered done function, but
198 * we leave the work item on the list as a barrier so
199 * that later work items that are done don't have their
200 * functions called before this one returns
201 */
202 if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags))
203 break;
204
Chris Mason4e3f9c52009-08-05 16:36:45 -0400205 spin_unlock(&workers->order_lock);
Chris Mason4a69a412008-11-06 22:03:00 -0500206
207 work->ordered_func(work);
208
209 /* now take the lock again and call the freeing code */
Chris Mason4e3f9c52009-08-05 16:36:45 -0400210 spin_lock(&workers->order_lock);
Chris Mason4a69a412008-11-06 22:03:00 -0500211 list_del(&work->order_list);
212 work->ordered_free(work);
213 }
214
Chris Mason4e3f9c52009-08-05 16:36:45 -0400215 spin_unlock(&workers->order_lock);
Chris Mason4a69a412008-11-06 22:03:00 -0500216 return 0;
217}
218
Chris Mason90428462009-08-04 16:56:34 -0400219static void put_worker(struct btrfs_worker_thread *worker)
220{
221 if (atomic_dec_and_test(&worker->refs))
222 kfree(worker);
223}
224
225static int try_worker_shutdown(struct btrfs_worker_thread *worker)
226{
227 int freeit = 0;
228
229 spin_lock_irq(&worker->lock);
Chris Mason627e4212009-09-15 20:00:36 -0400230 spin_lock(&worker->workers->lock);
Chris Mason90428462009-08-04 16:56:34 -0400231 if (worker->workers->num_workers > 1 &&
232 worker->idle &&
233 !worker->working &&
234 !list_empty(&worker->worker_list) &&
235 list_empty(&worker->prio_pending) &&
Chris Mason6e740572009-09-15 20:02:33 -0400236 list_empty(&worker->pending) &&
237 atomic_read(&worker->num_pending) == 0) {
Chris Mason90428462009-08-04 16:56:34 -0400238 freeit = 1;
239 list_del_init(&worker->worker_list);
240 worker->workers->num_workers--;
241 }
Chris Mason627e4212009-09-15 20:00:36 -0400242 spin_unlock(&worker->workers->lock);
Chris Mason90428462009-08-04 16:56:34 -0400243 spin_unlock_irq(&worker->lock);
244
245 if (freeit)
246 put_worker(worker);
247 return freeit;
248}
249
Chris Mason4f878e82009-08-07 09:27:38 -0400250static struct btrfs_work *get_next_work(struct btrfs_worker_thread *worker,
251 struct list_head *prio_head,
252 struct list_head *head)
253{
254 struct btrfs_work *work = NULL;
255 struct list_head *cur = NULL;
256
257 if(!list_empty(prio_head))
258 cur = prio_head->next;
259
260 smp_mb();
261 if (!list_empty(&worker->prio_pending))
262 goto refill;
263
264 if (!list_empty(head))
265 cur = head->next;
266
267 if (cur)
268 goto out;
269
270refill:
271 spin_lock_irq(&worker->lock);
272 list_splice_tail_init(&worker->prio_pending, prio_head);
273 list_splice_tail_init(&worker->pending, head);
274
275 if (!list_empty(prio_head))
276 cur = prio_head->next;
277 else if (!list_empty(head))
278 cur = head->next;
279 spin_unlock_irq(&worker->lock);
280
281 if (!cur)
282 goto out_fail;
283
284out:
285 work = list_entry(cur, struct btrfs_work, list);
286
287out_fail:
288 return work;
289}
290
Chris Mason35d8ba62008-06-11 20:21:24 -0400291/*
Chris Mason8b712842008-06-11 16:50:36 -0400292 * main loop for servicing work items
293 */
294static int worker_loop(void *arg)
295{
296 struct btrfs_worker_thread *worker = arg;
Chris Mason4f878e82009-08-07 09:27:38 -0400297 struct list_head head;
298 struct list_head prio_head;
Chris Mason8b712842008-06-11 16:50:36 -0400299 struct btrfs_work *work;
Chris Mason4f878e82009-08-07 09:27:38 -0400300
301 INIT_LIST_HEAD(&head);
302 INIT_LIST_HEAD(&prio_head);
303
Chris Mason8b712842008-06-11 16:50:36 -0400304 do {
Chris Mason4f878e82009-08-07 09:27:38 -0400305again:
Chris Masond313d7a2009-04-20 15:50:09 -0400306 while (1) {
Chris Mason4f878e82009-08-07 09:27:38 -0400307
308
309 work = get_next_work(worker, &prio_head, &head);
310 if (!work)
Chris Masond313d7a2009-04-20 15:50:09 -0400311 break;
312
Chris Mason8b712842008-06-11 16:50:36 -0400313 list_del(&work->list);
Chris Mason4a69a412008-11-06 22:03:00 -0500314 clear_bit(WORK_QUEUED_BIT, &work->flags);
Chris Mason8b712842008-06-11 16:50:36 -0400315
316 work->worker = worker;
Chris Mason8b712842008-06-11 16:50:36 -0400317
318 work->func(work);
319
320 atomic_dec(&worker->num_pending);
Chris Mason4a69a412008-11-06 22:03:00 -0500321 /*
322 * unless this is an ordered work queue,
323 * 'work' was probably freed by func above.
324 */
325 run_ordered_completions(worker->workers, work);
326
Chris Mason90428462009-08-04 16:56:34 -0400327 check_pending_worker_creates(worker);
Chris Mason8f3b65a2011-12-15 09:29:43 -0500328 cond_resched();
Chris Mason8b712842008-06-11 16:50:36 -0400329 }
Chris Mason4f878e82009-08-07 09:27:38 -0400330
331 spin_lock_irq(&worker->lock);
332 check_idle_worker(worker);
333
Chris Mason8b712842008-06-11 16:50:36 -0400334 if (freezing(current)) {
Chris Masonb51912c2009-02-04 09:23:24 -0500335 worker->working = 0;
336 spin_unlock_irq(&worker->lock);
Tejun Heoa0acae02011-11-21 12:32:22 -0800337 try_to_freeze();
Chris Mason8b712842008-06-11 16:50:36 -0400338 } else {
Chris Mason8b712842008-06-11 16:50:36 -0400339 spin_unlock_irq(&worker->lock);
Chris Masonb51912c2009-02-04 09:23:24 -0500340 if (!kthread_should_stop()) {
341 cpu_relax();
342 /*
343 * we've dropped the lock, did someone else
344 * jump_in?
345 */
346 smp_mb();
Chris Masond313d7a2009-04-20 15:50:09 -0400347 if (!list_empty(&worker->pending) ||
348 !list_empty(&worker->prio_pending))
Chris Masonb51912c2009-02-04 09:23:24 -0500349 continue;
350
351 /*
352 * this short schedule allows more work to
353 * come in without the queue functions
354 * needing to go through wake_up_process()
355 *
356 * worker->working is still 1, so nobody
357 * is going to try and wake us up
358 */
359 schedule_timeout(1);
360 smp_mb();
Chris Masond313d7a2009-04-20 15:50:09 -0400361 if (!list_empty(&worker->pending) ||
362 !list_empty(&worker->prio_pending))
Chris Masonb51912c2009-02-04 09:23:24 -0500363 continue;
364
Amit Gudb5555f72009-04-02 17:01:27 -0400365 if (kthread_should_stop())
366 break;
367
Chris Masonb51912c2009-02-04 09:23:24 -0500368 /* still no more work?, sleep for real */
369 spin_lock_irq(&worker->lock);
370 set_current_state(TASK_INTERRUPTIBLE);
Chris Masond313d7a2009-04-20 15:50:09 -0400371 if (!list_empty(&worker->pending) ||
Chris Mason4f878e82009-08-07 09:27:38 -0400372 !list_empty(&worker->prio_pending)) {
373 spin_unlock_irq(&worker->lock);
Chris Masoned3b3d3142010-05-25 10:12:41 -0400374 set_current_state(TASK_RUNNING);
Chris Mason4f878e82009-08-07 09:27:38 -0400375 goto again;
376 }
Chris Masonb51912c2009-02-04 09:23:24 -0500377
378 /*
379 * this makes sure we get a wakeup when someone
380 * adds something new to the queue
381 */
382 worker->working = 0;
383 spin_unlock_irq(&worker->lock);
384
Chris Mason90428462009-08-04 16:56:34 -0400385 if (!kthread_should_stop()) {
386 schedule_timeout(HZ * 120);
387 if (!worker->working &&
388 try_worker_shutdown(worker)) {
389 return 0;
390 }
391 }
Chris Masonb51912c2009-02-04 09:23:24 -0500392 }
Chris Mason8b712842008-06-11 16:50:36 -0400393 __set_current_state(TASK_RUNNING);
394 }
395 } while (!kthread_should_stop());
396 return 0;
397}
398
399/*
400 * this will wait for all the worker threads to shutdown
401 */
402int btrfs_stop_workers(struct btrfs_workers *workers)
403{
404 struct list_head *cur;
405 struct btrfs_worker_thread *worker;
Chris Mason90428462009-08-04 16:56:34 -0400406 int can_stop;
Chris Mason8b712842008-06-11 16:50:36 -0400407
Chris Mason90428462009-08-04 16:56:34 -0400408 spin_lock_irq(&workers->lock);
Chris Mason35d8ba62008-06-11 20:21:24 -0400409 list_splice_init(&workers->idle_list, &workers->worker_list);
Chris Masond3977122009-01-05 21:25:51 -0500410 while (!list_empty(&workers->worker_list)) {
Chris Mason8b712842008-06-11 16:50:36 -0400411 cur = workers->worker_list.next;
412 worker = list_entry(cur, struct btrfs_worker_thread,
413 worker_list);
Chris Mason90428462009-08-04 16:56:34 -0400414
415 atomic_inc(&worker->refs);
416 workers->num_workers -= 1;
417 if (!list_empty(&worker->worker_list)) {
418 list_del_init(&worker->worker_list);
419 put_worker(worker);
420 can_stop = 1;
421 } else
422 can_stop = 0;
423 spin_unlock_irq(&workers->lock);
424 if (can_stop)
425 kthread_stop(worker->task);
426 spin_lock_irq(&workers->lock);
427 put_worker(worker);
Chris Mason8b712842008-06-11 16:50:36 -0400428 }
Chris Mason90428462009-08-04 16:56:34 -0400429 spin_unlock_irq(&workers->lock);
Chris Mason8b712842008-06-11 16:50:36 -0400430 return 0;
431}
432
433/*
434 * simple init on struct btrfs_workers
435 */
Chris Mason61d92c32009-10-02 19:11:56 -0400436void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max,
437 struct btrfs_workers *async_helper)
Chris Mason8b712842008-06-11 16:50:36 -0400438{
439 workers->num_workers = 0;
Chris Mason61d92c32009-10-02 19:11:56 -0400440 workers->num_workers_starting = 0;
Chris Mason8b712842008-06-11 16:50:36 -0400441 INIT_LIST_HEAD(&workers->worker_list);
Chris Mason35d8ba62008-06-11 20:21:24 -0400442 INIT_LIST_HEAD(&workers->idle_list);
Chris Mason4a69a412008-11-06 22:03:00 -0500443 INIT_LIST_HEAD(&workers->order_list);
Chris Masond313d7a2009-04-20 15:50:09 -0400444 INIT_LIST_HEAD(&workers->prio_order_list);
Chris Mason8b712842008-06-11 16:50:36 -0400445 spin_lock_init(&workers->lock);
Chris Mason4e3f9c52009-08-05 16:36:45 -0400446 spin_lock_init(&workers->order_lock);
Chris Mason8b712842008-06-11 16:50:36 -0400447 workers->max_workers = max;
Chris Mason61b49442008-07-31 15:42:53 -0400448 workers->idle_thresh = 32;
Chris Mason5443be42008-08-15 15:34:16 -0400449 workers->name = name;
Chris Mason4a69a412008-11-06 22:03:00 -0500450 workers->ordered = 0;
Chris Mason90428462009-08-04 16:56:34 -0400451 workers->atomic_start_pending = 0;
Chris Mason61d92c32009-10-02 19:11:56 -0400452 workers->atomic_worker_start = async_helper;
Chris Mason8b712842008-06-11 16:50:36 -0400453}
454
455/*
456 * starts new worker threads. This does not enforce the max worker
457 * count in case you need to temporarily go past it.
458 */
Josef Bacik0dc3b842011-11-18 14:37:27 -0500459static int __btrfs_start_workers(struct btrfs_workers *workers)
Chris Mason8b712842008-06-11 16:50:36 -0400460{
461 struct btrfs_worker_thread *worker;
462 int ret = 0;
Chris Mason8b712842008-06-11 16:50:36 -0400463
Josef Bacik0dc3b842011-11-18 14:37:27 -0500464 worker = kzalloc(sizeof(*worker), GFP_NOFS);
465 if (!worker) {
466 ret = -ENOMEM;
467 goto fail;
Chris Mason8b712842008-06-11 16:50:36 -0400468 }
Josef Bacik0dc3b842011-11-18 14:37:27 -0500469
470 INIT_LIST_HEAD(&worker->pending);
471 INIT_LIST_HEAD(&worker->prio_pending);
472 INIT_LIST_HEAD(&worker->worker_list);
473 spin_lock_init(&worker->lock);
474
475 atomic_set(&worker->num_pending, 0);
476 atomic_set(&worker->refs, 1);
477 worker->workers = workers;
478 worker->task = kthread_run(worker_loop, worker,
479 "btrfs-%s-%d", workers->name,
480 workers->num_workers + 1);
481 if (IS_ERR(worker->task)) {
482 ret = PTR_ERR(worker->task);
483 kfree(worker);
484 goto fail;
485 }
486 spin_lock_irq(&workers->lock);
487 list_add_tail(&worker->worker_list, &workers->idle_list);
488 worker->idle = 1;
489 workers->num_workers++;
490 workers->num_workers_starting--;
491 WARN_ON(workers->num_workers_starting < 0);
492 spin_unlock_irq(&workers->lock);
493
Chris Mason8b712842008-06-11 16:50:36 -0400494 return 0;
495fail:
Josef Bacik0dc3b842011-11-18 14:37:27 -0500496 spin_lock_irq(&workers->lock);
497 workers->num_workers_starting--;
498 spin_unlock_irq(&workers->lock);
Chris Mason8b712842008-06-11 16:50:36 -0400499 return ret;
500}
501
Josef Bacik0dc3b842011-11-18 14:37:27 -0500502int btrfs_start_workers(struct btrfs_workers *workers)
Chris Mason61d92c32009-10-02 19:11:56 -0400503{
504 spin_lock_irq(&workers->lock);
Josef Bacik0dc3b842011-11-18 14:37:27 -0500505 workers->num_workers_starting++;
Chris Mason61d92c32009-10-02 19:11:56 -0400506 spin_unlock_irq(&workers->lock);
Josef Bacik0dc3b842011-11-18 14:37:27 -0500507 return __btrfs_start_workers(workers);
Chris Mason61d92c32009-10-02 19:11:56 -0400508}
509
Chris Mason8b712842008-06-11 16:50:36 -0400510/*
511 * run through the list and find a worker thread that doesn't have a lot
512 * to do right now. This can return null if we aren't yet at the thread
513 * count limit and all of the threads are busy.
514 */
515static struct btrfs_worker_thread *next_worker(struct btrfs_workers *workers)
516{
517 struct btrfs_worker_thread *worker;
518 struct list_head *next;
Chris Mason61d92c32009-10-02 19:11:56 -0400519 int enforce_min;
520
521 enforce_min = (workers->num_workers + workers->num_workers_starting) <
522 workers->max_workers;
Chris Mason8b712842008-06-11 16:50:36 -0400523
Chris Mason8b712842008-06-11 16:50:36 -0400524 /*
Chris Mason35d8ba62008-06-11 20:21:24 -0400525 * if we find an idle thread, don't move it to the end of the
526 * idle list. This improves the chance that the next submission
527 * will reuse the same thread, and maybe catch it while it is still
528 * working
Chris Mason8b712842008-06-11 16:50:36 -0400529 */
Chris Mason35d8ba62008-06-11 20:21:24 -0400530 if (!list_empty(&workers->idle_list)) {
531 next = workers->idle_list.next;
Chris Mason8b712842008-06-11 16:50:36 -0400532 worker = list_entry(next, struct btrfs_worker_thread,
533 worker_list);
Chris Mason35d8ba62008-06-11 20:21:24 -0400534 return worker;
Chris Mason8b712842008-06-11 16:50:36 -0400535 }
Chris Mason35d8ba62008-06-11 20:21:24 -0400536 if (enforce_min || list_empty(&workers->worker_list))
537 return NULL;
538
Chris Mason8b712842008-06-11 16:50:36 -0400539 /*
Chris Mason35d8ba62008-06-11 20:21:24 -0400540 * if we pick a busy task, move the task to the end of the list.
Chris Masond352ac62008-09-29 15:18:18 -0400541 * hopefully this will keep things somewhat evenly balanced.
542 * Do the move in batches based on the sequence number. This groups
543 * requests submitted at roughly the same time onto the same worker.
Chris Mason8b712842008-06-11 16:50:36 -0400544 */
Chris Mason35d8ba62008-06-11 20:21:24 -0400545 next = workers->worker_list.next;
546 worker = list_entry(next, struct btrfs_worker_thread, worker_list);
Chris Mason4854ddd2008-08-15 15:34:17 -0400547 worker->sequence++;
Chris Masond352ac62008-09-29 15:18:18 -0400548
Chris Mason53863232008-08-15 15:34:18 -0400549 if (worker->sequence % workers->idle_thresh == 0)
Chris Mason4854ddd2008-08-15 15:34:17 -0400550 list_move_tail(next, &workers->worker_list);
Chris Mason8b712842008-06-11 16:50:36 -0400551 return worker;
552}
553
Chris Masond352ac62008-09-29 15:18:18 -0400554/*
555 * selects a worker thread to take the next job. This will either find
556 * an idle worker, start a new worker up to the max count, or just return
557 * one of the existing busy workers.
558 */
Chris Mason8b712842008-06-11 16:50:36 -0400559static struct btrfs_worker_thread *find_worker(struct btrfs_workers *workers)
560{
561 struct btrfs_worker_thread *worker;
562 unsigned long flags;
Chris Mason90428462009-08-04 16:56:34 -0400563 struct list_head *fallback;
Josef Bacik0dc3b842011-11-18 14:37:27 -0500564 int ret;
Chris Mason8b712842008-06-11 16:50:36 -0400565
Chris Mason8b712842008-06-11 16:50:36 -0400566 spin_lock_irqsave(&workers->lock, flags);
Chris Mason8d532b22011-12-23 07:53:00 -0500567again:
Chris Mason8b712842008-06-11 16:50:36 -0400568 worker = next_worker(workers);
Chris Mason8b712842008-06-11 16:50:36 -0400569
570 if (!worker) {
Chris Mason61d92c32009-10-02 19:11:56 -0400571 if (workers->num_workers + workers->num_workers_starting >=
572 workers->max_workers) {
Chris Mason90428462009-08-04 16:56:34 -0400573 goto fallback;
574 } else if (workers->atomic_worker_start) {
575 workers->atomic_start_pending = 1;
576 goto fallback;
Chris Mason8b712842008-06-11 16:50:36 -0400577 } else {
Chris Mason61d92c32009-10-02 19:11:56 -0400578 workers->num_workers_starting++;
Chris Mason8b712842008-06-11 16:50:36 -0400579 spin_unlock_irqrestore(&workers->lock, flags);
580 /* we're below the limit, start another worker */
Josef Bacik0dc3b842011-11-18 14:37:27 -0500581 ret = __btrfs_start_workers(workers);
Chris Mason8d532b22011-12-23 07:53:00 -0500582 spin_lock_irqsave(&workers->lock, flags);
Josef Bacik0dc3b842011-11-18 14:37:27 -0500583 if (ret)
584 goto fallback;
Chris Mason8b712842008-06-11 16:50:36 -0400585 goto again;
586 }
587 }
Chris Mason6e740572009-09-15 20:02:33 -0400588 goto found;
Chris Mason90428462009-08-04 16:56:34 -0400589
590fallback:
591 fallback = NULL;
592 /*
593 * we have failed to find any workers, just
594 * return the first one we can find.
595 */
596 if (!list_empty(&workers->worker_list))
597 fallback = workers->worker_list.next;
598 if (!list_empty(&workers->idle_list))
599 fallback = workers->idle_list.next;
600 BUG_ON(!fallback);
601 worker = list_entry(fallback,
602 struct btrfs_worker_thread, worker_list);
Chris Mason6e740572009-09-15 20:02:33 -0400603found:
604 /*
605 * this makes sure the worker doesn't exit before it is placed
606 * onto a busy/idle list
607 */
608 atomic_inc(&worker->num_pending);
Chris Mason90428462009-08-04 16:56:34 -0400609 spin_unlock_irqrestore(&workers->lock, flags);
610 return worker;
Chris Mason8b712842008-06-11 16:50:36 -0400611}
612
613/*
614 * btrfs_requeue_work just puts the work item back on the tail of the list
615 * it was taken from. It is intended for use with long running work functions
616 * that make some progress and want to give the cpu up for others.
617 */
618int btrfs_requeue_work(struct btrfs_work *work)
619{
620 struct btrfs_worker_thread *worker = work->worker;
621 unsigned long flags;
Chris Masona6837052009-02-04 09:19:41 -0500622 int wake = 0;
Chris Mason8b712842008-06-11 16:50:36 -0400623
Chris Mason4a69a412008-11-06 22:03:00 -0500624 if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
Chris Mason8b712842008-06-11 16:50:36 -0400625 goto out;
626
627 spin_lock_irqsave(&worker->lock, flags);
Chris Masond313d7a2009-04-20 15:50:09 -0400628 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags))
629 list_add_tail(&work->list, &worker->prio_pending);
630 else
631 list_add_tail(&work->list, &worker->pending);
Chris Masonb51912c2009-02-04 09:23:24 -0500632 atomic_inc(&worker->num_pending);
Chris Mason75ccf472008-09-30 19:24:06 -0400633
634 /* by definition we're busy, take ourselves off the idle
635 * list
636 */
637 if (worker->idle) {
Julia Lawall29c5e8c2009-07-22 16:49:00 -0400638 spin_lock(&worker->workers->lock);
Chris Mason75ccf472008-09-30 19:24:06 -0400639 worker->idle = 0;
640 list_move_tail(&worker->worker_list,
Chris Mason6e740572009-09-15 20:02:33 -0400641 &worker->workers->worker_list);
Julia Lawall29c5e8c2009-07-22 16:49:00 -0400642 spin_unlock(&worker->workers->lock);
Chris Mason75ccf472008-09-30 19:24:06 -0400643 }
Chris Masona6837052009-02-04 09:19:41 -0500644 if (!worker->working) {
645 wake = 1;
646 worker->working = 1;
647 }
Chris Mason75ccf472008-09-30 19:24:06 -0400648
Chris Masona6837052009-02-04 09:19:41 -0500649 if (wake)
650 wake_up_process(worker->task);
Chris Mason90428462009-08-04 16:56:34 -0400651 spin_unlock_irqrestore(&worker->lock, flags);
Chris Mason8b712842008-06-11 16:50:36 -0400652out:
Chris Masona6837052009-02-04 09:19:41 -0500653
Chris Mason8b712842008-06-11 16:50:36 -0400654 return 0;
655}
656
Chris Masond313d7a2009-04-20 15:50:09 -0400657void btrfs_set_work_high_prio(struct btrfs_work *work)
658{
659 set_bit(WORK_HIGH_PRIO_BIT, &work->flags);
660}
661
Chris Mason8b712842008-06-11 16:50:36 -0400662/*
663 * places a struct btrfs_work into the pending queue of one of the kthreads
664 */
Josef Bacik0dc3b842011-11-18 14:37:27 -0500665void btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work)
Chris Mason8b712842008-06-11 16:50:36 -0400666{
667 struct btrfs_worker_thread *worker;
668 unsigned long flags;
669 int wake = 0;
670
671 /* don't requeue something already on a list */
Chris Mason4a69a412008-11-06 22:03:00 -0500672 if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
Josef Bacik0dc3b842011-11-18 14:37:27 -0500673 return;
Chris Mason8b712842008-06-11 16:50:36 -0400674
675 worker = find_worker(workers);
Chris Mason4a69a412008-11-06 22:03:00 -0500676 if (workers->ordered) {
Chris Mason4e3f9c52009-08-05 16:36:45 -0400677 /*
678 * you're not allowed to do ordered queues from an
679 * interrupt handler
680 */
681 spin_lock(&workers->order_lock);
Chris Masond313d7a2009-04-20 15:50:09 -0400682 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) {
683 list_add_tail(&work->order_list,
684 &workers->prio_order_list);
685 } else {
686 list_add_tail(&work->order_list, &workers->order_list);
687 }
Chris Mason4e3f9c52009-08-05 16:36:45 -0400688 spin_unlock(&workers->order_lock);
Chris Mason4a69a412008-11-06 22:03:00 -0500689 } else {
690 INIT_LIST_HEAD(&work->order_list);
691 }
Chris Mason8b712842008-06-11 16:50:36 -0400692
693 spin_lock_irqsave(&worker->lock, flags);
Chris Masona6837052009-02-04 09:19:41 -0500694
Chris Masond313d7a2009-04-20 15:50:09 -0400695 if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags))
696 list_add_tail(&work->list, &worker->prio_pending);
697 else
698 list_add_tail(&work->list, &worker->pending);
Chris Mason35d8ba62008-06-11 20:21:24 -0400699 check_busy_worker(worker);
Chris Mason8b712842008-06-11 16:50:36 -0400700
701 /*
702 * avoid calling into wake_up_process if this thread has already
703 * been kicked
704 */
705 if (!worker->working)
706 wake = 1;
707 worker->working = 1;
708
Chris Mason8b712842008-06-11 16:50:36 -0400709 if (wake)
710 wake_up_process(worker->task);
Chris Mason90428462009-08-04 16:56:34 -0400711 spin_unlock_irqrestore(&worker->lock, flags);
Chris Mason8b712842008-06-11 16:50:36 -0400712}