Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * mm/pdflush.c - worker threads for writing back filesystem data |
| 3 | * |
| 4 | * Copyright (C) 2002, Linus Torvalds. |
| 5 | * |
Francois Cami | e1f8e87 | 2008-10-15 22:01:59 -0700 | [diff] [blame] | 6 | * 09Apr2002 Andrew Morton |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * Initial version |
| 8 | * 29Feb2004 kaos@sgi.com |
| 9 | * Move worker thread creation to kthread to avoid chewing |
| 10 | * up stack space with nested calls to kernel_thread. |
| 11 | */ |
| 12 | |
| 13 | #include <linux/sched.h> |
| 14 | #include <linux/list.h> |
| 15 | #include <linux/signal.h> |
| 16 | #include <linux/spinlock.h> |
| 17 | #include <linux/gfp.h> |
| 18 | #include <linux/init.h> |
| 19 | #include <linux/module.h> |
Pavel Machek | f526448 | 2008-04-21 22:15:06 +0000 | [diff] [blame] | 20 | #include <linux/fs.h> /* Needed by writeback.h */ |
| 21 | #include <linux/writeback.h> /* Prototypes pdflush_operation() */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | #include <linux/kthread.h> |
Paul Jackson | 28a42b9 | 2005-10-30 15:02:32 -0800 | [diff] [blame] | 23 | #include <linux/cpuset.h> |
Nigel Cunningham | 7dfb710 | 2006-12-06 20:34:23 -0800 | [diff] [blame] | 24 | #include <linux/freezer.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | |
| 27 | /* |
| 28 | * Minimum and maximum number of pdflush instances |
| 29 | */ |
| 30 | #define MIN_PDFLUSH_THREADS 2 |
| 31 | #define MAX_PDFLUSH_THREADS 8 |
| 32 | |
| 33 | static void start_one_pdflush_thread(void); |
| 34 | |
| 35 | |
| 36 | /* |
| 37 | * The pdflush threads are worker threads for writing back dirty data. |
| 38 | * Ideally, we'd like one thread per active disk spindle. But the disk |
| 39 | * topology is very hard to divine at this level. Instead, we take |
| 40 | * care in various places to prevent more than one pdflush thread from |
| 41 | * performing writeback against a single filesystem. pdflush threads |
| 42 | * have the PF_FLUSHER flag set in current->flags to aid in this. |
| 43 | */ |
| 44 | |
| 45 | /* |
| 46 | * All the pdflush threads. Protected by pdflush_lock |
| 47 | */ |
| 48 | static LIST_HEAD(pdflush_list); |
| 49 | static DEFINE_SPINLOCK(pdflush_lock); |
| 50 | |
| 51 | /* |
| 52 | * The count of currently-running pdflush threads. Protected |
| 53 | * by pdflush_lock. |
| 54 | * |
| 55 | * Readable by sysctl, but not writable. Published to userspace at |
| 56 | * /proc/sys/vm/nr_pdflush_threads. |
| 57 | */ |
| 58 | int nr_pdflush_threads = 0; |
| 59 | |
| 60 | /* |
| 61 | * The time at which the pdflush thread pool last went empty |
| 62 | */ |
| 63 | static unsigned long last_empty_jifs; |
| 64 | |
| 65 | /* |
| 66 | * The pdflush thread. |
| 67 | * |
| 68 | * Thread pool management algorithm: |
| 69 | * |
| 70 | * - The minimum and maximum number of pdflush instances are bound |
Jens Axboe | cd17cbf | 2009-05-15 11:32:24 +0200 | [diff] [blame] | 71 | * by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | * |
| 73 | * - If there have been no idle pdflush instances for 1 second, create |
| 74 | * a new one. |
| 75 | * |
| 76 | * - If the least-recently-went-to-sleep pdflush thread has been asleep |
| 77 | * for more than one second, terminate a thread. |
| 78 | */ |
| 79 | |
| 80 | /* |
| 81 | * A structure for passing work to a pdflush thread. Also for passing |
| 82 | * state information between pdflush threads. Protected by pdflush_lock. |
| 83 | */ |
| 84 | struct pdflush_work { |
| 85 | struct task_struct *who; /* The thread */ |
| 86 | void (*fn)(unsigned long); /* A callback function */ |
| 87 | unsigned long arg0; /* An argument to the callback */ |
| 88 | struct list_head list; /* On pdflush_list, when idle */ |
| 89 | unsigned long when_i_went_to_sleep; |
| 90 | }; |
| 91 | |
| 92 | static int __pdflush(struct pdflush_work *my_work) |
| 93 | { |
Christoph Lameter | 930d915 | 2006-01-08 01:00:47 -0800 | [diff] [blame] | 94 | current->flags |= PF_FLUSHER | PF_SWAPWRITE; |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 95 | set_freezable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | my_work->fn = NULL; |
| 97 | my_work->who = current; |
| 98 | INIT_LIST_HEAD(&my_work->list); |
| 99 | |
| 100 | spin_lock_irq(&pdflush_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | for ( ; ; ) { |
| 102 | struct pdflush_work *pdf; |
| 103 | |
| 104 | set_current_state(TASK_INTERRUPTIBLE); |
| 105 | list_move(&my_work->list, &pdflush_list); |
| 106 | my_work->when_i_went_to_sleep = jiffies; |
| 107 | spin_unlock_irq(&pdflush_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | schedule(); |
Andrew Morton | d616e09 | 2006-06-25 05:47:46 -0700 | [diff] [blame] | 109 | try_to_freeze(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | spin_lock_irq(&pdflush_lock); |
| 111 | if (!list_empty(&my_work->list)) { |
Andrew Morton | d616e09 | 2006-06-25 05:47:46 -0700 | [diff] [blame] | 112 | /* |
| 113 | * Someone woke us up, but without removing our control |
| 114 | * structure from the global list. swsusp will do this |
| 115 | * in try_to_freeze()->refrigerator(). Handle it. |
| 116 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | my_work->fn = NULL; |
| 118 | continue; |
| 119 | } |
| 120 | if (my_work->fn == NULL) { |
Andrew Morton | d616e09 | 2006-06-25 05:47:46 -0700 | [diff] [blame] | 121 | printk("pdflush: bogus wakeup\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | continue; |
| 123 | } |
| 124 | spin_unlock_irq(&pdflush_lock); |
| 125 | |
| 126 | (*my_work->fn)(my_work->arg0); |
| 127 | |
Peter W Morreale | a56ed66 | 2009-04-06 19:00:28 -0700 | [diff] [blame] | 128 | spin_lock_irq(&pdflush_lock); |
| 129 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | /* |
| 131 | * Thread creation: For how long have there been zero |
| 132 | * available threads? |
Peter W Morreale | a56ed66 | 2009-04-06 19:00:28 -0700 | [diff] [blame] | 133 | * |
| 134 | * To throttle creation, we reset last_empty_jifs. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 135 | */ |
OGAWA Hirofumi | 2b4bc46 | 2008-07-25 01:45:42 -0700 | [diff] [blame] | 136 | if (time_after(jiffies, last_empty_jifs + 1 * HZ)) { |
Jens Axboe | cd17cbf | 2009-05-15 11:32:24 +0200 | [diff] [blame] | 137 | if (list_empty(&pdflush_list)) { |
| 138 | if (nr_pdflush_threads < MAX_PDFLUSH_THREADS) { |
| 139 | last_empty_jifs = jiffies; |
| 140 | nr_pdflush_threads++; |
| 141 | spin_unlock_irq(&pdflush_lock); |
| 142 | start_one_pdflush_thread(); |
| 143 | spin_lock_irq(&pdflush_lock); |
| 144 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | } |
| 146 | } |
| 147 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 148 | my_work->fn = NULL; |
| 149 | |
| 150 | /* |
| 151 | * Thread destruction: For how long has the sleepiest |
| 152 | * thread slept? |
| 153 | */ |
| 154 | if (list_empty(&pdflush_list)) |
| 155 | continue; |
Jens Axboe | cd17cbf | 2009-05-15 11:32:24 +0200 | [diff] [blame] | 156 | if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | continue; |
| 158 | pdf = list_entry(pdflush_list.prev, struct pdflush_work, list); |
OGAWA Hirofumi | 2b4bc46 | 2008-07-25 01:45:42 -0700 | [diff] [blame] | 159 | if (time_after(jiffies, pdf->when_i_went_to_sleep + 1 * HZ)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 160 | /* Limit exit rate */ |
| 161 | pdf->when_i_went_to_sleep = jiffies; |
| 162 | break; /* exeunt */ |
| 163 | } |
| 164 | } |
| 165 | nr_pdflush_threads--; |
| 166 | spin_unlock_irq(&pdflush_lock); |
| 167 | return 0; |
| 168 | } |
| 169 | |
| 170 | /* |
| 171 | * Of course, my_work wants to be just a local in __pdflush(). It is |
| 172 | * separated out in this manner to hopefully prevent the compiler from |
| 173 | * performing unfortunate optimisations against the auto variables. Because |
| 174 | * these are visible to other tasks and CPUs. (No problem has actually |
| 175 | * been observed. This is just paranoia). |
| 176 | */ |
| 177 | static int pdflush(void *dummy) |
| 178 | { |
| 179 | struct pdflush_work my_work; |
Rusty Russell | 174596a | 2009-01-01 10:12:29 +1030 | [diff] [blame] | 180 | cpumask_var_t cpus_allowed; |
| 181 | |
| 182 | /* |
| 183 | * Since the caller doesn't even check kthread_run() worked, let's not |
| 184 | * freak out too much if this fails. |
| 185 | */ |
| 186 | if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { |
| 187 | printk(KERN_WARNING "pdflush failed to allocate cpumask\n"); |
| 188 | return 0; |
| 189 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | |
| 191 | /* |
| 192 | * pdflush can spend a lot of time doing encryption via dm-crypt. We |
| 193 | * don't want to do that at keventd's priority. |
| 194 | */ |
| 195 | set_user_nice(current, 0); |
Paul Jackson | 28a42b9 | 2005-10-30 15:02:32 -0800 | [diff] [blame] | 196 | |
| 197 | /* |
| 198 | * Some configs put our parent kthread in a limited cpuset, |
Rusty Russell | 1a2142a | 2009-03-30 22:05:10 -0600 | [diff] [blame] | 199 | * which kthread() overrides, forcing cpus_allowed == cpu_all_mask. |
Paul Jackson | 28a42b9 | 2005-10-30 15:02:32 -0800 | [diff] [blame] | 200 | * Our needs are more modest - cut back to our cpusets cpus_allowed. |
| 201 | * This is needed as pdflush's are dynamically created and destroyed. |
| 202 | * The boottime pdflush's are easily placed w/o these 2 lines. |
| 203 | */ |
Rusty Russell | 174596a | 2009-01-01 10:12:29 +1030 | [diff] [blame] | 204 | cpuset_cpus_allowed(current, cpus_allowed); |
| 205 | set_cpus_allowed_ptr(current, cpus_allowed); |
| 206 | free_cpumask_var(cpus_allowed); |
Paul Jackson | 28a42b9 | 2005-10-30 15:02:32 -0800 | [diff] [blame] | 207 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | return __pdflush(&my_work); |
| 209 | } |
| 210 | |
| 211 | /* |
| 212 | * Attempt to wake up a pdflush thread, and get it to do some work for you. |
| 213 | * Returns zero if it indeed managed to find a worker thread, and passed your |
| 214 | * payload to it. |
| 215 | */ |
| 216 | int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0) |
| 217 | { |
| 218 | unsigned long flags; |
| 219 | int ret = 0; |
| 220 | |
Eric Sesterhenn | 125e187 | 2006-06-23 02:06:06 -0700 | [diff] [blame] | 221 | BUG_ON(fn == NULL); /* Hard to diagnose if it's deferred */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | |
| 223 | spin_lock_irqsave(&pdflush_lock, flags); |
| 224 | if (list_empty(&pdflush_list)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | ret = -1; |
| 226 | } else { |
| 227 | struct pdflush_work *pdf; |
| 228 | |
| 229 | pdf = list_entry(pdflush_list.next, struct pdflush_work, list); |
| 230 | list_del_init(&pdf->list); |
| 231 | if (list_empty(&pdflush_list)) |
| 232 | last_empty_jifs = jiffies; |
| 233 | pdf->fn = fn; |
| 234 | pdf->arg0 = arg0; |
| 235 | wake_up_process(pdf->who); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | } |
Denis Cheng | 5aecd55 | 2008-05-12 14:02:14 -0700 | [diff] [blame] | 237 | spin_unlock_irqrestore(&pdflush_lock, flags); |
| 238 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 239 | return ret; |
| 240 | } |
| 241 | |
| 242 | static void start_one_pdflush_thread(void) |
| 243 | { |
Peter W Morreale | a56ed66 | 2009-04-06 19:00:28 -0700 | [diff] [blame] | 244 | struct task_struct *k; |
| 245 | |
| 246 | k = kthread_run(pdflush, NULL, "pdflush"); |
| 247 | if (unlikely(IS_ERR(k))) { |
| 248 | spin_lock_irq(&pdflush_lock); |
| 249 | nr_pdflush_threads--; |
| 250 | spin_unlock_irq(&pdflush_lock); |
| 251 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | } |
| 253 | |
| 254 | static int __init pdflush_init(void) |
| 255 | { |
| 256 | int i; |
| 257 | |
Peter W Morreale | a56ed66 | 2009-04-06 19:00:28 -0700 | [diff] [blame] | 258 | /* |
| 259 | * Pre-set nr_pdflush_threads... If we fail to create, |
| 260 | * the count will be decremented. |
| 261 | */ |
Jens Axboe | cd17cbf | 2009-05-15 11:32:24 +0200 | [diff] [blame] | 262 | nr_pdflush_threads = MIN_PDFLUSH_THREADS; |
Peter W Morreale | a56ed66 | 2009-04-06 19:00:28 -0700 | [diff] [blame] | 263 | |
Jens Axboe | cd17cbf | 2009-05-15 11:32:24 +0200 | [diff] [blame] | 264 | for (i = 0; i < MIN_PDFLUSH_THREADS; i++) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 265 | start_one_pdflush_thread(); |
| 266 | return 0; |
| 267 | } |
| 268 | |
| 269 | module_init(pdflush_init); |