Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* Kernel thread helper functions. |
| 2 | * Copyright (C) 2004 IBM Corporation, Rusty Russell. |
| 3 | * |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 4 | * Creation is done via kthreadd, so that we get a clean environment |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * even if we're invoked from userspace (think modprobe, hotplug cpu, |
| 6 | * etc.). |
| 7 | */ |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/kthread.h> |
| 10 | #include <linux/completion.h> |
| 11 | #include <linux/err.h> |
Miao Xie | 58568d2 | 2009-06-16 15:31:49 -0700 | [diff] [blame] | 12 | #include <linux/cpuset.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/unistd.h> |
| 14 | #include <linux/file.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 15 | #include <linux/export.h> |
Arjan van de Ven | 97d1f15 | 2006-03-23 03:00:24 -0800 | [diff] [blame] | 16 | #include <linux/mutex.h> |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 17 | #include <linux/slab.h> |
| 18 | #include <linux/freezer.h> |
Al Viro | a74fb73 | 2012-10-10 21:28:25 -0400 | [diff] [blame^] | 19 | #include <linux/ptrace.h> |
Steven Rostedt | ad8d75f | 2009-04-14 19:39:12 -0400 | [diff] [blame] | 20 | #include <trace/events/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 22 | static DEFINE_SPINLOCK(kthread_create_lock); |
| 23 | static LIST_HEAD(kthread_create_list); |
| 24 | struct task_struct *kthreadd_task; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | struct kthread_create_info |
| 27 | { |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 28 | /* Information passed to kthread() from kthreadd. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | int (*threadfn)(void *data); |
| 30 | void *data; |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 31 | int node; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 33 | /* Result passed back to kthread_create() from kthreadd. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | struct task_struct *result; |
| 35 | struct completion done; |
David Howells | 65f27f3 | 2006-11-22 14:55:48 +0000 | [diff] [blame] | 36 | |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 37 | struct list_head list; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | }; |
| 39 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 40 | struct kthread { |
| 41 | int should_stop; |
Tejun Heo | 82805ab | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 42 | void *data; |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 43 | struct completion exited; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | }; |
| 45 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 46 | #define to_kthread(tsk) \ |
| 47 | container_of((tsk)->vfork_done, struct kthread, exited) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 49 | /** |
| 50 | * kthread_should_stop - should this kthread return now? |
| 51 | * |
Robert P. J. Day | 72fd4a3 | 2007-02-10 01:45:59 -0800 | [diff] [blame] | 52 | * When someone calls kthread_stop() on your kthread, it will be woken |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 53 | * and this will return true. You should then return, and your return |
| 54 | * value will be passed through to kthread_stop(). |
| 55 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 56 | int kthread_should_stop(void) |
| 57 | { |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 58 | return to_kthread(current)->should_stop; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | } |
| 60 | EXPORT_SYMBOL(kthread_should_stop); |
| 61 | |
Tejun Heo | 82805ab | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 62 | /** |
Tejun Heo | 8a32c44 | 2011-11-21 12:32:23 -0800 | [diff] [blame] | 63 | * kthread_freezable_should_stop - should this freezable kthread return now? |
| 64 | * @was_frozen: optional out parameter, indicates whether %current was frozen |
| 65 | * |
| 66 | * kthread_should_stop() for freezable kthreads, which will enter |
| 67 | * refrigerator if necessary. This function is safe from kthread_stop() / |
| 68 | * freezer deadlock and freezable kthreads should use this function instead |
| 69 | * of calling try_to_freeze() directly. |
| 70 | */ |
| 71 | bool kthread_freezable_should_stop(bool *was_frozen) |
| 72 | { |
| 73 | bool frozen = false; |
| 74 | |
| 75 | might_sleep(); |
| 76 | |
| 77 | if (unlikely(freezing(current))) |
| 78 | frozen = __refrigerator(true); |
| 79 | |
| 80 | if (was_frozen) |
| 81 | *was_frozen = frozen; |
| 82 | |
| 83 | return kthread_should_stop(); |
| 84 | } |
| 85 | EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); |
| 86 | |
| 87 | /** |
Tejun Heo | 82805ab | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 88 | * kthread_data - return data value specified on kthread creation |
| 89 | * @task: kthread task in question |
| 90 | * |
| 91 | * Return the data value specified when kthread @task was created. |
| 92 | * The caller is responsible for ensuring the validity of @task when |
| 93 | * calling this function. |
| 94 | */ |
| 95 | void *kthread_data(struct task_struct *task) |
| 96 | { |
| 97 | return to_kthread(task)->data; |
| 98 | } |
| 99 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | static int kthread(void *_create) |
| 101 | { |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 102 | /* Copy data: it's on kthread's stack */ |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 103 | struct kthread_create_info *create = _create; |
| 104 | int (*threadfn)(void *data) = create->threadfn; |
| 105 | void *data = create->data; |
| 106 | struct kthread self; |
| 107 | int ret; |
| 108 | |
| 109 | self.should_stop = 0; |
Tejun Heo | 82805ab | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 110 | self.data = data; |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 111 | init_completion(&self.exited); |
| 112 | current->vfork_done = &self.exited; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 113 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | /* OK, tell user we're spawned, wait for stop or wakeup */ |
Oleg Nesterov | a076e4b | 2007-05-23 13:57:27 -0700 | [diff] [blame] | 115 | __set_current_state(TASK_UNINTERRUPTIBLE); |
Vitaliy Gusev | 3217ab9 | 2009-04-09 09:50:35 -0600 | [diff] [blame] | 116 | create->result = current; |
Oleg Nesterov | cdd140b | 2009-06-17 16:27:43 -0700 | [diff] [blame] | 117 | complete(&create->done); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | schedule(); |
| 119 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 120 | ret = -EINTR; |
| 121 | if (!self.should_stop) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | ret = threadfn(data); |
| 123 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 124 | /* we can't just return, we must preserve "self" on stack */ |
| 125 | do_exit(ret); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | } |
| 127 | |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 128 | /* called from do_fork() to get node information for about to be created task */ |
| 129 | int tsk_fork_get_node(struct task_struct *tsk) |
| 130 | { |
| 131 | #ifdef CONFIG_NUMA |
| 132 | if (tsk == kthreadd_task) |
| 133 | return tsk->pref_node_fork; |
| 134 | #endif |
| 135 | return numa_node_id(); |
| 136 | } |
| 137 | |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 138 | static void create_kthread(struct kthread_create_info *create) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | int pid; |
| 141 | |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 142 | #ifdef CONFIG_NUMA |
| 143 | current->pref_node_fork = create->node; |
| 144 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | /* We want our own signal handler (we take no signals by default). */ |
| 146 | pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD); |
Oleg Nesterov | cdd140b | 2009-06-17 16:27:43 -0700 | [diff] [blame] | 147 | if (pid < 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 148 | create->result = ERR_PTR(pid); |
Oleg Nesterov | cdd140b | 2009-06-17 16:27:43 -0700 | [diff] [blame] | 149 | complete(&create->done); |
| 150 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | } |
| 152 | |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 153 | /** |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 154 | * kthread_create_on_node - create a kthread. |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 155 | * @threadfn: the function to run until signal_pending(current). |
| 156 | * @data: data ptr for @threadfn. |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 157 | * @node: memory node number. |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 158 | * @namefmt: printf-style name for the thread. |
| 159 | * |
| 160 | * Description: This helper function creates and names a kernel |
| 161 | * thread. The thread will be stopped: use wake_up_process() to start |
Anton Blanchard | 301ba04 | 2010-02-09 15:07:40 +1100 | [diff] [blame] | 162 | * it. See also kthread_run(). |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 163 | * |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 164 | * If thread is going to be bound on a particular cpu, give its node |
| 165 | * in @node, to get NUMA affinity for kthread stack, or else give -1. |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 166 | * When woken, the thread will run @threadfn() with @data as its |
Robert P. J. Day | 72fd4a3 | 2007-02-10 01:45:59 -0800 | [diff] [blame] | 167 | * argument. @threadfn() can either call do_exit() directly if it is a |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 168 | * standalone thread for which no one will call kthread_stop(), or |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 169 | * return when 'kthread_should_stop()' is true (which means |
| 170 | * kthread_stop() has been called). The return value should be zero |
| 171 | * or a negative error number; it will be passed to kthread_stop(). |
| 172 | * |
| 173 | * Returns a task_struct or ERR_PTR(-ENOMEM). |
| 174 | */ |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 175 | struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), |
| 176 | void *data, |
| 177 | int node, |
| 178 | const char namefmt[], |
| 179 | ...) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 180 | { |
| 181 | struct kthread_create_info create; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | |
| 183 | create.threadfn = threadfn; |
| 184 | create.data = data; |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 185 | create.node = node; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | init_completion(&create.done); |
| 187 | |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 188 | spin_lock(&kthread_create_lock); |
| 189 | list_add_tail(&create.list, &kthread_create_list); |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 190 | spin_unlock(&kthread_create_lock); |
| 191 | |
Dmitry Adamushko | cbd9b67 | 2008-04-29 00:59:23 -0700 | [diff] [blame] | 192 | wake_up_process(kthreadd_task); |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 193 | wait_for_completion(&create.done); |
| 194 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | if (!IS_ERR(create.result)) { |
Peter Zijlstra | c9b5f50 | 2011-01-07 13:41:40 +0100 | [diff] [blame] | 196 | static const struct sched_param param = { .sched_priority = 0 }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | va_list args; |
Oleg Nesterov | 1c99315 | 2009-04-09 09:50:36 -0600 | [diff] [blame] | 198 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | va_start(args, namefmt); |
| 200 | vsnprintf(create.result->comm, sizeof(create.result->comm), |
| 201 | namefmt, args); |
| 202 | va_end(args); |
Oleg Nesterov | 1c99315 | 2009-04-09 09:50:36 -0600 | [diff] [blame] | 203 | /* |
| 204 | * root may have changed our (kthreadd's) priority or CPU mask. |
| 205 | * The kernel thread should not inherit these properties. |
| 206 | */ |
| 207 | sched_setscheduler_nocheck(create.result, SCHED_NORMAL, ¶m); |
Oleg Nesterov | 1c99315 | 2009-04-09 09:50:36 -0600 | [diff] [blame] | 208 | set_cpus_allowed_ptr(create.result, cpu_all_mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | return create.result; |
| 211 | } |
Eric Dumazet | 207205a | 2011-03-22 16:30:44 -0700 | [diff] [blame] | 212 | EXPORT_SYMBOL(kthread_create_on_node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 214 | /** |
Peter Zijlstra | 881232b | 2009-12-16 18:04:39 +0100 | [diff] [blame] | 215 | * kthread_bind - bind a just-created kthread to a cpu. |
| 216 | * @p: thread created by kthread_create(). |
| 217 | * @cpu: cpu (might not be online, must be possible) for @k to run on. |
| 218 | * |
| 219 | * Description: This function is equivalent to set_cpus_allowed(), |
| 220 | * except that @cpu doesn't need to be online, and the thread must be |
| 221 | * stopped (i.e., just returned from kthread_create()). |
| 222 | */ |
| 223 | void kthread_bind(struct task_struct *p, unsigned int cpu) |
| 224 | { |
| 225 | /* Must have done schedule() in kthread() before we set_task_cpu */ |
| 226 | if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) { |
| 227 | WARN_ON(1); |
| 228 | return; |
| 229 | } |
| 230 | |
KOSAKI Motohiro | 1e1b6c5 | 2011-05-19 15:08:58 +0900 | [diff] [blame] | 231 | /* It's safe because the task is inactive. */ |
| 232 | do_set_cpus_allowed(p, cpumask_of(cpu)); |
Peter Zijlstra | 881232b | 2009-12-16 18:04:39 +0100 | [diff] [blame] | 233 | p->flags |= PF_THREAD_BOUND; |
| 234 | } |
| 235 | EXPORT_SYMBOL(kthread_bind); |
| 236 | |
| 237 | /** |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 238 | * kthread_stop - stop a thread created by kthread_create(). |
| 239 | * @k: thread created by kthread_create(). |
| 240 | * |
| 241 | * Sets kthread_should_stop() for @k to return true, wakes it, and |
Oleg Nesterov | 9ae2602 | 2009-06-19 02:51:13 +0200 | [diff] [blame] | 242 | * waits for it to exit. This can also be called after kthread_create() |
| 243 | * instead of calling wake_up_process(): the thread will exit without |
| 244 | * calling threadfn(). |
| 245 | * |
| 246 | * If threadfn() may call do_exit() itself, the caller must ensure |
| 247 | * task_struct can't go away. |
Randy Dunlap | 9e37bd3 | 2006-06-25 05:49:19 -0700 | [diff] [blame] | 248 | * |
| 249 | * Returns the result of threadfn(), or %-EINTR if wake_up_process() |
| 250 | * was never called. |
| 251 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | int kthread_stop(struct task_struct *k) |
| 253 | { |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 254 | struct kthread *kthread; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | int ret; |
| 256 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 257 | trace_sched_kthread_stop(k); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 258 | get_task_struct(k); |
| 259 | |
Oleg Nesterov | 6370617 | 2009-06-17 16:27:45 -0700 | [diff] [blame] | 260 | kthread = to_kthread(k); |
| 261 | barrier(); /* it might have exited */ |
| 262 | if (k->vfork_done != NULL) { |
| 263 | kthread->should_stop = 1; |
| 264 | wake_up_process(k); |
| 265 | wait_for_completion(&kthread->exited); |
| 266 | } |
| 267 | ret = k->exit_code; |
Mathieu Desnoyers | 0a16b60 | 2008-07-18 12:16:17 -0400 | [diff] [blame] | 268 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | put_task_struct(k); |
Mathieu Desnoyers | 0a16b60 | 2008-07-18 12:16:17 -0400 | [diff] [blame] | 270 | trace_sched_kthread_stop_ret(ret); |
| 271 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 272 | return ret; |
| 273 | } |
Adrian Bunk | 52e92e5 | 2006-07-14 00:24:05 -0700 | [diff] [blame] | 274 | EXPORT_SYMBOL(kthread_stop); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | |
Satyam Sharma | e804a4a | 2007-07-31 00:39:16 -0700 | [diff] [blame] | 276 | int kthreadd(void *unused) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | { |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 278 | struct task_struct *tsk = current; |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 279 | |
Satyam Sharma | e804a4a | 2007-07-31 00:39:16 -0700 | [diff] [blame] | 280 | /* Setup a clean context for our children to inherit. */ |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 281 | set_task_comm(tsk, "kthreadd"); |
Oleg Nesterov | 10ab825 | 2007-05-09 02:34:37 -0700 | [diff] [blame] | 282 | ignore_signals(tsk); |
Rusty Russell | 1a2142a | 2009-03-30 22:05:10 -0600 | [diff] [blame] | 283 | set_cpus_allowed_ptr(tsk, cpu_all_mask); |
Miao Xie | 5ab116c | 2010-03-23 13:35:34 -0700 | [diff] [blame] | 284 | set_mems_allowed(node_states[N_HIGH_MEMORY]); |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 285 | |
Tejun Heo | 34b087e | 2011-11-23 09:28:17 -0800 | [diff] [blame] | 286 | current->flags |= PF_NOFREEZE; |
Eric W. Biederman | 73c2799 | 2007-05-09 02:34:32 -0700 | [diff] [blame] | 287 | |
| 288 | for (;;) { |
| 289 | set_current_state(TASK_INTERRUPTIBLE); |
| 290 | if (list_empty(&kthread_create_list)) |
| 291 | schedule(); |
| 292 | __set_current_state(TASK_RUNNING); |
| 293 | |
| 294 | spin_lock(&kthread_create_lock); |
| 295 | while (!list_empty(&kthread_create_list)) { |
| 296 | struct kthread_create_info *create; |
| 297 | |
| 298 | create = list_entry(kthread_create_list.next, |
| 299 | struct kthread_create_info, list); |
| 300 | list_del_init(&create->list); |
| 301 | spin_unlock(&kthread_create_lock); |
| 302 | |
| 303 | create_kthread(create); |
| 304 | |
| 305 | spin_lock(&kthread_create_lock); |
| 306 | } |
| 307 | spin_unlock(&kthread_create_lock); |
| 308 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 309 | |
| 310 | return 0; |
| 311 | } |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 312 | |
Yong Zhang | 4f32e9b | 2010-12-22 10:27:53 +0100 | [diff] [blame] | 313 | void __init_kthread_worker(struct kthread_worker *worker, |
| 314 | const char *name, |
| 315 | struct lock_class_key *key) |
| 316 | { |
| 317 | spin_lock_init(&worker->lock); |
| 318 | lockdep_set_class_and_name(&worker->lock, key, name); |
| 319 | INIT_LIST_HEAD(&worker->work_list); |
| 320 | worker->task = NULL; |
| 321 | } |
| 322 | EXPORT_SYMBOL_GPL(__init_kthread_worker); |
| 323 | |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 324 | /** |
| 325 | * kthread_worker_fn - kthread function to process kthread_worker |
| 326 | * @worker_ptr: pointer to initialized kthread_worker |
| 327 | * |
| 328 | * This function can be used as @threadfn to kthread_create() or |
| 329 | * kthread_run() with @worker_ptr argument pointing to an initialized |
| 330 | * kthread_worker. The started kthread will process work_list until |
| 331 | * the it is stopped with kthread_stop(). A kthread can also call |
| 332 | * this function directly after extra initialization. |
| 333 | * |
| 334 | * Different kthreads can be used for the same kthread_worker as long |
| 335 | * as there's only one kthread attached to it at any given time. A |
| 336 | * kthread_worker without an attached kthread simply collects queued |
| 337 | * kthread_works. |
| 338 | */ |
| 339 | int kthread_worker_fn(void *worker_ptr) |
| 340 | { |
| 341 | struct kthread_worker *worker = worker_ptr; |
| 342 | struct kthread_work *work; |
| 343 | |
| 344 | WARN_ON(worker->task); |
| 345 | worker->task = current; |
| 346 | repeat: |
| 347 | set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ |
| 348 | |
| 349 | if (kthread_should_stop()) { |
| 350 | __set_current_state(TASK_RUNNING); |
| 351 | spin_lock_irq(&worker->lock); |
| 352 | worker->task = NULL; |
| 353 | spin_unlock_irq(&worker->lock); |
| 354 | return 0; |
| 355 | } |
| 356 | |
| 357 | work = NULL; |
| 358 | spin_lock_irq(&worker->lock); |
| 359 | if (!list_empty(&worker->work_list)) { |
| 360 | work = list_first_entry(&worker->work_list, |
| 361 | struct kthread_work, node); |
| 362 | list_del_init(&work->node); |
| 363 | } |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 364 | worker->current_work = work; |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 365 | spin_unlock_irq(&worker->lock); |
| 366 | |
| 367 | if (work) { |
| 368 | __set_current_state(TASK_RUNNING); |
| 369 | work->func(work); |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 370 | } else if (!freezing(current)) |
| 371 | schedule(); |
| 372 | |
| 373 | try_to_freeze(); |
| 374 | goto repeat; |
| 375 | } |
| 376 | EXPORT_SYMBOL_GPL(kthread_worker_fn); |
| 377 | |
Tejun Heo | 9a2e03d | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 378 | /* insert @work before @pos in @worker */ |
| 379 | static void insert_kthread_work(struct kthread_worker *worker, |
| 380 | struct kthread_work *work, |
| 381 | struct list_head *pos) |
| 382 | { |
| 383 | lockdep_assert_held(&worker->lock); |
| 384 | |
| 385 | list_add_tail(&work->node, pos); |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 386 | work->worker = worker; |
Tejun Heo | 9a2e03d | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 387 | if (likely(worker->task)) |
| 388 | wake_up_process(worker->task); |
| 389 | } |
| 390 | |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 391 | /** |
| 392 | * queue_kthread_work - queue a kthread_work |
| 393 | * @worker: target kthread_worker |
| 394 | * @work: kthread_work to queue |
| 395 | * |
| 396 | * Queue @work to work processor @task for async execution. @task |
| 397 | * must have been created with kthread_worker_create(). Returns %true |
| 398 | * if @work was successfully queued, %false if it was already pending. |
| 399 | */ |
| 400 | bool queue_kthread_work(struct kthread_worker *worker, |
| 401 | struct kthread_work *work) |
| 402 | { |
| 403 | bool ret = false; |
| 404 | unsigned long flags; |
| 405 | |
| 406 | spin_lock_irqsave(&worker->lock, flags); |
| 407 | if (list_empty(&work->node)) { |
Tejun Heo | 9a2e03d | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 408 | insert_kthread_work(worker, work, &worker->work_list); |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 409 | ret = true; |
| 410 | } |
| 411 | spin_unlock_irqrestore(&worker->lock, flags); |
| 412 | return ret; |
| 413 | } |
| 414 | EXPORT_SYMBOL_GPL(queue_kthread_work); |
| 415 | |
Tejun Heo | 9a2e03d | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 416 | struct kthread_flush_work { |
| 417 | struct kthread_work work; |
| 418 | struct completion done; |
| 419 | }; |
| 420 | |
| 421 | static void kthread_flush_work_fn(struct kthread_work *work) |
| 422 | { |
| 423 | struct kthread_flush_work *fwork = |
| 424 | container_of(work, struct kthread_flush_work, work); |
| 425 | complete(&fwork->done); |
| 426 | } |
| 427 | |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 428 | /** |
| 429 | * flush_kthread_work - flush a kthread_work |
| 430 | * @work: work to flush |
| 431 | * |
| 432 | * If @work is queued or executing, wait for it to finish execution. |
| 433 | */ |
| 434 | void flush_kthread_work(struct kthread_work *work) |
| 435 | { |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 436 | struct kthread_flush_work fwork = { |
| 437 | KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), |
| 438 | COMPLETION_INITIALIZER_ONSTACK(fwork.done), |
| 439 | }; |
| 440 | struct kthread_worker *worker; |
| 441 | bool noop = false; |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 442 | |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 443 | retry: |
| 444 | worker = work->worker; |
| 445 | if (!worker) |
| 446 | return; |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 447 | |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 448 | spin_lock_irq(&worker->lock); |
| 449 | if (work->worker != worker) { |
| 450 | spin_unlock_irq(&worker->lock); |
| 451 | goto retry; |
| 452 | } |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 453 | |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 454 | if (!list_empty(&work->node)) |
| 455 | insert_kthread_work(worker, &fwork.work, work->node.next); |
| 456 | else if (worker->current_work == work) |
| 457 | insert_kthread_work(worker, &fwork.work, worker->work_list.next); |
| 458 | else |
| 459 | noop = true; |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 460 | |
Tejun Heo | 46f3d97 | 2012-07-19 13:52:53 -0700 | [diff] [blame] | 461 | spin_unlock_irq(&worker->lock); |
| 462 | |
| 463 | if (!noop) |
| 464 | wait_for_completion(&fwork.done); |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 465 | } |
| 466 | EXPORT_SYMBOL_GPL(flush_kthread_work); |
| 467 | |
Tejun Heo | b56c0d8 | 2010-06-29 10:07:09 +0200 | [diff] [blame] | 468 | /** |
| 469 | * flush_kthread_worker - flush all current works on a kthread_worker |
| 470 | * @worker: worker to flush |
| 471 | * |
| 472 | * Wait until all currently executing or pending works on @worker are |
| 473 | * finished. |
| 474 | */ |
| 475 | void flush_kthread_worker(struct kthread_worker *worker) |
| 476 | { |
| 477 | struct kthread_flush_work fwork = { |
| 478 | KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), |
| 479 | COMPLETION_INITIALIZER_ONSTACK(fwork.done), |
| 480 | }; |
| 481 | |
| 482 | queue_kthread_work(worker, &fwork.work); |
| 483 | wait_for_completion(&fwork.done); |
| 484 | } |
| 485 | EXPORT_SYMBOL_GPL(flush_kthread_worker); |