blob: d2edd6efec56295d3dcfefa5587266915657d9d3 [file] [log] [blame]
Arjan van de Ven22a9d642009-01-07 08:45:46 -08001/*
2 * async.c: Asynchronous function calls for boot performance
3 *
4 * (C) Copyright 2009 Intel Corporation
5 * Author: Arjan van de Ven <arjan@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 */
12
13
14/*
15
16Goals and Theory of Operation
17
18The primary goal of this feature is to reduce the kernel boot time,
19by doing various independent hardware delays and discovery operations
20decoupled and not strictly serialized.
21
22More specifically, the asynchronous function call concept allows
23certain operations (primarily during system boot) to happen
24asynchronously, out of order, while these operations still
25have their externally visible parts happen sequentially and in-order.
26(not unlike how out-of-order CPUs retire their instructions in order)
27
28Key to the asynchronous function call implementation is the concept of
29a "sequence cookie" (which, although it has an abstracted type, can be
30thought of as a monotonically incrementing number).
31
32The async core will assign each scheduled event such a sequence cookie and
33pass this to the called functions.
34
35The asynchronously called function should before doing a globally visible
36operation, such as registering device numbers, call the
37async_synchronize_cookie() function and pass in its own cookie. The
38async_synchronize_cookie() function will make sure that all asynchronous
39operations that were scheduled prior to the operation corresponding with the
40cookie have completed.
41
42Subsystem/driver initialization code that scheduled asynchronous probe
43functions, but which shares global resources with other drivers/subsystems
44that do not use the asynchronous call feature, need to do a full
45synchronization with the async_synchronize_full() function, before returning
46from their init function. This is to maintain strict ordering between the
47asynchronous and synchronous parts of the kernel.
48
49*/
50
51#include <linux/async.h>
Paul McQuade84c15022011-05-31 20:51:55 +010052#include <linux/atomic.h>
53#include <linux/ktime.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040054#include <linux/export.h>
Arjan van de Ven22a9d642009-01-07 08:45:46 -080055#include <linux/wait.h>
56#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090057#include <linux/slab.h>
Tejun Heo083b8042010-07-02 10:03:52 +020058#include <linux/workqueue.h>
Arjan van de Ven22a9d642009-01-07 08:45:46 -080059
Tejun Heo84b233a2013-01-18 14:05:56 -080060#include "workqueue_internal.h"
61
Arjan van de Ven22a9d642009-01-07 08:45:46 -080062static async_cookie_t next_cookie = 1;
63
Tejun Heoc68eee12013-01-23 09:32:30 -080064#define MAX_WORK 32768
65#define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
Arjan van de Ven22a9d642009-01-07 08:45:46 -080066
Tejun Heo9fdb04c2013-01-23 09:32:30 -080067static LIST_HEAD(async_global_pending); /* pending from all registered doms */
Tejun Heo8723d502013-01-23 09:32:30 -080068static ASYNC_DOMAIN(async_dfl_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -080069static DEFINE_SPINLOCK(async_lock);
70
71struct async_entry {
Tejun Heo9fdb04c2013-01-23 09:32:30 -080072 struct list_head domain_list;
73 struct list_head global_list;
Tejun Heo083b8042010-07-02 10:03:52 +020074 struct work_struct work;
75 async_cookie_t cookie;
Lai Jiangshan362f2b02013-03-12 13:59:14 -070076 async_func_t func;
Tejun Heo083b8042010-07-02 10:03:52 +020077 void *data;
Tejun Heo8723d502013-01-23 09:32:30 -080078 struct async_domain *domain;
Arjan van de Ven22a9d642009-01-07 08:45:46 -080079};
80
81static DECLARE_WAIT_QUEUE_HEAD(async_done);
Arjan van de Ven22a9d642009-01-07 08:45:46 -080082
83static atomic_t entry_count;
Arjan van de Ven22a9d642009-01-07 08:45:46 -080084
Tejun Heo8723d502013-01-23 09:32:30 -080085static async_cookie_t lowest_in_progress(struct async_domain *domain)
Arjan van de Ven37a76bd2009-01-11 15:35:01 +000086{
Lai Jiangshan92266d62013-03-12 13:59:13 -070087 struct list_head *pending;
Tejun Heo52722792013-01-23 09:32:30 -080088 async_cookie_t ret = ASYNC_COOKIE_MAX;
Arjan van de Ven37a76bd2009-01-11 15:35:01 +000089 unsigned long flags;
Arjan van de Ven37a76bd2009-01-11 15:35:01 +000090
91 spin_lock_irqsave(&async_lock, flags);
Tejun Heo9fdb04c2013-01-23 09:32:30 -080092
Lai Jiangshan92266d62013-03-12 13:59:13 -070093 if (domain)
94 pending = &domain->pending;
95 else
96 pending = &async_global_pending;
Tejun Heo9fdb04c2013-01-23 09:32:30 -080097
Lai Jiangshan92266d62013-03-12 13:59:13 -070098 if (!list_empty(pending))
99 ret = list_first_entry(pending, struct async_entry,
100 domain_list)->cookie;
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800101
Arjan van de Ven37a76bd2009-01-11 15:35:01 +0000102 spin_unlock_irqrestore(&async_lock, flags);
103 return ret;
104}
Tejun Heo083b8042010-07-02 10:03:52 +0200105
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800106/*
107 * pick the first pending entry and run it
108 */
Tejun Heo083b8042010-07-02 10:03:52 +0200109static void async_run_entry_fn(struct work_struct *work)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800110{
Tejun Heo083b8042010-07-02 10:03:52 +0200111 struct async_entry *entry =
112 container_of(work, struct async_entry, work);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800113 unsigned long flags;
Vitaliy Ivanov124ff4e2011-07-07 14:10:40 +0300114 ktime_t uninitialized_var(calltime), delta, rettime;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800115
Tejun Heo52722792013-01-23 09:32:30 -0800116 /* 1) run (and print duration) */
Arjan van de Venad160d22009-01-07 09:28:53 -0800117 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Ionut Alexa27fb10e2014-10-09 15:30:19 -0700118 pr_debug("calling %lli_%pF @ %i\n",
Paul McQuade84c15022011-05-31 20:51:55 +0100119 (long long)entry->cookie,
Andrew Morton58763a22009-02-04 15:11:58 -0800120 entry->func, task_pid_nr(current));
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800121 calltime = ktime_get();
122 }
123 entry->func(entry->data, entry->cookie);
Arjan van de Venad160d22009-01-07 09:28:53 -0800124 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800125 rettime = ktime_get();
126 delta = ktime_sub(rettime, calltime);
Ionut Alexa27fb10e2014-10-09 15:30:19 -0700127 pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n",
Andrew Morton58763a22009-02-04 15:11:58 -0800128 (long long)entry->cookie,
129 entry->func,
130 (long long)ktime_to_ns(delta) >> 10);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800131 }
132
Tejun Heo52722792013-01-23 09:32:30 -0800133 /* 2) remove self from the pending queues */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800134 spin_lock_irqsave(&async_lock, flags);
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800135 list_del_init(&entry->domain_list);
136 list_del_init(&entry->global_list);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800137
Tejun Heo52722792013-01-23 09:32:30 -0800138 /* 3) free the entry */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800139 kfree(entry);
140 atomic_dec(&entry_count);
141
142 spin_unlock_irqrestore(&async_lock, flags);
143
Tejun Heo52722792013-01-23 09:32:30 -0800144 /* 4) wake up any waiters */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800145 wake_up(&async_done);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800146}
147
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700148static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800149{
150 struct async_entry *entry;
151 unsigned long flags;
152 async_cookie_t newcookie;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800153
154 /* allow irq-off callers */
155 entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
156
157 /*
158 * If we're out of memory or if there's too much work
159 * pending already, we execute synchronously.
160 */
Tejun Heo083b8042010-07-02 10:03:52 +0200161 if (!entry || atomic_read(&entry_count) > MAX_WORK) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800162 kfree(entry);
163 spin_lock_irqsave(&async_lock, flags);
164 newcookie = next_cookie++;
165 spin_unlock_irqrestore(&async_lock, flags);
166
167 /* low on memory.. run synchronously */
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700168 func(data, newcookie);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800169 return newcookie;
170 }
James Hogana0327ff2013-01-25 10:13:59 +0000171 INIT_LIST_HEAD(&entry->domain_list);
172 INIT_LIST_HEAD(&entry->global_list);
Tejun Heo083b8042010-07-02 10:03:52 +0200173 INIT_WORK(&entry->work, async_run_entry_fn);
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700174 entry->func = func;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800175 entry->data = data;
Tejun Heo8723d502013-01-23 09:32:30 -0800176 entry->domain = domain;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800177
178 spin_lock_irqsave(&async_lock, flags);
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800179
180 /* allocate cookie and queue */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800181 newcookie = entry->cookie = next_cookie++;
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800182
183 list_add_tail(&entry->domain_list, &domain->pending);
184 if (domain->registered)
185 list_add_tail(&entry->global_list, &async_global_pending);
186
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800187 atomic_inc(&entry_count);
188 spin_unlock_irqrestore(&async_lock, flags);
Tejun Heo083b8042010-07-02 10:03:52 +0200189
Tejun Heo774a1222013-01-15 18:52:51 -0800190 /* mark that this task has queued an async job, used by module init */
191 current->flags |= PF_USED_ASYNC;
192
Tejun Heo083b8042010-07-02 10:03:52 +0200193 /* schedule for execution */
194 queue_work(system_unbound_wq, &entry->work);
195
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800196 return newcookie;
197}
198
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100199/**
200 * async_schedule - schedule a function for asynchronous execution
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700201 * @func: function to execute asynchronously
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100202 * @data: data pointer to pass to the function
203 *
204 * Returns an async_cookie_t that may be used for checkpointing later.
205 * Note: This function may be called from atomic or non-atomic contexts.
206 */
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700207async_cookie_t async_schedule(async_func_t func, void *data)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800208{
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700209 return __async_schedule(func, data, &async_dfl_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800210}
211EXPORT_SYMBOL_GPL(async_schedule);
212
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100213/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100214 * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700215 * @func: function to execute asynchronously
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100216 * @data: data pointer to pass to the function
Tejun Heo8723d502013-01-23 09:32:30 -0800217 * @domain: the domain
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100218 *
219 * Returns an async_cookie_t that may be used for checkpointing later.
Tejun Heo8723d502013-01-23 09:32:30 -0800220 * @domain may be used in the async_synchronize_*_domain() functions to
221 * wait within a certain synchronization domain rather than globally. A
222 * synchronization domain is specified via @domain. Note: This function
223 * may be called from atomic or non-atomic contexts.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100224 */
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700225async_cookie_t async_schedule_domain(async_func_t func, void *data,
Tejun Heo8723d502013-01-23 09:32:30 -0800226 struct async_domain *domain)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800227{
Lai Jiangshan362f2b02013-03-12 13:59:14 -0700228 return __async_schedule(func, data, domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800229}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100230EXPORT_SYMBOL_GPL(async_schedule_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800231
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100232/**
233 * async_synchronize_full - synchronize all asynchronous function calls
234 *
235 * This function waits until all asynchronous function calls have been done.
236 */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800237void async_synchronize_full(void)
238{
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800239 async_synchronize_full_domain(NULL);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800240}
241EXPORT_SYMBOL_GPL(async_synchronize_full);
242
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100243/**
Dan Williamsa4683482012-07-09 19:33:30 -0700244 * async_unregister_domain - ensure no more anonymous waiters on this domain
245 * @domain: idle domain to flush out of any async_synchronize_full instances
246 *
247 * async_synchronize_{cookie|full}_domain() are not flushed since callers
248 * of these routines should know the lifetime of @domain
249 *
250 * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
251 */
252void async_unregister_domain(struct async_domain *domain)
253{
Dan Williamsa4683482012-07-09 19:33:30 -0700254 spin_lock_irq(&async_lock);
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800255 WARN_ON(!domain->registered || !list_empty(&domain->pending));
Dan Williamsa4683482012-07-09 19:33:30 -0700256 domain->registered = 0;
257 spin_unlock_irq(&async_lock);
Dan Williamsa4683482012-07-09 19:33:30 -0700258}
259EXPORT_SYMBOL_GPL(async_unregister_domain);
260
261/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100262 * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
Tejun Heo8723d502013-01-23 09:32:30 -0800263 * @domain: the domain to synchronize
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100264 *
Cornelia Huck766ccb92009-01-20 15:31:31 +0100265 * This function waits until all asynchronous function calls for the
Tejun Heo8723d502013-01-23 09:32:30 -0800266 * synchronization domain specified by @domain have been done.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100267 */
Dan Williams2955b472012-07-09 19:33:25 -0700268void async_synchronize_full_domain(struct async_domain *domain)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800269{
Tejun Heoc68eee12013-01-23 09:32:30 -0800270 async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800271}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100272EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800273
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100274/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100275 * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100276 * @cookie: async_cookie_t to use as checkpoint
Tejun Heo9fdb04c2013-01-23 09:32:30 -0800277 * @domain: the domain to synchronize (%NULL for all registered domains)
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100278 *
Cornelia Huck766ccb92009-01-20 15:31:31 +0100279 * This function waits until all asynchronous function calls for the
Tejun Heo8723d502013-01-23 09:32:30 -0800280 * synchronization domain specified by @domain submitted prior to @cookie
281 * have been done.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100282 */
Tejun Heo8723d502013-01-23 09:32:30 -0800283void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800284{
Vitaliy Ivanov124ff4e2011-07-07 14:10:40 +0300285 ktime_t uninitialized_var(starttime), delta, endtime;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800286
Arjan van de Venad160d22009-01-07 09:28:53 -0800287 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Ionut Alexa27fb10e2014-10-09 15:30:19 -0700288 pr_debug("async_waiting @ %i\n", task_pid_nr(current));
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800289 starttime = ktime_get();
290 }
291
Tejun Heo8723d502013-01-23 09:32:30 -0800292 wait_event(async_done, lowest_in_progress(domain) >= cookie);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800293
Arjan van de Venad160d22009-01-07 09:28:53 -0800294 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800295 endtime = ktime_get();
296 delta = ktime_sub(endtime, starttime);
297
Ionut Alexa27fb10e2014-10-09 15:30:19 -0700298 pr_debug("async_continuing @ %i after %lli usec\n",
Andrew Morton58763a22009-02-04 15:11:58 -0800299 task_pid_nr(current),
300 (long long)ktime_to_ns(delta) >> 10);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800301 }
302}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100303EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800304
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100305/**
306 * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
307 * @cookie: async_cookie_t to use as checkpoint
308 *
309 * This function waits until all asynchronous function calls prior to @cookie
310 * have been done.
311 */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800312void async_synchronize_cookie(async_cookie_t cookie)
313{
Tejun Heo8723d502013-01-23 09:32:30 -0800314 async_synchronize_cookie_domain(cookie, &async_dfl_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800315}
316EXPORT_SYMBOL_GPL(async_synchronize_cookie);
Tejun Heo84b233a2013-01-18 14:05:56 -0800317
318/**
319 * current_is_async - is %current an async worker task?
320 *
321 * Returns %true if %current is an async worker task.
322 */
323bool current_is_async(void)
324{
325 struct worker *worker = current_wq_worker();
326
327 return worker && worker->current_func == async_run_entry_fn;
328}
Lukas Wunner581da2c2015-11-19 16:31:11 +0100329EXPORT_SYMBOL_GPL(current_is_async);