blob: c73207abb5a44a77ed0819101ab75cd626ceb353 [file] [log] [blame]
Len Brown4f86d3a2007-10-03 18:58:00 -04001/*
2 * cpuidle.c - core cpuidle infrastructure
3 *
4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 * Shaohua Li <shaohua.li@intel.com>
6 * Adam Belay <abelay@novell.com>
7 *
8 * This code is licenced under the GPL.
9 */
10
Daniel Lezcanob60e6a02013-03-21 12:21:31 +000011#include <linux/clockchips.h>
Len Brown4f86d3a2007-10-03 18:58:00 -040012#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/notifier.h>
Jean Pihete8db0be2011-08-25 15:35:03 +020016#include <linux/pm_qos.h>
Len Brown4f86d3a2007-10-03 18:58:00 -040017#include <linux/cpu.h>
18#include <linux/cpuidle.h>
venkatesh.pallipadi@intel.com9a0b8412008-01-31 17:35:06 -080019#include <linux/ktime.h>
Arjan van de Ven2e94d1f2008-09-10 16:06:00 -070020#include <linux/hrtimer.h>
Paul Gortmaker884b17e2011-08-29 17:52:39 -040021#include <linux/module.h>
Rafael J. Wysocki38106312015-02-12 23:33:15 +010022#include <linux/suspend.h>
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +010023#include <linux/tick.h>
Arjan van de Ven288f0232009-09-19 13:35:33 +020024#include <trace/events/power.h>
Len Brown4f86d3a2007-10-03 18:58:00 -040025
26#include "cpuidle.h"
27
28DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
Daniel Lezcano4c637b22013-04-23 08:54:33 +000029DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
Len Brown4f86d3a2007-10-03 18:58:00 -040030
31DEFINE_MUTEX(cpuidle_lock);
32LIST_HEAD(cpuidle_detected_devices);
Len Brown4f86d3a2007-10-03 18:58:00 -040033
34static int enabled_devices;
Len Brown62027ae2011-04-01 18:13:10 -040035static int off __read_mostly;
Len Browna0bfa132011-04-01 19:34:59 -040036static int initialized __read_mostly;
Len Brown62027ae2011-04-01 18:13:10 -040037
38int cpuidle_disabled(void)
39{
40 return off;
41}
Len Brownd91ee582011-04-01 18:28:35 -040042void disable_cpuidle(void)
43{
44 off = 1;
45}
Len Brown4f86d3a2007-10-03 18:58:00 -040046
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +010047bool cpuidle_not_available(struct cpuidle_driver *drv,
48 struct cpuidle_device *dev)
Rafael J. Wysocki31a34092015-02-27 00:39:56 +010049{
50 return off || !initialized || !drv || !dev || !dev->enabled;
51}
52
Len Brown4f86d3a2007-10-03 18:58:00 -040053/**
Boris Ostrovsky1a022e32012-03-13 19:55:09 +010054 * cpuidle_play_dead - cpu off-lining
55 *
Toshi Kaniee01e662012-03-31 21:37:02 -060056 * Returns in case of an error or no driver
Boris Ostrovsky1a022e32012-03-13 19:55:09 +010057 */
58int cpuidle_play_dead(void)
59{
60 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +000061 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
Daniel Lezcano8aef33a2013-01-15 14:18:04 +010062 int i;
Boris Ostrovsky1a022e32012-03-13 19:55:09 +010063
Toshi Kaniee01e662012-03-31 21:37:02 -060064 if (!drv)
65 return -ENODEV;
66
Boris Ostrovsky1a022e32012-03-13 19:55:09 +010067 /* Find lowest-power state that supports long-term idle */
Rafael J. Wysocki7d51d9792015-05-28 04:09:24 +020068 for (i = drv->state_count - 1; i >= 0; i--)
Daniel Lezcano8aef33a2013-01-15 14:18:04 +010069 if (drv->states[i].enter_dead)
70 return drv->states[i].enter_dead(dev, i);
Boris Ostrovsky1a022e32012-03-13 19:55:09 +010071
72 return -ENODEV;
73}
74
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +010075static int find_deepest_state(struct cpuidle_driver *drv,
Rafael J. Wysocki0d940392015-05-10 01:19:52 +020076 struct cpuidle_device *dev,
77 unsigned int max_latency,
78 unsigned int forbidden_flags,
79 bool freeze)
Rafael J. Wysockia6220fc2014-05-05 00:51:54 +020080{
81 unsigned int latency_req = 0;
Rafael J. Wysocki51164252016-01-16 00:54:53 +010082 int i, ret = 0;
Rafael J. Wysockia6220fc2014-05-05 00:51:54 +020083
Rafael J. Wysocki51164252016-01-16 00:54:53 +010084 for (i = 1; i < drv->state_count; i++) {
Rafael J. Wysockia6220fc2014-05-05 00:51:54 +020085 struct cpuidle_state *s = &drv->states[i];
86 struct cpuidle_state_usage *su = &dev->states_usage[i];
87
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +010088 if (s->disabled || su->disable || s->exit_latency <= latency_req
Rafael J. Wysocki0d940392015-05-10 01:19:52 +020089 || s->exit_latency > max_latency
90 || (s->flags & forbidden_flags)
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +010091 || (freeze && !s->enter_freeze))
Rafael J. Wysockia6220fc2014-05-05 00:51:54 +020092 continue;
93
94 latency_req = s->exit_latency;
95 ret = i;
96 }
97 return ret;
98}
99
Rafael J. Wysocki87e9b9f2015-05-16 01:38:15 +0200100#ifdef CONFIG_SUSPEND
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100101/**
102 * cpuidle_find_deepest_state - Find the deepest available idle state.
103 * @drv: cpuidle driver for the given CPU.
104 * @dev: cpuidle device for the given CPU.
105 */
106int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
107 struct cpuidle_device *dev)
108{
Rafael J. Wysocki0d940392015-05-10 01:19:52 +0200109 return find_deepest_state(drv, dev, UINT_MAX, 0, false);
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100110}
111
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100112static void enter_freeze_proper(struct cpuidle_driver *drv,
113 struct cpuidle_device *dev, int index)
114{
Rafael J. Wysockiae0afb42015-07-09 22:59:49 +0200115 /*
116 * trace_suspend_resume() called by tick_freeze() for the last CPU
117 * executing it contains RCU usage regarded as invalid in the idle
118 * context, so tell RCU about that.
119 */
120 RCU_NONIDLE(tick_freeze());
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100121 /*
122 * The state used here cannot be a "coupled" one, because the "coupled"
123 * cpuidle mechanism enables interrupts and doing that with timekeeping
124 * suspended is generally unsafe.
125 */
Lucas Stach63caae82015-07-20 18:34:50 +0200126 stop_critical_timings();
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100127 drv->states[index].enter_freeze(dev, drv, index);
128 WARN_ON(!irqs_disabled());
129 /*
130 * timekeeping_resume() that will be called by tick_unfreeze() for the
Rafael J. Wysockiae0afb42015-07-09 22:59:49 +0200131 * first CPU executing it calls functions containing RCU read-side
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100132 * critical sections, so tell RCU about that.
133 */
134 RCU_NONIDLE(tick_unfreeze());
Lucas Stach63caae82015-07-20 18:34:50 +0200135 start_critical_timings();
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100136}
137
Rafael J. Wysockia6220fc2014-05-05 00:51:54 +0200138/**
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100139 * cpuidle_enter_freeze - Enter an idle state suitable for suspend-to-idle.
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100140 * @drv: cpuidle driver for the given CPU.
141 * @dev: cpuidle device for the given CPU.
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100142 *
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100143 * If there are states with the ->enter_freeze callback, find the deepest of
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100144 * them and enter it with frozen tick.
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100145 */
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100146int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev)
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100147{
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100148 int index;
149
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100150 /*
151 * Find the deepest state with ->enter_freeze present, which guarantees
152 * that interrupts won't be enabled when it exits and allows the tick to
153 * be frozen safely.
154 */
Rafael J. Wysocki0d940392015-05-10 01:19:52 +0200155 index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
Sudeep Holla6f168862016-01-21 11:19:29 +0000156 if (index > 0)
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100157 enter_freeze_proper(drv, dev, index);
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +0100158
Rafael J. Wysockief2b22a2015-03-02 22:26:55 +0100159 return index;
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100160}
Rafael J. Wysocki87e9b9f2015-05-16 01:38:15 +0200161#endif /* CONFIG_SUSPEND */
Rafael J. Wysocki38106312015-02-12 23:33:15 +0100162
163/**
Colin Cross56cfbf72012-05-07 17:57:39 -0700164 * cpuidle_enter_state - enter the state and update stats
165 * @dev: cpuidle device for this cpu
166 * @drv: cpuidle driver for this cpu
Rafael J. Wysocki73122802015-05-09 21:50:32 +0200167 * @index: index into the states table in @drv of the state to enter
Colin Cross56cfbf72012-05-07 17:57:39 -0700168 */
169int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000170 int index)
Colin Cross56cfbf72012-05-07 17:57:39 -0700171{
172 int entered_state;
173
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000174 struct cpuidle_state *target_state = &drv->states[index];
Rafael J. Wysockidf8d9ee2015-04-29 15:19:21 +0200175 bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
Shreyas B. Prabhudbd1b8e2016-07-01 09:24:14 -0500176 ktime_t time_start, time_end;
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000177 s64 diff;
178
Rafael J. Wysockidf8d9ee2015-04-29 15:19:21 +0200179 /*
180 * Tell the time framework to switch to a broadcast timer because our
181 * local timer will be shut down. If a local timer is used from another
182 * CPU as a broadcast timer, this call may fail if it is not available.
183 */
Rafael J. Wysocki827a5ae2015-05-10 01:18:46 +0200184 if (broadcast && tick_broadcast_enter()) {
Rafael J. Wysocki0d940392015-05-10 01:19:52 +0200185 index = find_deepest_state(drv, dev, target_state->exit_latency,
186 CPUIDLE_FLAG_TIMER_STOP, false);
187 if (index < 0) {
188 default_idle_call();
189 return -EBUSY;
190 }
191 target_state = &drv->states[index];
Rafael J. Wysocki827a5ae2015-05-10 01:18:46 +0200192 }
Rafael J. Wysockidf8d9ee2015-04-29 15:19:21 +0200193
Rafael J. Wysockifaad3842015-05-10 01:18:03 +0200194 /* Take note of the planned idle state. */
195 sched_idle_set_state(target_state);
196
Sandeep Tripathy30fe6882014-07-02 15:00:58 +0530197 trace_cpu_idle_rcuidle(index, dev->cpu);
Shreyas B. Prabhudbd1b8e2016-07-01 09:24:14 -0500198 time_start = ns_to_ktime(local_clock());
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000199
Lucas Stach63caae82015-07-20 18:34:50 +0200200 stop_critical_timings();
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000201 entered_state = target_state->enter(dev, drv, index);
Lucas Stach63caae82015-07-20 18:34:50 +0200202 start_critical_timings();
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000203
Shreyas B. Prabhudbd1b8e2016-07-01 09:24:14 -0500204 time_end = ns_to_ktime(local_clock());
Sandeep Tripathy30fe6882014-07-02 15:00:58 +0530205 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000206
Rafael J. Wysockifaad3842015-05-10 01:18:03 +0200207 /* The cpu is no longer idle or about to enter idle. */
208 sched_idle_set_state(NULL);
209
Rafael J. Wysockidf8d9ee2015-04-29 15:19:21 +0200210 if (broadcast) {
211 if (WARN_ON_ONCE(!irqs_disabled()))
212 local_irq_disable();
213
214 tick_broadcast_exit();
215 }
216
Daniel Lezcanoe7387da2016-05-17 16:54:00 +0200217 if (!cpuidle_state_is_coupled(drv, index))
Paul Burton0b89e9a2014-03-06 11:02:01 +0000218 local_irq_enable();
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000219
Shreyas B. Prabhudbd1b8e2016-07-01 09:24:14 -0500220 diff = ktime_us_delta(time_end, time_start);
Daniel Lezcano554c06b2013-04-23 08:54:31 +0000221 if (diff > INT_MAX)
222 diff = INT_MAX;
223
224 dev->last_residency = (int) diff;
Colin Cross56cfbf72012-05-07 17:57:39 -0700225
226 if (entered_state >= 0) {
227 /* Update cpuidle counters */
228 /* This can be moved to within driver enter routine
229 * but that results in multiple copies of same code.
230 */
Julius Wernera474a512012-11-27 14:17:58 +0100231 dev->states_usage[entered_state].time += dev->last_residency;
Colin Cross56cfbf72012-05-07 17:57:39 -0700232 dev->states_usage[entered_state].usage++;
233 } else {
234 dev->last_residency = 0;
235 }
236
237 return entered_state;
238}
239
240/**
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100241 * cpuidle_select - ask the cpuidle framework to choose an idle state
Len Brown4f86d3a2007-10-03 18:58:00 -0400242 *
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100243 * @drv: the cpuidle driver
244 * @dev: the cpuidle device
245 *
Rafael J. Wysocki51164252016-01-16 00:54:53 +0100246 * Returns the index of the idle state. The return value must not be negative.
Len Brown4f86d3a2007-10-03 18:58:00 -0400247 */
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100248int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
Len Brown4f86d3a2007-10-03 18:58:00 -0400249{
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100250 return cpuidle_curr_governor->select(drv, dev);
251}
Len Brown4f86d3a2007-10-03 18:58:00 -0400252
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100253/**
254 * cpuidle_enter - enter into the specified idle state
255 *
256 * @drv: the cpuidle driver tied with the cpu
257 * @dev: the cpuidle device
258 * @index: the index in the idle state table
259 *
260 * Returns the index in the idle state, < 0 in case of error.
261 * The error code depends on the backend driver
262 */
263int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
264 int index)
265{
Xunlei Pang4c1ed5a2015-08-04 13:48:56 +0800266 if (cpuidle_state_is_coupled(drv, index))
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100267 return cpuidle_enter_state_coupled(dev, drv, index);
268 return cpuidle_enter_state(dev, drv, index);
269}
Len Browna0bfa132011-04-01 19:34:59 -0400270
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100271/**
272 * cpuidle_reflect - tell the underlying governor what was the state
273 * we were in
274 *
275 * @dev : the cpuidle device
276 * @index: the index in the idle state table
277 *
278 */
279void cpuidle_reflect(struct cpuidle_device *dev, int index)
280{
Rafael J. Wysockia802ea92015-05-04 22:53:28 +0200281 if (cpuidle_curr_governor->reflect && index >= 0)
Daniel Lezcano907e30f2014-03-03 08:48:50 +0100282 cpuidle_curr_governor->reflect(dev, index);
Len Brown4f86d3a2007-10-03 18:58:00 -0400283}
284
285/**
286 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
287 */
288void cpuidle_install_idle_handler(void)
289{
Len Browna0bfa132011-04-01 19:34:59 -0400290 if (enabled_devices) {
Len Brown4f86d3a2007-10-03 18:58:00 -0400291 /* Make sure all changes finished before we switch to new idle */
292 smp_wmb();
Len Browna0bfa132011-04-01 19:34:59 -0400293 initialized = 1;
Len Brown4f86d3a2007-10-03 18:58:00 -0400294 }
295}
296
297/**
298 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
299 */
300void cpuidle_uninstall_idle_handler(void)
301{
Len Browna0bfa132011-04-01 19:34:59 -0400302 if (enabled_devices) {
303 initialized = 0;
Chuansheng Liu2ed903c2014-09-04 15:17:55 +0800304 wake_up_all_idle_cpus();
Len Brown4f86d3a2007-10-03 18:58:00 -0400305 }
Daniel Lezcano442bf3a2014-09-04 11:32:09 -0400306
307 /*
308 * Make sure external observers (such as the scheduler)
309 * are done looking at pointed idle states.
310 */
311 synchronize_rcu();
Len Brown4f86d3a2007-10-03 18:58:00 -0400312}
313
314/**
315 * cpuidle_pause_and_lock - temporarily disables CPUIDLE
316 */
317void cpuidle_pause_and_lock(void)
318{
319 mutex_lock(&cpuidle_lock);
320 cpuidle_uninstall_idle_handler();
321}
322
323EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
324
325/**
326 * cpuidle_resume_and_unlock - resumes CPUIDLE operation
327 */
328void cpuidle_resume_and_unlock(void)
329{
330 cpuidle_install_idle_handler();
331 mutex_unlock(&cpuidle_lock);
332}
333
334EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
335
Preeti U Murthy8651f972012-07-09 10:12:56 +0200336/* Currently used in suspend/resume path to suspend cpuidle */
337void cpuidle_pause(void)
338{
339 mutex_lock(&cpuidle_lock);
340 cpuidle_uninstall_idle_handler();
341 mutex_unlock(&cpuidle_lock);
342}
343
344/* Currently used in suspend/resume path to resume cpuidle */
345void cpuidle_resume(void)
346{
347 mutex_lock(&cpuidle_lock);
348 cpuidle_install_idle_handler();
349 mutex_unlock(&cpuidle_lock);
350}
351
Len Brown4f86d3a2007-10-03 18:58:00 -0400352/**
353 * cpuidle_enable_device - enables idle PM for a CPU
354 * @dev: the CPU
355 *
356 * This function must be called between cpuidle_pause_and_lock and
357 * cpuidle_resume_and_unlock when used externally.
358 */
359int cpuidle_enable_device(struct cpuidle_device *dev)
360{
Daniel Lezcano5df0aa72013-06-12 15:08:54 +0200361 int ret;
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000362 struct cpuidle_driver *drv;
Len Brown4f86d3a2007-10-03 18:58:00 -0400363
Srivatsa S. Bhat1b0a0e92012-05-04 14:06:02 -0700364 if (!dev)
365 return -EINVAL;
366
Len Brown4f86d3a2007-10-03 18:58:00 -0400367 if (dev->enabled)
368 return 0;
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000369
370 drv = cpuidle_get_cpu_driver(dev);
371
Robert Leee1689792012-03-20 15:22:42 -0500372 if (!drv || !cpuidle_curr_governor)
Len Brown4f86d3a2007-10-03 18:58:00 -0400373 return -EIO;
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000374
Daniel Lezcano10b9d3f2013-06-12 15:08:49 +0200375 if (!dev->registered)
376 return -EINVAL;
377
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000378 ret = cpuidle_add_device_sysfs(dev);
379 if (ret)
Len Brown4f86d3a2007-10-03 18:58:00 -0400380 return ret;
381
382 if (cpuidle_curr_governor->enable &&
Robert Leee1689792012-03-20 15:22:42 -0500383 (ret = cpuidle_curr_governor->enable(drv, dev)))
Len Brown4f86d3a2007-10-03 18:58:00 -0400384 goto fail_sysfs;
385
Len Brown4f86d3a2007-10-03 18:58:00 -0400386 smp_wmb();
387
388 dev->enabled = 1;
389
390 enabled_devices++;
391 return 0;
392
393fail_sysfs:
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000394 cpuidle_remove_device_sysfs(dev);
Len Brown4f86d3a2007-10-03 18:58:00 -0400395
396 return ret;
397}
398
399EXPORT_SYMBOL_GPL(cpuidle_enable_device);
400
401/**
402 * cpuidle_disable_device - disables idle PM for a CPU
403 * @dev: the CPU
404 *
405 * This function must be called between cpuidle_pause_and_lock and
406 * cpuidle_resume_and_unlock when used externally.
407 */
408void cpuidle_disable_device(struct cpuidle_device *dev)
409{
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000410 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
411
Srivatsa S. Bhatcf31cd12012-10-08 13:43:08 +0530412 if (!dev || !dev->enabled)
Len Brown4f86d3a2007-10-03 18:58:00 -0400413 return;
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000414
415 if (!drv || !cpuidle_curr_governor)
Len Brown4f86d3a2007-10-03 18:58:00 -0400416 return;
417
418 dev->enabled = 0;
419
420 if (cpuidle_curr_governor->disable)
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000421 cpuidle_curr_governor->disable(drv, dev);
Len Brown4f86d3a2007-10-03 18:58:00 -0400422
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000423 cpuidle_remove_device_sysfs(dev);
Len Brown4f86d3a2007-10-03 18:58:00 -0400424 enabled_devices--;
425}
426
427EXPORT_SYMBOL_GPL(cpuidle_disable_device);
428
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200429static void __cpuidle_unregister_device(struct cpuidle_device *dev)
430{
431 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
432
433 list_del(&dev->device_list);
434 per_cpu(cpuidle_devices, dev->cpu) = NULL;
435 module_put(drv->owner);
Dave Gerlachc998c072016-04-05 14:05:38 -0500436
437 dev->registered = 0;
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200438}
439
Viresh Kumar267d4bf2013-10-03 21:26:43 +0530440static void __cpuidle_device_init(struct cpuidle_device *dev)
Daniel Lezcano5df0aa72013-06-12 15:08:54 +0200441{
442 memset(dev->states_usage, 0, sizeof(dev->states_usage));
443 dev->last_residency = 0;
Daniel Lezcano5df0aa72013-06-12 15:08:54 +0200444}
445
Len Brown4f86d3a2007-10-03 18:58:00 -0400446/**
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400447 * __cpuidle_register_device - internal register function called before register
448 * and enable routines
Len Brown4f86d3a2007-10-03 18:58:00 -0400449 * @dev: the cpu
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400450 *
451 * cpuidle_lock mutex must be held before this is called
Len Brown4f86d3a2007-10-03 18:58:00 -0400452 */
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400453static int __cpuidle_register_device(struct cpuidle_device *dev)
Len Brown4f86d3a2007-10-03 18:58:00 -0400454{
455 int ret;
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000456 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
Len Brown4f86d3a2007-10-03 18:58:00 -0400457
Daniel Lezcanobf4d1b52012-10-31 16:44:48 +0000458 if (!try_module_get(drv->owner))
Len Brown4f86d3a2007-10-03 18:58:00 -0400459 return -EINVAL;
460
Len Brown4f86d3a2007-10-03 18:58:00 -0400461 per_cpu(cpuidle_devices, dev->cpu) = dev;
462 list_add(&dev->device_list, &cpuidle_detected_devices);
Len Brown4f86d3a2007-10-03 18:58:00 -0400463
Colin Cross4126c012012-05-07 17:57:41 -0700464 ret = cpuidle_coupled_register_device(dev);
Viresh Kumar47182662013-10-03 21:26:46 +0530465 if (ret)
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200466 __cpuidle_unregister_device(dev);
Viresh Kumar47182662013-10-03 21:26:46 +0530467 else
468 dev->registered = 1;
Len Brown4f86d3a2007-10-03 18:58:00 -0400469
Viresh Kumar47182662013-10-03 21:26:46 +0530470 return ret;
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400471}
472
473/**
474 * cpuidle_register_device - registers a CPU's idle PM feature
475 * @dev: the cpu
476 */
477int cpuidle_register_device(struct cpuidle_device *dev)
478{
Daniel Lezcanoc878a522013-06-12 15:08:55 +0200479 int ret = -EBUSY;
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400480
Srivatsa S. Bhat1b0a0e92012-05-04 14:06:02 -0700481 if (!dev)
482 return -EINVAL;
483
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400484 mutex_lock(&cpuidle_lock);
485
Daniel Lezcanoc878a522013-06-12 15:08:55 +0200486 if (dev->registered)
487 goto out_unlock;
488
Viresh Kumar267d4bf2013-10-03 21:26:43 +0530489 __cpuidle_device_init(dev);
Daniel Lezcano5df0aa72013-06-12 15:08:54 +0200490
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200491 ret = __cpuidle_register_device(dev);
492 if (ret)
493 goto out_unlock;
494
495 ret = cpuidle_add_sysfs(dev);
496 if (ret)
497 goto out_unregister;
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400498
Daniel Lezcano10b9d3f2013-06-12 15:08:49 +0200499 ret = cpuidle_enable_device(dev);
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200500 if (ret)
501 goto out_sysfs;
Daniel Lezcano10b9d3f2013-06-12 15:08:49 +0200502
Len Brown4f86d3a2007-10-03 18:58:00 -0400503 cpuidle_install_idle_handler();
504
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200505out_unlock:
Len Brown4f86d3a2007-10-03 18:58:00 -0400506 mutex_unlock(&cpuidle_lock);
507
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200508 return ret;
509
510out_sysfs:
511 cpuidle_remove_sysfs(dev);
512out_unregister:
513 __cpuidle_unregister_device(dev);
514 goto out_unlock;
Len Brown4f86d3a2007-10-03 18:58:00 -0400515}
516
517EXPORT_SYMBOL_GPL(cpuidle_register_device);
518
519/**
520 * cpuidle_unregister_device - unregisters a CPU's idle PM feature
521 * @dev: the cpu
522 */
523void cpuidle_unregister_device(struct cpuidle_device *dev)
524{
Konrad Rzeszutek Wilk813e8e32013-12-03 10:59:58 -0500525 if (!dev || dev->registered == 0)
Venkatesh Pallipadidcb84f32008-05-19 19:09:27 -0400526 return;
527
Len Brown4f86d3a2007-10-03 18:58:00 -0400528 cpuidle_pause_and_lock();
529
530 cpuidle_disable_device(dev);
531
Daniel Lezcano1aef40e2012-10-26 12:26:24 +0200532 cpuidle_remove_sysfs(dev);
Daniel Lezcanof6bb51a2013-06-12 15:08:53 +0200533
534 __cpuidle_unregister_device(dev);
Len Brown4f86d3a2007-10-03 18:58:00 -0400535
Colin Cross4126c012012-05-07 17:57:41 -0700536 cpuidle_coupled_unregister_device(dev);
537
Len Brown4f86d3a2007-10-03 18:58:00 -0400538 cpuidle_resume_and_unlock();
Len Brown4f86d3a2007-10-03 18:58:00 -0400539}
540
541EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
542
Daniel Lezcano1c192d02013-04-23 15:28:44 +0000543/**
Daniel Lezcano4c637b22013-04-23 08:54:33 +0000544 * cpuidle_unregister: unregister a driver and the devices. This function
545 * can be used only if the driver has been previously registered through
546 * the cpuidle_register function.
547 *
548 * @drv: a valid pointer to a struct cpuidle_driver
549 */
550void cpuidle_unregister(struct cpuidle_driver *drv)
551{
552 int cpu;
553 struct cpuidle_device *device;
554
Daniel Lezcano82467a52013-06-07 21:53:09 +0000555 for_each_cpu(cpu, drv->cpumask) {
Daniel Lezcano4c637b22013-04-23 08:54:33 +0000556 device = &per_cpu(cpuidle_dev, cpu);
557 cpuidle_unregister_device(device);
558 }
559
560 cpuidle_unregister_driver(drv);
561}
562EXPORT_SYMBOL_GPL(cpuidle_unregister);
563
564/**
565 * cpuidle_register: registers the driver and the cpu devices with the
566 * coupled_cpus passed as parameter. This function is used for all common
567 * initialization pattern there are in the arch specific drivers. The
568 * devices is globally defined in this file.
569 *
570 * @drv : a valid pointer to a struct cpuidle_driver
571 * @coupled_cpus: a cpumask for the coupled states
572 *
573 * Returns 0 on success, < 0 otherwise
574 */
575int cpuidle_register(struct cpuidle_driver *drv,
576 const struct cpumask *const coupled_cpus)
577{
578 int ret, cpu;
579 struct cpuidle_device *device;
580
581 ret = cpuidle_register_driver(drv);
582 if (ret) {
583 pr_err("failed to register cpuidle driver\n");
584 return ret;
585 }
586
Daniel Lezcano82467a52013-06-07 21:53:09 +0000587 for_each_cpu(cpu, drv->cpumask) {
Daniel Lezcano4c637b22013-04-23 08:54:33 +0000588 device = &per_cpu(cpuidle_dev, cpu);
589 device->cpu = cpu;
590
591#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
592 /*
Viresh Kumarcaf4a362013-10-03 21:26:41 +0530593 * On multiplatform for ARM, the coupled idle states could be
Daniel Lezcano4c637b22013-04-23 08:54:33 +0000594 * enabled in the kernel even if the cpuidle driver does not
595 * use it. Note, coupled_cpus is a struct copy.
596 */
597 if (coupled_cpus)
598 device->coupled_cpus = *coupled_cpus;
599#endif
600 ret = cpuidle_register_device(device);
601 if (!ret)
602 continue;
603
604 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
605
606 cpuidle_unregister(drv);
607 break;
608 }
609
610 return ret;
611}
612EXPORT_SYMBOL_GPL(cpuidle_register);
613
Len Brown4f86d3a2007-10-03 18:58:00 -0400614#ifdef CONFIG_SMP
615
Len Brown4f86d3a2007-10-03 18:58:00 -0400616/*
617 * This function gets called when a part of the kernel has a new latency
618 * requirement. This means we need to get all processors out of their C-state,
619 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
620 * wakes them all right up.
621 */
622static int cpuidle_latency_notify(struct notifier_block *b,
623 unsigned long l, void *v)
624{
Chuansheng Liu2ed903c2014-09-04 15:17:55 +0800625 wake_up_all_idle_cpus();
Len Brown4f86d3a2007-10-03 18:58:00 -0400626 return NOTIFY_OK;
627}
628
629static struct notifier_block cpuidle_latency_notifier = {
630 .notifier_call = cpuidle_latency_notify,
631};
632
Mark Grossd82b3512008-02-04 22:30:08 -0800633static inline void latency_notifier_init(struct notifier_block *n)
634{
635 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
636}
Len Brown4f86d3a2007-10-03 18:58:00 -0400637
638#else /* CONFIG_SMP */
639
640#define latency_notifier_init(x) do { } while (0)
641
642#endif /* CONFIG_SMP */
643
644/**
645 * cpuidle_init - core initializer
646 */
647static int __init cpuidle_init(void)
648{
649 int ret;
650
Len Brown62027ae2011-04-01 18:13:10 -0400651 if (cpuidle_disabled())
652 return -ENODEV;
653
Kay Sievers8a25a2f2011-12-21 14:29:42 -0800654 ret = cpuidle_add_interface(cpu_subsys.dev_root);
Len Brown4f86d3a2007-10-03 18:58:00 -0400655 if (ret)
656 return ret;
657
658 latency_notifier_init(&cpuidle_latency_notifier);
659
660 return 0;
661}
662
Len Brown62027ae2011-04-01 18:13:10 -0400663module_param(off, int, 0444);
Len Brown4f86d3a2007-10-03 18:58:00 -0400664core_initcall(cpuidle_init);