blob: d42bbc2c3f213567362b0e51e927957ea8ea0953 [file] [log] [blame]
Mike Chan9d49b702010-06-22 11:26:45 -07001/*
2 * drivers/cpufreq/cpufreq_interactive.c
3 *
4 * Copyright (C) 2010 Google, Inc.
5 *
6 * This software is licensed under the terms of the GNU General Public
7 * License version 2, as published by the Free Software Foundation, and
8 * may be copied, distributed, and modified under those terms.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * Author: Mike Chan (mike@android.com)
16 *
17 */
18
19#include <linux/cpu.h>
20#include <linux/cpumask.h>
21#include <linux/cpufreq.h>
22#include <linux/module.h>
Lianwei Wangba6c6bb2012-11-01 09:59:52 +080023#include <linux/moduleparam.h>
Mike Chan9d49b702010-06-22 11:26:45 -070024#include <linux/mutex.h>
25#include <linux/sched.h>
26#include <linux/tick.h>
27#include <linux/time.h>
28#include <linux/timer.h>
29#include <linux/workqueue.h>
30#include <linux/kthread.h>
31#include <linux/mutex.h>
Todd Poynor7820a652012-04-02 17:17:14 -070032#include <linux/slab.h>
Todd Poynor9fb15312012-04-23 20:42:41 -070033#include <asm/cputime.h>
Mike Chan9d49b702010-06-22 11:26:45 -070034
Todd Poynora1e19512012-02-16 16:27:59 -080035#define CREATE_TRACE_POINTS
36#include <trace/events/cpufreq_interactive.h>
37
Mike Chan9d49b702010-06-22 11:26:45 -070038static atomic_t active_count = ATOMIC_INIT(0);
39
40struct cpufreq_interactive_cpuinfo {
41 struct timer_list cpu_timer;
42 int timer_idlecancel;
43 u64 time_in_idle;
Todd Poynor22b5c3a2012-10-08 20:14:34 -070044 u64 time_in_idle_timestamp;
Mike Chan9d49b702010-06-22 11:26:45 -070045 struct cpufreq_policy *policy;
46 struct cpufreq_frequency_table *freq_table;
47 unsigned int target_freq;
Todd Poynoraad27322012-04-26 21:41:40 -070048 unsigned int floor_freq;
49 u64 floor_validate_time;
Todd Poynor5a5aa702012-05-10 23:28:06 -070050 u64 hispeed_validate_time;
Mike Chan9d49b702010-06-22 11:26:45 -070051 int governor_enabled;
52};
53
54static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
55
Todd Poynor8a37bb72012-07-16 17:07:15 -070056/* realtime thread handles frequency scaling */
57static struct task_struct *speedchange_task;
58static cpumask_t speedchange_cpumask;
59static spinlock_t speedchange_cpumask_lock;
Mike Chan9d49b702010-06-22 11:26:45 -070060
61/* Hi speed to bump to from lo speed when load burst (default max) */
Todd Poynoracfaec92012-10-03 00:39:56 -070062static unsigned int hispeed_freq;
Mike Chan9d49b702010-06-22 11:26:45 -070063
64/* Go to hi speed when CPU load at or above this value. */
Todd Poynora0ec4362012-04-17 17:39:34 -070065#define DEFAULT_GO_HISPEED_LOAD 85
Mike Chan9d49b702010-06-22 11:26:45 -070066static unsigned long go_hispeed_load;
67
Todd Poynorbc51d672012-11-28 17:58:17 -080068/* Target load. Lower values result in higher CPU speeds. */
69#define DEFAULT_TARGET_LOAD 90
Todd Poynor2fbf5e12012-11-14 11:41:21 -080070static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
71static spinlock_t target_loads_lock;
72static unsigned int *target_loads = default_target_loads;
73static int ntarget_loads = ARRAY_SIZE(default_target_loads);
Todd Poynorbc51d672012-11-28 17:58:17 -080074
Mike Chan9d49b702010-06-22 11:26:45 -070075/*
76 * The minimum amount of time to spend at a frequency before we can ramp down.
77 */
Todd Poynora0ec4362012-04-17 17:39:34 -070078#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
Mike Chan9d49b702010-06-22 11:26:45 -070079static unsigned long min_sample_time;
80
81/*
82 * The sample rate of the timer used to increase frequency
83 */
Todd Poynora0ec4362012-04-17 17:39:34 -070084#define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
Mike Chan9d49b702010-06-22 11:26:45 -070085static unsigned long timer_rate;
86
Todd Poynor596cf1f2012-04-13 20:18:02 -070087/*
88 * Wait this long before raising speed above hispeed, by default a single
89 * timer interval.
90 */
91#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
92static unsigned long above_hispeed_delay_val;
93
Todd Poynor7820a652012-04-02 17:17:14 -070094/*
Todd Poynor9fb15312012-04-23 20:42:41 -070095 * Non-zero means longer-term speed boost active.
96 */
97
98static int boost_val;
99
Lianwei Wangba6c6bb2012-11-01 09:59:52 +0800100static bool governidle;
101module_param(governidle, bool, S_IWUSR | S_IRUGO);
102MODULE_PARM_DESC(governidle,
103 "Set to 1 to wake up CPUs from idle to reduce speed (default 0)");
104
Mike Chan9d49b702010-06-22 11:26:45 -0700105static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
106 unsigned int event);
107
108#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
109static
110#endif
111struct cpufreq_governor cpufreq_gov_interactive = {
112 .name = "interactive",
113 .governor = cpufreq_governor_interactive,
114 .max_transition_latency = 10000000,
115 .owner = THIS_MODULE,
116};
117
Todd Poynor22b5c3a2012-10-08 20:14:34 -0700118static void cpufreq_interactive_timer_resched(
119 struct cpufreq_interactive_cpuinfo *pcpu)
120{
121 mod_timer_pinned(&pcpu->cpu_timer,
122 jiffies + usecs_to_jiffies(timer_rate));
123 pcpu->time_in_idle =
124 get_cpu_idle_time_us(smp_processor_id(),
125 &pcpu->time_in_idle_timestamp);
126}
127
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800128static unsigned int freq_to_targetload(unsigned int freq)
129{
130 int i;
131 unsigned int ret;
132
133 spin_lock(&target_loads_lock);
134
135 for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
136 ;
137
138 ret = target_loads[i];
139 spin_unlock(&target_loads_lock);
140 return ret;
141}
142
143/*
144 * If increasing frequencies never map to a lower target load then
145 * choose_freq() will find the minimum frequency that does not exceed its
146 * target load given the current load.
147 */
148
149static unsigned int choose_freq(
150 struct cpufreq_interactive_cpuinfo *pcpu, unsigned int curload)
151{
152 unsigned int freq = pcpu->policy->cur;
153 unsigned int loadadjfreq = freq * curload;
154 unsigned int prevfreq, freqmin, freqmax;
155 unsigned int tl;
156 int index;
157
158 freqmin = 0;
159 freqmax = UINT_MAX;
160
161 do {
162 prevfreq = freq;
163 tl = freq_to_targetload(freq);
164
165 /*
166 * Find the lowest frequency where the computed load is less
167 * than or equal to the target load.
168 */
169
170 cpufreq_frequency_table_target(
171 pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
172 CPUFREQ_RELATION_L, &index);
173 freq = pcpu->freq_table[index].frequency;
174
175 if (freq > prevfreq) {
176 /* The previous frequency is too low. */
177 freqmin = prevfreq;
178
179 if (freq >= freqmax) {
180 /*
181 * Find the highest frequency that is less
182 * than freqmax.
183 */
184 cpufreq_frequency_table_target(
185 pcpu->policy, pcpu->freq_table,
186 freqmax - 1, CPUFREQ_RELATION_H,
187 &index);
188 freq = pcpu->freq_table[index].frequency;
189
190 if (freq == freqmin) {
191 /*
192 * The first frequency below freqmax
193 * has already been found to be too
194 * low. freqmax is the lowest speed
195 * we found that is fast enough.
196 */
197 freq = freqmax;
198 break;
199 }
200 }
201 } else if (freq < prevfreq) {
202 /* The previous frequency is high enough. */
203 freqmax = prevfreq;
204
205 if (freq <= freqmin) {
206 /*
207 * Find the lowest frequency that is higher
208 * than freqmin.
209 */
210 cpufreq_frequency_table_target(
211 pcpu->policy, pcpu->freq_table,
212 freqmin + 1, CPUFREQ_RELATION_L,
213 &index);
214 freq = pcpu->freq_table[index].frequency;
215
216 /*
217 * If freqmax is the first frequency above
218 * freqmin then we have already found that
219 * this speed is fast enough.
220 */
221 if (freq == freqmax)
222 break;
223 }
224 }
225
226 /* If same frequency chosen as previous then done. */
227 } while (freq != prevfreq);
228
229 return freq;
230}
231
Mike Chan9d49b702010-06-22 11:26:45 -0700232static void cpufreq_interactive_timer(unsigned long data)
233{
Todd Poynor7aa95c82012-11-05 13:09:03 -0800234 u64 now;
Mike Chan9d49b702010-06-22 11:26:45 -0700235 unsigned int delta_idle;
236 unsigned int delta_time;
237 int cpu_load;
Mike Chan9d49b702010-06-22 11:26:45 -0700238 struct cpufreq_interactive_cpuinfo *pcpu =
239 &per_cpu(cpuinfo, data);
240 u64 now_idle;
241 unsigned int new_freq;
242 unsigned int index;
243 unsigned long flags;
244
245 smp_rmb();
246
247 if (!pcpu->governor_enabled)
248 goto exit;
249
Todd Poynor7aa95c82012-11-05 13:09:03 -0800250 now_idle = get_cpu_idle_time_us(data, &now);
Todd Poynor22b5c3a2012-10-08 20:14:34 -0700251 delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
252 delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
Mike Chan9d49b702010-06-22 11:26:45 -0700253
254 /*
255 * If timer ran less than 1ms after short-term sample started, retry.
256 */
257 if (delta_time < 1000)
258 goto rearm;
259
260 if (delta_idle > delta_time)
261 cpu_load = 0;
262 else
263 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
264
Todd Poynorf96f2c82012-11-08 15:06:55 -0800265 if ((cpu_load >= go_hispeed_load || boost_val) &&
266 pcpu->target_freq < hispeed_freq)
267 new_freq = hispeed_freq;
268 else
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800269 new_freq = choose_freq(pcpu, cpu_load);
Todd Poynorf96f2c82012-11-08 15:06:55 -0800270
271 if (pcpu->target_freq >= hispeed_freq &&
272 new_freq > pcpu->target_freq &&
273 now - pcpu->hispeed_validate_time < above_hispeed_delay_val) {
274 trace_cpufreq_interactive_notyet(
275 data, cpu_load, pcpu->target_freq,
276 pcpu->policy->cur, new_freq);
277 goto rearm;
Mike Chan9d49b702010-06-22 11:26:45 -0700278 }
279
Todd Poynorf96f2c82012-11-08 15:06:55 -0800280 pcpu->hispeed_validate_time = now;
Todd Poynor5a5aa702012-05-10 23:28:06 -0700281
Mike Chan9d49b702010-06-22 11:26:45 -0700282 if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
Todd Poynorbc51d672012-11-28 17:58:17 -0800283 new_freq, CPUFREQ_RELATION_L,
Mike Chan9d49b702010-06-22 11:26:45 -0700284 &index)) {
285 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
286 (int) data);
287 goto rearm;
288 }
289
290 new_freq = pcpu->freq_table[index].frequency;
291
Mike Chan9d49b702010-06-22 11:26:45 -0700292 /*
Todd Poynoraad27322012-04-26 21:41:40 -0700293 * Do not scale below floor_freq unless we have been at or above the
294 * floor frequency for the minimum sample time since last validated.
Mike Chan9d49b702010-06-22 11:26:45 -0700295 */
Todd Poynoraad27322012-04-26 21:41:40 -0700296 if (new_freq < pcpu->floor_freq) {
Todd Poynor7aa95c82012-11-05 13:09:03 -0800297 if (now - pcpu->floor_validate_time < min_sample_time) {
Todd Poynor46660b02012-11-28 17:56:09 -0800298 trace_cpufreq_interactive_notyet(
299 data, cpu_load, pcpu->target_freq,
300 pcpu->policy->cur, new_freq);
Mike Chan9d49b702010-06-22 11:26:45 -0700301 goto rearm;
Todd Poynora1e19512012-02-16 16:27:59 -0800302 }
Mike Chan9d49b702010-06-22 11:26:45 -0700303 }
304
Todd Poynoraad27322012-04-26 21:41:40 -0700305 pcpu->floor_freq = new_freq;
Todd Poynor7aa95c82012-11-05 13:09:03 -0800306 pcpu->floor_validate_time = now;
Todd Poynor0a92d482012-04-06 19:59:36 -0700307
308 if (pcpu->target_freq == new_freq) {
Todd Poynor46660b02012-11-28 17:56:09 -0800309 trace_cpufreq_interactive_already(
310 data, cpu_load, pcpu->target_freq,
311 pcpu->policy->cur, new_freq);
Todd Poynor0a92d482012-04-06 19:59:36 -0700312 goto rearm_if_notmax;
313 }
314
Todd Poynora1e19512012-02-16 16:27:59 -0800315 trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
Todd Poynor46660b02012-11-28 17:56:09 -0800316 pcpu->policy->cur, new_freq);
Todd Poynora1e19512012-02-16 16:27:59 -0800317
Todd Poynor8a37bb72012-07-16 17:07:15 -0700318 pcpu->target_freq = new_freq;
319 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
320 cpumask_set_cpu(data, &speedchange_cpumask);
321 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
322 wake_up_process(speedchange_task);
Mike Chan9d49b702010-06-22 11:26:45 -0700323
324rearm_if_notmax:
325 /*
326 * Already set max speed and don't see a need to change that,
327 * wait until next idle to re-evaluate, don't need timer.
328 */
329 if (pcpu->target_freq == pcpu->policy->max)
330 goto exit;
331
332rearm:
333 if (!timer_pending(&pcpu->cpu_timer)) {
334 /*
Lianwei Wangba6c6bb2012-11-01 09:59:52 +0800335 * If governing speed in idle and already at min, cancel the
336 * timer if that CPU goes idle. We don't need to re-evaluate
337 * speed until the next idle exit.
Mike Chan9d49b702010-06-22 11:26:45 -0700338 */
Lianwei Wangba6c6bb2012-11-01 09:59:52 +0800339 if (governidle && pcpu->target_freq == pcpu->policy->min)
Mike Chan9d49b702010-06-22 11:26:45 -0700340 pcpu->timer_idlecancel = 1;
Mike Chan9d49b702010-06-22 11:26:45 -0700341
Todd Poynor22b5c3a2012-10-08 20:14:34 -0700342 cpufreq_interactive_timer_resched(pcpu);
Mike Chan9d49b702010-06-22 11:26:45 -0700343 }
344
345exit:
346 return;
347}
348
349static void cpufreq_interactive_idle_start(void)
350{
351 struct cpufreq_interactive_cpuinfo *pcpu =
352 &per_cpu(cpuinfo, smp_processor_id());
353 int pending;
354
355 if (!pcpu->governor_enabled)
356 return;
357
Mike Chan9d49b702010-06-22 11:26:45 -0700358 pending = timer_pending(&pcpu->cpu_timer);
359
360 if (pcpu->target_freq != pcpu->policy->min) {
Mike Chan9d49b702010-06-22 11:26:45 -0700361 /*
362 * Entering idle while not at lowest speed. On some
363 * platforms this can hold the other CPU(s) at that speed
364 * even though the CPU is idle. Set a timer to re-evaluate
365 * speed so this idle CPU doesn't hold the other CPUs above
366 * min indefinitely. This should probably be a quirk of
367 * the CPUFreq driver.
368 */
369 if (!pending) {
Mike Chan9d49b702010-06-22 11:26:45 -0700370 pcpu->timer_idlecancel = 0;
Todd Poynor22b5c3a2012-10-08 20:14:34 -0700371 cpufreq_interactive_timer_resched(pcpu);
Mike Chan9d49b702010-06-22 11:26:45 -0700372 }
Lianwei Wangba6c6bb2012-11-01 09:59:52 +0800373 } else if (governidle) {
Mike Chan9d49b702010-06-22 11:26:45 -0700374 /*
375 * If at min speed and entering idle after load has
376 * already been evaluated, and a timer has been set just in
377 * case the CPU suddenly goes busy, cancel that timer. The
378 * CPU didn't go busy; we'll recheck things upon idle exit.
379 */
380 if (pending && pcpu->timer_idlecancel) {
381 del_timer(&pcpu->cpu_timer);
Mike Chan9d49b702010-06-22 11:26:45 -0700382 pcpu->timer_idlecancel = 0;
383 }
384 }
385
386}
387
388static void cpufreq_interactive_idle_end(void)
389{
390 struct cpufreq_interactive_cpuinfo *pcpu =
391 &per_cpu(cpuinfo, smp_processor_id());
392
Sam Lefflerae0d23f2012-06-27 10:12:04 -0700393 if (!pcpu->governor_enabled)
394 return;
395
Todd Poynor7aa95c82012-11-05 13:09:03 -0800396 /* Arm the timer for 1-2 ticks later if not already. */
397 if (!timer_pending(&pcpu->cpu_timer)) {
Mike Chan9d49b702010-06-22 11:26:45 -0700398 pcpu->timer_idlecancel = 0;
Todd Poynor22b5c3a2012-10-08 20:14:34 -0700399 cpufreq_interactive_timer_resched(pcpu);
400 } else if (!governidle &&
401 time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
402 del_timer(&pcpu->cpu_timer);
403 cpufreq_interactive_timer(smp_processor_id());
Mike Chan9d49b702010-06-22 11:26:45 -0700404 }
Mike Chan9d49b702010-06-22 11:26:45 -0700405}
406
Todd Poynor8a37bb72012-07-16 17:07:15 -0700407static int cpufreq_interactive_speedchange_task(void *data)
Mike Chan9d49b702010-06-22 11:26:45 -0700408{
409 unsigned int cpu;
410 cpumask_t tmp_mask;
411 unsigned long flags;
412 struct cpufreq_interactive_cpuinfo *pcpu;
413
414 while (1) {
415 set_current_state(TASK_INTERRUPTIBLE);
Todd Poynor8a37bb72012-07-16 17:07:15 -0700416 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
Mike Chan9d49b702010-06-22 11:26:45 -0700417
Todd Poynor8a37bb72012-07-16 17:07:15 -0700418 if (cpumask_empty(&speedchange_cpumask)) {
419 spin_unlock_irqrestore(&speedchange_cpumask_lock,
420 flags);
Mike Chan9d49b702010-06-22 11:26:45 -0700421 schedule();
422
423 if (kthread_should_stop())
424 break;
425
Todd Poynor8a37bb72012-07-16 17:07:15 -0700426 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
Mike Chan9d49b702010-06-22 11:26:45 -0700427 }
428
429 set_current_state(TASK_RUNNING);
Todd Poynor8a37bb72012-07-16 17:07:15 -0700430 tmp_mask = speedchange_cpumask;
431 cpumask_clear(&speedchange_cpumask);
432 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
Mike Chan9d49b702010-06-22 11:26:45 -0700433
434 for_each_cpu(cpu, &tmp_mask) {
435 unsigned int j;
436 unsigned int max_freq = 0;
437
438 pcpu = &per_cpu(cpuinfo, cpu);
439 smp_rmb();
440
441 if (!pcpu->governor_enabled)
442 continue;
443
Mike Chan9d49b702010-06-22 11:26:45 -0700444 for_each_cpu(j, pcpu->policy->cpus) {
445 struct cpufreq_interactive_cpuinfo *pjcpu =
446 &per_cpu(cpuinfo, j);
447
448 if (pjcpu->target_freq > max_freq)
449 max_freq = pjcpu->target_freq;
450 }
451
452 if (max_freq != pcpu->policy->cur)
453 __cpufreq_driver_target(pcpu->policy,
454 max_freq,
455 CPUFREQ_RELATION_H);
Todd Poynor8a37bb72012-07-16 17:07:15 -0700456 trace_cpufreq_interactive_setspeed(cpu,
457 pcpu->target_freq,
Todd Poynora1e19512012-02-16 16:27:59 -0800458 pcpu->policy->cur);
Mike Chan9d49b702010-06-22 11:26:45 -0700459 }
460 }
461
462 return 0;
463}
464
Todd Poynor7820a652012-04-02 17:17:14 -0700465static void cpufreq_interactive_boost(void)
466{
467 int i;
468 int anyboost = 0;
469 unsigned long flags;
470 struct cpufreq_interactive_cpuinfo *pcpu;
471
Todd Poynor8a37bb72012-07-16 17:07:15 -0700472 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
Todd Poynor7820a652012-04-02 17:17:14 -0700473
474 for_each_online_cpu(i) {
475 pcpu = &per_cpu(cpuinfo, i);
476
477 if (pcpu->target_freq < hispeed_freq) {
478 pcpu->target_freq = hispeed_freq;
Todd Poynor8a37bb72012-07-16 17:07:15 -0700479 cpumask_set_cpu(i, &speedchange_cpumask);
Todd Poynor3c081182012-12-07 20:08:45 -0800480 pcpu->hispeed_validate_time =
481 ktime_to_us(ktime_get());
Todd Poynor7820a652012-04-02 17:17:14 -0700482 anyboost = 1;
483 }
484
485 /*
Todd Poynoraad27322012-04-26 21:41:40 -0700486 * Set floor freq and (re)start timer for when last
487 * validated.
Todd Poynor7820a652012-04-02 17:17:14 -0700488 */
489
Todd Poynoraad27322012-04-26 21:41:40 -0700490 pcpu->floor_freq = hispeed_freq;
491 pcpu->floor_validate_time = ktime_to_us(ktime_get());
Todd Poynor7820a652012-04-02 17:17:14 -0700492 }
493
Todd Poynor8a37bb72012-07-16 17:07:15 -0700494 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
Todd Poynor7820a652012-04-02 17:17:14 -0700495
496 if (anyboost)
Todd Poynor8a37bb72012-07-16 17:07:15 -0700497 wake_up_process(speedchange_task);
Todd Poynor7820a652012-04-02 17:17:14 -0700498}
499
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800500static ssize_t show_target_loads(
Todd Poynorbc51d672012-11-28 17:58:17 -0800501 struct kobject *kobj, struct attribute *attr, char *buf)
502{
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800503 int i;
504 ssize_t ret = 0;
505
506 spin_lock(&target_loads_lock);
507
508 for (i = 0; i < ntarget_loads; i++)
509 ret += sprintf(buf + ret, "%u%s", target_loads[i],
510 i & 0x1 ? ":" : " ");
511
512 ret += sprintf(buf + ret, "\n");
513 spin_unlock(&target_loads_lock);
514 return ret;
Todd Poynorbc51d672012-11-28 17:58:17 -0800515}
516
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800517static ssize_t store_target_loads(
Todd Poynorbc51d672012-11-28 17:58:17 -0800518 struct kobject *kobj, struct attribute *attr, const char *buf,
519 size_t count)
520{
521 int ret;
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800522 const char *cp;
523 unsigned int *new_target_loads = NULL;
524 int ntokens = 1;
525 int i;
Todd Poynorbc51d672012-11-28 17:58:17 -0800526
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800527 cp = buf;
528 while ((cp = strpbrk(cp + 1, " :")))
529 ntokens++;
530
531 if (!(ntokens & 0x1))
532 goto err_inval;
533
534 new_target_loads = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
535 if (!new_target_loads) {
536 ret = -ENOMEM;
537 goto err;
538 }
539
540 cp = buf;
541 i = 0;
542 while (i < ntokens) {
543 if (sscanf(cp, "%u", &new_target_loads[i++]) != 1)
544 goto err_inval;
545
546 cp = strpbrk(cp, " :");
547 if (!cp)
548 break;
549 cp++;
550 }
551
552 if (i != ntokens)
553 goto err_inval;
554
555 spin_lock(&target_loads_lock);
556 if (target_loads != default_target_loads)
557 kfree(target_loads);
558 target_loads = new_target_loads;
559 ntarget_loads = ntokens;
560 spin_unlock(&target_loads_lock);
Todd Poynorbc51d672012-11-28 17:58:17 -0800561 return count;
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800562
563err_inval:
564 ret = -EINVAL;
565err:
566 kfree(new_target_loads);
567 return ret;
Todd Poynorbc51d672012-11-28 17:58:17 -0800568}
569
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800570static struct global_attr target_loads_attr =
571 __ATTR(target_loads, S_IRUGO | S_IWUSR,
572 show_target_loads, store_target_loads);
Todd Poynorbc51d672012-11-28 17:58:17 -0800573
Mike Chan9d49b702010-06-22 11:26:45 -0700574static ssize_t show_hispeed_freq(struct kobject *kobj,
575 struct attribute *attr, char *buf)
576{
Todd Poynoracfaec92012-10-03 00:39:56 -0700577 return sprintf(buf, "%u\n", hispeed_freq);
Mike Chan9d49b702010-06-22 11:26:45 -0700578}
579
580static ssize_t store_hispeed_freq(struct kobject *kobj,
581 struct attribute *attr, const char *buf,
582 size_t count)
583{
584 int ret;
Todd Poynoracfaec92012-10-03 00:39:56 -0700585 long unsigned int val;
Mike Chan9d49b702010-06-22 11:26:45 -0700586
Todd Poynoracfaec92012-10-03 00:39:56 -0700587 ret = strict_strtoul(buf, 0, &val);
Mike Chan9d49b702010-06-22 11:26:45 -0700588 if (ret < 0)
589 return ret;
590 hispeed_freq = val;
591 return count;
592}
593
594static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
595 show_hispeed_freq, store_hispeed_freq);
596
597
598static ssize_t show_go_hispeed_load(struct kobject *kobj,
599 struct attribute *attr, char *buf)
600{
601 return sprintf(buf, "%lu\n", go_hispeed_load);
602}
603
604static ssize_t store_go_hispeed_load(struct kobject *kobj,
605 struct attribute *attr, const char *buf, size_t count)
606{
607 int ret;
608 unsigned long val;
609
610 ret = strict_strtoul(buf, 0, &val);
611 if (ret < 0)
612 return ret;
613 go_hispeed_load = val;
614 return count;
615}
616
617static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
618 show_go_hispeed_load, store_go_hispeed_load);
619
620static ssize_t show_min_sample_time(struct kobject *kobj,
621 struct attribute *attr, char *buf)
622{
623 return sprintf(buf, "%lu\n", min_sample_time);
624}
625
626static ssize_t store_min_sample_time(struct kobject *kobj,
627 struct attribute *attr, const char *buf, size_t count)
628{
629 int ret;
630 unsigned long val;
631
632 ret = strict_strtoul(buf, 0, &val);
633 if (ret < 0)
634 return ret;
635 min_sample_time = val;
636 return count;
637}
638
639static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
640 show_min_sample_time, store_min_sample_time);
641
Todd Poynor596cf1f2012-04-13 20:18:02 -0700642static ssize_t show_above_hispeed_delay(struct kobject *kobj,
643 struct attribute *attr, char *buf)
644{
645 return sprintf(buf, "%lu\n", above_hispeed_delay_val);
646}
647
648static ssize_t store_above_hispeed_delay(struct kobject *kobj,
649 struct attribute *attr,
650 const char *buf, size_t count)
651{
652 int ret;
653 unsigned long val;
654
655 ret = strict_strtoul(buf, 0, &val);
656 if (ret < 0)
657 return ret;
658 above_hispeed_delay_val = val;
659 return count;
660}
661
662define_one_global_rw(above_hispeed_delay);
663
Mike Chan9d49b702010-06-22 11:26:45 -0700664static ssize_t show_timer_rate(struct kobject *kobj,
665 struct attribute *attr, char *buf)
666{
667 return sprintf(buf, "%lu\n", timer_rate);
668}
669
670static ssize_t store_timer_rate(struct kobject *kobj,
671 struct attribute *attr, const char *buf, size_t count)
672{
673 int ret;
674 unsigned long val;
675
676 ret = strict_strtoul(buf, 0, &val);
677 if (ret < 0)
678 return ret;
679 timer_rate = val;
680 return count;
681}
682
683static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
684 show_timer_rate, store_timer_rate);
685
Todd Poynor9fb15312012-04-23 20:42:41 -0700686static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
687 char *buf)
688{
689 return sprintf(buf, "%d\n", boost_val);
690}
691
692static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
693 const char *buf, size_t count)
694{
695 int ret;
696 unsigned long val;
697
698 ret = kstrtoul(buf, 0, &val);
699 if (ret < 0)
700 return ret;
701
702 boost_val = val;
703
Todd Poynor2e739a02012-05-03 00:16:55 -0700704 if (boost_val) {
705 trace_cpufreq_interactive_boost("on");
Todd Poynor9fb15312012-04-23 20:42:41 -0700706 cpufreq_interactive_boost();
Todd Poynor2e739a02012-05-03 00:16:55 -0700707 } else {
708 trace_cpufreq_interactive_unboost("off");
709 }
Todd Poynor9fb15312012-04-23 20:42:41 -0700710
711 return count;
712}
713
714define_one_global_rw(boost);
715
Todd Poynor2e739a02012-05-03 00:16:55 -0700716static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
717 const char *buf, size_t count)
718{
719 int ret;
720 unsigned long val;
721
722 ret = kstrtoul(buf, 0, &val);
723 if (ret < 0)
724 return ret;
725
726 trace_cpufreq_interactive_boost("pulse");
727 cpufreq_interactive_boost();
728 return count;
729}
730
731static struct global_attr boostpulse =
732 __ATTR(boostpulse, 0200, NULL, store_boostpulse);
733
Mike Chan9d49b702010-06-22 11:26:45 -0700734static struct attribute *interactive_attributes[] = {
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800735 &target_loads_attr.attr,
Mike Chan9d49b702010-06-22 11:26:45 -0700736 &hispeed_freq_attr.attr,
737 &go_hispeed_load_attr.attr,
Todd Poynor596cf1f2012-04-13 20:18:02 -0700738 &above_hispeed_delay.attr,
Mike Chan9d49b702010-06-22 11:26:45 -0700739 &min_sample_time_attr.attr,
740 &timer_rate_attr.attr,
Todd Poynor9fb15312012-04-23 20:42:41 -0700741 &boost.attr,
Todd Poynor2e739a02012-05-03 00:16:55 -0700742 &boostpulse.attr,
Mike Chan9d49b702010-06-22 11:26:45 -0700743 NULL,
744};
745
746static struct attribute_group interactive_attr_group = {
747 .attrs = interactive_attributes,
748 .name = "interactive",
749};
750
Sam Lefflerae0d23f2012-06-27 10:12:04 -0700751static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
752 unsigned long val,
753 void *data)
754{
755 switch (val) {
756 case IDLE_START:
757 cpufreq_interactive_idle_start();
758 break;
759 case IDLE_END:
760 cpufreq_interactive_idle_end();
761 break;
762 }
763
764 return 0;
765}
766
767static struct notifier_block cpufreq_interactive_idle_nb = {
768 .notifier_call = cpufreq_interactive_idle_notifier,
769};
770
Mike Chan9d49b702010-06-22 11:26:45 -0700771static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
772 unsigned int event)
773{
774 int rc;
775 unsigned int j;
776 struct cpufreq_interactive_cpuinfo *pcpu;
777 struct cpufreq_frequency_table *freq_table;
778
779 switch (event) {
780 case CPUFREQ_GOV_START:
781 if (!cpu_online(policy->cpu))
782 return -EINVAL;
783
784 freq_table =
785 cpufreq_frequency_get_table(policy->cpu);
Todd Poynor7aa95c82012-11-05 13:09:03 -0800786 if (!hispeed_freq)
787 hispeed_freq = policy->max;
Mike Chan9d49b702010-06-22 11:26:45 -0700788
789 for_each_cpu(j, policy->cpus) {
790 pcpu = &per_cpu(cpuinfo, j);
791 pcpu->policy = policy;
792 pcpu->target_freq = policy->cur;
793 pcpu->freq_table = freq_table;
Todd Poynoraad27322012-04-26 21:41:40 -0700794 pcpu->floor_freq = pcpu->target_freq;
795 pcpu->floor_validate_time =
Todd Poynor3c081182012-12-07 20:08:45 -0800796 ktime_to_us(ktime_get());
Todd Poynor5a5aa702012-05-10 23:28:06 -0700797 pcpu->hispeed_validate_time =
Todd Poynor3c081182012-12-07 20:08:45 -0800798 pcpu->floor_validate_time;
Mike Chan9d49b702010-06-22 11:26:45 -0700799 pcpu->governor_enabled = 1;
800 smp_wmb();
Todd Poynor7aa95c82012-11-05 13:09:03 -0800801 pcpu->cpu_timer.expires =
802 jiffies + usecs_to_jiffies(timer_rate);
803 add_timer_on(&pcpu->cpu_timer, j);
Mike Chan9d49b702010-06-22 11:26:45 -0700804 }
805
Mike Chan9d49b702010-06-22 11:26:45 -0700806 /*
807 * Do not register the idle hook and create sysfs
808 * entries if we have already done so.
809 */
810 if (atomic_inc_return(&active_count) > 1)
811 return 0;
812
813 rc = sysfs_create_group(cpufreq_global_kobject,
814 &interactive_attr_group);
815 if (rc)
816 return rc;
817
Sam Lefflerae0d23f2012-06-27 10:12:04 -0700818 idle_notifier_register(&cpufreq_interactive_idle_nb);
Mike Chan9d49b702010-06-22 11:26:45 -0700819 break;
820
821 case CPUFREQ_GOV_STOP:
822 for_each_cpu(j, policy->cpus) {
823 pcpu = &per_cpu(cpuinfo, j);
824 pcpu->governor_enabled = 0;
825 smp_wmb();
826 del_timer_sync(&pcpu->cpu_timer);
Mike Chan9d49b702010-06-22 11:26:45 -0700827 }
828
Mike Chan9d49b702010-06-22 11:26:45 -0700829 if (atomic_dec_return(&active_count) > 0)
830 return 0;
831
Sam Lefflerae0d23f2012-06-27 10:12:04 -0700832 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
Mike Chan9d49b702010-06-22 11:26:45 -0700833 sysfs_remove_group(cpufreq_global_kobject,
834 &interactive_attr_group);
835
836 break;
837
838 case CPUFREQ_GOV_LIMITS:
839 if (policy->max < policy->cur)
840 __cpufreq_driver_target(policy,
841 policy->max, CPUFREQ_RELATION_H);
842 else if (policy->min > policy->cur)
843 __cpufreq_driver_target(policy,
844 policy->min, CPUFREQ_RELATION_L);
845 break;
846 }
847 return 0;
848}
849
Mike Chan9d49b702010-06-22 11:26:45 -0700850static int __init cpufreq_interactive_init(void)
851{
852 unsigned int i;
853 struct cpufreq_interactive_cpuinfo *pcpu;
854 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
855
856 go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
857 min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
Todd Poynor596cf1f2012-04-13 20:18:02 -0700858 above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
Mike Chan9d49b702010-06-22 11:26:45 -0700859 timer_rate = DEFAULT_TIMER_RATE;
860
861 /* Initalize per-cpu timers */
862 for_each_possible_cpu(i) {
863 pcpu = &per_cpu(cpuinfo, i);
Lianwei Wangba6c6bb2012-11-01 09:59:52 +0800864 if (governidle)
865 init_timer(&pcpu->cpu_timer);
866 else
867 init_timer_deferrable(&pcpu->cpu_timer);
Mike Chan9d49b702010-06-22 11:26:45 -0700868 pcpu->cpu_timer.function = cpufreq_interactive_timer;
869 pcpu->cpu_timer.data = i;
870 }
871
Todd Poynor2fbf5e12012-11-14 11:41:21 -0800872 spin_lock_init(&target_loads_lock);
Todd Poynor8a37bb72012-07-16 17:07:15 -0700873 spin_lock_init(&speedchange_cpumask_lock);
874 speedchange_task =
875 kthread_create(cpufreq_interactive_speedchange_task, NULL,
876 "cfinteractive");
877 if (IS_ERR(speedchange_task))
878 return PTR_ERR(speedchange_task);
Sam Leffler9f1dcd62012-06-27 12:55:56 -0700879
Todd Poynor8a37bb72012-07-16 17:07:15 -0700880 sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
881 get_task_struct(speedchange_task);
Mike Chan9d49b702010-06-22 11:26:45 -0700882
Sam Leffler9f1dcd62012-06-27 12:55:56 -0700883 /* NB: wake up so the thread does not look hung to the freezer */
Todd Poynor8a37bb72012-07-16 17:07:15 -0700884 wake_up_process(speedchange_task);
Sam Leffler9f1dcd62012-06-27 12:55:56 -0700885
Mike Chan9d49b702010-06-22 11:26:45 -0700886 return cpufreq_register_governor(&cpufreq_gov_interactive);
Mike Chan9d49b702010-06-22 11:26:45 -0700887}
888
889#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
890fs_initcall(cpufreq_interactive_init);
891#else
892module_init(cpufreq_interactive_init);
893#endif
894
895static void __exit cpufreq_interactive_exit(void)
896{
897 cpufreq_unregister_governor(&cpufreq_gov_interactive);
Todd Poynor8a37bb72012-07-16 17:07:15 -0700898 kthread_stop(speedchange_task);
899 put_task_struct(speedchange_task);
Mike Chan9d49b702010-06-22 11:26:45 -0700900}
901
902module_exit(cpufreq_interactive_exit);
903
904MODULE_AUTHOR("Mike Chan <mike@android.com>");
905MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
906 "Latency sensitive workloads");
907MODULE_LICENSE("GPL");