blob: 5c5f2777a62f92bf1fc53c4e6f51666a0d3caf94 [file] [log] [blame]
/*
* drivers/cpufreq/cpufreq_interactive.c
*
* Copyright (C) 2010 Google, Inc.
*
* This software is licensed under the terms of the GNU General Public
* License version 2, as published by the Free Software Foundation, and
* may be copied, distributed, and modified under those terms.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* Author: Mike Chan (mike@android.com)
*
*/
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/cpufreq.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/tick.h>
#include <linux/time.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/slab.h>
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
struct cpufreq_interactive_cpuinfo {
struct timer_list cpu_timer;
struct timer_list cpu_slack_timer;
spinlock_t load_lock; /* protects the next 4 fields */
u64 time_in_idle;
u64 time_in_idle_timestamp;
u64 cputime_speedadj;
u64 cputime_speedadj_timestamp;
u64 last_evaluated_jiffy;
struct cpufreq_policy *policy;
struct cpufreq_frequency_table *freq_table;
spinlock_t target_freq_lock; /*protects target freq */
unsigned int target_freq;
unsigned int floor_freq;
u64 pol_floor_val_time; /* policy floor_validate_time */
u64 loc_floor_val_time; /* per-cpu floor_validate_time */
u64 pol_hispeed_val_time; /* policy hispeed_validate_time */
u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */
u64 max_freq_hyst_start_time;
unsigned int min_freq;
struct rw_semaphore enable_sem;
bool reject_notification;
int governor_enabled;
struct cpufreq_interactive_tunables *cached_tunables;
int first_cpu;
};
static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
/* realtime thread handles frequency scaling */
static struct task_struct *speedchange_task;
static cpumask_t speedchange_cpumask;
static spinlock_t speedchange_cpumask_lock;
static struct mutex gov_lock;
static int set_window_count;
static int migration_register_count;
static struct mutex sched_lock;
/* Target load. Lower values result in higher CPU speeds. */
#define DEFAULT_TARGET_LOAD 90
static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
#define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
static unsigned int default_above_hispeed_delay[] = {
DEFAULT_ABOVE_HISPEED_DELAY };
struct cpufreq_interactive_tunables {
int usage_count;
/* Hi speed to bump to from lo speed when load burst (default max) */
unsigned int hispeed_freq;
/* Go to hi speed when CPU load at or above this value. */
#define DEFAULT_GO_HISPEED_LOAD 99
unsigned long go_hispeed_load;
/* Target load. Lower values result in higher CPU speeds. */
spinlock_t target_loads_lock;
unsigned int *target_loads;
int ntarget_loads;
/*
* The minimum amount of time to spend at a frequency before we can ramp
* down.
*/
#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
unsigned long min_sample_time;
/*
* The sample rate of the timer used to increase frequency
*/
unsigned long timer_rate;
/*
* Wait this long before raising speed above hispeed, by default a
* single timer interval.
*/
spinlock_t above_hispeed_delay_lock;
unsigned int *above_hispeed_delay;
int nabove_hispeed_delay;
/* Non-zero means indefinite speed boost active */
int boost_val;
/* Duration of a boot pulse in usecs */
int boostpulse_duration_val;
/* End time of boost pulse in ktime converted to usecs */
u64 boostpulse_endtime;
bool boosted;
/*
* Max additional time to wait in idle, beyond timer_rate, at speeds
* above minimum before wakeup to reduce speed, or -1 if unnecessary.
*/
#define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
int timer_slack_val;
bool io_is_busy;
/* scheduler input related flags */
bool use_sched_load;
bool use_migration_notif;
/*
* Whether to align timer windows across all CPUs. When
* use_sched_load is true, this flag is ignored and windows
* will always be aligned.
*/
bool align_windows;
/*
* Stay at max freq for at least max_freq_hysteresis before dropping
* frequency.
*/
unsigned int max_freq_hysteresis;
};
/*
* HACK: FIXME: Bring back cpufreq_{get,put}_global_kobject()
* definition removed by upstream commit 8eec1020f0c0 "cpufreq:
* create cpu/cpufreq at boot time" to fix build failures.
*/
static int cpufreq_global_kobject_usage;
int cpufreq_get_global_kobject(void)
{
if (!cpufreq_global_kobject_usage++)
return kobject_add(cpufreq_global_kobject,
&cpu_subsys.dev_root->kobj, "%s", "cpufreq");
return 0;
}
void cpufreq_put_global_kobject(void)
{
if (!--cpufreq_global_kobject_usage)
kobject_del(cpufreq_global_kobject);
}
/* For cases where we have single governor instance for system */
static struct cpufreq_interactive_tunables *common_tunables;
static struct attribute_group *get_sysfs_attr(void);
/* Round to starting jiffy of next evaluation window */
static u64 round_to_nw_start(u64 jif,
struct cpufreq_interactive_tunables *tunables)
{
unsigned long step = usecs_to_jiffies(tunables->timer_rate);
u64 ret;
if (tunables->use_sched_load || tunables->align_windows) {
do_div(jif, step);
ret = (jif + 1) * step;
} else {
ret = jiffies + usecs_to_jiffies(tunables->timer_rate);
}
return ret;
}
static inline int set_window_helper(
struct cpufreq_interactive_tunables *tunables)
{
return sched_set_window(round_to_nw_start(get_jiffies_64(), tunables),
usecs_to_jiffies(tunables->timer_rate));
}
static void cpufreq_interactive_timer_resched(unsigned long cpu,
bool slack_only)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
struct cpufreq_interactive_tunables *tunables =
pcpu->policy->governor_data;
u64 expires;
unsigned long flags;
spin_lock_irqsave(&pcpu->load_lock, flags);
expires = round_to_nw_start(pcpu->last_evaluated_jiffy, tunables);
if (!slack_only) {
pcpu->time_in_idle =
get_cpu_idle_time(smp_processor_id(),
&pcpu->time_in_idle_timestamp,
tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
del_timer(&pcpu->cpu_timer);
pcpu->cpu_timer.expires = expires;
add_timer_on(&pcpu->cpu_timer, cpu);
}
if (tunables->timer_slack_val >= 0 &&
pcpu->target_freq > pcpu->policy->min) {
expires += usecs_to_jiffies(tunables->timer_slack_val);
del_timer(&pcpu->cpu_slack_timer);
pcpu->cpu_slack_timer.expires = expires;
add_timer_on(&pcpu->cpu_slack_timer, cpu);
}
spin_unlock_irqrestore(&pcpu->load_lock, flags);
}
/* The caller shall take enable_sem write semaphore to avoid any timer race.
* The cpu_timer and cpu_slack_timer must be deactivated when calling this
* function.
*/
static void cpufreq_interactive_timer_start(
struct cpufreq_interactive_tunables *tunables, int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
u64 expires = round_to_nw_start(pcpu->last_evaluated_jiffy, tunables);
unsigned long flags;
spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->cpu_timer.expires = expires;
add_timer_on(&pcpu->cpu_timer, cpu);
if (tunables->timer_slack_val >= 0 &&
pcpu->target_freq > pcpu->policy->min) {
expires += usecs_to_jiffies(tunables->timer_slack_val);
pcpu->cpu_slack_timer.expires = expires;
add_timer_on(&pcpu->cpu_slack_timer, cpu);
}
pcpu->time_in_idle =
get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp,
tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
spin_unlock_irqrestore(&pcpu->load_lock, flags);
}
static unsigned int freq_to_above_hispeed_delay(
struct cpufreq_interactive_tunables *tunables,
unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;
spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
for (i = 0; i < tunables->nabove_hispeed_delay - 1 &&
freq >= tunables->above_hispeed_delay[i+1]; i += 2)
;
ret = tunables->above_hispeed_delay[i];
spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}
static unsigned int freq_to_targetload(
struct cpufreq_interactive_tunables *tunables, unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;
spin_lock_irqsave(&tunables->target_loads_lock, flags);
for (i = 0; i < tunables->ntarget_loads - 1 &&
freq >= tunables->target_loads[i+1]; i += 2)
;
ret = tunables->target_loads[i];
spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}
/*
* If increasing frequencies never map to a lower target load then
* choose_freq() will find the minimum frequency that does not exceed its
* target load given the current load.
*/
static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu,
unsigned int loadadjfreq)
{
unsigned int freq = pcpu->policy->cur;
unsigned int prevfreq, freqmin, freqmax;
unsigned int tl;
int index;
freqmin = 0;
freqmax = UINT_MAX;
do {
prevfreq = freq;
tl = freq_to_targetload(pcpu->policy->governor_data, freq);
/*
* Find the lowest frequency where the computed load is less
* than or equal to the target load.
*/
if (cpufreq_frequency_table_target(
pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
CPUFREQ_RELATION_L, &index))
break;
freq = pcpu->freq_table[index].frequency;
if (freq > prevfreq) {
/* The previous frequency is too low. */
freqmin = prevfreq;
if (freq >= freqmax) {
/*
* Find the highest frequency that is less
* than freqmax.
*/
if (cpufreq_frequency_table_target(
pcpu->policy, pcpu->freq_table,
freqmax - 1, CPUFREQ_RELATION_H,
&index))
break;
freq = pcpu->freq_table[index].frequency;
if (freq == freqmin) {
/*
* The first frequency below freqmax
* has already been found to be too
* low. freqmax is the lowest speed
* we found that is fast enough.
*/
freq = freqmax;
break;
}
}
} else if (freq < prevfreq) {
/* The previous frequency is high enough. */
freqmax = prevfreq;
if (freq <= freqmin) {
/*
* Find the lowest frequency that is higher
* than freqmin.
*/
if (cpufreq_frequency_table_target(
pcpu->policy, pcpu->freq_table,
freqmin + 1, CPUFREQ_RELATION_L,
&index))
break;
freq = pcpu->freq_table[index].frequency;
/*
* If freqmax is the first frequency above
* freqmin then we have already found that
* this speed is fast enough.
*/
if (freq == freqmax)
break;
}
}
/* If same frequency chosen as previous then done. */
} while (freq != prevfreq);
return freq;
}
static u64 update_load(int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
struct cpufreq_interactive_tunables *tunables =
pcpu->policy->governor_data;
u64 now;
u64 now_idle;
unsigned int delta_idle;
unsigned int delta_time;
u64 active_time;
now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy);
delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
if (delta_time <= delta_idle)
active_time = 0;
else
active_time = delta_time - delta_idle;
pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
pcpu->time_in_idle = now_idle;
pcpu->time_in_idle_timestamp = now;
return now;
}
#define MAX_LOCAL_LOAD 100
static void __cpufreq_interactive_timer(unsigned long data, bool is_notif)
{
u64 now;
unsigned int delta_time;
u64 cputime_speedadj;
int cpu_load;
struct cpufreq_interactive_cpuinfo *pcpu =
&per_cpu(cpuinfo, data);
struct cpufreq_interactive_tunables *tunables =
pcpu->policy->governor_data;
unsigned int new_freq;
unsigned int loadadjfreq;
unsigned int index;
unsigned long flags;
u64 max_fvtime;
struct cpufreq_govinfo int_info;
if (!down_read_trylock(&pcpu->enable_sem))
return;
if (!pcpu->governor_enabled)
goto exit;
spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->last_evaluated_jiffy = get_jiffies_64();
now = update_load(data);
if (tunables->use_sched_load) {
/*
* Unlock early to avoid deadlock.
*
* load_change_callback() for thread migration already
* holds rq lock. Then it locks load_lock to avoid racing
* with cpufreq_interactive_timer_resched/start().
* sched_get_busy() will also acquire rq lock. Thus we
* can't hold load_lock when calling sched_get_busy().
*
* load_lock used in this function protects time
* and load information. These stats are not used when
* scheduler input is available. Thus unlocking load_lock
* early is perfectly OK.
*/
spin_unlock_irqrestore(&pcpu->load_lock, flags);
cputime_speedadj = (u64)sched_get_busy(data) *
pcpu->policy->cpuinfo.max_freq;
do_div(cputime_speedadj, tunables->timer_rate);
} else {
delta_time = (unsigned int)
(now - pcpu->cputime_speedadj_timestamp);
cputime_speedadj = pcpu->cputime_speedadj;
spin_unlock_irqrestore(&pcpu->load_lock, flags);
if (WARN_ON_ONCE(!delta_time))
goto rearm;
do_div(cputime_speedadj, delta_time);
}
loadadjfreq = (unsigned int)cputime_speedadj * 100;
int_info.cpu = data;
int_info.load = loadadjfreq / pcpu->policy->max;
int_info.sampling_rate_us = tunables->timer_rate;
atomic_notifier_call_chain(&cpufreq_govinfo_notifier_list,
CPUFREQ_LOAD_CHANGE, &int_info);
spin_lock_irqsave(&pcpu->target_freq_lock, flags);
cpu_load = loadadjfreq / pcpu->policy->cur;
tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) {
if (pcpu->policy->cur < tunables->hispeed_freq &&
cpu_load <= MAX_LOCAL_LOAD) {
new_freq = tunables->hispeed_freq;
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
if (new_freq < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
}
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
if (new_freq > tunables->hispeed_freq &&
pcpu->policy->cur < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
}
if (cpu_load <= MAX_LOCAL_LOAD &&
pcpu->policy->cur >= tunables->hispeed_freq &&
new_freq > pcpu->policy->cur &&
now - pcpu->pol_hispeed_val_time <
freq_to_above_hispeed_delay(tunables, pcpu->policy->cur)) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
pcpu->loc_hispeed_val_time = now;
if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
new_freq, CPUFREQ_RELATION_L,
&index)) {
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
new_freq = pcpu->freq_table[index].frequency;
if (!is_notif && pcpu->target_freq >= pcpu->policy->max
&& new_freq < pcpu->target_freq
&& now - pcpu->max_freq_hyst_start_time <
tunables->max_freq_hysteresis) {
trace_cpufreq_interactive_notyet(data, cpu_load,
pcpu->target_freq, pcpu->policy->cur, new_freq);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
/*
* Do not scale below floor_freq unless we have been at or above the
* floor frequency for the minimum sample time since last validated.
*/
max_fvtime = max(pcpu->pol_floor_val_time, pcpu->loc_floor_val_time);
if (!is_notif && new_freq < pcpu->floor_freq &&
pcpu->target_freq >= pcpu->policy->cur) {
if (now - max_fvtime < tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
}
/*
* Update the timestamp for checking whether speed has been held at
* or above the selected frequency for a minimum of min_sample_time,
* if not boosted to hispeed_freq. If boosted to hispeed_freq then we
* allow the speed to drop as soon as the boostpulse duration expires
* (or the indefinite boost is turned off).
*/
if (!tunables->boosted || new_freq > tunables->hispeed_freq) {
pcpu->floor_freq = new_freq;
if (pcpu->target_freq >= pcpu->policy->cur ||
new_freq >= pcpu->policy->cur)
pcpu->loc_floor_val_time = now;
}
if (pcpu->target_freq == new_freq &&
pcpu->target_freq <= pcpu->policy->cur) {
trace_cpufreq_interactive_already(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
pcpu->target_freq = new_freq;
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
cpumask_set_cpu(data, &speedchange_cpumask);
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
wake_up_process(speedchange_task);
rearm:
if (!timer_pending(&pcpu->cpu_timer))
cpufreq_interactive_timer_resched(data, false);
exit:
up_read(&pcpu->enable_sem);
return;
}
static void cpufreq_interactive_timer(unsigned long data)
{
__cpufreq_interactive_timer(data, false);
}
static void cpufreq_interactive_idle_end(void)
{
struct cpufreq_interactive_cpuinfo *pcpu =
&per_cpu(cpuinfo, smp_processor_id());
if (!down_read_trylock(&pcpu->enable_sem))
return;
if (!pcpu->governor_enabled) {
up_read(&pcpu->enable_sem);
return;
}
/* Arm the timer for 1-2 ticks later if not already. */
if (!timer_pending(&pcpu->cpu_timer)) {
cpufreq_interactive_timer_resched(smp_processor_id(), false);
} else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
del_timer(&pcpu->cpu_timer);
del_timer(&pcpu->cpu_slack_timer);
cpufreq_interactive_timer(smp_processor_id());
}
up_read(&pcpu->enable_sem);
}
static int cpufreq_interactive_speedchange_task(void *data)
{
unsigned int cpu;
cpumask_t tmp_mask;
unsigned long flags;
struct cpufreq_interactive_cpuinfo *pcpu;
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
if (cpumask_empty(&speedchange_cpumask)) {
spin_unlock_irqrestore(&speedchange_cpumask_lock,
flags);
schedule();
if (kthread_should_stop())
break;
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
}
set_current_state(TASK_RUNNING);
tmp_mask = speedchange_cpumask;
cpumask_clear(&speedchange_cpumask);
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
for_each_cpu(cpu, &tmp_mask) {
unsigned int j;
unsigned int max_freq = 0;
struct cpufreq_interactive_cpuinfo *pjcpu;
u64 hvt = ~0ULL, fvt = 0;
pcpu = &per_cpu(cpuinfo, cpu);
if (!down_read_trylock(&pcpu->enable_sem))
continue;
if (!pcpu->governor_enabled) {
up_read(&pcpu->enable_sem);
continue;
}
for_each_cpu(j, pcpu->policy->cpus) {
pjcpu = &per_cpu(cpuinfo, j);
fvt = max(fvt, pjcpu->loc_floor_val_time);
if (pjcpu->target_freq > max_freq) {
max_freq = pjcpu->target_freq;
hvt = pjcpu->loc_hispeed_val_time;
} else if (pjcpu->target_freq == max_freq) {
hvt = min(hvt, pjcpu->loc_hispeed_val_time);
}
}
for_each_cpu(j, pcpu->policy->cpus) {
pjcpu = &per_cpu(cpuinfo, j);
pjcpu->pol_floor_val_time = fvt;
}
if (max_freq != pcpu->policy->cur) {
__cpufreq_driver_target(pcpu->policy,
max_freq,
CPUFREQ_RELATION_H);
for_each_cpu(j, pcpu->policy->cpus) {
pjcpu = &per_cpu(cpuinfo, j);
pjcpu->pol_hispeed_val_time = hvt;
}
}
trace_cpufreq_interactive_setspeed(cpu,
pcpu->target_freq,
pcpu->policy->cur);
up_read(&pcpu->enable_sem);
}
}
return 0;
}
static void cpufreq_interactive_boost(struct cpufreq_interactive_tunables *tunables)
{
int i;
int anyboost = 0;
unsigned long flags[2];
struct cpufreq_interactive_cpuinfo *pcpu;
tunables->boosted = true;
spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]);
for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
if (tunables != pcpu->policy->governor_data)
continue;
spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]);
if (pcpu->target_freq < tunables->hispeed_freq) {
pcpu->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
pcpu->pol_hispeed_val_time =
ktime_to_us(ktime_get());
anyboost = 1;
}
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]);
}
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]);
if (anyboost)
wake_up_process(speedchange_task);
}
static int load_change_callback(struct notifier_block *nb, unsigned long val,
void *data)
{
unsigned long cpu = (unsigned long) data;
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
struct cpufreq_interactive_tunables *tunables;
if (speedchange_task == current)
return 0;
if (pcpu->reject_notification)
return 0;
if (!down_read_trylock(&pcpu->enable_sem))
return 0;
if (!pcpu->governor_enabled) {
up_read(&pcpu->enable_sem);
return 0;
}
tunables = pcpu->policy->governor_data;
if (!tunables->use_sched_load || !tunables->use_migration_notif) {
up_read(&pcpu->enable_sem);
return 0;
}
trace_cpufreq_interactive_load_change(cpu);
del_timer(&pcpu->cpu_timer);
del_timer(&pcpu->cpu_slack_timer);
__cpufreq_interactive_timer(cpu, true);
up_read(&pcpu->enable_sem);
return 0;
}
static struct notifier_block load_notifier_block = {
.notifier_call = load_change_callback,
};
static int cpufreq_interactive_notifier(
struct notifier_block *nb, unsigned long val, void *data)
{
struct cpufreq_freqs *freq = data;
struct cpufreq_interactive_cpuinfo *pcpu;
int cpu;
unsigned long flags;
if (val == CPUFREQ_POSTCHANGE) {
pcpu = &per_cpu(cpuinfo, freq->cpu);
if (!down_read_trylock(&pcpu->enable_sem))
return 0;
if (!pcpu->governor_enabled) {
up_read(&pcpu->enable_sem);
return 0;
}
for_each_cpu(cpu, pcpu->policy->cpus) {
struct cpufreq_interactive_cpuinfo *pjcpu =
&per_cpu(cpuinfo, cpu);
if (cpu != freq->cpu) {
if (!down_read_trylock(&pjcpu->enable_sem))
continue;
if (!pjcpu->governor_enabled) {
up_read(&pjcpu->enable_sem);
continue;
}
}
spin_lock_irqsave(&pjcpu->load_lock, flags);
update_load(cpu);
spin_unlock_irqrestore(&pjcpu->load_lock, flags);
if (cpu != freq->cpu)
up_read(&pjcpu->enable_sem);
}
up_read(&pcpu->enable_sem);
}
return 0;
}
static struct notifier_block cpufreq_notifier_block = {
.notifier_call = cpufreq_interactive_notifier,
};
static unsigned int *get_tokenized_data(const char *buf, int *num_tokens)
{
const char *cp;
int i;
int ntokens = 1;
unsigned int *tokenized_data;
int err = -EINVAL;
cp = buf;
while ((cp = strpbrk(cp + 1, " :")))
ntokens++;
if (!(ntokens & 0x1))
goto err;
tokenized_data = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
if (!tokenized_data) {
err = -ENOMEM;
goto err;
}
cp = buf;
i = 0;
while (i < ntokens) {
if (sscanf(cp, "%u", &tokenized_data[i++]) != 1)
goto err_kfree;
cp = strpbrk(cp, " :");
if (!cp)
break;
cp++;
}
if (i != ntokens)
goto err_kfree;
*num_tokens = ntokens;
return tokenized_data;
err_kfree:
kfree(tokenized_data);
err:
return ERR_PTR(err);
}
static ssize_t show_target_loads(
struct cpufreq_interactive_tunables *tunables,
char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;
spin_lock_irqsave(&tunables->target_loads_lock, flags);
for (i = 0; i < tunables->ntarget_loads; i++)
ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i],
i & 0x1 ? ":" : " ");
sprintf(buf + ret - 1, "\n");
spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}
static ssize_t store_target_loads(
struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ntokens;
unsigned int *new_target_loads = NULL;
unsigned long flags;
new_target_loads = get_tokenized_data(buf, &ntokens);
if (IS_ERR(new_target_loads))
return PTR_RET(new_target_loads);
spin_lock_irqsave(&tunables->target_loads_lock, flags);
if (tunables->target_loads != default_target_loads)
kfree(tunables->target_loads);
tunables->target_loads = new_target_loads;
tunables->ntarget_loads = ntokens;
spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return count;
}
static ssize_t show_above_hispeed_delay(
struct cpufreq_interactive_tunables *tunables, char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;
spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
for (i = 0; i < tunables->nabove_hispeed_delay; i++)
ret += sprintf(buf + ret, "%u%s",
tunables->above_hispeed_delay[i],
i & 0x1 ? ":" : " ");
sprintf(buf + ret - 1, "\n");
spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}
static ssize_t store_above_hispeed_delay(
struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ntokens;
unsigned int *new_above_hispeed_delay = NULL;
unsigned long flags;
new_above_hispeed_delay = get_tokenized_data(buf, &ntokens);
if (IS_ERR(new_above_hispeed_delay))
return PTR_RET(new_above_hispeed_delay);
spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
if (tunables->above_hispeed_delay != default_above_hispeed_delay)
kfree(tunables->above_hispeed_delay);
tunables->above_hispeed_delay = new_above_hispeed_delay;
tunables->nabove_hispeed_delay = ntokens;
spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return count;
}
static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
return sprintf(buf, "%u\n", tunables->hispeed_freq);
}
static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
long unsigned int val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->hispeed_freq = val;
return count;
}
#define show_store_one(file_name) \
static ssize_t show_##file_name( \
struct cpufreq_interactive_tunables *tunables, char *buf) \
{ \
return snprintf(buf, PAGE_SIZE, "%u\n", tunables->file_name); \
} \
static ssize_t store_##file_name( \
struct cpufreq_interactive_tunables *tunables, \
const char *buf, size_t count) \
{ \
int ret; \
unsigned long int val; \
\
ret = kstrtoul(buf, 0, &val); \
if (ret < 0) \
return ret; \
tunables->file_name = val; \
return count; \
}
show_store_one(max_freq_hysteresis);
show_store_one(align_windows);
static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunables
*tunables, char *buf)
{
return sprintf(buf, "%lu\n", tunables->go_hispeed_load);
}
static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunables
*tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->go_hispeed_load = val;
return count;
}
static ssize_t show_min_sample_time(struct cpufreq_interactive_tunables
*tunables, char *buf)
{
return sprintf(buf, "%lu\n", tunables->min_sample_time);
}
static ssize_t store_min_sample_time(struct cpufreq_interactive_tunables
*tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->min_sample_time = val;
return count;
}
static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
return sprintf(buf, "%lu\n", tunables->timer_rate);
}
static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val, val_round;
struct cpufreq_interactive_tunables *t;
int cpu;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
val_round = jiffies_to_usecs(usecs_to_jiffies(val));
if (val != val_round)
pr_warn("timer_rate not aligned to jiffy. Rounded up to %lu\n",
val_round);
tunables->timer_rate = val_round;
if (!tunables->use_sched_load)
return count;
for_each_possible_cpu(cpu) {
t = per_cpu(cpuinfo, cpu).cached_tunables;
if (t && t->use_sched_load)
t->timer_rate = val_round;
}
set_window_helper(tunables);
return count;
}
static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
return sprintf(buf, "%d\n", tunables->timer_slack_val);
}
static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtol(buf, 10, &val);
if (ret < 0)
return ret;
tunables->timer_slack_val = val;
return count;
}
static ssize_t show_boost(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
return sprintf(buf, "%d\n", tunables->boost_val);
}
static ssize_t store_boost(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->boost_val = val;
if (tunables->boost_val) {
trace_cpufreq_interactive_boost("on");
if (!tunables->boosted)
cpufreq_interactive_boost(tunables);
} else {
tunables->boostpulse_endtime = ktime_to_us(ktime_get());
trace_cpufreq_interactive_unboost("off");
}
return count;
}
static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->boostpulse_endtime = ktime_to_us(ktime_get()) +
tunables->boostpulse_duration_val;
trace_cpufreq_interactive_boost("pulse");
if (!tunables->boosted)
cpufreq_interactive_boost(tunables);
return count;
}
static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tunables
*tunables, char *buf)
{
return sprintf(buf, "%d\n", tunables->boostpulse_duration_val);
}
static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tunables
*tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->boostpulse_duration_val = val;
return count;
}
static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
return sprintf(buf, "%u\n", tunables->io_is_busy);
}
static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
struct cpufreq_interactive_tunables *t;
int cpu;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->io_is_busy = val;
if (!tunables->use_sched_load)
return count;
for_each_possible_cpu(cpu) {
t = per_cpu(cpuinfo, cpu).cached_tunables;
if (t && t->use_sched_load)
t->io_is_busy = val;
}
sched_set_io_is_busy(val);
return count;
}
static int cpufreq_interactive_enable_sched_input(
struct cpufreq_interactive_tunables *tunables)
{
int rc = 0, j;
struct cpufreq_interactive_tunables *t;
mutex_lock(&sched_lock);
set_window_count++;
if (set_window_count > 1) {
for_each_possible_cpu(j) {
t = per_cpu(cpuinfo, j).cached_tunables;
if (t && t->use_sched_load) {
tunables->timer_rate = t->timer_rate;
tunables->io_is_busy = t->io_is_busy;
break;
}
}
} else {
rc = set_window_helper(tunables);
if (rc) {
pr_err("%s: Failed to set sched window\n", __func__);
set_window_count--;
goto out;
}
sched_set_io_is_busy(tunables->io_is_busy);
}
if (!tunables->use_migration_notif)
goto out;
migration_register_count++;
if (migration_register_count > 1)
goto out;
else
atomic_notifier_chain_register(&load_alert_notifier_head,
&load_notifier_block);
out:
mutex_unlock(&sched_lock);
return rc;
}
static int cpufreq_interactive_disable_sched_input(
struct cpufreq_interactive_tunables *tunables)
{
mutex_lock(&sched_lock);
if (tunables->use_migration_notif) {
migration_register_count--;
if (migration_register_count < 1)
atomic_notifier_chain_unregister(
&load_alert_notifier_head,
&load_notifier_block);
}
set_window_count--;
mutex_unlock(&sched_lock);
return 0;
}
static ssize_t show_use_sched_load(
struct cpufreq_interactive_tunables *tunables, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%d\n", tunables->use_sched_load);
}
static ssize_t store_use_sched_load(
struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
if (tunables->use_sched_load == (bool) val)
return count;
tunables->use_sched_load = val;
if (val)
ret = cpufreq_interactive_enable_sched_input(tunables);
else
ret = cpufreq_interactive_disable_sched_input(tunables);
if (ret) {
tunables->use_sched_load = !val;
return ret;
}
return count;
}
static ssize_t show_use_migration_notif(
struct cpufreq_interactive_tunables *tunables, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%d\n",
tunables->use_migration_notif);
}
static ssize_t store_use_migration_notif(
struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
if (tunables->use_migration_notif == (bool) val)
return count;
tunables->use_migration_notif = val;
if (!tunables->use_sched_load)
return count;
mutex_lock(&sched_lock);
if (val) {
migration_register_count++;
if (migration_register_count == 1)
atomic_notifier_chain_register(
&load_alert_notifier_head,
&load_notifier_block);
} else {
migration_register_count--;
if (!migration_register_count)
atomic_notifier_chain_unregister(
&load_alert_notifier_head,
&load_notifier_block);
}
mutex_unlock(&sched_lock);
return count;
}
/*
* Create show/store routines
* - sys: One governor instance for complete SYSTEM
* - pol: One governor instance per struct cpufreq_policy
*/
#define show_gov_pol_sys(file_name) \
static ssize_t show_##file_name##_gov_sys \
(struct kobject *kobj, struct attribute *attr, char *buf) \
{ \
return show_##file_name(common_tunables, buf); \
} \
\
static ssize_t show_##file_name##_gov_pol \
(struct cpufreq_policy *policy, char *buf) \
{ \
return show_##file_name(policy->governor_data, buf); \
}
#define store_gov_pol_sys(file_name) \
static ssize_t store_##file_name##_gov_sys \
(struct kobject *kobj, struct attribute *attr, const char *buf, \
size_t count) \
{ \
return store_##file_name(common_tunables, buf, count); \
} \
\
static ssize_t store_##file_name##_gov_pol \
(struct cpufreq_policy *policy, const char *buf, size_t count) \
{ \
return store_##file_name(policy->governor_data, buf, count); \
}
#define show_store_gov_pol_sys(file_name) \
show_gov_pol_sys(file_name); \
store_gov_pol_sys(file_name)
show_store_gov_pol_sys(target_loads);
show_store_gov_pol_sys(above_hispeed_delay);
show_store_gov_pol_sys(hispeed_freq);
show_store_gov_pol_sys(go_hispeed_load);
show_store_gov_pol_sys(min_sample_time);
show_store_gov_pol_sys(timer_rate);
show_store_gov_pol_sys(timer_slack);
show_store_gov_pol_sys(boost);
store_gov_pol_sys(boostpulse);
show_store_gov_pol_sys(boostpulse_duration);
show_store_gov_pol_sys(io_is_busy);
show_store_gov_pol_sys(use_sched_load);
show_store_gov_pol_sys(use_migration_notif);
show_store_gov_pol_sys(max_freq_hysteresis);
show_store_gov_pol_sys(align_windows);
#define gov_sys_attr_rw(_name) \
static struct global_attr _name##_gov_sys = \
__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys)
#define gov_pol_attr_rw(_name) \
static struct freq_attr _name##_gov_pol = \
__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
#define gov_sys_pol_attr_rw(_name) \
gov_sys_attr_rw(_name); \
gov_pol_attr_rw(_name)
gov_sys_pol_attr_rw(target_loads);
gov_sys_pol_attr_rw(above_hispeed_delay);
gov_sys_pol_attr_rw(hispeed_freq);
gov_sys_pol_attr_rw(go_hispeed_load);
gov_sys_pol_attr_rw(min_sample_time);
gov_sys_pol_attr_rw(timer_rate);
gov_sys_pol_attr_rw(timer_slack);
gov_sys_pol_attr_rw(boost);
gov_sys_pol_attr_rw(boostpulse_duration);
gov_sys_pol_attr_rw(io_is_busy);
gov_sys_pol_attr_rw(use_sched_load);
gov_sys_pol_attr_rw(use_migration_notif);
gov_sys_pol_attr_rw(max_freq_hysteresis);
gov_sys_pol_attr_rw(align_windows);
static struct global_attr boostpulse_gov_sys =
__ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys);
static struct freq_attr boostpulse_gov_pol =
__ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol);
/* One Governor instance for entire system */
static struct attribute *interactive_attributes_gov_sys[] = {
&target_loads_gov_sys.attr,
&above_hispeed_delay_gov_sys.attr,
&hispeed_freq_gov_sys.attr,
&go_hispeed_load_gov_sys.attr,
&min_sample_time_gov_sys.attr,
&timer_rate_gov_sys.attr,
&timer_slack_gov_sys.attr,
&boost_gov_sys.attr,
&boostpulse_gov_sys.attr,
&boostpulse_duration_gov_sys.attr,
&io_is_busy_gov_sys.attr,
&use_sched_load_gov_sys.attr,
&use_migration_notif_gov_sys.attr,
&max_freq_hysteresis_gov_sys.attr,
&align_windows_gov_sys.attr,
NULL,
};
static struct attribute_group interactive_attr_group_gov_sys = {
.attrs = interactive_attributes_gov_sys,
.name = "interactive",
};
/* Per policy governor instance */
static struct attribute *interactive_attributes_gov_pol[] = {
&target_loads_gov_pol.attr,
&above_hispeed_delay_gov_pol.attr,
&hispeed_freq_gov_pol.attr,
&go_hispeed_load_gov_pol.attr,
&min_sample_time_gov_pol.attr,
&timer_rate_gov_pol.attr,
&timer_slack_gov_pol.attr,
&boost_gov_pol.attr,
&boostpulse_gov_pol.attr,
&boostpulse_duration_gov_pol.attr,
&io_is_busy_gov_pol.attr,
&use_sched_load_gov_pol.attr,
&use_migration_notif_gov_pol.attr,
&max_freq_hysteresis_gov_pol.attr,
&align_windows_gov_pol.attr,
NULL,
};
static struct attribute_group interactive_attr_group_gov_pol = {
.attrs = interactive_attributes_gov_pol,
.name = "interactive",
};
static struct attribute_group *get_sysfs_attr(void)
{
if (have_governor_per_policy())
return &interactive_attr_group_gov_pol;
else
return &interactive_attr_group_gov_sys;
}
static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
unsigned long val,
void *data)
{
if (val == IDLE_END)
cpufreq_interactive_idle_end();
return 0;
}
static struct notifier_block cpufreq_interactive_idle_nb = {
.notifier_call = cpufreq_interactive_idle_notifier,
};
static void save_tunables(struct cpufreq_policy *policy,
struct cpufreq_interactive_tunables *tunables)
{
int cpu;
struct cpufreq_interactive_cpuinfo *pcpu;
if (have_governor_per_policy())
cpu = cpumask_first(policy->related_cpus);
else
cpu = 0;
pcpu = &per_cpu(cpuinfo, cpu);
WARN_ON(pcpu->cached_tunables && pcpu->cached_tunables != tunables);
pcpu->cached_tunables = tunables;
}
static struct cpufreq_interactive_tunables *alloc_tunable(
struct cpufreq_policy *policy)
{
struct cpufreq_interactive_tunables *tunables;
tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
if (!tunables)
return ERR_PTR(-ENOMEM);
tunables->above_hispeed_delay = default_above_hispeed_delay;
tunables->nabove_hispeed_delay =
ARRAY_SIZE(default_above_hispeed_delay);
tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
tunables->target_loads = default_target_loads;
tunables->ntarget_loads = ARRAY_SIZE(default_target_loads);
tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
tunables->timer_rate = DEFAULT_TIMER_RATE;
tunables->boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
tunables->timer_slack_val = DEFAULT_TIMER_SLACK;
spin_lock_init(&tunables->target_loads_lock);
spin_lock_init(&tunables->above_hispeed_delay_lock);
save_tunables(policy, tunables);
return tunables;
}
static struct cpufreq_interactive_tunables *restore_tunables(
struct cpufreq_policy *policy)
{
int cpu;
if (have_governor_per_policy())
cpu = cpumask_first(policy->related_cpus);
else
cpu = 0;
return per_cpu(cpuinfo, cpu).cached_tunables;
}
static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
unsigned int event)
{
int rc;
unsigned int j;
struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_frequency_table *freq_table;
struct cpufreq_interactive_tunables *tunables;
unsigned long flags;
if (have_governor_per_policy())
tunables = policy->governor_data;
else
tunables = common_tunables;
BUG_ON(!tunables && (event != CPUFREQ_GOV_POLICY_INIT));
switch (event) {
case CPUFREQ_GOV_POLICY_INIT:
if (have_governor_per_policy()) {
WARN_ON(tunables);
} else if (tunables) {
tunables->usage_count++;
policy->governor_data = tunables;
return 0;
}
tunables = restore_tunables(policy);
if (!tunables) {
tunables = alloc_tunable(policy);
if (IS_ERR(tunables))
return PTR_ERR(tunables);
}
tunables->usage_count = 1;
policy->governor_data = tunables;
if (!have_governor_per_policy())
common_tunables = tunables;
rc = sysfs_create_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
if (rc) {
kfree(tunables);
policy->governor_data = NULL;
if (!have_governor_per_policy())
common_tunables = NULL;
return rc;
}
if (!policy->governor->initialized) {
idle_notifier_register(&cpufreq_interactive_idle_nb);
cpufreq_register_notifier(&cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
}
if (tunables->use_sched_load)
cpufreq_interactive_enable_sched_input(tunables);
break;
case CPUFREQ_GOV_POLICY_EXIT:
if (!--tunables->usage_count) {
if (policy->governor->initialized == 1) {
cpufreq_unregister_notifier(&cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
idle_notifier_unregister(&cpufreq_interactive_idle_nb);
}
sysfs_remove_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
common_tunables = NULL;
}
policy->governor_data = NULL;
if (tunables->use_sched_load)
cpufreq_interactive_disable_sched_input(tunables);
break;
case CPUFREQ_GOV_START:
mutex_lock(&gov_lock);
freq_table = cpufreq_frequency_get_table(policy->cpu);
if (!tunables->hispeed_freq)
tunables->hispeed_freq = policy->max;
for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
pcpu->policy = policy;
pcpu->target_freq = policy->cur;
pcpu->freq_table = freq_table;
pcpu->floor_freq = pcpu->target_freq;
pcpu->pol_floor_val_time =
ktime_to_us(ktime_get());
pcpu->loc_floor_val_time = pcpu->pol_floor_val_time;
pcpu->pol_hispeed_val_time = pcpu->pol_floor_val_time;
pcpu->loc_hispeed_val_time = pcpu->pol_floor_val_time;
pcpu->min_freq = policy->min;
pcpu->reject_notification = true;
down_write(&pcpu->enable_sem);
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
pcpu->last_evaluated_jiffy = get_jiffies_64();
cpufreq_interactive_timer_start(tunables, j);
pcpu->governor_enabled = 1;
up_write(&pcpu->enable_sem);
pcpu->reject_notification = false;
}
mutex_unlock(&gov_lock);
break;
case CPUFREQ_GOV_STOP:
mutex_lock(&gov_lock);
for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
pcpu->reject_notification = true;
down_write(&pcpu->enable_sem);
pcpu->governor_enabled = 0;
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
up_write(&pcpu->enable_sem);
pcpu->reject_notification = false;
}
mutex_unlock(&gov_lock);
break;
case CPUFREQ_GOV_LIMITS:
__cpufreq_driver_target(policy,
policy->cur, CPUFREQ_RELATION_L);
for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
down_read(&pcpu->enable_sem);
if (pcpu->governor_enabled == 0) {
up_read(&pcpu->enable_sem);
continue;
}
spin_lock_irqsave(&pcpu->target_freq_lock, flags);
if (policy->max < pcpu->target_freq)
pcpu->target_freq = policy->max;
else if (policy->min > pcpu->target_freq)
pcpu->target_freq = policy->min;
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
if (policy->min < pcpu->min_freq)
cpufreq_interactive_timer_resched(j, true);
pcpu->min_freq = policy->min;
up_read(&pcpu->enable_sem);
}
break;
}
return 0;
}
#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
static
#endif
struct cpufreq_governor cpufreq_gov_interactive = {
.name = "interactive",
.governor = cpufreq_governor_interactive,
.max_transition_latency = 10000000,
.owner = THIS_MODULE,
};
static void cpufreq_interactive_nop_timer(unsigned long data)
{
}
static int __init cpufreq_interactive_init(void)
{
unsigned int i;
struct cpufreq_interactive_cpuinfo *pcpu;
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
/* Initalize per-cpu timers */
for_each_possible_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
init_timer_deferrable(&pcpu->cpu_timer);
pcpu->cpu_timer.function = cpufreq_interactive_timer;
pcpu->cpu_timer.data = i;
init_timer(&pcpu->cpu_slack_timer);
pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
spin_lock_init(&pcpu->load_lock);
spin_lock_init(&pcpu->target_freq_lock);
init_rwsem(&pcpu->enable_sem);
}
spin_lock_init(&speedchange_cpumask_lock);
mutex_init(&gov_lock);
mutex_init(&sched_lock);
speedchange_task =
kthread_create(cpufreq_interactive_speedchange_task, NULL,
"cfinteractive");
if (IS_ERR(speedchange_task))
return PTR_ERR(speedchange_task);
sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
get_task_struct(speedchange_task);
/* NB: wake up so the thread does not look hung to the freezer */
wake_up_process(speedchange_task);
return cpufreq_register_governor(&cpufreq_gov_interactive);
}
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
fs_initcall(cpufreq_interactive_init);
#else
module_init(cpufreq_interactive_init);
#endif
static void __exit cpufreq_interactive_exit(void)
{
int cpu;
struct cpufreq_interactive_cpuinfo *pcpu;
cpufreq_unregister_governor(&cpufreq_gov_interactive);
kthread_stop(speedchange_task);
put_task_struct(speedchange_task);
for_each_possible_cpu(cpu) {
pcpu = &per_cpu(cpuinfo, cpu);
kfree(pcpu->cached_tunables);
pcpu->cached_tunables = NULL;
}
}
module_exit(cpufreq_interactive_exit);
MODULE_AUTHOR("Mike Chan <mike@android.com>");
MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
"Latency sensitive workloads");
MODULE_LICENSE("GPL");