cpufreq: Interactive: Implement per policy instances of governor
If we have a multi-package system, where we have multiple instances of struct
policy (per package), currently we can't have multiple instances of same
governor. i.e. We can't have multiple instances of Interactive governor for
multiple packages.
This is a bottleneck for multicluster system, where we want different packages
to use Interactive governor, but with different tunables.
This patch uses the infrastructure provided by earlier patches pushed in
Mainline in v3.10-rc1/rc2 and implements per policy instances of Interactive
governor.
Change-Id: I70436d4a5a45c6cb6edf37f3e46d0b9fbc930982
[toddpoynor@google.com: merge with later code, minor changes]
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Stephen Boyd <sboyd@codeaurora.org>
diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c
index b0685c7..66c096d 100644
--- a/drivers/cpufreq/cpufreq_interactive.c
+++ b/drivers/cpufreq/cpufreq_interactive.c
@@ -36,8 +36,6 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
-static int active_count;
-
struct cpufreq_interactive_cpuinfo {
struct timer_list cpu_timer;
struct timer_list cpu_slack_timer;
@@ -64,58 +62,62 @@
static spinlock_t speedchange_cpumask_lock;
static struct mutex gov_lock;
-/* Hi speed to bump to from lo speed when load burst (default max) */
-static unsigned int hispeed_freq;
-
-/* Go to hi speed when CPU load at or above this value. */
-#define DEFAULT_GO_HISPEED_LOAD 99
-static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
-
/* Target load. Lower values result in higher CPU speeds. */
#define DEFAULT_TARGET_LOAD 90
static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
-static spinlock_t target_loads_lock;
-static unsigned int *target_loads = default_target_loads;
-static int ntarget_loads = ARRAY_SIZE(default_target_loads);
-/*
- * The minimum amount of time to spend at a frequency before we can ramp down.
- */
-#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
-static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
-
-/*
- * The sample rate of the timer used to increase frequency
- */
#define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
-static unsigned long timer_rate = DEFAULT_TIMER_RATE;
-
-/*
- * Wait this long before raising speed above hispeed, by default a single
- * timer interval.
- */
#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
static unsigned int default_above_hispeed_delay[] = {
DEFAULT_ABOVE_HISPEED_DELAY };
-static spinlock_t above_hispeed_delay_lock;
-static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
-static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
-/* Non-zero means indefinite speed boost active */
-static int boost_val;
-/* Duration of a boot pulse in usecs */
-static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
-/* End time of boost pulse in ktime converted to usecs */
-static u64 boostpulse_endtime;
-
-/*
- * Max additional time to wait in idle, beyond timer_rate, at speeds above
- * minimum before wakeup to reduce speed, or -1 if unnecessary.
- */
+struct cpufreq_interactive_tunables {
+ int usage_count;
+ /* Hi speed to bump to from lo speed when load burst (default max) */
+ unsigned int hispeed_freq;
+ /* Go to hi speed when CPU load at or above this value. */
+#define DEFAULT_GO_HISPEED_LOAD 99
+ unsigned long go_hispeed_load;
+ /* Target load. Lower values result in higher CPU speeds. */
+ spinlock_t target_loads_lock;
+ unsigned int *target_loads;
+ int ntarget_loads;
+ /*
+ * The minimum amount of time to spend at a frequency before we can ramp
+ * down.
+ */
+#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
+ unsigned long min_sample_time;
+ /*
+ * The sample rate of the timer used to increase frequency
+ */
+ unsigned long timer_rate;
+ /*
+ * Wait this long before raising speed above hispeed, by default a
+ * single timer interval.
+ */
+ spinlock_t above_hispeed_delay_lock;
+ unsigned int *above_hispeed_delay;
+ int nabove_hispeed_delay;
+ /* Non-zero means indefinite speed boost active */
+ int boost_val;
+ /* Duration of a boot pulse in usecs */
+ int boostpulse_duration_val;
+ /* End time of boost pulse in ktime converted to usecs */
+ u64 boostpulse_endtime;
+ /*
+ * Max additional time to wait in idle, beyond timer_rate, at speeds
+ * above minimum before wakeup to reduce speed, or -1 if unnecessary.
+ */
#define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
-static int timer_slack_val = DEFAULT_TIMER_SLACK;
+ int timer_slack_val;
+ bool io_is_busy;
+};
-static bool io_is_busy;
+/* For cases where we have single governor instance for system */
+struct cpufreq_interactive_tunables *common_tunables;
+
+static struct attribute_group *get_sysfs_attr(void);
static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
cputime64_t *wall)
@@ -140,8 +142,10 @@
return jiffies_to_usecs(idle_time);
}
-static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
- cputime64_t *wall)
+static inline cputime64_t get_cpu_idle_time(
+ unsigned int cpu,
+ cputime64_t *wall,
+ bool io_is_busy)
{
u64 idle_time = get_cpu_idle_time_us(cpu, wall);
@@ -156,20 +160,24 @@
static void cpufreq_interactive_timer_resched(
struct cpufreq_interactive_cpuinfo *pcpu)
{
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
unsigned long expires;
unsigned long flags;
spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->time_in_idle =
get_cpu_idle_time(smp_processor_id(),
- &pcpu->time_in_idle_timestamp);
+ &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
- expires = jiffies + usecs_to_jiffies(timer_rate);
+ expires = jiffies + usecs_to_jiffies(tunables->timer_rate);
mod_timer_pinned(&pcpu->cpu_timer, expires);
- if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
- expires += usecs_to_jiffies(timer_slack_val);
+ if (tunables->timer_slack_val >= 0 &&
+ pcpu->target_freq > pcpu->policy->min) {
+ expires += usecs_to_jiffies(tunables->timer_slack_val);
mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
}
@@ -180,58 +188,66 @@
* The cpu_timer and cpu_slack_timer must be deactivated when calling this
* function.
*/
-static void cpufreq_interactive_timer_start(int cpu)
+static void cpufreq_interactive_timer_start(
+ struct cpufreq_interactive_tunables *tunables, int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
- unsigned long expires = jiffies + usecs_to_jiffies(timer_rate);
+ unsigned long expires = jiffies +
+ usecs_to_jiffies(tunables->timer_rate);
unsigned long flags;
pcpu->cpu_timer.expires = expires;
add_timer_on(&pcpu->cpu_timer, cpu);
- if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
- expires += usecs_to_jiffies(timer_slack_val);
+ if (tunables->timer_slack_val >= 0 &&
+ pcpu->target_freq > pcpu->policy->min) {
+ expires += usecs_to_jiffies(tunables->timer_slack_val);
pcpu->cpu_slack_timer.expires = expires;
add_timer_on(&pcpu->cpu_slack_timer, cpu);
}
spin_lock_irqsave(&pcpu->load_lock, flags);
pcpu->time_in_idle =
- get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp);
+ get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp,
+ tunables->io_is_busy);
pcpu->cputime_speedadj = 0;
pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
spin_unlock_irqrestore(&pcpu->load_lock, flags);
}
-static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
+static unsigned int freq_to_above_hispeed_delay(
+ struct cpufreq_interactive_tunables *tunables,
+ unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;
- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
- for (i = 0; i < nabove_hispeed_delay - 1 &&
- freq >= above_hispeed_delay[i+1]; i += 2)
+ for (i = 0; i < tunables->nabove_hispeed_delay - 1 &&
+ freq >= tunables->above_hispeed_delay[i+1]; i += 2)
;
- ret = above_hispeed_delay[i];
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ ret = tunables->above_hispeed_delay[i];
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}
-static unsigned int freq_to_targetload(unsigned int freq)
+static unsigned int freq_to_targetload(
+ struct cpufreq_interactive_tunables *tunables, unsigned int freq)
{
int i;
unsigned int ret;
unsigned long flags;
- spin_lock_irqsave(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);
- for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
+ for (i = 0; i < tunables->ntarget_loads - 1 &&
+ freq >= tunables->target_loads[i+1]; i += 2)
;
- ret = target_loads[i];
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ ret = tunables->target_loads[i];
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}
@@ -240,9 +256,8 @@
* choose_freq() will find the minimum frequency that does not exceed its
* target load given the current load.
*/
-
-static unsigned int choose_freq(
- struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
+static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu,
+ unsigned int loadadjfreq)
{
unsigned int freq = pcpu->policy->cur;
unsigned int prevfreq, freqmin, freqmax;
@@ -254,7 +269,7 @@
do {
prevfreq = freq;
- tl = freq_to_targetload(freq);
+ tl = freq_to_targetload(pcpu->policy->governor_data, freq);
/*
* Find the lowest frequency where the computed load is less
@@ -329,13 +344,15 @@
static u64 update_load(int cpu)
{
struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
u64 now;
u64 now_idle;
unsigned int delta_idle;
unsigned int delta_time;
u64 active_time;
- now_idle = get_cpu_idle_time(cpu, &now);
+ now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy);
delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
@@ -359,6 +376,8 @@
int cpu_load;
struct cpufreq_interactive_cpuinfo *pcpu =
&per_cpu(cpuinfo, data);
+ struct cpufreq_interactive_tunables *tunables =
+ pcpu->policy->governor_data;
unsigned int new_freq;
unsigned int loadadjfreq;
unsigned int index;
@@ -382,25 +401,25 @@
do_div(cputime_speedadj, delta_time);
loadadjfreq = (unsigned int)cputime_speedadj * 100;
cpu_load = loadadjfreq / pcpu->target_freq;
- boosted = boost_val || now < boostpulse_endtime;
+ boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
- if (cpu_load >= go_hispeed_load || boosted) {
- if (pcpu->target_freq < hispeed_freq) {
- new_freq = hispeed_freq;
+ if (cpu_load >= tunables->go_hispeed_load || boosted) {
+ if (pcpu->target_freq < tunables->hispeed_freq) {
+ new_freq = tunables->hispeed_freq;
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
- if (new_freq < hispeed_freq)
- new_freq = hispeed_freq;
+ if (new_freq < tunables->hispeed_freq)
+ new_freq = tunables->hispeed_freq;
}
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
}
- if (pcpu->target_freq >= hispeed_freq &&
+ if (pcpu->target_freq >= tunables->hispeed_freq &&
new_freq > pcpu->target_freq &&
now - pcpu->hispeed_validate_time <
- freq_to_above_hispeed_delay(pcpu->target_freq)) {
+ freq_to_above_hispeed_delay(tunables, pcpu->target_freq)) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
@@ -421,7 +440,8 @@
* floor frequency for the minimum sample time since last validated.
*/
if (new_freq < pcpu->floor_freq) {
- if (now - pcpu->floor_validate_time < min_sample_time) {
+ if (now - pcpu->floor_validate_time <
+ tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
@@ -437,7 +457,7 @@
* (or the indefinite boost is turned off).
*/
- if (!boosted || new_freq > hispeed_freq) {
+ if (!boosted || new_freq > tunables->hispeed_freq) {
pcpu->floor_freq = new_freq;
pcpu->floor_validate_time = now;
}
@@ -598,14 +618,16 @@
int anyboost = 0;
unsigned long flags;
struct cpufreq_interactive_cpuinfo *pcpu;
+ struct cpufreq_interactive_tunables *tunables;
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
+ tunables = pcpu->policy->governor_data;
- if (pcpu->target_freq < hispeed_freq) {
- pcpu->target_freq = hispeed_freq;
+ if (pcpu->target_freq < tunables->hispeed_freq) {
+ pcpu->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
pcpu->hispeed_validate_time =
ktime_to_us(ktime_get());
@@ -617,7 +639,7 @@
* validated.
*/
- pcpu->floor_freq = hispeed_freq;
+ pcpu->floor_freq = tunables->hispeed_freq;
pcpu->floor_validate_time = ktime_to_us(ktime_get());
}
@@ -717,26 +739,27 @@
}
static ssize_t show_target_loads(
- struct kobject *kobj, struct attribute *attr, char *buf)
+ struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;
- spin_lock_irqsave(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);
- for (i = 0; i < ntarget_loads; i++)
- ret += sprintf(buf + ret, "%u%s", target_loads[i],
+ for (i = 0; i < tunables->ntarget_loads; i++)
+ ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i],
i & 0x1 ? ":" : " ");
ret += sprintf(buf + --ret, "\n");
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}
static ssize_t store_target_loads(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+ struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ntokens;
unsigned int *new_target_loads = NULL;
@@ -746,40 +769,37 @@
if (IS_ERR(new_target_loads))
return PTR_RET(new_target_loads);
- spin_lock_irqsave(&target_loads_lock, flags);
- if (target_loads != default_target_loads)
- kfree(target_loads);
- target_loads = new_target_loads;
- ntarget_loads = ntokens;
- spin_unlock_irqrestore(&target_loads_lock, flags);
+ spin_lock_irqsave(&tunables->target_loads_lock, flags);
+ if (tunables->target_loads != default_target_loads)
+ kfree(tunables->target_loads);
+ tunables->target_loads = new_target_loads;
+ tunables->ntarget_loads = ntokens;
+ spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return count;
}
-static struct global_attr target_loads_attr =
- __ATTR(target_loads, S_IRUGO | S_IWUSR,
- show_target_loads, store_target_loads);
-
static ssize_t show_above_hispeed_delay(
- struct kobject *kobj, struct attribute *attr, char *buf)
+ struct cpufreq_interactive_tunables *tunables, char *buf)
{
int i;
ssize_t ret = 0;
unsigned long flags;
- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
- for (i = 0; i < nabove_hispeed_delay; i++)
- ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
+ for (i = 0; i < tunables->nabove_hispeed_delay; i++)
+ ret += sprintf(buf + ret, "%u%s",
+ tunables->above_hispeed_delay[i],
i & 0x1 ? ":" : " ");
ret += sprintf(buf + --ret, "\n");
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}
static ssize_t store_above_hispeed_delay(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+ struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ntokens;
unsigned int *new_above_hispeed_delay = NULL;
@@ -789,29 +809,24 @@
if (IS_ERR(new_above_hispeed_delay))
return PTR_RET(new_above_hispeed_delay);
- spin_lock_irqsave(&above_hispeed_delay_lock, flags);
- if (above_hispeed_delay != default_above_hispeed_delay)
- kfree(above_hispeed_delay);
- above_hispeed_delay = new_above_hispeed_delay;
- nabove_hispeed_delay = ntokens;
- spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
+ spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags);
+ if (tunables->above_hispeed_delay != default_above_hispeed_delay)
+ kfree(tunables->above_hispeed_delay);
+ tunables->above_hispeed_delay = new_above_hispeed_delay;
+ tunables->nabove_hispeed_delay = ntokens;
+ spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return count;
}
-static struct global_attr above_hispeed_delay_attr =
- __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
- show_above_hispeed_delay, store_above_hispeed_delay);
-
-static ssize_t show_hispeed_freq(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%u\n", hispeed_freq);
+ return sprintf(buf, "%u\n", tunables->hispeed_freq);
}
-static ssize_t store_hispeed_freq(struct kobject *kobj,
- struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
long unsigned int val;
@@ -819,22 +834,18 @@
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- hispeed_freq = val;
+ tunables->hispeed_freq = val;
return count;
}
-static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
- show_hispeed_freq, store_hispeed_freq);
-
-
-static ssize_t show_go_hispeed_load(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%lu\n", go_hispeed_load);
+ return sprintf(buf, "%lu\n", tunables->go_hispeed_load);
}
-static ssize_t store_go_hispeed_load(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -842,21 +853,18 @@
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- go_hispeed_load = val;
+ tunables->go_hispeed_load = val;
return count;
}
-static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
- show_go_hispeed_load, store_go_hispeed_load);
-
-static ssize_t show_min_sample_time(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_min_sample_time(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%lu\n", min_sample_time);
+ return sprintf(buf, "%lu\n", tunables->min_sample_time);
}
-static ssize_t store_min_sample_time(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_min_sample_time(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -864,21 +872,18 @@
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- min_sample_time = val;
+ tunables->min_sample_time = val;
return count;
}
-static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
- show_min_sample_time, store_min_sample_time);
-
-static ssize_t show_timer_rate(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%lu\n", timer_rate);
+ return sprintf(buf, "%lu\n", tunables->timer_rate);
}
-static ssize_t store_timer_rate(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -886,22 +891,18 @@
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- timer_rate = val;
+ tunables->timer_rate = val;
return count;
}
-static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
- show_timer_rate, store_timer_rate);
-
-static ssize_t show_timer_slack(
- struct kobject *kobj, struct attribute *attr, char *buf)
+static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%d\n", timer_slack_val);
+ return sprintf(buf, "%d\n", tunables->timer_slack_val);
}
-static ssize_t store_timer_slack(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -910,19 +911,17 @@
if (ret < 0)
return ret;
- timer_slack_val = val;
+ tunables->timer_slack_val = val;
return count;
}
-define_one_global_rw(timer_slack);
-
-static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
+static ssize_t show_boost(struct cpufreq_interactive_tunables *tunables,
char *buf)
{
- return sprintf(buf, "%d\n", boost_val);
+ return sprintf(buf, "%d\n", tunables->boost_val);
}
-static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
+static ssize_t store_boost(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
@@ -932,9 +931,9 @@
if (ret < 0)
return ret;
- boost_val = val;
+ tunables->boost_val = val;
- if (boost_val) {
+ if (tunables->boost_val) {
trace_cpufreq_interactive_boost("on");
cpufreq_interactive_boost();
} else {
@@ -944,9 +943,7 @@
return count;
}
-define_one_global_rw(boost);
-
-static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
+static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
@@ -956,24 +953,21 @@
if (ret < 0)
return ret;
- boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
+ tunables->boostpulse_endtime = ktime_to_us(ktime_get()) +
+ tunables->boostpulse_duration_val;
trace_cpufreq_interactive_boost("pulse");
cpufreq_interactive_boost();
return count;
}
-static struct global_attr boostpulse =
- __ATTR(boostpulse, 0200, NULL, store_boostpulse);
-
-static ssize_t show_boostpulse_duration(
- struct kobject *kobj, struct attribute *attr, char *buf)
+static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tunables
+ *tunables, char *buf)
{
- return sprintf(buf, "%d\n", boostpulse_duration_val);
+ return sprintf(buf, "%d\n", tunables->boostpulse_duration_val);
}
-static ssize_t store_boostpulse_duration(
- struct kobject *kobj, struct attribute *attr, const char *buf,
- size_t count)
+static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tunables
+ *tunables, const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -982,20 +976,18 @@
if (ret < 0)
return ret;
- boostpulse_duration_val = val;
+ tunables->boostpulse_duration_val = val;
return count;
}
-define_one_global_rw(boostpulse_duration);
-
-static ssize_t show_io_is_busy(struct kobject *kobj,
- struct attribute *attr, char *buf)
+static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tunables,
+ char *buf)
{
- return sprintf(buf, "%u\n", io_is_busy);
+ return sprintf(buf, "%u\n", tunables->io_is_busy);
}
-static ssize_t store_io_is_busy(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
+static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables,
+ const char *buf, size_t count)
{
int ret;
unsigned long val;
@@ -1003,33 +995,137 @@
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
- io_is_busy = val;
+ tunables->io_is_busy = val;
return count;
}
-static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
- show_io_is_busy, store_io_is_busy);
+/*
+ * Create show/store routines
+ * - sys: One governor instance for complete SYSTEM
+ * - pol: One governor instance per struct cpufreq_policy
+ */
+#define show_gov_pol_sys(file_name) \
+static ssize_t show_##file_name##_gov_sys \
+(struct kobject *kobj, struct attribute *attr, char *buf) \
+{ \
+ return show_##file_name(common_tunables, buf); \
+} \
+ \
+static ssize_t show_##file_name##_gov_pol \
+(struct cpufreq_policy *policy, char *buf) \
+{ \
+ return show_##file_name(policy->governor_data, buf); \
+}
-static struct attribute *interactive_attributes[] = {
- &target_loads_attr.attr,
- &above_hispeed_delay_attr.attr,
- &hispeed_freq_attr.attr,
- &go_hispeed_load_attr.attr,
- &min_sample_time_attr.attr,
- &timer_rate_attr.attr,
- &timer_slack.attr,
- &boost.attr,
- &boostpulse.attr,
- &boostpulse_duration.attr,
- &io_is_busy_attr.attr,
+#define store_gov_pol_sys(file_name) \
+static ssize_t store_##file_name##_gov_sys \
+(struct kobject *kobj, struct attribute *attr, const char *buf, \
+ size_t count) \
+{ \
+ return store_##file_name(common_tunables, buf, count); \
+} \
+ \
+static ssize_t store_##file_name##_gov_pol \
+(struct cpufreq_policy *policy, const char *buf, size_t count) \
+{ \
+ return store_##file_name(policy->governor_data, buf, count); \
+}
+
+#define show_store_gov_pol_sys(file_name) \
+show_gov_pol_sys(file_name); \
+store_gov_pol_sys(file_name)
+
+show_store_gov_pol_sys(target_loads);
+show_store_gov_pol_sys(above_hispeed_delay);
+show_store_gov_pol_sys(hispeed_freq);
+show_store_gov_pol_sys(go_hispeed_load);
+show_store_gov_pol_sys(min_sample_time);
+show_store_gov_pol_sys(timer_rate);
+show_store_gov_pol_sys(timer_slack);
+show_store_gov_pol_sys(boost);
+store_gov_pol_sys(boostpulse);
+show_store_gov_pol_sys(boostpulse_duration);
+show_store_gov_pol_sys(io_is_busy);
+
+#define gov_sys_attr_rw(_name) \
+static struct global_attr _name##_gov_sys = \
+__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys)
+
+#define gov_pol_attr_rw(_name) \
+static struct freq_attr _name##_gov_pol = \
+__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
+
+#define gov_sys_pol_attr_rw(_name) \
+ gov_sys_attr_rw(_name); \
+ gov_pol_attr_rw(_name)
+
+gov_sys_pol_attr_rw(target_loads);
+gov_sys_pol_attr_rw(above_hispeed_delay);
+gov_sys_pol_attr_rw(hispeed_freq);
+gov_sys_pol_attr_rw(go_hispeed_load);
+gov_sys_pol_attr_rw(min_sample_time);
+gov_sys_pol_attr_rw(timer_rate);
+gov_sys_pol_attr_rw(timer_slack);
+gov_sys_pol_attr_rw(boost);
+gov_sys_pol_attr_rw(boostpulse_duration);
+gov_sys_pol_attr_rw(io_is_busy);
+
+static struct global_attr boostpulse_gov_sys =
+ __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys);
+
+static struct freq_attr boostpulse_gov_pol =
+ __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol);
+
+/* One Governor instance for entire system */
+static struct attribute *interactive_attributes_gov_sys[] = {
+ &target_loads_gov_sys.attr,
+ &above_hispeed_delay_gov_sys.attr,
+ &hispeed_freq_gov_sys.attr,
+ &go_hispeed_load_gov_sys.attr,
+ &min_sample_time_gov_sys.attr,
+ &timer_rate_gov_sys.attr,
+ &timer_slack_gov_sys.attr,
+ &boost_gov_sys.attr,
+ &boostpulse_gov_sys.attr,
+ &boostpulse_duration_gov_sys.attr,
+ &io_is_busy_gov_sys.attr,
NULL,
};
-static struct attribute_group interactive_attr_group = {
- .attrs = interactive_attributes,
+static struct attribute_group interactive_attr_group_gov_sys = {
+ .attrs = interactive_attributes_gov_sys,
.name = "interactive",
};
+/* Per policy governor instance */
+static struct attribute *interactive_attributes_gov_pol[] = {
+ &target_loads_gov_pol.attr,
+ &above_hispeed_delay_gov_pol.attr,
+ &hispeed_freq_gov_pol.attr,
+ &go_hispeed_load_gov_pol.attr,
+ &min_sample_time_gov_pol.attr,
+ &timer_rate_gov_pol.attr,
+ &timer_slack_gov_pol.attr,
+ &boost_gov_pol.attr,
+ &boostpulse_gov_pol.attr,
+ &boostpulse_duration_gov_pol.attr,
+ &io_is_busy_gov_pol.attr,
+ NULL,
+};
+
+static struct attribute_group interactive_attr_group_gov_pol = {
+ .attrs = interactive_attributes_gov_pol,
+ .name = "interactive",
+};
+
+static struct attribute_group *get_sysfs_attr(void)
+{
+ if (have_governor_per_policy())
+ return &interactive_attr_group_gov_pol;
+ else
+ return &interactive_attr_group_gov_sys;
+}
+
static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
unsigned long val,
void *data)
@@ -1057,15 +1153,88 @@
unsigned int j;
struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_frequency_table *freq_table;
+ struct cpufreq_interactive_tunables *tunables;
+
+ if (have_governor_per_policy())
+ tunables = policy->governor_data;
+ else
+ tunables = common_tunables;
+
+ WARN_ON(!tunables && (event != CPUFREQ_GOV_POLICY_INIT));
switch (event) {
+ case CPUFREQ_GOV_POLICY_INIT:
+ if (have_governor_per_policy()) {
+ WARN_ON(tunables);
+ } else if (tunables) {
+ tunables->usage_count++;
+ policy->governor_data = tunables;
+ return 0;
+ }
+
+ tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
+ if (!tunables) {
+ pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__);
+ return -ENOMEM;
+ }
+
+ rc = sysfs_create_group(get_governor_parent_kobj(policy),
+ get_sysfs_attr());
+ if (rc) {
+ kfree(tunables);
+ return rc;
+ }
+
+ tunables->usage_count = 1;
+ tunables->above_hispeed_delay = default_above_hispeed_delay;
+ tunables->nabove_hispeed_delay =
+ ARRAY_SIZE(default_above_hispeed_delay);
+ tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
+ tunables->target_loads = default_target_loads;
+ tunables->ntarget_loads = ARRAY_SIZE(default_target_loads);
+ tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
+ tunables->timer_rate = DEFAULT_TIMER_RATE;
+ tunables->boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
+ tunables->timer_slack_val = DEFAULT_TIMER_SLACK;
+
+ spin_lock_init(&tunables->target_loads_lock);
+ spin_lock_init(&tunables->above_hispeed_delay_lock);
+
+ if (!policy->governor->initialized) {
+ idle_notifier_register(&cpufreq_interactive_idle_nb);
+ cpufreq_register_notifier(&cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+ }
+
+ policy->governor_data = tunables;
+ if (!have_governor_per_policy())
+ common_tunables = tunables;
+
+ break;
+
+ case CPUFREQ_GOV_POLICY_EXIT:
+ if (!--tunables->usage_count) {
+ if (policy->governor->initialized == 1) {
+ cpufreq_unregister_notifier(&cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+ idle_notifier_unregister(&cpufreq_interactive_idle_nb);
+ }
+
+ sysfs_remove_group(get_governor_parent_kobj(policy),
+ get_sysfs_attr());
+ kfree(tunables);
+ common_tunables = NULL;
+ }
+
+ policy->governor_data = NULL;
+ break;
+
case CPUFREQ_GOV_START:
mutex_lock(&gov_lock);
- freq_table =
- cpufreq_frequency_get_table(policy->cpu);
- if (!hispeed_freq)
- hispeed_freq = policy->max;
+ freq_table = cpufreq_frequency_get_table(policy->cpu);
+ if (!tunables->hispeed_freq)
+ tunables->hispeed_freq = policy->max;
for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
@@ -1078,30 +1247,11 @@
pcpu->hispeed_validate_time =
pcpu->floor_validate_time;
down_write(&pcpu->enable_sem);
- cpufreq_interactive_timer_start(j);
+ cpufreq_interactive_timer_start(tunables, j);
pcpu->governor_enabled = 1;
up_write(&pcpu->enable_sem);
}
- /*
- * Do not register the idle hook and create sysfs
- * entries if we have already done so.
- */
- if (++active_count > 1) {
- mutex_unlock(&gov_lock);
- return 0;
- }
-
- rc = sysfs_create_group(cpufreq_global_kobject,
- &interactive_attr_group);
- if (rc) {
- mutex_unlock(&gov_lock);
- return rc;
- }
-
- idle_notifier_register(&cpufreq_interactive_idle_nb);
- cpufreq_register_notifier(
- &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
mutex_unlock(&gov_lock);
break;
@@ -1116,18 +1266,7 @@
up_write(&pcpu->enable_sem);
}
- if (--active_count > 0) {
- mutex_unlock(&gov_lock);
- return 0;
- }
-
- cpufreq_unregister_notifier(
- &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
- idle_notifier_unregister(&cpufreq_interactive_idle_nb);
- sysfs_remove_group(cpufreq_global_kobject,
- &interactive_attr_group);
mutex_unlock(&gov_lock);
-
break;
case CPUFREQ_GOV_LIMITS:
@@ -1161,7 +1300,7 @@
*/
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
- cpufreq_interactive_timer_start(j);
+ cpufreq_interactive_timer_start(tunables, j);
up_write(&pcpu->enable_sem);
}
break;
@@ -1201,9 +1340,7 @@
init_rwsem(&pcpu->enable_sem);
}
- spin_lock_init(&target_loads_lock);
spin_lock_init(&speedchange_cpumask_lock);
- spin_lock_init(&above_hispeed_delay_lock);
mutex_init(&gov_lock);
speedchange_task =
kthread_create(cpufreq_interactive_speedchange_task, NULL,