sched/core_ctl: Refactor cpu data
Refactor cpu data into cpu data and cluster data to improve readability and
ease of understanding the code.
Change-Id: I96505aeb9d07a6fa3a2c28648ffa299e0cfa2e41
Signed-off-by: Olav Haugan <ohaugan@codeaurora.org>
diff --git a/kernel/sched/core_ctl.c b/kernel/sched/core_ctl.c
index 213bd6e..9b37e29 100644
--- a/kernel/sched/core_ctl.c
+++ b/kernel/sched/core_ctl.c
@@ -22,26 +22,16 @@
#include <trace/events/sched.h>
-#define MAX_CPUS_PER_GROUP 4
+#define MAX_CPUS_PER_CLUSTER 4
+#define MAX_CLUSTERS 2
-struct cpu_data {
- /* Per CPU data. */
- bool inited;
- bool online;
- bool rejected;
- bool is_busy;
- bool not_preferred;
- unsigned int busy;
- unsigned int cpu;
- struct list_head sib;
- unsigned int first_cpu;
-
- /* Per cluster data set only on first CPU */
+struct cluster_data {
+ bool inited;
unsigned int min_cpus;
unsigned int max_cpus;
unsigned int offline_delay_ms;
- unsigned int busy_up_thres[MAX_CPUS_PER_GROUP];
- unsigned int busy_down_thres[MAX_CPUS_PER_GROUP];
+ unsigned int busy_up_thres[MAX_CPUS_PER_CLUSTER];
+ unsigned int busy_down_thres[MAX_CPUS_PER_CLUSTER];
unsigned int online_cpus;
unsigned int avail_cpus;
unsigned int num_cpus;
@@ -56,17 +46,36 @@
bool nrrun_changed;
struct timer_list timer;
struct task_struct *hotplug_thread;
+ unsigned int first_cpu;
struct kobject kobj;
};
+struct cpu_data {
+ bool online;
+ bool rejected;
+ bool is_busy;
+ unsigned int busy;
+ unsigned int cpu;
+ bool not_preferred;
+ struct cluster_data *cluster;
+ struct list_head sib;
+};
+
static DEFINE_PER_CPU(struct cpu_data, cpu_state);
+static struct cluster_data cluster_state[MAX_CLUSTERS];
+static unsigned int num_clusters;
+
+#define for_each_cluster(cluster, idx) \
+ for ((cluster) = &cluster_state[idx]; (idx) < num_clusters;\
+ (idx)++, (cluster) = &cluster_state[idx])
+
static DEFINE_SPINLOCK(state_lock);
-static void apply_need(struct cpu_data *f);
-static void wake_up_hotplug_thread(struct cpu_data *state);
+static void apply_need(struct cluster_data *state);
+static void wake_up_hotplug_thread(struct cluster_data *state);
/* ========================= sysfs interface =========================== */
-static ssize_t store_min_cpus(struct cpu_data *state,
+static ssize_t store_min_cpus(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@@ -80,12 +89,12 @@
return count;
}
-static ssize_t show_min_cpus(struct cpu_data *state, char *buf)
+static ssize_t show_min_cpus(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->min_cpus);
}
-static ssize_t store_max_cpus(struct cpu_data *state,
+static ssize_t store_max_cpus(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@@ -101,12 +110,12 @@
return count;
}
-static ssize_t show_max_cpus(struct cpu_data *state, char *buf)
+static ssize_t show_max_cpus(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->max_cpus);
}
-static ssize_t store_offline_delay_ms(struct cpu_data *state,
+static ssize_t store_offline_delay_ms(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@@ -120,13 +129,13 @@
return count;
}
-static ssize_t show_task_thres(struct cpu_data *state, char *buf)
+static ssize_t show_task_thres(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->task_thres);
}
-static ssize_t store_task_thres(struct cpu_data *state,
- const char *buf, size_t count)
+static ssize_t store_task_thres(struct cluster_data *state,
+ const char *buf, size_t count)
{
unsigned int val;
@@ -142,15 +151,16 @@
return count;
}
-static ssize_t show_offline_delay_ms(struct cpu_data *state, char *buf)
+static ssize_t show_offline_delay_ms(const struct cluster_data *state,
+ char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->offline_delay_ms);
}
-static ssize_t store_busy_up_thres(struct cpu_data *state,
+static ssize_t store_busy_up_thres(struct cluster_data *state,
const char *buf, size_t count)
{
- unsigned int val[MAX_CPUS_PER_GROUP];
+ unsigned int val[MAX_CPUS_PER_CLUSTER];
int ret, i;
ret = sscanf(buf, "%u %u %u %u\n", &val[0], &val[1], &val[2], &val[3]);
@@ -168,21 +178,22 @@
return count;
}
-static ssize_t show_busy_up_thres(struct cpu_data *state, char *buf)
+static ssize_t show_busy_up_thres(const struct cluster_data *state, char *buf)
{
int i, count = 0;
for (i = 0; i < state->num_cpus; i++)
count += snprintf(buf + count, PAGE_SIZE - count, "%u ",
state->busy_up_thres[i]);
+
count += snprintf(buf + count, PAGE_SIZE - count, "\n");
return count;
}
-static ssize_t store_busy_down_thres(struct cpu_data *state,
+static ssize_t store_busy_down_thres(struct cluster_data *state,
const char *buf, size_t count)
{
- unsigned int val[MAX_CPUS_PER_GROUP];
+ unsigned int val[MAX_CPUS_PER_CLUSTER];
int ret, i;
ret = sscanf(buf, "%u %u %u %u\n", &val[0], &val[1], &val[2], &val[3]);
@@ -200,18 +211,19 @@
return count;
}
-static ssize_t show_busy_down_thres(struct cpu_data *state, char *buf)
+static ssize_t show_busy_down_thres(const struct cluster_data *state, char *buf)
{
int i, count = 0;
for (i = 0; i < state->num_cpus; i++)
count += snprintf(buf + count, PAGE_SIZE - count, "%u ",
state->busy_down_thres[i]);
+
count += snprintf(buf + count, PAGE_SIZE - count, "\n");
return count;
}
-static ssize_t store_is_big_cluster(struct cpu_data *state,
+static ssize_t store_is_big_cluster(struct cluster_data *state,
const char *buf, size_t count)
{
unsigned int val;
@@ -223,12 +235,12 @@
return count;
}
-static ssize_t show_is_big_cluster(struct cpu_data *state, char *buf)
+static ssize_t show_is_big_cluster(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->is_big_cluster);
}
-static ssize_t show_cpus(struct cpu_data *state, char *buf)
+static ssize_t show_cpus(const struct cluster_data *state, char *buf)
{
struct cpu_data *c;
ssize_t count = 0;
@@ -237,26 +249,27 @@
spin_lock_irqsave(&state_lock, flags);
list_for_each_entry(c, &state->lru, sib) {
count += snprintf(buf + count, PAGE_SIZE - count,
- "CPU%u (%s)\n", c->cpu,
- c->online ? "Online" : "Offline");
+ "CPU%u (%s)\n", c->cpu,
+ c->online ? "Online" : "Offline");
}
spin_unlock_irqrestore(&state_lock, flags);
return count;
}
-static ssize_t show_need_cpus(struct cpu_data *state, char *buf)
+static ssize_t show_need_cpus(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->need_cpus);
}
-static ssize_t show_online_cpus(struct cpu_data *state, char *buf)
+static ssize_t show_online_cpus(const struct cluster_data *state, char *buf)
{
return snprintf(buf, PAGE_SIZE, "%u\n", state->online_cpus);
}
-static ssize_t show_global_state(struct cpu_data *state, char *buf)
+static ssize_t show_global_state(const struct cluster_data *state, char *buf)
{
struct cpu_data *c;
+ struct cluster_data *cluster;
ssize_t count = 0;
unsigned int cpu;
@@ -264,8 +277,10 @@
count += snprintf(buf + count, PAGE_SIZE - count,
"CPU%u\n", cpu);
c = &per_cpu(cpu_state, cpu);
- if (!c->inited)
+ cluster = c->cluster;
+ if (!cluster || !cluster->inited)
continue;
+
count += snprintf(buf + count, PAGE_SIZE - count,
"\tCPU: %u\n", c->cpu);
count += snprintf(buf + count, PAGE_SIZE - count,
@@ -273,69 +288,65 @@
count += snprintf(buf + count, PAGE_SIZE - count,
"\tRejected: %u\n", c->rejected);
count += snprintf(buf + count, PAGE_SIZE - count,
- "\tFirst CPU: %u\n", c->first_cpu);
+ "\tFirst CPU: %u\n",
+ cluster->first_cpu);
count += snprintf(buf + count, PAGE_SIZE - count,
"\tBusy%%: %u\n", c->busy);
count += snprintf(buf + count, PAGE_SIZE - count,
"\tIs busy: %u\n", c->is_busy);
- if (c->cpu != c->first_cpu)
- continue;
count += snprintf(buf + count, PAGE_SIZE - count,
- "\tNr running: %u\n", c->nrrun);
+ "\tNr running: %u\n", cluster->nrrun);
count += snprintf(buf + count, PAGE_SIZE - count,
- "\tAvail CPUs: %u\n", c->avail_cpus);
+ "\tAvail CPUs: %u\n", cluster->avail_cpus);
count += snprintf(buf + count, PAGE_SIZE - count,
- "\tNeed CPUs: %u\n", c->need_cpus);
+ "\tNeed CPUs: %u\n", cluster->need_cpus);
}
return count;
}
-static ssize_t store_not_preferred(struct cpu_data *state,
- const char *buf, size_t count)
+static ssize_t store_not_preferred(struct cluster_data *state,
+ const char *buf, size_t count)
{
struct cpu_data *c;
- unsigned int i, first_cpu;
- unsigned int val[MAX_CPUS_PER_GROUP];
+ unsigned int i;
+ unsigned int val[MAX_CPUS_PER_CLUSTER];
+ unsigned long flags;
int ret;
ret = sscanf(buf, "%u %u %u %u\n", &val[0], &val[1], &val[2], &val[3]);
if (ret != 1 && ret != state->num_cpus)
return -EINVAL;
- first_cpu = state->first_cpu;
-
- for (i = 0; i < state->num_cpus; i++) {
- c = &per_cpu(cpu_state, first_cpu);
- c->not_preferred = val[i];
- first_cpu++;
- }
+ i = 0;
+ spin_lock_irqsave(&state_lock, flags);
+ list_for_each_entry(c, &state->lru, sib)
+ c->not_preferred = val[i++];
+ spin_unlock_irqrestore(&state_lock, flags);
return count;
}
-static ssize_t show_not_preferred(struct cpu_data *state, char *buf)
+static ssize_t show_not_preferred(const struct cluster_data *state, char *buf)
{
struct cpu_data *c;
ssize_t count = 0;
- unsigned int i, first_cpu;
+ unsigned long flags;
- first_cpu = state->first_cpu;
-
- for (i = 0; i < state->num_cpus; i++) {
- c = &per_cpu(cpu_state, first_cpu);
+ spin_lock_irqsave(&state_lock, flags);
+ list_for_each_entry(c, &state->lru, sib)
count += snprintf(buf + count, PAGE_SIZE - count,
- "\tCPU:%d %u\n", first_cpu, c->not_preferred);
- first_cpu++;
- }
+ "\tCPU:%d %u\n", c->cpu, c->not_preferred);
+ spin_unlock_irqrestore(&state_lock, flags);
return count;
}
+
struct core_ctl_attr {
struct attribute attr;
- ssize_t (*show)(struct cpu_data *, char *);
- ssize_t (*store)(struct cpu_data *, const char *, size_t count);
+ ssize_t (*show)(const struct cluster_data *, char *);
+ ssize_t (*store)(struct cluster_data *, const char *, size_t count);
};
#define core_ctl_attr_ro(_name) \
@@ -375,11 +386,11 @@
NULL
};
-#define to_cpu_data(k) container_of(k, struct cpu_data, kobj)
+#define to_cluster_data(k) container_of(k, struct cluster_data, kobj)
#define to_attr(a) container_of(a, struct core_ctl_attr, attr)
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
{
- struct cpu_data *data = to_cpu_data(kobj);
+ struct cluster_data *data = to_cluster_data(kobj);
struct core_ctl_attr *cattr = to_attr(attr);
ssize_t ret = -EIO;
@@ -392,7 +403,7 @@
static ssize_t store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
- struct cpu_data *data = to_cpu_data(kobj);
+ struct cluster_data *data = to_cluster_data(kobj);
struct core_ctl_attr *cattr = to_attr(attr);
ssize_t ret = -EIO;
@@ -424,11 +435,11 @@
static void update_running_avg(bool trigger_update)
{
- int cpu;
- struct cpu_data *pcpu;
int avg, iowait_avg, big_avg, old_nrrun;
s64 now;
unsigned long flags;
+ struct cluster_data *cluster;
+ unsigned int index = 0;
spin_lock_irqsave(&state_lock, flags);
@@ -457,11 +468,10 @@
avg = (avg + NR_RUNNING_TOLERANCE) / 100;
big_avg = (big_avg + NR_RUNNING_TOLERANCE) / 100;
- for_each_possible_cpu(cpu) {
- pcpu = &per_cpu(cpu_state, cpu);
- if (!pcpu->inited || pcpu->first_cpu != cpu)
+ for_each_cluster(cluster, index) {
+ if (!cluster->inited)
continue;
- old_nrrun = pcpu->nrrun;
+ old_nrrun = cluster->nrrun;
/*
* Big cluster only need to take care of big tasks, but if
* there are not enough big cores, big tasks need to be run
@@ -471,25 +481,27 @@
* is not easy to get given core control reacts much slower
* than scheduler, and can't predict scheduler's behavior.
*/
- pcpu->nrrun = pcpu->is_big_cluster ? big_avg : avg;
- if (pcpu->nrrun != old_nrrun) {
+ cluster->nrrun = cluster->is_big_cluster ? big_avg : avg;
+ if (cluster->nrrun != old_nrrun) {
if (trigger_update)
- apply_need(pcpu);
+ apply_need(cluster);
else
- pcpu->nrrun_changed = true;
+ cluster->nrrun_changed = true;
}
}
+ return;
}
/* adjust needed CPUs based on current runqueue information */
-static unsigned int apply_task_need(struct cpu_data *f, unsigned int new_need)
+static unsigned int apply_task_need(const struct cluster_data *cluster,
+ unsigned int new_need)
{
/* Online all cores if there are enough tasks */
- if (f->nrrun >= f->task_thres)
- return f->num_cpus;
+ if (cluster->nrrun >= cluster->task_thres)
+ return cluster->num_cpus;
/* only online more cores if there are tasks to run */
- if (f->nrrun > new_need)
+ if (cluster->nrrun > new_need)
return new_need + 1;
return new_need;
@@ -512,12 +524,13 @@
/* ======================= load based core count ====================== */
-static unsigned int apply_limits(struct cpu_data *f, unsigned int need_cpus)
+static unsigned int apply_limits(const struct cluster_data *cluster,
+ unsigned int need_cpus)
{
- return min(max(f->min_cpus, need_cpus), f->max_cpus);
+ return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus);
}
-static bool eval_need(struct cpu_data *f)
+static bool eval_need(struct cluster_data *cluster)
{
unsigned long flags;
struct cpu_data *c;
@@ -526,26 +539,27 @@
bool need_flag = false;
s64 now;
- if (unlikely(!f->inited))
+ if (unlikely(!cluster->inited))
return 0;
spin_lock_irqsave(&state_lock, flags);
- thres_idx = f->online_cpus ? f->online_cpus - 1 : 0;
- list_for_each_entry(c, &f->lru, sib) {
- if (c->busy >= f->busy_up_thres[thres_idx])
+ thres_idx = cluster->online_cpus ? cluster->online_cpus - 1 : 0;
+ list_for_each_entry(c, &cluster->lru, sib) {
+ if (c->busy >= cluster->busy_up_thres[thres_idx])
c->is_busy = true;
- else if (c->busy < f->busy_down_thres[thres_idx])
+ else if (c->busy < cluster->busy_down_thres[thres_idx])
c->is_busy = false;
need_cpus += c->is_busy;
}
- need_cpus = apply_task_need(f, need_cpus);
- need_flag = apply_limits(f, need_cpus) != apply_limits(f, f->need_cpus);
- last_need = f->need_cpus;
+ need_cpus = apply_task_need(cluster, need_cpus);
+ need_flag = apply_limits(cluster, need_cpus) !=
+ apply_limits(cluster, cluster->need_cpus);
+ last_need = cluster->need_cpus;
now = ktime_to_ms(ktime_get());
if (need_cpus == last_need) {
- f->need_ts = now;
+ cluster->need_ts = now;
spin_unlock_irqrestore(&state_lock, flags);
return 0;
}
@@ -553,51 +567,50 @@
if (need_cpus > last_need) {
ret = 1;
} else if (need_cpus < last_need) {
- s64 elapsed = now - f->need_ts;
+ s64 elapsed = now - cluster->need_ts;
- if (elapsed >= f->offline_delay_ms) {
+ if (elapsed >= cluster->offline_delay_ms) {
ret = 1;
} else {
- mod_timer(&f->timer, jiffies +
- msecs_to_jiffies(f->offline_delay_ms));
+ mod_timer(&cluster->timer, jiffies +
+ msecs_to_jiffies(cluster->offline_delay_ms));
}
}
if (ret) {
- f->need_ts = now;
- f->need_cpus = need_cpus;
+ cluster->need_ts = now;
+ cluster->need_cpus = need_cpus;
}
- trace_core_ctl_eval_need(f->cpu, last_need, need_cpus,
+ trace_core_ctl_eval_need(cluster->first_cpu, last_need, need_cpus,
ret && need_flag);
spin_unlock_irqrestore(&state_lock, flags);
return ret && need_flag;
}
-static void apply_need(struct cpu_data *f)
+static void apply_need(struct cluster_data *cluster)
{
- if (eval_need(f))
- wake_up_hotplug_thread(f);
+ if (eval_need(cluster))
+ wake_up_hotplug_thread(cluster);
}
static int core_ctl_set_busy(unsigned int cpu, unsigned int busy)
{
struct cpu_data *c = &per_cpu(cpu_state, cpu);
- struct cpu_data *f;
+ struct cluster_data *cluster = c->cluster;
unsigned int old_is_busy = c->is_busy;
- if (!c->inited)
+ if (!cluster || !cluster->inited)
return 0;
- f = &per_cpu(cpu_state, c->first_cpu);
update_running_avg(false);
- if (c->busy == busy && !f->nrrun_changed)
+ if (c->busy == busy && !cluster->nrrun_changed)
return 0;
c->busy = busy;
- f->nrrun_changed = false;
+ cluster->nrrun_changed = false;
- apply_need(f);
+ apply_need(cluster);
trace_core_ctl_set_busy(cpu, busy, old_is_busy, c->is_busy);
return 0;
}
@@ -609,46 +622,44 @@
* itself or other hotplug threads because it will deadlock. Instead,
* schedule a timer to fire in next timer tick and wake up the thread.
*/
-static void wake_up_hotplug_thread(struct cpu_data *state)
+static void wake_up_hotplug_thread(struct cluster_data *cluster)
{
unsigned long flags;
- int cpu;
- struct cpu_data *pcpu;
bool no_wakeup = false;
+ struct cluster_data *cls;
+ unsigned long index = 0;
- for_each_possible_cpu(cpu) {
- pcpu = &per_cpu(cpu_state, cpu);
- if (cpu != pcpu->first_cpu)
- continue;
- if (pcpu->hotplug_thread == current) {
+ for_each_cluster(cls, index) {
+ if (cls->hotplug_thread == current) {
no_wakeup = true;
break;
}
}
- spin_lock_irqsave(&state->pending_lock, flags);
- state->pending = true;
- spin_unlock_irqrestore(&state->pending_lock, flags);
+ spin_lock_irqsave(&cluster->pending_lock, flags);
+ cluster->pending = true;
+ spin_unlock_irqrestore(&cluster->pending_lock, flags);
if (no_wakeup) {
spin_lock_irqsave(&state_lock, flags);
- mod_timer(&state->timer, jiffies);
+ mod_timer(&cluster->timer, jiffies);
spin_unlock_irqrestore(&state_lock, flags);
} else {
- wake_up_process(state->hotplug_thread);
+ wake_up_process(cluster->hotplug_thread);
}
}
-static void core_ctl_timer_func(unsigned long cpu)
+static void core_ctl_timer_func(unsigned long data)
{
- struct cpu_data *state = &per_cpu(cpu_state, cpu);
- unsigned long flags;
+ struct cluster_data *cluster = (struct cluster_data *) data;
- if (eval_need(state)) {
- spin_lock_irqsave(&state->pending_lock, flags);
- state->pending = true;
- spin_unlock_irqrestore(&state->pending_lock, flags);
- wake_up_process(state->hotplug_thread);
+ if (eval_need(cluster)) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&cluster->pending_lock, flags);
+ cluster->pending = true;
+ spin_unlock_irqrestore(&cluster->pending_lock, flags);
+ wake_up_process(cluster->hotplug_thread);
}
}
@@ -687,20 +698,20 @@
return ret;
}
-static void __ref do_hotplug(struct cpu_data *f)
+static void __ref do_hotplug(struct cluster_data *cluster)
{
unsigned int need;
struct cpu_data *c, *tmp;
- need = apply_limits(f, f->need_cpus);
- pr_debug("Trying to adjust group %u to %u\n", f->first_cpu, need);
+ need = apply_limits(cluster, cluster->need_cpus);
+ pr_debug("Trying to adjust group %u to %u\n", cluster->first_cpu, need);
- if (f->online_cpus > need) {
- list_for_each_entry_safe(c, tmp, &f->lru, sib) {
+ if (cluster->online_cpus > need) {
+ list_for_each_entry_safe(c, tmp, &cluster->lru, sib) {
if (!c->online)
continue;
- if (f->online_cpus == need)
+ if (cluster->online_cpus == need)
break;
/* Don't offline busy CPUs. */
@@ -716,25 +727,25 @@
* If the number of online CPUs is within the limits, then
* don't force any busy CPUs offline.
*/
- if (f->online_cpus <= f->max_cpus)
+ if (cluster->online_cpus <= cluster->max_cpus)
return;
- list_for_each_entry_safe(c, tmp, &f->lru, sib) {
+ list_for_each_entry_safe(c, tmp, &cluster->lru, sib) {
if (!c->online)
continue;
- if (f->online_cpus <= f->max_cpus)
+ if (cluster->online_cpus <= cluster->max_cpus)
break;
pr_debug("Trying to Offline CPU%u\n", c->cpu);
if (core_ctl_offline_core(c->cpu))
pr_debug("Unable to Offline CPU%u\n", c->cpu);
}
- } else if (f->online_cpus < need) {
- list_for_each_entry_safe(c, tmp, &f->lru, sib) {
+ } else if (cluster->online_cpus < need) {
+ list_for_each_entry_safe(c, tmp, &cluster->lru, sib) {
if (c->online || c->rejected || c->not_preferred)
continue;
- if (f->online_cpus == need)
+ if (cluster->online_cpus == need)
break;
pr_debug("Trying to Online CPU%u\n", c->cpu);
@@ -742,14 +753,14 @@
pr_debug("Unable to Online CPU%u\n", c->cpu);
}
- if (f->online_cpus == need)
+ if (cluster->online_cpus == need)
return;
- list_for_each_entry_safe(c, tmp, &f->lru, sib) {
+ list_for_each_entry_safe(c, tmp, &cluster->lru, sib) {
if (c->online || c->rejected || !c->not_preferred)
continue;
- if (f->online_cpus == need)
+ if (cluster->online_cpus == need)
break;
pr_debug("Trying to Online CPU%u\n", c->cpu);
@@ -762,24 +773,24 @@
static int __ref try_hotplug(void *data)
{
- struct cpu_data *f = data;
+ struct cluster_data *cluster = data;
unsigned long flags;
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
- spin_lock_irqsave(&f->pending_lock, flags);
- if (!f->pending) {
- spin_unlock_irqrestore(&f->pending_lock, flags);
+ spin_lock_irqsave(&cluster->pending_lock, flags);
+ if (!cluster->pending) {
+ spin_unlock_irqrestore(&cluster->pending_lock, flags);
schedule();
if (kthread_should_stop())
break;
- spin_lock_irqsave(&f->pending_lock, flags);
+ spin_lock_irqsave(&cluster->pending_lock, flags);
}
set_current_state(TASK_RUNNING);
- f->pending = false;
- spin_unlock_irqrestore(&f->pending_lock, flags);
+ cluster->pending = false;
+ spin_unlock_irqrestore(&cluster->pending_lock, flags);
- do_hotplug(f);
+ do_hotplug(cluster);
}
return 0;
@@ -790,7 +801,7 @@
{
uint32_t cpu = (uintptr_t)hcpu;
struct cpu_data *state = &per_cpu(cpu_state, cpu);
- struct cpu_data *f;
+ struct cluster_data *cluster = state->cluster;
int ret = NOTIFY_OK;
unsigned long flags;
@@ -798,24 +809,22 @@
if (action & CPU_TASKS_FROZEN)
return NOTIFY_OK;
- if (unlikely(!state->inited))
+ if (unlikely(!cluster || !cluster->inited))
return NOTIFY_OK;
- f = &per_cpu(cpu_state, state->first_cpu);
-
switch (action) {
case CPU_UP_PREPARE:
/* If online state of CPU somehow got out of sync, fix it. */
if (state->online) {
- f->online_cpus--;
+ cluster->online_cpus--;
state->online = false;
pr_warn("CPU%d offline when state is online\n", cpu);
}
if (state->rejected) {
state->rejected = false;
- f->avail_cpus++;
+ cluster->avail_cpus++;
}
/*
@@ -823,12 +832,13 @@
* so that there's no race with hotplug thread bringing up more
* CPUs than necessary.
*/
- if (apply_limits(f, f->need_cpus) <= f->online_cpus) {
+ if (apply_limits(cluster, cluster->need_cpus) <=
+ cluster->online_cpus) {
pr_debug("Prevent CPU%d onlining\n", cpu);
ret = NOTIFY_BAD;
} else {
state->online = true;
- f->online_cpus++;
+ cluster->online_cpus++;
}
break;
@@ -841,7 +851,7 @@
*/
spin_lock_irqsave(&state_lock, flags);
list_del(&state->sib);
- list_add_tail(&state->sib, &f->lru);
+ list_add_tail(&state->sib, &cluster->lru);
spin_unlock_irqrestore(&state_lock, flags);
break;
@@ -849,7 +859,7 @@
/* Move a CPU to the end of the LRU when it goes offline. */
spin_lock_irqsave(&state_lock, flags);
list_del(&state->sib);
- list_add_tail(&state->sib, &f->lru);
+ list_add_tail(&state->sib, &cluster->lru);
spin_unlock_irqrestore(&state_lock, flags);
/* Fall through */
@@ -858,25 +868,25 @@
/* If online state of CPU somehow got out of sync, fix it. */
if (!state->online) {
- f->online_cpus++;
+ cluster->online_cpus++;
pr_warn("CPU%d online when state is offline\n", cpu);
}
if (!state->rejected && action == CPU_UP_CANCELED) {
state->rejected = true;
- f->avail_cpus--;
+ cluster->avail_cpus--;
}
state->online = false;
state->busy = 0;
- f->online_cpus--;
+ cluster->online_cpus--;
break;
}
- if (f->online_cpus < apply_limits(f, f->need_cpus)
- && f->online_cpus < f->avail_cpus
+ if (cluster->online_cpus < apply_limits(cluster, cluster->need_cpus)
+ && cluster->online_cpus < cluster->avail_cpus
&& action == CPU_DEAD)
- wake_up_hotplug_thread(f);
+ wake_up_hotplug_thread(cluster);
return ret;
}
@@ -887,16 +897,28 @@
/* ============================ init code ============================== */
-static int group_init(struct cpumask *mask)
+static struct cluster_data *find_cluster_by_first_cpu(unsigned int first_cpu)
+{
+ unsigned int i;
+
+ for (i = 0; i < num_clusters; ++i) {
+ if (cluster_state[i].first_cpu == first_cpu)
+ return &cluster_state[i];
+ }
+
+ return NULL;
+}
+
+static int cluster_init(const struct cpumask *mask)
{
struct device *dev;
unsigned int first_cpu = cpumask_first(mask);
- struct cpu_data *f = &per_cpu(cpu_state, first_cpu);
+ struct cluster_data *cluster;
struct cpu_data *state;
unsigned int cpu;
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
- if (likely(f->inited))
+ if (find_cluster_by_first_cpu(first_cpu))
return 0;
dev = get_cpu_device(first_cpu);
@@ -905,50 +927,55 @@
pr_info("Creating CPU group %d\n", first_cpu);
- f->num_cpus = cpumask_weight(mask);
- if (f->num_cpus > MAX_CPUS_PER_GROUP) {
+ if (num_clusters == MAX_CLUSTERS) {
+ pr_err("Unsupported number of clusters. Only %u supported\n",
+ MAX_CLUSTERS);
+ return -EINVAL;
+ }
+ cluster = &cluster_state[num_clusters];
+ ++num_clusters;
+
+ cluster->num_cpus = cpumask_weight(mask);
+ if (cluster->num_cpus > MAX_CPUS_PER_CLUSTER) {
pr_err("HW configuration not supported\n");
return -EINVAL;
}
- f->min_cpus = 1;
- f->max_cpus = f->num_cpus;
- f->need_cpus = f->num_cpus;
- f->avail_cpus = f->num_cpus;
- f->offline_delay_ms = 100;
- f->task_thres = UINT_MAX;
- f->nrrun = f->num_cpus;
- INIT_LIST_HEAD(&f->lru);
- init_timer(&f->timer);
- spin_lock_init(&f->pending_lock);
- f->timer.function = core_ctl_timer_func;
- f->timer.data = first_cpu;
+ cluster->first_cpu = first_cpu;
+ cluster->min_cpus = 1;
+ cluster->max_cpus = cluster->num_cpus;
+ cluster->need_cpus = cluster->num_cpus;
+ cluster->avail_cpus = cluster->num_cpus;
+ cluster->offline_delay_ms = 100;
+ cluster->task_thres = UINT_MAX;
+ cluster->nrrun = cluster->num_cpus;
+ INIT_LIST_HEAD(&cluster->lru);
+ init_timer(&cluster->timer);
+ spin_lock_init(&cluster->pending_lock);
+ cluster->timer.function = core_ctl_timer_func;
+ cluster->timer.data = (unsigned long) cluster;
for_each_cpu(cpu, mask) {
pr_info("Init CPU%u state\n", cpu);
state = &per_cpu(cpu_state, cpu);
+ state->cluster = cluster;
state->cpu = cpu;
- state->first_cpu = first_cpu;
-
if (cpu_online(cpu)) {
- f->online_cpus++;
+ cluster->online_cpus++;
state->online = true;
}
-
- list_add_tail(&state->sib, &f->lru);
+ list_add_tail(&state->sib, &cluster->lru);
}
- f->hotplug_thread = kthread_run(try_hotplug, (void *) f,
+ cluster->hotplug_thread = kthread_run(try_hotplug, (void *) cluster,
"core_ctl/%d", first_cpu);
- sched_setscheduler_nocheck(f->hotplug_thread, SCHED_FIFO, ¶m);
+ sched_setscheduler_nocheck(cluster->hotplug_thread, SCHED_FIFO,
+ ¶m);
- for_each_cpu(cpu, mask) {
- state = &per_cpu(cpu_state, cpu);
- state->inited = true;
- }
+ cluster->inited = true;
- kobject_init(&f->kobj, &ktype_core_ctl);
- return kobject_add(&f->kobj, &dev->kobj, "core_ctl");
+ kobject_init(&cluster->kobj, &ktype_core_ctl);
+ return kobject_add(&cluster->kobj, &dev->kobj, "core_ctl");
}
static int cpufreq_policy_cb(struct notifier_block *nb, unsigned long val,
@@ -958,7 +985,7 @@
switch (val) {
case CPUFREQ_CREATE_POLICY:
- group_init(policy->related_cpus);
+ cluster_init(policy->related_cpus);
break;
}
@@ -1002,7 +1029,7 @@
for_each_online_cpu(cpu) {
policy = cpufreq_cpu_get(cpu);
if (policy) {
- group_init(policy->related_cpus);
+ cluster_init(policy->related_cpus);
cpufreq_cpu_put(policy);
}
}