Merge "sched/walt: improve the scheduler"
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index 5641c02..b059fd2 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -35,6 +35,7 @@
extern unsigned int sysctl_sched_group_downmigrate_pct;
extern unsigned int sysctl_sched_walt_rotate_big_tasks;
extern unsigned int sysctl_sched_min_task_util_for_boost_colocation;
+extern unsigned int sysctl_sched_little_cluster_coloc_fmin_khz;
extern int
walt_proc_update_handler(struct ctl_table *table, int write,
@@ -136,4 +137,9 @@
void __user *buffer, size_t *lenp,
loff_t *ppos);
+#ifdef CONFIG_SCHED_WALT
+extern int sched_little_cluster_coloc_fmin_khz_handler(struct ctl_table *table,
+ int write, void __user *buffer,
+ size_t *lenp, loff_t *ppos);
+#endif
#endif /* _SCHED_SYSCTL_H */
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index b3ec962..be3d8b2 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -596,8 +596,14 @@
TRACE_EVENT(sched_load_to_gov,
- TP_PROTO(struct rq *rq, u64 aggr_grp_load, u32 tt_load, u64 freq_aggr_thresh, u64 load, int policy, int big_task_rotation),
- TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr_thresh, load, policy, big_task_rotation),
+ TP_PROTO(struct rq *rq, u64 aggr_grp_load, u32 tt_load,
+ u64 freq_aggr_thresh, u64 load, int policy,
+ int big_task_rotation,
+ unsigned int sysctl_sched_little_cluster_coloc_fmin_khz,
+ u64 coloc_boost_load),
+ TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr_thresh, load, policy,
+ big_task_rotation, sysctl_sched_little_cluster_coloc_fmin_khz,
+ coloc_boost_load),
TP_STRUCT__entry(
__field( int, cpu )
@@ -613,6 +619,9 @@
__field( u64, pl )
__field( u64, load )
__field( int, big_task_rotation )
+ __field(unsigned int,
+ sysctl_sched_little_cluster_coloc_fmin_khz)
+ __field( u64, coloc_boost_load )
),
TP_fast_assign(
@@ -629,14 +638,19 @@
__entry->pl = rq->walt_stats.pred_demands_sum;
__entry->load = load;
__entry->big_task_rotation = big_task_rotation;
+ __entry->sysctl_sched_little_cluster_coloc_fmin_khz =
+ sysctl_sched_little_cluster_coloc_fmin_khz;
+ __entry->coloc_boost_load = coloc_boost_load;
),
- TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr_thresh=%llu tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d",
+ TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr_thresh=%llu tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d sysctl_sched_little_cluster_coloc_fmin_khz=%u coloc_boost_load=%llu",
__entry->cpu, __entry->policy, __entry->ed_task_pid,
__entry->aggr_grp_load, __entry->freq_aggr_thresh,
__entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps,
__entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load,
- __entry->big_task_rotation)
+ __entry->big_task_rotation,
+ __entry->sysctl_sched_little_cluster_coloc_fmin_khz,
+ __entry->coloc_boost_load)
);
#endif
diff --git a/kernel/sched/energy.c b/kernel/sched/energy.c
index 01daf82..08ada30 100644
--- a/kernel/sched/energy.c
+++ b/kernel/sched/energy.c
@@ -301,6 +301,8 @@
walt_sched_energy_populated_callback();
}
+ walt_map_freq_to_load();
+
dev_info(&pdev->dev, "Sched-energy-costs capacity updated\n");
return 0;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 6a7c485..9942e02 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -86,6 +86,7 @@
int notifier_sent;
bool wake_up_idle;
u64 aggr_grp_load;
+ u64 coloc_boost_load;
};
extern unsigned int sched_disable_window_stats;
@@ -2720,6 +2721,13 @@
return boost_on_big;
}
+extern void walt_map_freq_to_load(void);
+
+static inline bool is_min_capacity_cluster(struct sched_cluster *cluster)
+{
+ return is_min_capacity_cpu(cluster_first_cpu(cluster));
+}
+
#else /* CONFIG_SCHED_WALT */
struct walt_sched_stats;
@@ -2857,6 +2865,7 @@
static inline void walt_sched_energy_populated_callback(void) { }
static inline void walt_update_min_max_capacity(void) { }
+static inline void walt_map_freq_to_load(void) { }
#endif /* CONFIG_SCHED_WALT */
static inline bool energy_aware(void)
diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c
index 28a797b..48f64aa 100644
--- a/kernel/sched/walt.c
+++ b/kernel/sched/walt.c
@@ -523,6 +523,7 @@
struct sched_cluster *cluster = rq->cluster;
u64 aggr_grp_load = cluster->aggr_grp_load;
u64 load, tt_load = 0;
+ u64 coloc_boost_load = cluster->coloc_boost_load;
if (rq->ed_task != NULL) {
load = sched_ravg_window;
@@ -534,6 +535,9 @@
else
load = rq->prev_runnable_sum + rq->grp_time.prev_runnable_sum;
+ if (coloc_boost_load)
+ load = max_t(u64, load, coloc_boost_load);
+
tt_load = top_task_load(rq);
switch (reporting_policy) {
case FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK:
@@ -550,7 +554,9 @@
done:
trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, freq_aggr_thresh,
- load, reporting_policy, walt_rotation_enabled);
+ load, reporting_policy, walt_rotation_enabled,
+ sysctl_sched_little_cluster_coloc_fmin_khz,
+ coloc_boost_load);
return load;
}
@@ -2385,6 +2391,7 @@
.notifier_sent = 0,
.wake_up_idle = 0,
.aggr_grp_load = 0,
+ .coloc_boost_load = 0,
};
void init_clusters(void)
@@ -3160,6 +3167,70 @@
BUG_ON((s64)*src_nt_prev_runnable_sum < 0);
}
+unsigned int sysctl_sched_little_cluster_coloc_fmin_khz;
+static u64 coloc_boost_load;
+
+void walt_map_freq_to_load(void)
+{
+ struct sched_cluster *cluster;
+
+ for_each_sched_cluster(cluster) {
+ if (is_min_capacity_cluster(cluster)) {
+ int fcpu = cluster_first_cpu(cluster);
+
+ coloc_boost_load = div64_u64(
+ ((u64)sched_ravg_window *
+ arch_scale_cpu_capacity(NULL, fcpu) *
+ sysctl_sched_little_cluster_coloc_fmin_khz),
+ (u64)1024 * cpu_max_possible_freq(fcpu));
+ coloc_boost_load = div64_u64(coloc_boost_load << 2, 5);
+ break;
+ }
+ }
+}
+
+static void walt_update_coloc_boost_load(void)
+{
+ struct related_thread_group *grp;
+ struct sched_cluster *cluster;
+
+ if (!sysctl_sched_little_cluster_coloc_fmin_khz ||
+ sysctl_sched_boost == CONSERVATIVE_BOOST)
+ return;
+
+ grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID);
+ if (!grp || !grp->preferred_cluster ||
+ is_min_capacity_cluster(grp->preferred_cluster))
+ return;
+
+ for_each_sched_cluster(cluster) {
+ if (is_min_capacity_cluster(cluster)) {
+ cluster->coloc_boost_load = coloc_boost_load;
+ break;
+ }
+ }
+}
+
+int sched_little_cluster_coloc_fmin_khz_handler(struct ctl_table *table,
+ int write, void __user *buffer, size_t *lenp,
+ loff_t *ppos)
+{
+ int ret;
+ static DEFINE_MUTEX(mutex);
+
+ mutex_lock(&mutex);
+
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ if (ret || !write)
+ goto done;
+
+ walt_map_freq_to_load();
+
+done:
+ mutex_unlock(&mutex);
+ return ret;
+}
+
/*
* Runs in hard-irq context. This should ideally run just after the latest
* window roll-over.
@@ -3169,7 +3240,7 @@
struct sched_cluster *cluster;
struct rq *rq;
int cpu;
- u64 wc;
+ u64 wc, total_grp_load = 0;
int flag = SCHED_CPUFREQ_WALT;
bool is_migration = false;
int level = 0;
@@ -3205,10 +3276,15 @@
}
cluster->aggr_grp_load = aggr_grp_load;
+ total_grp_load = aggr_grp_load;
+ cluster->coloc_boost_load = 0;
raw_spin_unlock(&cluster->load_lock);
}
+ if (total_grp_load)
+ walt_update_coloc_boost_load();
+
for_each_sched_cluster(cluster) {
for_each_cpu(cpu, &cluster->cpus) {
int nflag = flag;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index d5e7fe5..1e49ef6 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -128,6 +128,9 @@
static unsigned long one_ul = 1;
static int one_hundred = 100;
static int __maybe_unused one_thousand = 1000;
+#ifdef CONFIG_SCHED_WALT
+static int two_million = 2000000;
+#endif
#ifdef CONFIG_PRINTK
static int ten_thousand = 10000;
#endif
@@ -364,6 +367,15 @@
.extra1 = &zero,
.extra2 = &one_thousand,
},
+ {
+ .procname = "sched_little_cluster_coloc_fmin_khz",
+ .data = &sysctl_sched_little_cluster_coloc_fmin_khz,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = sched_little_cluster_coloc_fmin_khz_handler,
+ .extra1 = &zero,
+ .extra2 = &two_million,
+ },
#endif
{
.procname = "sched_upmigrate",