cpufreq: interactive: Replace policy_timer with sched_util hook
Instead of scheduling a timer to run, let's setup an irq_work to
run every time the sched_util hook is called. Due to the way
sched_util works, the hook could be called on any CPU, so we need
to have a hook for each CPU, that funnels down to the right
policy for that CPU. We also need to ignore multiple CPU calls
for the same policy, so keep track of the irq_work that may be in
process and bail out early if one is already running.
Change-Id: I84054afe99edbbc44ac1fdfffaf9d8d4e7d1ab54
Signed-off-by: Stephen Boyd <sboyd@codeaurora.org>
diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c
index 1b8c739..12eb6d8 100644
--- a/drivers/cpufreq/cpufreq_interactive.c
+++ b/drivers/cpufreq/cpufreq_interactive.c
@@ -35,8 +35,12 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
+static DEFINE_PER_CPU(struct update_util_data, update_util);
+
struct cpufreq_interactive_policyinfo {
- struct timer_list policy_timer;
+ bool work_in_progress;
+ struct irq_work irq_work;
+ spinlock_t irq_work_lock; /* protects work_in_progress */
struct timer_list policy_slack_timer;
struct hrtimer notif_timer;
spinlock_t load_lock; /* protects load tracking stat */
@@ -215,9 +219,6 @@
pcpu->cputime_speedadj_timestamp =
pcpu->time_in_idle_timestamp;
}
- del_timer(&ppol->policy_timer);
- ppol->policy_timer.expires = expires;
- add_timer(&ppol->policy_timer);
}
if (tunables->timer_slack_val >= 0 &&
@@ -231,9 +232,51 @@
spin_unlock_irqrestore(&ppol->load_lock, flags);
}
+static void update_util_handler(struct update_util_data *data, u64 time,
+ unsigned int sched_flags)
+{
+ struct cpufreq_interactive_policyinfo *ppol;
+ unsigned long flags;
+
+ ppol = *this_cpu_ptr(&polinfo);
+ spin_lock_irqsave(&ppol->irq_work_lock, flags);
+ /*
+ * The irq-work may not be allowed to be queued up right now
+ * because work has already been queued up or is in progress.
+ */
+ if (ppol->work_in_progress ||
+ sched_flags & SCHED_CPUFREQ_INTERCLUSTER_MIG)
+ goto out;
+
+ ppol->work_in_progress = true;
+ irq_work_queue(&ppol->irq_work);
+out:
+ spin_unlock_irqrestore(&ppol->irq_work_lock, flags);
+}
+
+static inline void gov_clear_update_util(struct cpufreq_policy *policy)
+{
+ int i;
+
+ for_each_cpu(i, policy->cpus)
+ cpufreq_remove_update_util_hook(i);
+
+ synchronize_sched();
+}
+
+static void gov_set_update_util(struct cpufreq_policy *policy)
+{
+ struct update_util_data *util;
+ int cpu;
+
+ for_each_cpu(cpu, policy->cpus) {
+ util = &per_cpu(update_util, cpu);
+ cpufreq_add_update_util_hook(cpu, util, update_util_handler);
+ }
+}
+
/* The caller shall take enable_sem write semaphore to avoid any timer race.
- * The policy_timer and policy_slack_timer must be deactivated when calling
- * this function.
+ * The policy_slack_timer must be deactivated when calling this function.
*/
static void cpufreq_interactive_timer_start(
struct cpufreq_interactive_tunables *tunables, int cpu)
@@ -245,8 +288,7 @@
int i;
spin_lock_irqsave(&ppol->load_lock, flags);
- ppol->policy_timer.expires = expires;
- add_timer(&ppol->policy_timer);
+ gov_set_update_util(ppol->policy);
if (tunables->timer_slack_val >= 0 &&
ppol->target_freq > ppol->policy->min) {
expires += usecs_to_jiffies(tunables->timer_slack_val);
@@ -265,6 +307,7 @@
spin_unlock_irqrestore(&ppol->load_lock, flags);
}
+
static unsigned int freq_to_above_hispeed_delay(
struct cpufreq_interactive_tunables *tunables,
unsigned int freq)
@@ -448,7 +491,7 @@
#define NEW_TASK_RATIO 75
#define PRED_TOLERANCE_PCT 10
-static void cpufreq_interactive_timer(unsigned long data)
+static void cpufreq_interactive_timer(int data)
{
s64 now;
unsigned int delta_time;
@@ -467,7 +510,7 @@
unsigned int index;
unsigned long flags;
unsigned long max_cpu;
- int cpu, i;
+ int i, cpu;
int new_load_pct = 0;
int prev_l, pred_l = 0;
struct cpufreq_govinfo govinfo;
@@ -659,8 +702,7 @@
wake_up_process_no_notif(speedchange_task);
rearm:
- if (!timer_pending(&ppol->policy_timer))
- cpufreq_interactive_timer_resched(data, false);
+ cpufreq_interactive_timer_resched(data, false);
/*
* Send govinfo notification.
@@ -822,7 +864,6 @@
}
cpu = ppol->notif_cpu;
trace_cpufreq_interactive_load_change(cpu);
- del_timer(&ppol->policy_timer);
del_timer(&ppol->policy_slack_timer);
cpufreq_interactive_timer(cpu);
@@ -1569,6 +1610,20 @@
return tunables;
}
+static void irq_work(struct irq_work *irq_work)
+{
+ struct cpufreq_interactive_policyinfo *ppol;
+ unsigned long flags;
+
+ ppol = container_of(irq_work, struct cpufreq_interactive_policyinfo,
+ irq_work);
+
+ cpufreq_interactive_timer(smp_processor_id());
+ spin_lock_irqsave(&ppol->irq_work_lock, flags);
+ ppol->work_in_progress = false;
+ spin_unlock_irqrestore(&ppol->irq_work_lock, flags);
+}
+
static struct cpufreq_interactive_policyinfo *get_policyinfo(
struct cpufreq_policy *policy)
{
@@ -1593,12 +1648,12 @@
}
ppol->sl = sl;
- init_timer_deferrable(&ppol->policy_timer);
- ppol->policy_timer.function = cpufreq_interactive_timer;
init_timer(&ppol->policy_slack_timer);
ppol->policy_slack_timer.function = cpufreq_interactive_nop_timer;
hrtimer_init(&ppol->notif_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ppol->notif_timer.function = cpufreq_interactive_hrtimer;
+ init_irq_work(&ppol->irq_work, irq_work);
+ spin_lock_init(&ppol->irq_work_lock);
spin_lock_init(&ppol->load_lock);
spin_lock_init(&ppol->target_freq_lock);
init_rwsem(&ppol->enable_sem);
@@ -1775,9 +1830,7 @@
ppol->reject_notification = true;
ppol->notif_pending = false;
down_write(&ppol->enable_sem);
- del_timer_sync(&ppol->policy_timer);
del_timer_sync(&ppol->policy_slack_timer);
- ppol->policy_timer.data = policy->cpu;
ppol->last_evaluated_jiffy = get_jiffies_64();
cpufreq_interactive_timer_start(tunables, policy->cpu);
ppol->governor_enabled = 1;
@@ -1807,7 +1860,9 @@
down_write(&ppol->enable_sem);
ppol->governor_enabled = 0;
ppol->target_freq = 0;
- del_timer_sync(&ppol->policy_timer);
+ gov_clear_update_util(ppol->policy);
+ irq_work_sync(&ppol->irq_work);
+ ppol->work_in_progress = false;
del_timer_sync(&ppol->policy_slack_timer);
up_write(&ppol->enable_sem);
ppol->reject_notification = false;