drivers/cpufreq/cpufreq_interactive.c - kernel/msm-4.9 - Gitiles

 /*
  * drivers/cpufreq/cpufreq_interactive.c
  *
  * Copyright (C) 2010 Google, Inc.
  *
  * This software is licensed under the terms of the GNU General Public
  * License version 2, as published by the Free Software Foundation, and
  * may be copied, distributed, and modified under those terms.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * Author: Mike Chan (mike@android.com)
  *
  */

 #include <linux/cpu.h>
 #include <linux/cpumask.h>
 #include <linux/cpufreq.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/sched/rt.h>
 #include <linux/tick.h>
 #include <linux/time.h>
 #include <linux/timer.h>
 #include <linux/workqueue.h>
 #include <linux/kthread.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <asm/cputime.h>

 #define CREATE_TRACE_POINTS
 #include <trace/events/cpufreq_interactive.h>

 static atomic_t active_count = ATOMIC_INIT(0);

 struct cpufreq_interactive_cpuinfo {
 	struct timer_list cpu_timer;
 	int timer_idlecancel;
 	u64 time_in_idle;
 	u64 idle_exit_time;
 	u64 target_set_time;
 	u64 target_set_time_in_idle;
 	struct cpufreq_policy *policy;
 	struct cpufreq_frequency_table *freq_table;
 	unsigned int target_freq;
 	unsigned int floor_freq;
 	u64 floor_validate_time;
 	u64 hispeed_validate_time;
 	int governor_enabled;
 };

 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);

 /* realtime thread handles frequency scaling */
 static struct task_struct *speedchange_task;
 static cpumask_t speedchange_cpumask;
 static spinlock_t speedchange_cpumask_lock;

 /* Hi speed to bump to from lo speed when load burst (default max) */
 static unsigned int hispeed_freq;

 /* Go to hi speed when CPU load at or above this value. */
 #define DEFAULT_GO_HISPEED_LOAD 85
 static unsigned long go_hispeed_load;

 /*
  * The minimum amount of time to spend at a frequency before we can ramp down.
  */
 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
 static unsigned long min_sample_time;

 /*
  * The sample rate of the timer used to increase frequency
  */
 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
 static unsigned long timer_rate;

 /*
  * Wait this long before raising speed above hispeed, by default a single
  * timer interval.
  */
 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
 static unsigned long above_hispeed_delay_val;

 /*
  * Non-zero means longer-term speed boost active.
  */

 static int boost_val;

 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 		unsigned int event);

 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 static
 #endif
 struct cpufreq_governor cpufreq_gov_interactive = {
 	.name = "interactive",
 	.governor = cpufreq_governor_interactive,
 	.max_transition_latency = 10000000,
 	.owner = THIS_MODULE,
 };

 static void cpufreq_interactive_timer(unsigned long data)
 {
 	u64 now;
 	unsigned int delta_idle;
 	unsigned int delta_time;
 	int cpu_load;
 	int load_since_change;
 	u64 time_in_idle;
 	u64 idle_exit_time;
 	struct cpufreq_interactive_cpuinfo *pcpu =
 		&per_cpu(cpuinfo, data);
 	u64 now_idle;
 	unsigned int new_freq;
 	unsigned int index;
 	unsigned long flags;

 	smp_rmb();

 	if (!pcpu->governor_enabled)
 		goto exit;

 	time_in_idle = pcpu->time_in_idle;
 	idle_exit_time = pcpu->idle_exit_time;
 	now_idle = get_cpu_idle_time_us(data, &now);
 	delta_idle = (unsigned int)(now_idle - time_in_idle);
 	delta_time = (unsigned int)(now - idle_exit_time);

 	/*
 	 * If timer ran less than 1ms after short-term sample started, retry.
 	 */
 	if (delta_time < 1000)
 		goto rearm;

 	if (delta_idle > delta_time)
 		cpu_load = 0;
 	else
 		cpu_load = 100 * (delta_time - delta_idle) / delta_time;

 	delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 	delta_time = (unsigned int)(now - pcpu->target_set_time);

 	if ((delta_time == 0) || (delta_idle > delta_time))
 		load_since_change = 0;
 	else
 		load_since_change =
 			100 * (delta_time - delta_idle) / delta_time;

 	/*
 	 * Choose greater of short-term load (since last idle timer
 	 * started or timer function re-armed itself) or long-term load
 	 * (since last frequency change).
 	 */
 	if (load_since_change > cpu_load)
 		cpu_load = load_since_change;

 	if (cpu_load >= go_hispeed_load || boost_val) {
 		if (pcpu->target_freq < hispeed_freq &&
 		    hispeed_freq < pcpu->policy->max) {
 			new_freq = hispeed_freq;
 		} else {
 			new_freq = pcpu->policy->max * cpu_load / 100;

 			if (new_freq < hispeed_freq)
 				new_freq = hispeed_freq;

 			if (pcpu->target_freq == hispeed_freq &&
 			    new_freq > hispeed_freq &&
 			    now - pcpu->hispeed_validate_time
 			    < above_hispeed_delay_val) {
 				trace_cpufreq_interactive_notyet(data, cpu_load,
 								 pcpu->target_freq,
 								 new_freq);
 				goto rearm;
 			}
 		}
 	} else {
 		new_freq = hispeed_freq * cpu_load / 100;
 	}

 	if (new_freq <= hispeed_freq)
 		pcpu->hispeed_validate_time = now;

 	if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 					   new_freq, CPUFREQ_RELATION_H,
 					   &index)) {
 		pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 			     (int) data);
 		goto rearm;
 	}

 	new_freq = pcpu->freq_table[index].frequency;

 	/*
 	 * Do not scale below floor_freq unless we have been at or above the
 	 * floor frequency for the minimum sample time since last validated.
 	 */
 	if (new_freq < pcpu->floor_freq) {
 		if (now - pcpu->floor_validate_time < min_sample_time) {
 			trace_cpufreq_interactive_notyet(data, cpu_load,
 					 pcpu->target_freq, new_freq);
 			goto rearm;
 		}
 	}

 	pcpu->floor_freq = new_freq;
 	pcpu->floor_validate_time = now;

 	if (pcpu->target_freq == new_freq) {
 		trace_cpufreq_interactive_already(data, cpu_load,
 						  pcpu->target_freq, new_freq);
 		goto rearm_if_notmax;
 	}

 	trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 					 new_freq);
 	pcpu->target_set_time_in_idle = now_idle;
 	pcpu->target_set_time = now;

 	pcpu->target_freq = new_freq;
 	spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 	cpumask_set_cpu(data, &speedchange_cpumask);
 	spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 	wake_up_process(speedchange_task);

 rearm_if_notmax:
 	/*
 	 * Already set max speed and don't see a need to change that,
 	 * wait until next idle to re-evaluate, don't need timer.
 	 */
 	if (pcpu->target_freq == pcpu->policy->max)
 		goto exit;

 rearm:
 	if (!timer_pending(&pcpu->cpu_timer)) {
 		/*
 		 * If already at min, cancel the timer if that CPU goes idle.
 		 * We don't need to re-evaluate speed until the next idle exit.
 		 */
 		if (pcpu->target_freq == pcpu->policy->min)
 			pcpu->timer_idlecancel = 1;

 		pcpu->time_in_idle = get_cpu_idle_time_us(
 			data, &pcpu->idle_exit_time);
 		mod_timer_pinned(&pcpu->cpu_timer,
 				 jiffies + usecs_to_jiffies(timer_rate));
 	}

 exit:
 	return;
 }

 static void cpufreq_interactive_idle_start(void)
 {
 	struct cpufreq_interactive_cpuinfo *pcpu =
 		&per_cpu(cpuinfo, smp_processor_id());
 	int pending;

 	if (!pcpu->governor_enabled)
 		return;

 	pending = timer_pending(&pcpu->cpu_timer);

 	if (pcpu->target_freq != pcpu->policy->min) {
 #ifdef CONFIG_SMP
 		/*
 		 * Entering idle while not at lowest speed.  On some
 		 * platforms this can hold the other CPU(s) at that speed
 		 * even though the CPU is idle. Set a timer to re-evaluate
 		 * speed so this idle CPU doesn't hold the other CPUs above
 		 * min indefinitely.  This should probably be a quirk of
 		 * the CPUFreq driver.
 		 */
 		if (!pending) {
 			pcpu->time_in_idle = get_cpu_idle_time_us(
 				smp_processor_id(), &pcpu->idle_exit_time);
 			pcpu->timer_idlecancel = 0;
 			mod_timer_pinned(
 				&pcpu->cpu_timer,
 				jiffies + usecs_to_jiffies(timer_rate));
 		}
 #endif
 	} else {
 		/*
 		 * If at min speed and entering idle after load has
 		 * already been evaluated, and a timer has been set just in
 		 * case the CPU suddenly goes busy, cancel that timer.  The
 		 * CPU didn't go busy; we'll recheck things upon idle exit.
 		 */
 		if (pending && pcpu->timer_idlecancel) {
 			del_timer(&pcpu->cpu_timer);
 			pcpu->timer_idlecancel = 0;
 		}
 	}

 }

 static void cpufreq_interactive_idle_end(void)
 {
 	struct cpufreq_interactive_cpuinfo *pcpu =
 		&per_cpu(cpuinfo, smp_processor_id());

 	if (!pcpu->governor_enabled)
 		return;

 	/* Arm the timer for 1-2 ticks later if not already. */
 	if (!timer_pending(&pcpu->cpu_timer)) {
 		pcpu->time_in_idle =
 			get_cpu_idle_time_us(smp_processor_id(),
 					     &pcpu->idle_exit_time);
 		pcpu->timer_idlecancel = 0;
 		mod_timer_pinned(
 			&pcpu->cpu_timer,
 			jiffies + usecs_to_jiffies(timer_rate));
 	}

 }

 static int cpufreq_interactive_speedchange_task(void *data)
 {
 	unsigned int cpu;
 	cpumask_t tmp_mask;
 	unsigned long flags;
 	struct cpufreq_interactive_cpuinfo *pcpu;

 	while (1) {
 		set_current_state(TASK_INTERRUPTIBLE);
 		spin_lock_irqsave(&speedchange_cpumask_lock, flags);

 		if (cpumask_empty(&speedchange_cpumask)) {
 			spin_unlock_irqrestore(&speedchange_cpumask_lock,
 					       flags);
 			schedule();

 			if (kthread_should_stop())
 				break;

 			spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 		}

 		set_current_state(TASK_RUNNING);
 		tmp_mask = speedchange_cpumask;
 		cpumask_clear(&speedchange_cpumask);
 		spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);

 		for_each_cpu(cpu, &tmp_mask) {
 			unsigned int j;
 			unsigned int max_freq = 0;

 			pcpu = &per_cpu(cpuinfo, cpu);
 			smp_rmb();

 			if (!pcpu->governor_enabled)
 				continue;

 			for_each_cpu(j, pcpu->policy->cpus) {
 				struct cpufreq_interactive_cpuinfo *pjcpu =
 					&per_cpu(cpuinfo, j);

 				if (pjcpu->target_freq > max_freq)
 					max_freq = pjcpu->target_freq;
 			}

 			if (max_freq != pcpu->policy->cur)
 				__cpufreq_driver_target(pcpu->policy,
 							max_freq,
 							CPUFREQ_RELATION_H);
 			trace_cpufreq_interactive_setspeed(cpu,
 						     pcpu->target_freq,
 						     pcpu->policy->cur);
 		}
 	}

 	return 0;
 }

 static void cpufreq_interactive_boost(void)
 {
 	int i;
 	int anyboost = 0;
 	unsigned long flags;
 	struct cpufreq_interactive_cpuinfo *pcpu;

 	spin_lock_irqsave(&speedchange_cpumask_lock, flags);

 	for_each_online_cpu(i) {
 		pcpu = &per_cpu(cpuinfo, i);

 		if (pcpu->target_freq < hispeed_freq) {
 			pcpu->target_freq = hispeed_freq;
 			cpumask_set_cpu(i, &speedchange_cpumask);
 			pcpu->target_set_time_in_idle =
 				get_cpu_idle_time_us(i, &pcpu->target_set_time);
 			pcpu->hispeed_validate_time = pcpu->target_set_time;
 			anyboost = 1;
 		}

 		/*
 		 * Set floor freq and (re)start timer for when last
 		 * validated.
 		 */

 		pcpu->floor_freq = hispeed_freq;
 		pcpu->floor_validate_time = ktime_to_us(ktime_get());
 	}

 	spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);

 	if (anyboost)
 		wake_up_process(speedchange_task);
 }

 static ssize_t show_hispeed_freq(struct kobject *kobj,
 				 struct attribute *attr, char *buf)
 {
 	return sprintf(buf, "%u\n", hispeed_freq);
 }

 static ssize_t store_hispeed_freq(struct kobject *kobj,
 				  struct attribute *attr, const char *buf,
 				  size_t count)
 {
 	int ret;
 	long unsigned int val;

 	ret = strict_strtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;
 	hispeed_freq = val;
 	return count;
 }

 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 		show_hispeed_freq, store_hispeed_freq);


 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 				     struct attribute *attr, char *buf)
 {
 	return sprintf(buf, "%lu\n", go_hispeed_load);
 }

 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 			struct attribute *attr, const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = strict_strtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;
 	go_hispeed_load = val;
 	return count;
 }

 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 		show_go_hispeed_load, store_go_hispeed_load);

 static ssize_t show_min_sample_time(struct kobject *kobj,
 				struct attribute *attr, char *buf)
 {
 	return sprintf(buf, "%lu\n", min_sample_time);
 }

 static ssize_t store_min_sample_time(struct kobject *kobj,
 			struct attribute *attr, const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = strict_strtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;
 	min_sample_time = val;
 	return count;
 }

 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 		show_min_sample_time, store_min_sample_time);

 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 					struct attribute *attr, char *buf)
 {
 	return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 }

 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 					 struct attribute *attr,
 					 const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = strict_strtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;
 	above_hispeed_delay_val = val;
 	return count;
 }

 define_one_global_rw(above_hispeed_delay);

 static ssize_t show_timer_rate(struct kobject *kobj,
 			struct attribute *attr, char *buf)
 {
 	return sprintf(buf, "%lu\n", timer_rate);
 }

 static ssize_t store_timer_rate(struct kobject *kobj,
 			struct attribute *attr, const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = strict_strtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;
 	timer_rate = val;
 	return count;
 }

 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 		show_timer_rate, store_timer_rate);

 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 			  char *buf)
 {
 	return sprintf(buf, "%d\n", boost_val);
 }

 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 			   const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = kstrtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;

 	boost_val = val;

 	if (boost_val) {
 		trace_cpufreq_interactive_boost("on");
 		cpufreq_interactive_boost();
 	} else {
 		trace_cpufreq_interactive_unboost("off");
 	}

 	return count;
 }

 define_one_global_rw(boost);

 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 				const char *buf, size_t count)
 {
 	int ret;
 	unsigned long val;

 	ret = kstrtoul(buf, 0, &val);
 	if (ret < 0)
 		return ret;

 	trace_cpufreq_interactive_boost("pulse");
 	cpufreq_interactive_boost();
 	return count;
 }

 static struct global_attr boostpulse =
 	__ATTR(boostpulse, 0200, NULL, store_boostpulse);

 static struct attribute *interactive_attributes[] = {
 	&hispeed_freq_attr.attr,
 	&go_hispeed_load_attr.attr,
 	&above_hispeed_delay.attr,
 	&min_sample_time_attr.attr,
 	&timer_rate_attr.attr,
 	&boost.attr,
 	&boostpulse.attr,
 	NULL,
 };

 static struct attribute_group interactive_attr_group = {
 	.attrs = interactive_attributes,
 	.name = "interactive",
 };

 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 					     unsigned long val,
 					     void *data)
 {
 	switch (val) {
 	case IDLE_START:
 		cpufreq_interactive_idle_start();
 		break;
 	case IDLE_END:
 		cpufreq_interactive_idle_end();
 		break;
 	}

 	return 0;
 }

 static struct notifier_block cpufreq_interactive_idle_nb = {
 	.notifier_call = cpufreq_interactive_idle_notifier,
 };

 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 		unsigned int event)
 {
 	int rc;
 	unsigned int j;
 	struct cpufreq_interactive_cpuinfo *pcpu;
 	struct cpufreq_frequency_table *freq_table;

 	switch (event) {
 	case CPUFREQ_GOV_START:
 		if (!cpu_online(policy->cpu))
 			return -EINVAL;

 		freq_table =
 			cpufreq_frequency_get_table(policy->cpu);
 		if (!hispeed_freq)
 			hispeed_freq = policy->max;

 		for_each_cpu(j, policy->cpus) {
 			pcpu = &per_cpu(cpuinfo, j);
 			pcpu->policy = policy;
 			pcpu->target_freq = policy->cur;
 			pcpu->freq_table = freq_table;
 			pcpu->target_set_time_in_idle =
 				get_cpu_idle_time_us(j,
 					     &pcpu->target_set_time);
 			pcpu->floor_freq = pcpu->target_freq;
 			pcpu->floor_validate_time =
 				pcpu->target_set_time;
 			pcpu->hispeed_validate_time =
 				pcpu->target_set_time;
 			pcpu->governor_enabled = 1;
 			smp_wmb();
 			pcpu->cpu_timer.expires =
 				jiffies + usecs_to_jiffies(timer_rate);
 			add_timer_on(&pcpu->cpu_timer, j);
 		}

 		/*
 		 * Do not register the idle hook and create sysfs
 		 * entries if we have already done so.
 		 */
 		if (atomic_inc_return(&active_count) > 1)
 			return 0;

 		rc = sysfs_create_group(cpufreq_global_kobject,
 				&interactive_attr_group);
 		if (rc)
 			return rc;

 		idle_notifier_register(&cpufreq_interactive_idle_nb);
 		break;

 	case CPUFREQ_GOV_STOP:
 		for_each_cpu(j, policy->cpus) {
 			pcpu = &per_cpu(cpuinfo, j);
 			pcpu->governor_enabled = 0;
 			smp_wmb();
 			del_timer_sync(&pcpu->cpu_timer);
 		}

 		if (atomic_dec_return(&active_count) > 0)
 			return 0;

 		idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 		sysfs_remove_group(cpufreq_global_kobject,
 				&interactive_attr_group);

 		break;

 	case CPUFREQ_GOV_LIMITS:
 		if (policy->max < policy->cur)
 			__cpufreq_driver_target(policy,
 					policy->max, CPUFREQ_RELATION_H);
 		else if (policy->min > policy->cur)
 			__cpufreq_driver_target(policy,
 					policy->min, CPUFREQ_RELATION_L);
 		break;
 	}
 	return 0;
 }

 static int __init cpufreq_interactive_init(void)
 {
 	unsigned int i;
 	struct cpufreq_interactive_cpuinfo *pcpu;
 	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

 	go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 	min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 	above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 	timer_rate = DEFAULT_TIMER_RATE;

 	/* Initalize per-cpu timers */
 	for_each_possible_cpu(i) {
 		pcpu = &per_cpu(cpuinfo, i);
 		init_timer(&pcpu->cpu_timer);
 		pcpu->cpu_timer.function = cpufreq_interactive_timer;
 		pcpu->cpu_timer.data = i;
 	}

 	spin_lock_init(&speedchange_cpumask_lock);
 	speedchange_task =
 		kthread_create(cpufreq_interactive_speedchange_task, NULL,
 			       "cfinteractive");
 	if (IS_ERR(speedchange_task))
 		return PTR_ERR(speedchange_task);

 	sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 	get_task_struct(speedchange_task);

 	/* NB: wake up so the thread does not look hung to the freezer */
 	wake_up_process(speedchange_task);

 	return cpufreq_register_governor(&cpufreq_gov_interactive);
 }

 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 fs_initcall(cpufreq_interactive_init);
 #else
 module_init(cpufreq_interactive_init);
 #endif

 static void __exit cpufreq_interactive_exit(void)
 {
 	cpufreq_unregister_governor(&cpufreq_gov_interactive);
 	kthread_stop(speedchange_task);
 	put_task_struct(speedchange_task);
 }

 module_exit(cpufreq_interactive_exit);

 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 	"Latency sensitive workloads");
 MODULE_LICENSE("GPL");
	/*
	* drivers/cpufreq/cpufreq_interactive.c
	*
	* Copyright (C) 2010 Google, Inc.
	*
	* This software is licensed under the terms of the GNU General Public
	* License version 2, as published by the Free Software Foundation, and
	* may be copied, distributed, and modified under those terms.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* Author: Mike Chan (mike@android.com)
	*
	*/

	#include <linux/cpu.h>
	#include <linux/cpumask.h>
	#include <linux/cpufreq.h>
	#include <linux/module.h>
	#include <linux/mutex.h>
	#include <linux/sched.h>
	#include <linux/sched/rt.h>
	#include <linux/tick.h>
	#include <linux/time.h>
	#include <linux/timer.h>
	#include <linux/workqueue.h>
	#include <linux/kthread.h>
	#include <linux/mutex.h>
	#include <linux/slab.h>
	#include <asm/cputime.h>

	#define CREATE_TRACE_POINTS
	#include <trace/events/cpufreq_interactive.h>

	static atomic_t active_count = ATOMIC_INIT(0);

	struct cpufreq_interactive_cpuinfo {
	struct timer_list cpu_timer;
	int timer_idlecancel;
	u64 time_in_idle;
	u64 idle_exit_time;
	u64 target_set_time;
	u64 target_set_time_in_idle;
	struct cpufreq_policy *policy;
	struct cpufreq_frequency_table *freq_table;
	unsigned int target_freq;
	unsigned int floor_freq;
	u64 floor_validate_time;
	u64 hispeed_validate_time;
	int governor_enabled;
	};

	static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);

	/* realtime thread handles frequency scaling */
	static struct task_struct *speedchange_task;
	static cpumask_t speedchange_cpumask;
	static spinlock_t speedchange_cpumask_lock;

	/* Hi speed to bump to from lo speed when load burst (default max) */
	static unsigned int hispeed_freq;

	/* Go to hi speed when CPU load at or above this value. */
	#define DEFAULT_GO_HISPEED_LOAD 85
	static unsigned long go_hispeed_load;

	/*
	* The minimum amount of time to spend at a frequency before we can ramp down.
	*/
	#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
	static unsigned long min_sample_time;

	/*
	* The sample rate of the timer used to increase frequency
	*/
	#define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
	static unsigned long timer_rate;

	/*
	* Wait this long before raising speed above hispeed, by default a single
	* timer interval.
	*/
	#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
	static unsigned long above_hispeed_delay_val;

	/*
	* Non-zero means longer-term speed boost active.
	*/

	static int boost_val;

	static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
	unsigned int event);

	#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
	static
	#endif
	struct cpufreq_governor cpufreq_gov_interactive = {
	.name = "interactive",
	.governor = cpufreq_governor_interactive,
	.max_transition_latency = 10000000,
	.owner = THIS_MODULE,
	};

	static void cpufreq_interactive_timer(unsigned long data)
	{
	u64 now;
	unsigned int delta_idle;
	unsigned int delta_time;
	int cpu_load;
	int load_since_change;
	u64 time_in_idle;
	u64 idle_exit_time;
	struct cpufreq_interactive_cpuinfo *pcpu =
	&per_cpu(cpuinfo, data);
	u64 now_idle;
	unsigned int new_freq;
	unsigned int index;
	unsigned long flags;

	smp_rmb();

	if (!pcpu->governor_enabled)
	goto exit;

	time_in_idle = pcpu->time_in_idle;
	idle_exit_time = pcpu->idle_exit_time;
	now_idle = get_cpu_idle_time_us(data, &now);
	delta_idle = (unsigned int)(now_idle - time_in_idle);
	delta_time = (unsigned int)(now - idle_exit_time);

	/*
	* If timer ran less than 1ms after short-term sample started, retry.
	*/
	if (delta_time < 1000)
	goto rearm;

	if (delta_idle > delta_time)
	cpu_load = 0;
	else
	cpu_load = 100 * (delta_time - delta_idle) / delta_time;

	delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
	delta_time = (unsigned int)(now - pcpu->target_set_time);

	if ((delta_time == 0) \|\| (delta_idle > delta_time))
	load_since_change = 0;
	else
	load_since_change =
	100 * (delta_time - delta_idle) / delta_time;

	/*
	* Choose greater of short-term load (since last idle timer
	* started or timer function re-armed itself) or long-term load
	* (since last frequency change).
	*/
	if (load_since_change > cpu_load)
	cpu_load = load_since_change;

	if (cpu_load >= go_hispeed_load \|\| boost_val) {
	if (pcpu->target_freq < hispeed_freq &&
	hispeed_freq < pcpu->policy->max) {
	new_freq = hispeed_freq;
	} else {
	new_freq = pcpu->policy->max * cpu_load / 100;

	if (new_freq < hispeed_freq)
	new_freq = hispeed_freq;

	if (pcpu->target_freq == hispeed_freq &&
	new_freq > hispeed_freq &&
	now - pcpu->hispeed_validate_time
	< above_hispeed_delay_val) {
	trace_cpufreq_interactive_notyet(data, cpu_load,
	pcpu->target_freq,
	new_freq);
	goto rearm;
	}
	}
	} else {
	new_freq = hispeed_freq * cpu_load / 100;
	}

	if (new_freq <= hispeed_freq)
	pcpu->hispeed_validate_time = now;

	if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
	new_freq, CPUFREQ_RELATION_H,
	&index)) {
	pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
	(int) data);
	goto rearm;
	}

	new_freq = pcpu->freq_table[index].frequency;

	/*
	* Do not scale below floor_freq unless we have been at or above the
	* floor frequency for the minimum sample time since last validated.
	*/
	if (new_freq < pcpu->floor_freq) {
	if (now - pcpu->floor_validate_time < min_sample_time) {
	trace_cpufreq_interactive_notyet(data, cpu_load,
	pcpu->target_freq, new_freq);
	goto rearm;
	}
	}

	pcpu->floor_freq = new_freq;
	pcpu->floor_validate_time = now;

	if (pcpu->target_freq == new_freq) {
	trace_cpufreq_interactive_already(data, cpu_load,
	pcpu->target_freq, new_freq);
	goto rearm_if_notmax;
	}

	trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
	new_freq);
	pcpu->target_set_time_in_idle = now_idle;
	pcpu->target_set_time = now;

	pcpu->target_freq = new_freq;
	spin_lock_irqsave(&speedchange_cpumask_lock, flags);
	cpumask_set_cpu(data, &speedchange_cpumask);
	spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
	wake_up_process(speedchange_task);

	rearm_if_notmax:
	/*
	* Already set max speed and don't see a need to change that,
	* wait until next idle to re-evaluate, don't need timer.
	*/
	if (pcpu->target_freq == pcpu->policy->max)
	goto exit;

	rearm:
	if (!timer_pending(&pcpu->cpu_timer)) {
	/*
	* If already at min, cancel the timer if that CPU goes idle.
	* We don't need to re-evaluate speed until the next idle exit.
	*/
	if (pcpu->target_freq == pcpu->policy->min)
	pcpu->timer_idlecancel = 1;

	pcpu->time_in_idle = get_cpu_idle_time_us(
	data, &pcpu->idle_exit_time);
	mod_timer_pinned(&pcpu->cpu_timer,
	jiffies + usecs_to_jiffies(timer_rate));
	}

	exit:
	return;
	}

	static void cpufreq_interactive_idle_start(void)
	{
	struct cpufreq_interactive_cpuinfo *pcpu =
	&per_cpu(cpuinfo, smp_processor_id());
	int pending;

	if (!pcpu->governor_enabled)
	return;

	pending = timer_pending(&pcpu->cpu_timer);

	if (pcpu->target_freq != pcpu->policy->min) {
	#ifdef CONFIG_SMP
	/*
	* Entering idle while not at lowest speed. On some
	* platforms this can hold the other CPU(s) at that speed
	* even though the CPU is idle. Set a timer to re-evaluate
	* speed so this idle CPU doesn't hold the other CPUs above
	* min indefinitely. This should probably be a quirk of
	* the CPUFreq driver.
	*/
	if (!pending) {
	pcpu->time_in_idle = get_cpu_idle_time_us(
	smp_processor_id(), &pcpu->idle_exit_time);
	pcpu->timer_idlecancel = 0;
	mod_timer_pinned(
	&pcpu->cpu_timer,
	jiffies + usecs_to_jiffies(timer_rate));
	}
	#endif
	} else {
	/*
	* If at min speed and entering idle after load has
	* already been evaluated, and a timer has been set just in
	* case the CPU suddenly goes busy, cancel that timer. The
	* CPU didn't go busy; we'll recheck things upon idle exit.
	*/
	if (pending && pcpu->timer_idlecancel) {
	del_timer(&pcpu->cpu_timer);
	pcpu->timer_idlecancel = 0;
	}
	}

	}

	static void cpufreq_interactive_idle_end(void)
	{
	struct cpufreq_interactive_cpuinfo *pcpu =
	&per_cpu(cpuinfo, smp_processor_id());

	if (!pcpu->governor_enabled)
	return;

	/* Arm the timer for 1-2 ticks later if not already. */
	if (!timer_pending(&pcpu->cpu_timer)) {
	pcpu->time_in_idle =
	get_cpu_idle_time_us(smp_processor_id(),
	&pcpu->idle_exit_time);
	pcpu->timer_idlecancel = 0;
	mod_timer_pinned(
	&pcpu->cpu_timer,
	jiffies + usecs_to_jiffies(timer_rate));
	}

	}

	static int cpufreq_interactive_speedchange_task(void *data)
	{
	unsigned int cpu;
	cpumask_t tmp_mask;
	unsigned long flags;
	struct cpufreq_interactive_cpuinfo *pcpu;

	while (1) {
	set_current_state(TASK_INTERRUPTIBLE);
	spin_lock_irqsave(&speedchange_cpumask_lock, flags);

	if (cpumask_empty(&speedchange_cpumask)) {
	spin_unlock_irqrestore(&speedchange_cpumask_lock,
	flags);
	schedule();

	if (kthread_should_stop())
	break;

	spin_lock_irqsave(&speedchange_cpumask_lock, flags);
	}

	set_current_state(TASK_RUNNING);
	tmp_mask = speedchange_cpumask;
	cpumask_clear(&speedchange_cpumask);
	spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);

	for_each_cpu(cpu, &tmp_mask) {
	unsigned int j;
	unsigned int max_freq = 0;

	pcpu = &per_cpu(cpuinfo, cpu);
	smp_rmb();

	if (!pcpu->governor_enabled)
	continue;

	for_each_cpu(j, pcpu->policy->cpus) {
	struct cpufreq_interactive_cpuinfo *pjcpu =
	&per_cpu(cpuinfo, j);

	if (pjcpu->target_freq > max_freq)
	max_freq = pjcpu->target_freq;
	}

	if (max_freq != pcpu->policy->cur)
	__cpufreq_driver_target(pcpu->policy,
	max_freq,
	CPUFREQ_RELATION_H);
	trace_cpufreq_interactive_setspeed(cpu,
	pcpu->target_freq,
	pcpu->policy->cur);
	}
	}

	return 0;
	}

	static void cpufreq_interactive_boost(void)
	{
	int i;
	int anyboost = 0;
	unsigned long flags;
	struct cpufreq_interactive_cpuinfo *pcpu;

	spin_lock_irqsave(&speedchange_cpumask_lock, flags);

	for_each_online_cpu(i) {
	pcpu = &per_cpu(cpuinfo, i);

	if (pcpu->target_freq < hispeed_freq) {
	pcpu->target_freq = hispeed_freq;
	cpumask_set_cpu(i, &speedchange_cpumask);
	pcpu->target_set_time_in_idle =
	get_cpu_idle_time_us(i, &pcpu->target_set_time);
	pcpu->hispeed_validate_time = pcpu->target_set_time;
	anyboost = 1;
	}

	/*
	* Set floor freq and (re)start timer for when last
	* validated.
	*/

	pcpu->floor_freq = hispeed_freq;
	pcpu->floor_validate_time = ktime_to_us(ktime_get());
	}

	spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);

	if (anyboost)
	wake_up_process(speedchange_task);
	}

	static ssize_t show_hispeed_freq(struct kobject *kobj,
	struct attribute attr, char buf)
	{
	return sprintf(buf, "%u\n", hispeed_freq);
	}

	static ssize_t store_hispeed_freq(struct kobject *kobj,
	struct attribute attr, const char buf,
	size_t count)
	{
	int ret;
	long unsigned int val;

	ret = strict_strtoul(buf, 0, &val);
	if (ret < 0)
	return ret;
	hispeed_freq = val;
	return count;
	}

	static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
	show_hispeed_freq, store_hispeed_freq);


	static ssize_t show_go_hispeed_load(struct kobject *kobj,
	struct attribute attr, char buf)
	{
	return sprintf(buf, "%lu\n", go_hispeed_load);
	}

	static ssize_t store_go_hispeed_load(struct kobject *kobj,
	struct attribute attr, const char buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = strict_strtoul(buf, 0, &val);
	if (ret < 0)
	return ret;
	go_hispeed_load = val;
	return count;
	}

	static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
	show_go_hispeed_load, store_go_hispeed_load);

	static ssize_t show_min_sample_time(struct kobject *kobj,
	struct attribute attr, char buf)
	{
	return sprintf(buf, "%lu\n", min_sample_time);
	}

	static ssize_t store_min_sample_time(struct kobject *kobj,
	struct attribute attr, const char buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = strict_strtoul(buf, 0, &val);
	if (ret < 0)
	return ret;
	min_sample_time = val;
	return count;
	}

	static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
	show_min_sample_time, store_min_sample_time);

	static ssize_t show_above_hispeed_delay(struct kobject *kobj,
	struct attribute attr, char buf)
	{
	return sprintf(buf, "%lu\n", above_hispeed_delay_val);
	}

	static ssize_t store_above_hispeed_delay(struct kobject *kobj,
	struct attribute *attr,
	const char *buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = strict_strtoul(buf, 0, &val);
	if (ret < 0)
	return ret;
	above_hispeed_delay_val = val;
	return count;
	}

	define_one_global_rw(above_hispeed_delay);

	static ssize_t show_timer_rate(struct kobject *kobj,
	struct attribute attr, char buf)
	{
	return sprintf(buf, "%lu\n", timer_rate);
	}

	static ssize_t store_timer_rate(struct kobject *kobj,
	struct attribute attr, const char buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = strict_strtoul(buf, 0, &val);
	if (ret < 0)
	return ret;
	timer_rate = val;
	return count;
	}

	static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
	show_timer_rate, store_timer_rate);

	static ssize_t show_boost(struct kobject kobj, struct attribute attr,
	char *buf)
	{
	return sprintf(buf, "%d\n", boost_val);
	}

	static ssize_t store_boost(struct kobject kobj, struct attribute attr,
	const char *buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = kstrtoul(buf, 0, &val);
	if (ret < 0)
	return ret;

	boost_val = val;

	if (boost_val) {
	trace_cpufreq_interactive_boost("on");
	cpufreq_interactive_boost();
	} else {
	trace_cpufreq_interactive_unboost("off");
	}

	return count;
	}

	define_one_global_rw(boost);

	static ssize_t store_boostpulse(struct kobject kobj, struct attribute attr,
	const char *buf, size_t count)
	{
	int ret;
	unsigned long val;

	ret = kstrtoul(buf, 0, &val);
	if (ret < 0)
	return ret;

	trace_cpufreq_interactive_boost("pulse");
	cpufreq_interactive_boost();
	return count;
	}

	static struct global_attr boostpulse =
	__ATTR(boostpulse, 0200, NULL, store_boostpulse);

	static struct attribute *interactive_attributes[] = {
	&hispeed_freq_attr.attr,
	&go_hispeed_load_attr.attr,
	&above_hispeed_delay.attr,
	&min_sample_time_attr.attr,
	&timer_rate_attr.attr,
	&boost.attr,
	&boostpulse.attr,
	NULL,
	};

	static struct attribute_group interactive_attr_group = {
	.attrs = interactive_attributes,
	.name = "interactive",
	};

	static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
	unsigned long val,
	void *data)
	{
	switch (val) {
	case IDLE_START:
	cpufreq_interactive_idle_start();
	break;
	case IDLE_END:
	cpufreq_interactive_idle_end();
	break;
	}

	return 0;
	}

	static struct notifier_block cpufreq_interactive_idle_nb = {
	.notifier_call = cpufreq_interactive_idle_notifier,
	};

	static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
	unsigned int event)
	{
	int rc;
	unsigned int j;
	struct cpufreq_interactive_cpuinfo *pcpu;
	struct cpufreq_frequency_table *freq_table;

	switch (event) {
	case CPUFREQ_GOV_START:
	if (!cpu_online(policy->cpu))
	return -EINVAL;

	freq_table =
	cpufreq_frequency_get_table(policy->cpu);
	if (!hispeed_freq)
	hispeed_freq = policy->max;

	for_each_cpu(j, policy->cpus) {
	pcpu = &per_cpu(cpuinfo, j);
	pcpu->policy = policy;
	pcpu->target_freq = policy->cur;
	pcpu->freq_table = freq_table;
	pcpu->target_set_time_in_idle =
	get_cpu_idle_time_us(j,
	&pcpu->target_set_time);
	pcpu->floor_freq = pcpu->target_freq;
	pcpu->floor_validate_time =
	pcpu->target_set_time;
	pcpu->hispeed_validate_time =
	pcpu->target_set_time;
	pcpu->governor_enabled = 1;
	smp_wmb();
	pcpu->cpu_timer.expires =
	jiffies + usecs_to_jiffies(timer_rate);
	add_timer_on(&pcpu->cpu_timer, j);
	}

	/*
	* Do not register the idle hook and create sysfs
	* entries if we have already done so.
	*/
	if (atomic_inc_return(&active_count) > 1)
	return 0;

	rc = sysfs_create_group(cpufreq_global_kobject,
	&interactive_attr_group);
	if (rc)
	return rc;

	idle_notifier_register(&cpufreq_interactive_idle_nb);
	break;

	case CPUFREQ_GOV_STOP:
	for_each_cpu(j, policy->cpus) {
	pcpu = &per_cpu(cpuinfo, j);
	pcpu->governor_enabled = 0;
	smp_wmb();
	del_timer_sync(&pcpu->cpu_timer);
	}

	if (atomic_dec_return(&active_count) > 0)
	return 0;

	idle_notifier_unregister(&cpufreq_interactive_idle_nb);
	sysfs_remove_group(cpufreq_global_kobject,
	&interactive_attr_group);

	break;

	case CPUFREQ_GOV_LIMITS:
	if (policy->max < policy->cur)
	__cpufreq_driver_target(policy,
	policy->max, CPUFREQ_RELATION_H);
	else if (policy->min > policy->cur)
	__cpufreq_driver_target(policy,
	policy->min, CPUFREQ_RELATION_L);
	break;
	}
	return 0;
	}

	static int __init cpufreq_interactive_init(void)
	{
	unsigned int i;
	struct cpufreq_interactive_cpuinfo *pcpu;
	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

	go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
	min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
	above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
	timer_rate = DEFAULT_TIMER_RATE;

	/* Initalize per-cpu timers */
	for_each_possible_cpu(i) {
	pcpu = &per_cpu(cpuinfo, i);
	init_timer(&pcpu->cpu_timer);
	pcpu->cpu_timer.function = cpufreq_interactive_timer;
	pcpu->cpu_timer.data = i;
	}

	spin_lock_init(&speedchange_cpumask_lock);
	speedchange_task =
	kthread_create(cpufreq_interactive_speedchange_task, NULL,
	"cfinteractive");
	if (IS_ERR(speedchange_task))
	return PTR_ERR(speedchange_task);

	sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
	get_task_struct(speedchange_task);

	/* NB: wake up so the thread does not look hung to the freezer */
	wake_up_process(speedchange_task);

	return cpufreq_register_governor(&cpufreq_gov_interactive);
	}

	#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
	fs_initcall(cpufreq_interactive_init);
	#else
	module_init(cpufreq_interactive_init);
	#endif

	static void __exit cpufreq_interactive_exit(void)
	{
	cpufreq_unregister_governor(&cpufreq_gov_interactive);
	kthread_stop(speedchange_task);
	put_task_struct(speedchange_task);
	}

	module_exit(cpufreq_interactive_exit);

	MODULE_AUTHOR("Mike Chan <mike@android.com>");
	MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
	"Latency sensitive workloads");
	MODULE_LICENSE("GPL");