blob: 33855cb3cf16f1ecd30528f35343c2c029959b1e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * drivers/cpufreq/cpufreq_ondemand.c
3 *
4 * Copyright (C) 2001 Russell King
5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6 * Jun Nakajima <jun.nakajima@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/kernel.h>
14#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/cpufreq.h>
Andrew Morton138a01282006-06-23 03:31:19 -070017#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/jiffies.h>
19#include <linux/kernel_stat.h>
akpm@osdl.org3fc54d32006-01-13 15:54:22 -080020#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22/*
23 * dbs is used in this file as a shortform for demandbased switching
24 * It helps to keep variable names smaller, simpler
25 */
26
27#define DEF_FREQUENCY_UP_THRESHOLD (80)
Dave Jonesc29f1402005-05-31 19:03:50 -070028#define MIN_FREQUENCY_UP_THRESHOLD (11)
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#define MAX_FREQUENCY_UP_THRESHOLD (100)
30
Dave Jones32ee8c32006-02-28 00:43:23 -050031/*
32 * The polling frequency of this governor depends on the capability of
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 * the processor. Default polling frequency is 1000 times the transition
Dave Jones32ee8c32006-02-28 00:43:23 -050034 * latency of the processor. The governor will work on any processor with
35 * transition latency <= 10mS, using appropriate sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * rate.
37 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
38 * this governor will not work.
39 * All times here are in uS.
40 */
Dave Jones32ee8c32006-02-28 00:43:23 -050041static unsigned int def_sampling_rate;
Dave Jonesdf8b59b2005-09-20 12:39:35 -070042#define MIN_SAMPLING_RATE_RATIO (2)
43/* for correct statistics, we need at least 10 ticks between each measure */
Gautham R Shenoye08f5f52006-10-26 16:20:58 +053044#define MIN_STAT_SAMPLING_RATE \
45 (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
46#define MIN_SAMPLING_RATE \
47 (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#define MAX_SAMPLING_RATE (500 * def_sampling_rate)
49#define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER (1000)
Thomas Renninger1c256242007-10-02 13:28:12 -070050#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000)
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
David Howellsc4028952006-11-22 14:57:56 +000052static void do_dbs_timer(struct work_struct *work);
53
54/* Sampling types */
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -080055enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57struct cpu_dbs_info_s {
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -070058 cputime64_t prev_cpu_idle;
59 cputime64_t prev_cpu_wall;
Dave Jones32ee8c32006-02-28 00:43:23 -050060 struct cpufreq_policy *cur_policy;
David Howellsc4028952006-11-22 14:57:56 +000061 struct delayed_work work;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040062 struct cpufreq_frequency_table *freq_table;
63 unsigned int freq_lo;
64 unsigned int freq_lo_jiffies;
65 unsigned int freq_hi_jiffies;
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -080066 int cpu;
67 unsigned int enable:1,
68 sample_type:1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070069};
70static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
71
72static unsigned int dbs_enable; /* number of CPUs using this policy */
73
Venkatesh Pallipadi4ec223d2006-06-21 15:18:34 -070074/*
75 * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
76 * lock and dbs_mutex. cpu_hotplug lock should always be held before
77 * dbs_mutex. If any function that can potentially take cpu_hotplug lock
78 * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
79 * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
80 * is recursive for the same process. -Venki
81 */
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -070082static DEFINE_MUTEX(dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -070084static struct workqueue_struct *kondemand_wq;
Andi Kleen6810b542006-05-08 15:17:31 +020085
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040086static struct dbs_tuners {
Dave Jones32ee8c32006-02-28 00:43:23 -050087 unsigned int sampling_rate;
Dave Jones32ee8c32006-02-28 00:43:23 -050088 unsigned int up_threshold;
89 unsigned int ignore_nice;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040090 unsigned int powersave_bias;
91} dbs_tuners_ins = {
Dave Jones32ee8c32006-02-28 00:43:23 -050092 .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
Eric Piel9cbad612006-03-10 11:35:27 +020093 .ignore_nice = 0,
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040094 .powersave_bias = 0,
Linus Torvalds1da177e2005-04-16 15:20:36 -070095};
96
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -070097static inline cputime64_t get_cpu_idle_time(unsigned int cpu)
Dave Jonesdac1c1a2005-05-31 19:03:49 -070098{
Venki Pallipadiea487612007-06-20 14:26:24 -070099 cputime64_t idle_time;
100 cputime64_t cur_jiffies;
101 cputime64_t busy_time;
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700102
Venki Pallipadiea487612007-06-20 14:26:24 -0700103 cur_jiffies = jiffies64_to_cputime64(get_jiffies_64());
104 busy_time = cputime64_add(kstat_cpu(cpu).cpustat.user,
105 kstat_cpu(cpu).cpustat.system);
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700106
Venki Pallipadiea487612007-06-20 14:26:24 -0700107 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.irq);
108 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.softirq);
109 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.steal);
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700110
Venki Pallipadiea487612007-06-20 14:26:24 -0700111 if (!dbs_tuners_ins.ignore_nice) {
112 busy_time = cputime64_add(busy_time,
113 kstat_cpu(cpu).cpustat.nice);
114 }
115
116 idle_time = cputime64_sub(cur_jiffies, busy_time);
117 return idle_time;
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700118}
119
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400120/*
121 * Find right freq to be set now with powersave_bias on.
122 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
123 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
124 */
Adrian Bunkb5ecf602006-08-13 23:00:08 +0200125static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
126 unsigned int freq_next,
127 unsigned int relation)
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400128{
129 unsigned int freq_req, freq_reduc, freq_avg;
130 unsigned int freq_hi, freq_lo;
131 unsigned int index = 0;
132 unsigned int jiffies_total, jiffies_hi, jiffies_lo;
133 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, policy->cpu);
134
135 if (!dbs_info->freq_table) {
136 dbs_info->freq_lo = 0;
137 dbs_info->freq_lo_jiffies = 0;
138 return freq_next;
139 }
140
141 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
142 relation, &index);
143 freq_req = dbs_info->freq_table[index].frequency;
144 freq_reduc = freq_req * dbs_tuners_ins.powersave_bias / 1000;
145 freq_avg = freq_req - freq_reduc;
146
147 /* Find freq bounds for freq_avg in freq_table */
148 index = 0;
149 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
150 CPUFREQ_RELATION_H, &index);
151 freq_lo = dbs_info->freq_table[index].frequency;
152 index = 0;
153 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
154 CPUFREQ_RELATION_L, &index);
155 freq_hi = dbs_info->freq_table[index].frequency;
156
157 /* Find out how long we have to be in hi and lo freqs */
158 if (freq_hi == freq_lo) {
159 dbs_info->freq_lo = 0;
160 dbs_info->freq_lo_jiffies = 0;
161 return freq_lo;
162 }
163 jiffies_total = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
164 jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
165 jiffies_hi += ((freq_hi - freq_lo) / 2);
166 jiffies_hi /= (freq_hi - freq_lo);
167 jiffies_lo = jiffies_total - jiffies_hi;
168 dbs_info->freq_lo = freq_lo;
169 dbs_info->freq_lo_jiffies = jiffies_lo;
170 dbs_info->freq_hi_jiffies = jiffies_hi;
171 return freq_hi;
172}
173
174static void ondemand_powersave_bias_init(void)
175{
176 int i;
177 for_each_online_cpu(i) {
178 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, i);
179 dbs_info->freq_table = cpufreq_frequency_get_table(i);
180 dbs_info->freq_lo = 0;
181 }
182}
183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184/************************** sysfs interface ************************/
185static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
186{
187 return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
188}
189
190static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
191{
192 return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
193}
194
Dave Jones32ee8c32006-02-28 00:43:23 -0500195#define define_one_ro(_name) \
196static struct freq_attr _name = \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197__ATTR(_name, 0444, show_##_name, NULL)
198
199define_one_ro(sampling_rate_max);
200define_one_ro(sampling_rate_min);
201
202/* cpufreq_ondemand Governor Tunables */
203#define show_one(file_name, object) \
204static ssize_t show_##file_name \
205(struct cpufreq_policy *unused, char *buf) \
206{ \
207 return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
208}
209show_one(sampling_rate, sampling_rate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210show_one(up_threshold, up_threshold);
Alexander Clouter001893c2005-12-01 01:09:25 -0800211show_one(ignore_nice_load, ignore_nice);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400212show_one(powersave_bias, powersave_bias);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
Dave Jones32ee8c32006-02-28 00:43:23 -0500214static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 const char *buf, size_t count)
216{
217 unsigned int input;
218 int ret;
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700219 ret = sscanf(buf, "%u", &input);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800221 mutex_lock(&dbs_mutex);
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530222 if (ret != 1 || input > MAX_SAMPLING_RATE
223 || input < MIN_SAMPLING_RATE) {
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800224 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 return -EINVAL;
226 }
227
228 dbs_tuners_ins.sampling_rate = input;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800229 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
231 return count;
232}
233
Dave Jones32ee8c32006-02-28 00:43:23 -0500234static ssize_t store_up_threshold(struct cpufreq_policy *unused,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 const char *buf, size_t count)
236{
237 unsigned int input;
238 int ret;
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700239 ret = sscanf(buf, "%u", &input);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800241 mutex_lock(&dbs_mutex);
Dave Jones32ee8c32006-02-28 00:43:23 -0500242 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
Dave Jonesc29f1402005-05-31 19:03:50 -0700243 input < MIN_FREQUENCY_UP_THRESHOLD) {
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800244 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 return -EINVAL;
246 }
247
248 dbs_tuners_ins.up_threshold = input;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800249 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
251 return count;
252}
253
Alexander Clouter001893c2005-12-01 01:09:25 -0800254static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700255 const char *buf, size_t count)
256{
257 unsigned int input;
258 int ret;
259
260 unsigned int j;
Dave Jones32ee8c32006-02-28 00:43:23 -0500261
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700262 ret = sscanf(buf, "%u", &input);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700263 if ( ret != 1 )
264 return -EINVAL;
265
266 if ( input > 1 )
267 input = 1;
Dave Jones32ee8c32006-02-28 00:43:23 -0500268
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800269 mutex_lock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700270 if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800271 mutex_unlock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700272 return count;
273 }
274 dbs_tuners_ins.ignore_nice = input;
275
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700276 /* we need to re-evaluate prev_cpu_idle */
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700277 for_each_online_cpu(j) {
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700278 struct cpu_dbs_info_s *dbs_info;
279 dbs_info = &per_cpu(cpu_dbs_info, j);
280 dbs_info->prev_cpu_idle = get_cpu_idle_time(j);
281 dbs_info->prev_cpu_wall = get_jiffies_64();
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700282 }
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800283 mutex_unlock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700284
285 return count;
286}
287
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400288static ssize_t store_powersave_bias(struct cpufreq_policy *unused,
289 const char *buf, size_t count)
290{
291 unsigned int input;
292 int ret;
293 ret = sscanf(buf, "%u", &input);
294
295 if (ret != 1)
296 return -EINVAL;
297
298 if (input > 1000)
299 input = 1000;
300
301 mutex_lock(&dbs_mutex);
302 dbs_tuners_ins.powersave_bias = input;
303 ondemand_powersave_bias_init();
304 mutex_unlock(&dbs_mutex);
305
306 return count;
307}
308
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309#define define_one_rw(_name) \
310static struct freq_attr _name = \
311__ATTR(_name, 0644, show_##_name, store_##_name)
312
313define_one_rw(sampling_rate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314define_one_rw(up_threshold);
Alexander Clouter001893c2005-12-01 01:09:25 -0800315define_one_rw(ignore_nice_load);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400316define_one_rw(powersave_bias);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
318static struct attribute * dbs_attributes[] = {
319 &sampling_rate_max.attr,
320 &sampling_rate_min.attr,
321 &sampling_rate.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 &up_threshold.attr,
Alexander Clouter001893c2005-12-01 01:09:25 -0800323 &ignore_nice_load.attr,
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400324 &powersave_bias.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 NULL
326};
327
328static struct attribute_group dbs_attr_group = {
329 .attrs = dbs_attributes,
330 .name = "ondemand",
331};
332
333/************************** sysfs end ************************/
334
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700335static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700337 unsigned int idle_ticks, total_ticks;
Venki Pallipadi0af99b12007-06-20 14:24:52 -0700338 unsigned int load = 0;
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700339 cputime64_t cur_jiffies;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
341 struct cpufreq_policy *policy;
342 unsigned int j;
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 if (!this_dbs_info->enable)
345 return;
346
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400347 this_dbs_info->freq_lo = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 policy = this_dbs_info->cur_policy;
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700349 cur_jiffies = jiffies64_to_cputime64(get_jiffies_64());
350 total_ticks = (unsigned int) cputime64_sub(cur_jiffies,
351 this_dbs_info->prev_cpu_wall);
Venki Pallipadiea487612007-06-20 14:26:24 -0700352 this_dbs_info->prev_cpu_wall = get_jiffies_64();
353
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700354 if (!total_ticks)
355 return;
Dave Jones32ee8c32006-02-28 00:43:23 -0500356 /*
Dave Jonesc29f1402005-05-31 19:03:50 -0700357 * Every sampling_rate, we check, if current idle time is less
358 * than 20% (default), then we try to increase frequency
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700359 * Every sampling_rate, we look for a the lowest
Dave Jonesc29f1402005-05-31 19:03:50 -0700360 * frequency which can sustain the load while keeping idle time over
361 * 30%. If such a frequency exist, we try to decrease to this frequency.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 *
Dave Jones32ee8c32006-02-28 00:43:23 -0500363 * Any frequency increase takes it to the maximum frequency.
364 * Frequency reduction happens at minimum steps of
365 * 5% (default) of current frequency
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 */
367
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700368 /* Get Idle Time */
Dave Jones9c7d2692005-05-31 19:03:49 -0700369 idle_ticks = UINT_MAX;
Mike Travis068b1272008-05-12 21:21:13 +0200370 for_each_cpu_mask_nr(j, policy->cpus) {
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700371 cputime64_t total_idle_ticks;
372 unsigned int tmp_idle_ticks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 struct cpu_dbs_info_s *j_dbs_info;
374
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 j_dbs_info = &per_cpu(cpu_dbs_info, j);
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700376 total_idle_ticks = get_cpu_idle_time(j);
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700377 tmp_idle_ticks = (unsigned int) cputime64_sub(total_idle_ticks,
378 j_dbs_info->prev_cpu_idle);
379 j_dbs_info->prev_cpu_idle = total_idle_ticks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380
381 if (tmp_idle_ticks < idle_ticks)
382 idle_ticks = tmp_idle_ticks;
383 }
Venki Pallipadi0af99b12007-06-20 14:24:52 -0700384 if (likely(total_ticks > idle_ticks))
385 load = (100 * (total_ticks - idle_ticks)) / total_ticks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700387 /* Check for frequency increase */
388 if (load > dbs_tuners_ins.up_threshold) {
Dave Jonesc11420a2005-05-31 19:03:48 -0700389 /* if we are already at full speed then break out early */
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400390 if (!dbs_tuners_ins.powersave_bias) {
391 if (policy->cur == policy->max)
392 return;
Dave Jones32ee8c32006-02-28 00:43:23 -0500393
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400394 __cpufreq_driver_target(policy, policy->max,
395 CPUFREQ_RELATION_H);
396 } else {
397 int freq = powersave_bias_target(policy, policy->max,
398 CPUFREQ_RELATION_H);
399 __cpufreq_driver_target(policy, freq,
400 CPUFREQ_RELATION_L);
401 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402 return;
403 }
404
405 /* Check for frequency decrease */
Dave Jonesc29f1402005-05-31 19:03:50 -0700406 /* if we cannot reduce the frequency anymore, break out early */
407 if (policy->cur == policy->min)
408 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
Dave Jonesc29f1402005-05-31 19:03:50 -0700410 /*
411 * The optimal frequency is the frequency that is the lowest that
412 * can support the current CPU usage without triggering the up
413 * policy. To be safe, we focus 10 points under the threshold.
414 */
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700415 if (load < (dbs_tuners_ins.up_threshold - 10)) {
Venkatesh Pallipadidfde5d62006-10-03 12:38:45 -0700416 unsigned int freq_next, freq_cur;
417
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800418 freq_cur = __cpufreq_driver_getavg(policy);
Venkatesh Pallipadidfde5d62006-10-03 12:38:45 -0700419 if (!freq_cur)
420 freq_cur = policy->cur;
421
422 freq_next = (freq_cur * load) /
Dave Jonesc29f1402005-05-31 19:03:50 -0700423 (dbs_tuners_ins.up_threshold - 10);
Venkatesh Pallipadidfde5d62006-10-03 12:38:45 -0700424
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400425 if (!dbs_tuners_ins.powersave_bias) {
426 __cpufreq_driver_target(policy, freq_next,
427 CPUFREQ_RELATION_L);
428 } else {
429 int freq = powersave_bias_target(policy, freq_next,
430 CPUFREQ_RELATION_L);
431 __cpufreq_driver_target(policy, freq,
432 CPUFREQ_RELATION_L);
433 }
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700434 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435}
436
David Howellsc4028952006-11-22 14:57:56 +0000437static void do_dbs_timer(struct work_struct *work)
Dave Jones32ee8c32006-02-28 00:43:23 -0500438{
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800439 struct cpu_dbs_info_s *dbs_info =
440 container_of(work, struct cpu_dbs_info_s, work.work);
441 unsigned int cpu = dbs_info->cpu;
442 int sample_type = dbs_info->sample_type;
443
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400444 /* We want all CPUs to do sampling nearly on same jiffy */
445 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
David Howellsc4028952006-11-22 14:57:56 +0000446
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400447 delay -= jiffies % delay;
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700448
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800449 if (lock_policy_rwsem_write(cpu) < 0)
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700450 return;
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800451
452 if (!dbs_info->enable) {
453 unlock_policy_rwsem_write(cpu);
454 return;
455 }
456
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400457 /* Common NORMAL_SAMPLE setup */
David Howellsc4028952006-11-22 14:57:56 +0000458 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400459 if (!dbs_tuners_ins.powersave_bias ||
David Howellsc4028952006-11-22 14:57:56 +0000460 sample_type == DBS_NORMAL_SAMPLE) {
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400461 dbs_check_cpu(dbs_info);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400462 if (dbs_info->freq_lo) {
463 /* Setup timer for SUB_SAMPLE */
David Howellsc4028952006-11-22 14:57:56 +0000464 dbs_info->sample_type = DBS_SUB_SAMPLE;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400465 delay = dbs_info->freq_hi_jiffies;
466 }
467 } else {
468 __cpufreq_driver_target(dbs_info->cur_policy,
469 dbs_info->freq_lo,
470 CPUFREQ_RELATION_H);
471 }
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400472 queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work, delay);
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800473 unlock_policy_rwsem_write(cpu);
Dave Jones32ee8c32006-02-28 00:43:23 -0500474}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800476static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477{
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400478 /* We want all CPUs to do sampling nearly on same jiffy */
479 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
480 delay -= jiffies % delay;
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700481
Dave Jonesc18a1482007-02-10 20:03:51 -0500482 dbs_info->enable = 1;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400483 ondemand_powersave_bias_init();
David Howellsc4028952006-11-22 14:57:56 +0000484 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
Venki Pallipadi28287032007-05-08 00:27:47 -0700485 INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800486 queue_delayed_work_on(dbs_info->cpu, kondemand_wq, &dbs_info->work,
487 delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488}
489
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700490static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491{
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700492 dbs_info->enable = 0;
493 cancel_delayed_work(&dbs_info->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494}
495
496static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
497 unsigned int event)
498{
499 unsigned int cpu = policy->cpu;
500 struct cpu_dbs_info_s *this_dbs_info;
501 unsigned int j;
Jeff Garzik914f7c32006-10-20 14:31:00 -0700502 int rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
504 this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
505
506 switch (event) {
507 case CPUFREQ_GOV_START:
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700508 if ((!cpu_online(cpu)) || (!policy->cur))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 return -EINVAL;
510
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 if (this_dbs_info->enable) /* Already enabled */
512 break;
Dave Jones32ee8c32006-02-28 00:43:23 -0500513
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800514 mutex_lock(&dbs_mutex);
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700515 dbs_enable++;
Jeff Garzik914f7c32006-10-20 14:31:00 -0700516
517 rc = sysfs_create_group(&policy->kobj, &dbs_attr_group);
518 if (rc) {
Jeff Garzik914f7c32006-10-20 14:31:00 -0700519 dbs_enable--;
520 mutex_unlock(&dbs_mutex);
521 return rc;
522 }
523
Mike Travis068b1272008-05-12 21:21:13 +0200524 for_each_cpu_mask_nr(j, policy->cpus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 struct cpu_dbs_info_s *j_dbs_info;
526 j_dbs_info = &per_cpu(cpu_dbs_info, j);
527 j_dbs_info->cur_policy = policy;
Dave Jones32ee8c32006-02-28 00:43:23 -0500528
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700529 j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j);
530 j_dbs_info->prev_cpu_wall = get_jiffies_64();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 }
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800532 this_dbs_info->cpu = cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 /*
534 * Start the timerschedule work, when this governor
535 * is used for first time
536 */
537 if (dbs_enable == 1) {
538 unsigned int latency;
539 /* policy latency is in nS. Convert it to uS first */
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700540 latency = policy->cpuinfo.transition_latency / 1000;
541 if (latency == 0)
542 latency = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700544 def_sampling_rate = latency *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700546
547 if (def_sampling_rate < MIN_STAT_SAMPLING_RATE)
548 def_sampling_rate = MIN_STAT_SAMPLING_RATE;
549
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 dbs_tuners_ins.sampling_rate = def_sampling_rate;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 }
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800552 dbs_timer_init(this_dbs_info);
Dave Jones32ee8c32006-02-28 00:43:23 -0500553
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800554 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 break;
556
557 case CPUFREQ_GOV_STOP:
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800558 mutex_lock(&dbs_mutex);
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700559 dbs_timer_exit(this_dbs_info);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 sysfs_remove_group(&policy->kobj, &dbs_attr_group);
561 dbs_enable--;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800562 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
564 break;
565
566 case CPUFREQ_GOV_LIMITS:
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800567 mutex_lock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 if (policy->max < this_dbs_info->cur_policy->cur)
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700569 __cpufreq_driver_target(this_dbs_info->cur_policy,
570 policy->max,
571 CPUFREQ_RELATION_H);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 else if (policy->min > this_dbs_info->cur_policy->cur)
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700573 __cpufreq_driver_target(this_dbs_info->cur_policy,
574 policy->min,
575 CPUFREQ_RELATION_L);
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800576 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 break;
578 }
579 return 0;
580}
581
Thomas Renninger1c256242007-10-02 13:28:12 -0700582struct cpufreq_governor cpufreq_gov_ondemand = {
583 .name = "ondemand",
584 .governor = cpufreq_governor_dbs,
585 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
586 .owner = THIS_MODULE,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587};
Thomas Renninger1c256242007-10-02 13:28:12 -0700588EXPORT_SYMBOL(cpufreq_gov_ondemand);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
590static int __init cpufreq_gov_dbs_init(void)
591{
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800592 kondemand_wq = create_workqueue("kondemand");
593 if (!kondemand_wq) {
594 printk(KERN_ERR "Creation of kondemand failed\n");
595 return -EFAULT;
596 }
Thomas Renninger1c256242007-10-02 13:28:12 -0700597 return cpufreq_register_governor(&cpufreq_gov_ondemand);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
599
600static void __exit cpufreq_gov_dbs_exit(void)
601{
Thomas Renninger1c256242007-10-02 13:28:12 -0700602 cpufreq_unregister_governor(&cpufreq_gov_ondemand);
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800603 destroy_workqueue(kondemand_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604}
605
606
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700607MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
608MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
609MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
610 "Low Latency Frequency Transition capable processors");
611MODULE_LICENSE("GPL");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612
Johannes Weiner69157192008-01-17 15:21:08 -0800613#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
614fs_initcall(cpufreq_gov_dbs_init);
615#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616module_init(cpufreq_gov_dbs_init);
Johannes Weiner69157192008-01-17 15:21:08 -0800617#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618module_exit(cpufreq_gov_dbs_exit);