blob: 3ff5160451b436ec48511a9428ab611b2291b1f9 [file] [log] [blame]
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301/*
2 * POWERNV cpufreq driver for the IBM POWER processors
3 *
4 * (C) Copyright IBM 2014
5 *
6 * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 */
19
20#define pr_fmt(fmt) "powernv-cpufreq: " fmt
21
22#include <linux/kernel.h>
23#include <linux/sysfs.h>
24#include <linux/cpumask.h>
25#include <linux/module.h>
26#include <linux/cpufreq.h>
27#include <linux/smp.h>
28#include <linux/of.h>
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +020029#include <linux/reboot.h>
Shilpasri G Bhat053819e2015-07-16 13:34:18 +053030#include <linux/slab.h>
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +053031#include <linux/cpu.h>
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +053032#include <trace/events/power.h>
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053033
34#include <asm/cputhreads.h>
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +053035#include <asm/firmware.h>
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053036#include <asm/reg.h>
Srivatsa S. Bhatf3cae352014-04-16 11:35:38 +053037#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +053038#include <asm/opal.h>
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053039#include <linux/timer.h>
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053040
41#define POWERNV_MAX_PSTATES 256
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +053042#define PMSR_PSAFE_ENABLE (1UL << 30)
43#define PMSR_SPR_EM_DISABLE (1UL << 31)
44#define PMSR_MAX(x) ((x >> 32) & 0xFF)
Akshay Adiga20b15b72016-11-08 19:03:28 +053045#define LPSTATE_SHIFT 48
46#define GPSTATE_SHIFT 56
47#define GET_LPSTATE(x) (((x) >> LPSTATE_SHIFT) & 0xFF)
48#define GET_GPSTATE(x) (((x) >> GPSTATE_SHIFT) & 0xFF)
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053049
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053050#define MAX_RAMP_DOWN_TIME 5120
51/*
52 * On an idle system we want the global pstate to ramp-down from max value to
53 * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and
54 * then ramp-down rapidly later on.
55 *
56 * This gives a percentage rampdown for time elapsed in milliseconds.
57 * ramp_down_percentage = ((ms * ms) >> 18)
58 * ~= 3.8 * (sec * sec)
59 *
60 * At 0 ms ramp_down_percent = 0
61 * At 5120 ms ramp_down_percent = 100
62 */
63#define ramp_down_percent(time) ((time * time) >> 18)
64
65/* Interval after which the timer is queued to bring down global pstate */
66#define GPSTATE_TIMER_INTERVAL 2000
67
68/**
69 * struct global_pstate_info - Per policy data structure to maintain history of
70 * global pstates
Akshay Adiga09ca4c92016-06-30 11:53:07 +053071 * @highest_lpstate_idx: The local pstate index from which we are
72 * ramping down
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053073 * @elapsed_time: Time in ms spent in ramping down from
Akshay Adiga09ca4c92016-06-30 11:53:07 +053074 * highest_lpstate_idx
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053075 * @last_sampled_time: Time from boot in ms when global pstates were
76 * last set
Akshay Adiga09ca4c92016-06-30 11:53:07 +053077 * @last_lpstate_idx, Last set value of local pstate and global
78 * last_gpstate_idx pstate in terms of cpufreq table index
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053079 * @timer: Is used for ramping down if cpu goes idle for
80 * a long time with global pstate held high
81 * @gpstate_lock: A spinlock to maintain synchronization between
82 * routines called by the timer handler and
83 * governer's target_index calls
84 */
85struct global_pstate_info {
Akshay Adiga09ca4c92016-06-30 11:53:07 +053086 int highest_lpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053087 unsigned int elapsed_time;
88 unsigned int last_sampled_time;
Akshay Adiga09ca4c92016-06-30 11:53:07 +053089 int last_lpstate_idx;
90 int last_gpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +053091 spinlock_t gpstate_lock;
92 struct timer_list timer;
93};
94
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053095static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +053096static bool rebooting, throttled, occ_reset;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +053097
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +053098static const char * const throttle_reason[] = {
99 "No throttling",
100 "Power Cap",
101 "Processor Over Temperature",
102 "Power Supply Failure",
103 "Over Current",
104 "OCC Reset"
105};
106
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530107enum throttle_reason_type {
108 NO_THROTTLE = 0,
109 POWERCAP,
110 CPU_OVERTEMP,
111 POWER_SUPPLY_FAILURE,
112 OVERCURRENT,
113 OCC_RESET_THROTTLE,
114 OCC_MAX_REASON
115};
116
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530117static struct chip {
118 unsigned int id;
119 bool throttled;
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530120 bool restore;
121 u8 throttle_reason;
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530122 cpumask_t mask;
123 struct work_struct throttle;
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530124 int throttle_turbo;
125 int throttle_sub_turbo;
126 int reason[OCC_MAX_REASON];
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530127} *chips;
128
129static int nr_chips;
Michael Neuling3e5963b2016-03-21 22:24:52 +0530130static DEFINE_PER_CPU(struct chip *, chip_info);
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530131
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530132/*
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530133 * Note:
134 * The set of pstates consists of contiguous integers.
135 * powernv_pstate_info stores the index of the frequency table for
136 * max, min and nominal frequencies. It also stores number of
137 * available frequencies.
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530138 *
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530139 * powernv_pstate_info.nominal indicates the index to the highest
140 * non-turbo frequency.
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530141 */
142static struct powernv_pstate_info {
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530143 unsigned int min;
144 unsigned int max;
145 unsigned int nominal;
146 unsigned int nr_pstates;
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530147 bool wof_enabled;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530148} powernv_pstate_info;
149
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530150/* Use following macros for conversions between pstate_id and index */
151static inline int idx_to_pstate(unsigned int i)
152{
Akshay Adiga8e859462016-08-04 20:59:17 +0530153 if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
154 pr_warn_once("index %u is out of bound\n", i);
155 return powernv_freqs[powernv_pstate_info.nominal].driver_data;
156 }
157
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530158 return powernv_freqs[i].driver_data;
159}
160
161static inline unsigned int pstate_to_idx(int pstate)
162{
Akshay Adiga8e859462016-08-04 20:59:17 +0530163 int min = powernv_freqs[powernv_pstate_info.min].driver_data;
164 int max = powernv_freqs[powernv_pstate_info.max].driver_data;
165
166 if (min > 0) {
167 if (unlikely((pstate < max) || (pstate > min))) {
168 pr_warn_once("pstate %d is out of bound\n", pstate);
169 return powernv_pstate_info.nominal;
170 }
171 } else {
172 if (unlikely((pstate > max) || (pstate < min))) {
173 pr_warn_once("pstate %d is out of bound\n", pstate);
174 return powernv_pstate_info.nominal;
175 }
176 }
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530177 /*
178 * abs() is deliberately used so that is works with
179 * both monotonically increasing and decreasing
180 * pstate values
181 */
182 return abs(pstate - idx_to_pstate(powernv_pstate_info.max));
183}
184
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530185static inline void reset_gpstates(struct cpufreq_policy *policy)
186{
187 struct global_pstate_info *gpstates = policy->driver_data;
188
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530189 gpstates->highest_lpstate_idx = 0;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530190 gpstates->elapsed_time = 0;
191 gpstates->last_sampled_time = 0;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530192 gpstates->last_lpstate_idx = 0;
193 gpstates->last_gpstate_idx = 0;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530194}
195
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530196/*
197 * Initialize the freq table based on data obtained
198 * from the firmware passed via device-tree
199 */
200static int init_powernv_pstates(void)
201{
202 struct device_node *power_mgt;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530203 int i, nr_pstates = 0;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530204 const __be32 *pstate_ids, *pstate_freqs;
205 u32 len_ids, len_freqs;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530206 u32 pstate_min, pstate_max, pstate_nominal;
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530207 u32 pstate_turbo, pstate_ultra_turbo;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530208
209 power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
210 if (!power_mgt) {
211 pr_warn("power-mgt node not found\n");
212 return -ENODEV;
213 }
214
215 if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
216 pr_warn("ibm,pstate-min node not found\n");
217 return -ENODEV;
218 }
219
220 if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
221 pr_warn("ibm,pstate-max node not found\n");
222 return -ENODEV;
223 }
224
225 if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
226 &pstate_nominal)) {
227 pr_warn("ibm,pstate-nominal not found\n");
228 return -ENODEV;
229 }
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530230
231 if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo",
232 &pstate_ultra_turbo)) {
233 powernv_pstate_info.wof_enabled = false;
234 goto next;
235 }
236
237 if (of_property_read_u32(power_mgt, "ibm,pstate-turbo",
238 &pstate_turbo)) {
239 powernv_pstate_info.wof_enabled = false;
240 goto next;
241 }
242
243 if (pstate_turbo == pstate_ultra_turbo)
244 powernv_pstate_info.wof_enabled = false;
245 else
246 powernv_pstate_info.wof_enabled = true;
247
248next:
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530249 pr_info("cpufreq pstate min %d nominal %d max %d\n", pstate_min,
250 pstate_nominal, pstate_max);
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530251 pr_info("Workload Optimized Frequency is %s in the platform\n",
252 (powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530253
254 pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
255 if (!pstate_ids) {
256 pr_warn("ibm,pstate-ids not found\n");
257 return -ENODEV;
258 }
259
260 pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
261 &len_freqs);
262 if (!pstate_freqs) {
263 pr_warn("ibm,pstate-frequencies-mhz not found\n");
264 return -ENODEV;
265 }
266
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +0530267 if (len_ids != len_freqs) {
268 pr_warn("Entries in ibm,pstate-ids and "
269 "ibm,pstate-frequencies-mhz does not match\n");
270 }
271
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530272 nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
273 if (!nr_pstates) {
274 pr_warn("No PStates found\n");
275 return -ENODEV;
276 }
277
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530278 powernv_pstate_info.nr_pstates = nr_pstates;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530279 pr_debug("NR PStates %d\n", nr_pstates);
280 for (i = 0; i < nr_pstates; i++) {
281 u32 id = be32_to_cpu(pstate_ids[i]);
282 u32 freq = be32_to_cpu(pstate_freqs[i]);
283
284 pr_debug("PState id %d freq %d MHz\n", id, freq);
285 powernv_freqs[i].frequency = freq * 1000; /* kHz */
Gautham R. Shenoy0692c692014-04-01 12:43:27 +0530286 powernv_freqs[i].driver_data = id;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530287
288 if (id == pstate_max)
289 powernv_pstate_info.max = i;
290 else if (id == pstate_nominal)
291 powernv_pstate_info.nominal = i;
292 else if (id == pstate_min)
293 powernv_pstate_info.min = i;
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530294
295 if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {
296 int j;
297
298 for (j = i - 1; j >= (int)powernv_pstate_info.max; j--)
299 powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ;
300 }
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530301 }
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530302
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530303 /* End of list marker entry */
304 powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530305 return 0;
306}
307
308/* Returns the CPU frequency corresponding to the pstate_id. */
309static unsigned int pstate_id_to_freq(int pstate_id)
310{
311 int i;
312
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530313 i = pstate_to_idx(pstate_id);
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +0530314 if (i >= powernv_pstate_info.nr_pstates || i < 0) {
315 pr_warn("PState id %d outside of PState table, "
316 "reporting nominal id %d instead\n",
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530317 pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
318 i = powernv_pstate_info.nominal;
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +0530319 }
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530320
321 return powernv_freqs[i].frequency;
322}
323
324/*
325 * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
326 * the firmware
327 */
328static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
329 char *buf)
330{
331 return sprintf(buf, "%u\n",
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530332 powernv_freqs[powernv_pstate_info.nominal].frequency);
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530333}
334
335struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
336 __ATTR_RO(cpuinfo_nominal_freq);
337
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530338#define SCALING_BOOST_FREQS_ATTR_INDEX 2
339
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530340static struct freq_attr *powernv_cpu_freq_attr[] = {
341 &cpufreq_freq_attr_scaling_available_freqs,
342 &cpufreq_freq_attr_cpuinfo_nominal_freq,
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +0530343 &cpufreq_freq_attr_scaling_boost_freqs,
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530344 NULL,
345};
346
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530347#define throttle_attr(name, member) \
348static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \
349{ \
350 struct chip *chip = per_cpu(chip_info, policy->cpu); \
351 \
352 return sprintf(buf, "%u\n", chip->member); \
353} \
354 \
355static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \
356
357throttle_attr(unthrottle, reason[NO_THROTTLE]);
358throttle_attr(powercap, reason[POWERCAP]);
359throttle_attr(overtemp, reason[CPU_OVERTEMP]);
360throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);
361throttle_attr(overcurrent, reason[OVERCURRENT]);
362throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);
363throttle_attr(turbo_stat, throttle_turbo);
364throttle_attr(sub_turbo_stat, throttle_sub_turbo);
365
366static struct attribute *throttle_attrs[] = {
367 &throttle_attr_unthrottle.attr,
368 &throttle_attr_powercap.attr,
369 &throttle_attr_overtemp.attr,
370 &throttle_attr_supply_fault.attr,
371 &throttle_attr_overcurrent.attr,
372 &throttle_attr_occ_reset.attr,
373 &throttle_attr_turbo_stat.attr,
374 &throttle_attr_sub_turbo_stat.attr,
375 NULL,
376};
377
378static const struct attribute_group throttle_attr_grp = {
379 .name = "throttle_stats",
380 .attrs = throttle_attrs,
381};
382
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530383/* Helper routines */
384
385/* Access helpers to power mgt SPR */
386
387static inline unsigned long get_pmspr(unsigned long sprn)
388{
389 switch (sprn) {
390 case SPRN_PMCR:
391 return mfspr(SPRN_PMCR);
392
393 case SPRN_PMICR:
394 return mfspr(SPRN_PMICR);
395
396 case SPRN_PMSR:
397 return mfspr(SPRN_PMSR);
398 }
399 BUG();
400}
401
402static inline void set_pmspr(unsigned long sprn, unsigned long val)
403{
404 switch (sprn) {
405 case SPRN_PMCR:
406 mtspr(SPRN_PMCR, val);
407 return;
408
409 case SPRN_PMICR:
410 mtspr(SPRN_PMICR, val);
411 return;
412 }
413 BUG();
414}
415
416/*
417 * Use objects of this type to query/update
418 * pstates on a remote CPU via smp_call_function.
419 */
420struct powernv_smp_call_data {
421 unsigned int freq;
422 int pstate_id;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530423 int gpstate_id;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530424};
425
426/*
427 * powernv_read_cpu_freq: Reads the current frequency on this CPU.
428 *
429 * Called via smp_call_function.
430 *
431 * Note: The caller of the smp_call_function should pass an argument of
432 * the type 'struct powernv_smp_call_data *' along with this function.
433 *
434 * The current frequency on this CPU will be returned via
435 * ((struct powernv_smp_call_data *)arg)->freq;
436 */
437static void powernv_read_cpu_freq(void *arg)
438{
439 unsigned long pmspr_val;
440 s8 local_pstate_id;
441 struct powernv_smp_call_data *freq_data = arg;
442
443 pmspr_val = get_pmspr(SPRN_PMSR);
444
445 /*
446 * The local pstate id corresponds bits 48..55 in the PMSR.
447 * Note: Watch out for the sign!
448 */
449 local_pstate_id = (pmspr_val >> 48) & 0xFF;
450 freq_data->pstate_id = local_pstate_id;
451 freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
452
453 pr_debug("cpu %d pmsr %016lX pstate_id %d frequency %d kHz\n",
454 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
455 freq_data->freq);
456}
457
458/*
459 * powernv_cpufreq_get: Returns the CPU frequency as reported by the
460 * firmware for CPU 'cpu'. This value is reported through the sysfs
461 * file cpuinfo_cur_freq.
462 */
Brian Norris60d1ea42014-05-11 00:51:20 -0700463static unsigned int powernv_cpufreq_get(unsigned int cpu)
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530464{
465 struct powernv_smp_call_data freq_data;
466
467 smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
468 &freq_data, 1);
469
470 return freq_data.freq;
471}
472
473/*
474 * set_pstate: Sets the pstate on this CPU.
475 *
476 * This is called via an smp_call_function.
477 *
478 * The caller must ensure that freq_data is of the type
479 * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
480 * on this CPU should be present in freq_data->pstate_id.
481 */
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530482static void set_pstate(void *data)
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530483{
484 unsigned long val;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530485 struct powernv_smp_call_data *freq_data = data;
486 unsigned long pstate_ul = freq_data->pstate_id;
487 unsigned long gpstate_ul = freq_data->gpstate_id;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530488
489 val = get_pmspr(SPRN_PMCR);
490 val = val & 0x0000FFFFFFFFFFFFULL;
491
492 pstate_ul = pstate_ul & 0xFF;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530493 gpstate_ul = gpstate_ul & 0xFF;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530494
495 /* Set both global(bits 56..63) and local(bits 48..55) PStates */
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530496 val = val | (gpstate_ul << 56) | (pstate_ul << 48);
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530497
498 pr_debug("Setting cpu %d pmcr to %016lX\n",
499 raw_smp_processor_id(), val);
500 set_pmspr(SPRN_PMCR, val);
501}
502
503/*
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +0200504 * get_nominal_index: Returns the index corresponding to the nominal
505 * pstate in the cpufreq table
506 */
507static inline unsigned int get_nominal_index(void)
508{
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530509 return powernv_pstate_info.nominal;
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +0200510}
511
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530512static void powernv_cpufreq_throttle_check(void *data)
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530513{
Michael Neuling3e5963b2016-03-21 22:24:52 +0530514 struct chip *chip;
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530515 unsigned int cpu = smp_processor_id();
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530516 unsigned long pmsr;
Michael Neuling3e5963b2016-03-21 22:24:52 +0530517 int pmsr_pmax;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530518 unsigned int pmsr_pmax_idx;
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530519
520 pmsr = get_pmspr(SPRN_PMSR);
Michael Neuling3e5963b2016-03-21 22:24:52 +0530521 chip = this_cpu_read(chip_info);
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530522
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530523 /* Check for Pmax Capping */
524 pmsr_pmax = (s8)PMSR_MAX(pmsr);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530525 pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
526 if (pmsr_pmax_idx != powernv_pstate_info.max) {
Michael Neuling3e5963b2016-03-21 22:24:52 +0530527 if (chip->throttled)
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530528 goto next;
Michael Neuling3e5963b2016-03-21 22:24:52 +0530529 chip->throttled = true;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530530 if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
531 pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n",
Michael Neuling3e5963b2016-03-21 22:24:52 +0530532 cpu, chip->id, pmsr_pmax,
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530533 idx_to_pstate(powernv_pstate_info.nominal));
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530534 chip->throttle_sub_turbo++;
535 } else {
536 chip->throttle_turbo++;
537 }
Michael Neuling3e5963b2016-03-21 22:24:52 +0530538 trace_powernv_throttle(chip->id,
539 throttle_reason[chip->throttle_reason],
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530540 pmsr_pmax);
Michael Neuling3e5963b2016-03-21 22:24:52 +0530541 } else if (chip->throttled) {
542 chip->throttled = false;
543 trace_powernv_throttle(chip->id,
544 throttle_reason[chip->throttle_reason],
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530545 pmsr_pmax);
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530546 }
547
Shilpasri G Bhat3dd3ebe2015-07-16 13:34:22 +0530548 /* Check if Psafe_mode_active is set in PMSR. */
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530549next:
Shilpasri G Bhat3dd3ebe2015-07-16 13:34:22 +0530550 if (pmsr & PMSR_PSAFE_ENABLE) {
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530551 throttled = true;
552 pr_info("Pstate set to safe frequency\n");
553 }
554
555 /* Check if SPR_EM_DISABLE is set in PMSR */
556 if (pmsr & PMSR_SPR_EM_DISABLE) {
557 throttled = true;
558 pr_info("Frequency Control disabled from OS\n");
559 }
560
561 if (throttled) {
562 pr_info("PMSR = %16lx\n", pmsr);
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530563 pr_warn("CPU Frequency could be throttled\n");
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530564 }
565}
566
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530567/**
568 * calc_global_pstate - Calculate global pstate
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530569 * @elapsed_time: Elapsed time in milliseconds
570 * @local_pstate_idx: New local pstate
571 * @highest_lpstate_idx: pstate from which its ramping down
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530572 *
573 * Finds the appropriate global pstate based on the pstate from which its
574 * ramping down and the time elapsed in ramping down. It follows a quadratic
575 * equation which ensures that it reaches ramping down to pmin in 5sec.
576 */
577static inline int calc_global_pstate(unsigned int elapsed_time,
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530578 int highest_lpstate_idx,
579 int local_pstate_idx)
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530580{
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530581 int index_diff;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530582
583 /*
584 * Using ramp_down_percent we get the percentage of rampdown
585 * that we are expecting to be dropping. Difference between
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530586 * highest_lpstate_idx and powernv_pstate_info.min will give a absolute
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530587 * number of how many pstates we will drop eventually by the end of
588 * 5 seconds, then just scale it get the number pstates to be dropped.
589 */
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530590 index_diff = ((int)ramp_down_percent(elapsed_time) *
591 (powernv_pstate_info.min - highest_lpstate_idx)) / 100;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530592
593 /* Ensure that global pstate is >= to local pstate */
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530594 if (highest_lpstate_idx + index_diff >= local_pstate_idx)
595 return local_pstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530596 else
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530597 return highest_lpstate_idx + index_diff;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530598}
599
600static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)
601{
602 unsigned int timer_interval;
603
604 /*
605 * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But
606 * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.
607 * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME
608 * seconds of ramp down time.
609 */
610 if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)
611 > MAX_RAMP_DOWN_TIME)
612 timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;
613 else
614 timer_interval = GPSTATE_TIMER_INTERVAL;
615
Thomas Gleixner7bc54b62016-07-04 09:50:18 +0000616 mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530617}
618
619/**
620 * gpstate_timer_handler
621 *
622 * @data: pointer to cpufreq_policy on which timer was queued
623 *
624 * This handler brings down the global pstate closer to the local pstate
625 * according quadratic equation. Queues a new timer if it is still not equal
626 * to local pstate
627 */
628void gpstate_timer_handler(unsigned long data)
629{
630 struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
631 struct global_pstate_info *gpstates = policy->driver_data;
Akshay Adiga20b15b72016-11-08 19:03:28 +0530632 int gpstate_idx, lpstate_idx;
633 unsigned long val;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530634 unsigned int time_diff = jiffies_to_msecs(jiffies)
635 - gpstates->last_sampled_time;
636 struct powernv_smp_call_data freq_data;
637
638 if (!spin_trylock(&gpstates->gpstate_lock))
639 return;
640
Akshay Adiga20b15b72016-11-08 19:03:28 +0530641 /*
642 * If PMCR was last updated was using fast_swtich then
643 * We may have wrong in gpstate->last_lpstate_idx
644 * value. Hence, read from PMCR to get correct data.
645 */
646 val = get_pmspr(SPRN_PMCR);
647 freq_data.gpstate_id = (s8)GET_GPSTATE(val);
648 freq_data.pstate_id = (s8)GET_LPSTATE(val);
649 if (freq_data.gpstate_id == freq_data.pstate_id) {
650 reset_gpstates(policy);
651 spin_unlock(&gpstates->gpstate_lock);
652 return;
653 }
654
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530655 gpstates->last_sampled_time += time_diff;
656 gpstates->elapsed_time += time_diff;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530657
Akshay Adiga20b15b72016-11-08 19:03:28 +0530658 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530659 gpstate_idx = pstate_to_idx(freq_data.pstate_id);
Akshay Adigac9a81e62016-11-14 17:29:27 +0530660 lpstate_idx = gpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530661 reset_gpstates(policy);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530662 gpstates->highest_lpstate_idx = gpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530663 } else {
Akshay Adiga20b15b72016-11-08 19:03:28 +0530664 lpstate_idx = pstate_to_idx(freq_data.pstate_id);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530665 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
666 gpstates->highest_lpstate_idx,
Akshay Adiga20b15b72016-11-08 19:03:28 +0530667 lpstate_idx);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530668 }
Akshay Adiga20b15b72016-11-08 19:03:28 +0530669 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
670 gpstates->last_gpstate_idx = gpstate_idx;
671 gpstates->last_lpstate_idx = lpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530672 /*
673 * If local pstate is equal to global pstate, rampdown is over
674 * So timer is not required to be queued.
675 */
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530676 if (gpstate_idx != gpstates->last_lpstate_idx)
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530677 queue_gpstate_timer(gpstates);
678
Akshay Adiga1fd3ff22016-05-03 20:49:35 +0530679 spin_unlock(&gpstates->gpstate_lock);
680
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530681 /* Timer may get migrated to a different cpu on cpu hot unplug */
682 smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530683}
684
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +0200685/*
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530686 * powernv_cpufreq_target_index: Sets the frequency corresponding to
687 * the cpufreq table entry indexed by new_index on the cpus in the
688 * mask policy->cpus
689 */
690static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
691 unsigned int new_index)
692{
693 struct powernv_smp_call_data freq_data;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530694 unsigned int cur_msec, gpstate_idx;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530695 struct global_pstate_info *gpstates = policy->driver_data;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530696
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +0200697 if (unlikely(rebooting) && new_index != get_nominal_index())
698 return 0;
699
Denis Kirjanov8a10c062016-11-08 05:39:28 -0500700 if (!throttled) {
701 /* we don't want to be preempted while
702 * checking if the CPU frequency has been throttled
703 */
704 preempt_disable();
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530705 powernv_cpufreq_throttle_check(NULL);
Denis Kirjanov8a10c062016-11-08 05:39:28 -0500706 preempt_enable();
707 }
Shilpasri G Bhat09a972d2015-04-01 15:16:34 +0530708
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530709 cur_msec = jiffies_to_msecs(get_jiffies_64());
710
Akshay Adiga1fd3ff22016-05-03 20:49:35 +0530711 spin_lock(&gpstates->gpstate_lock);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530712 freq_data.pstate_id = idx_to_pstate(new_index);
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530713
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530714 if (!gpstates->last_sampled_time) {
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530715 gpstate_idx = new_index;
716 gpstates->highest_lpstate_idx = new_index;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530717 goto gpstates_done;
718 }
719
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530720 if (gpstates->last_gpstate_idx < new_index) {
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530721 gpstates->elapsed_time += cur_msec -
722 gpstates->last_sampled_time;
723
724 /*
725 * If its has been ramping down for more than MAX_RAMP_DOWN_TIME
726 * we should be resetting all global pstate related data. Set it
727 * equal to local pstate to start fresh.
728 */
729 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
730 reset_gpstates(policy);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530731 gpstates->highest_lpstate_idx = new_index;
732 gpstate_idx = new_index;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530733 } else {
734 /* Elaspsed_time is less than 5 seconds, continue to rampdown */
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530735 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
736 gpstates->highest_lpstate_idx,
737 new_index);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530738 }
739 } else {
740 reset_gpstates(policy);
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530741 gpstates->highest_lpstate_idx = new_index;
742 gpstate_idx = new_index;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530743 }
744
745 /*
746 * If local pstate is equal to global pstate, rampdown is over
747 * So timer is not required to be queued.
748 */
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530749 if (gpstate_idx != new_index)
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530750 queue_gpstate_timer(gpstates);
Akshay Adiga0bc10b92016-05-03 20:49:36 +0530751 else
752 del_timer_sync(&gpstates->timer);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530753
754gpstates_done:
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530755 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530756 gpstates->last_sampled_time = cur_msec;
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530757 gpstates->last_gpstate_idx = gpstate_idx;
758 gpstates->last_lpstate_idx = new_index;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530759
Akshay Adiga1fd3ff22016-05-03 20:49:35 +0530760 spin_unlock(&gpstates->gpstate_lock);
761
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530762 /*
763 * Use smp_call_function to send IPI and execute the
764 * mtspr on target CPU. We could do that without IPI
765 * if current CPU is within policy->cpus (core)
766 */
767 smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530768 return 0;
769}
770
771static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
772{
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530773 int base, i, ret;
Shilpasri G Bhat2920e9c2016-04-19 15:28:00 +0530774 struct kernfs_node *kn;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530775 struct global_pstate_info *gpstates;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530776
777 base = cpu_first_thread_sibling(policy->cpu);
778
779 for (i = 0; i < threads_per_core; i++)
780 cpumask_set_cpu(base + i, policy->cpus);
781
Shilpasri G Bhat2920e9c2016-04-19 15:28:00 +0530782 kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);
783 if (!kn) {
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530784 int ret;
785
786 ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);
787 if (ret) {
788 pr_info("Failed to create throttle stats directory for cpu %d\n",
789 policy->cpu);
790 return ret;
791 }
Shilpasri G Bhat2920e9c2016-04-19 15:28:00 +0530792 } else {
793 kernfs_put(kn);
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530794 }
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530795
796 gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL);
797 if (!gpstates)
798 return -ENOMEM;
799
800 policy->driver_data = gpstates;
801
802 /* initialize timer */
Thomas Gleixner7bc54b62016-07-04 09:50:18 +0000803 init_timer_pinned_deferrable(&gpstates->timer);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530804 gpstates->timer.data = (unsigned long)policy;
805 gpstates->timer.function = gpstate_timer_handler;
806 gpstates->timer.expires = jiffies +
807 msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);
808 spin_lock_init(&gpstates->gpstate_lock);
809 ret = cpufreq_table_validate_and_show(policy, powernv_freqs);
810
Akshay Adiga60c9efb2016-11-08 19:03:27 +0530811 if (ret < 0) {
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530812 kfree(policy->driver_data);
Akshay Adiga60c9efb2016-11-08 19:03:27 +0530813 return ret;
814 }
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530815
Akshay Adiga60c9efb2016-11-08 19:03:27 +0530816 policy->fast_switch_possible = true;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530817 return ret;
818}
819
820static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)
821{
822 /* timer is deleted in cpufreq_cpu_stop() */
823 kfree(policy->driver_data);
824
825 return 0;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530826}
827
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +0200828static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
829 unsigned long action, void *unused)
830{
831 int cpu;
832 struct cpufreq_policy cpu_policy;
833
834 rebooting = true;
835 for_each_online_cpu(cpu) {
836 cpufreq_get_policy(&cpu_policy, cpu);
837 powernv_cpufreq_target_index(&cpu_policy, get_nominal_index());
838 }
839
840 return NOTIFY_DONE;
841}
842
843static struct notifier_block powernv_cpufreq_reboot_nb = {
844 .notifier_call = powernv_cpufreq_reboot_notifier,
845};
846
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530847void powernv_cpufreq_work_fn(struct work_struct *work)
848{
849 struct chip *chip = container_of(work, struct chip, throttle);
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530850 unsigned int cpu;
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530851 cpumask_t mask;
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530852
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530853 get_online_cpus();
854 cpumask_and(&mask, &chip->mask, cpu_online_mask);
855 smp_call_function_any(&mask,
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530856 powernv_cpufreq_throttle_check, NULL, 0);
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530857
858 if (!chip->restore)
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530859 goto out;
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530860
861 chip->restore = false;
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530862 for_each_cpu(cpu, &mask) {
863 int index;
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530864 struct cpufreq_policy policy;
865
866 cpufreq_get_policy(&policy, cpu);
Viresh Kumar82577362016-06-27 09:59:34 +0530867 index = cpufreq_table_find_index_c(&policy, policy.cur);
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530868 powernv_cpufreq_target_index(&policy, index);
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530869 cpumask_andnot(&mask, &mask, policy.cpus);
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530870 }
Shilpasri G Bhat6d167a42016-02-03 01:11:38 +0530871out:
872 put_online_cpus();
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530873}
874
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530875static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
876 unsigned long msg_type, void *_msg)
877{
878 struct opal_msg *msg = _msg;
879 struct opal_occ_msg omsg;
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530880 int i;
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530881
882 if (msg_type != OPAL_MSG_OCC)
883 return 0;
884
885 omsg.type = be64_to_cpu(msg->params[0]);
886
887 switch (omsg.type) {
888 case OCC_RESET:
889 occ_reset = true;
Shilpasri G Bhat309d0632015-08-27 14:41:44 +0530890 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530891 /*
892 * powernv_cpufreq_throttle_check() is called in
893 * target() callback which can detect the throttle state
894 * for governors like ondemand.
895 * But static governors will not call target() often thus
896 * report throttling here.
897 */
898 if (!throttled) {
899 throttled = true;
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530900 pr_warn("CPU frequency is throttled for duration\n");
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530901 }
Shilpasri G Bhat309d0632015-08-27 14:41:44 +0530902
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530903 break;
904 case OCC_LOAD:
Shilpasri G Bhat309d0632015-08-27 14:41:44 +0530905 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530906 break;
907 case OCC_THROTTLE:
908 omsg.chip = be64_to_cpu(msg->params[1]);
909 omsg.throttle_status = be64_to_cpu(msg->params[2]);
910
911 if (occ_reset) {
912 occ_reset = false;
913 throttled = false;
Shilpasri G Bhat309d0632015-08-27 14:41:44 +0530914 pr_info("OCC Active, CPU frequency is no longer throttled\n");
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530915
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530916 for (i = 0; i < nr_chips; i++) {
917 chips[i].restore = true;
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530918 schedule_work(&chips[i].throttle);
Shilpasri G Bhat227942802015-07-16 13:34:23 +0530919 }
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530920
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530921 return 0;
922 }
923
Shilpasri G Bhat735366f2015-07-16 13:34:21 +0530924 for (i = 0; i < nr_chips; i++)
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530925 if (chips[i].id == omsg.chip)
926 break;
927
928 if (omsg.throttle_status >= 0 &&
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530929 omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530930 chips[i].throttle_reason = omsg.throttle_status;
Shilpasri G Bhat1b028982016-03-22 18:57:09 +0530931 chips[i].reason[omsg.throttle_status]++;
932 }
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +0530933
934 if (!omsg.throttle_status)
935 chips[i].restore = true;
936
937 schedule_work(&chips[i].throttle);
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +0530938 }
939 return 0;
940}
941
942static struct notifier_block powernv_cpufreq_opal_nb = {
943 .notifier_call = powernv_cpufreq_occ_msg,
944 .next = NULL,
945 .priority = 0,
946};
947
Preeti U Murthyb1203392014-09-29 15:47:53 +0200948static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
949{
950 struct powernv_smp_call_data freq_data;
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530951 struct global_pstate_info *gpstates = policy->driver_data;
Preeti U Murthyb1203392014-09-29 15:47:53 +0200952
Akshay Adiga09ca4c92016-06-30 11:53:07 +0530953 freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);
954 freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);
Preeti U Murthyb1203392014-09-29 15:47:53 +0200955 smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530956 del_timer_sync(&gpstates->timer);
Preeti U Murthyb1203392014-09-29 15:47:53 +0200957}
958
Akshay Adiga60c9efb2016-11-08 19:03:27 +0530959static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,
960 unsigned int target_freq)
961{
962 int index;
963 struct powernv_smp_call_data freq_data;
964
965 index = cpufreq_table_find_index_dl(policy, target_freq);
966 freq_data.pstate_id = powernv_freqs[index].driver_data;
967 freq_data.gpstate_id = powernv_freqs[index].driver_data;
968 set_pstate(&freq_data);
969
970 return powernv_freqs[index].frequency;
971}
972
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530973static struct cpufreq_driver powernv_cpufreq_driver = {
974 .name = "powernv-cpufreq",
975 .flags = CPUFREQ_CONST_LOOPS,
976 .init = powernv_cpufreq_cpu_init,
Akshay Adigaeaa2c3a2016-04-19 15:28:01 +0530977 .exit = powernv_cpufreq_cpu_exit,
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530978 .verify = cpufreq_generic_frequency_table_verify,
979 .target_index = powernv_cpufreq_target_index,
Akshay Adiga60c9efb2016-11-08 19:03:27 +0530980 .fast_switch = powernv_fast_switch,
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530981 .get = powernv_cpufreq_get,
Preeti U Murthyb1203392014-09-29 15:47:53 +0200982 .stop_cpu = powernv_cpufreq_stop_cpu,
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +0530983 .attr = powernv_cpu_freq_attr,
984};
985
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530986static int init_chip_info(void)
987{
988 unsigned int chip[256];
989 unsigned int cpu, i;
990 unsigned int prev_chip_id = UINT_MAX;
991
Michael Neuling3e5963b2016-03-21 22:24:52 +0530992 for_each_possible_cpu(cpu) {
Shilpasri G Bhat053819e2015-07-16 13:34:18 +0530993 unsigned int id = cpu_to_chip_id(cpu);
994
995 if (prev_chip_id != id) {
996 prev_chip_id = id;
997 chip[nr_chips++] = id;
998 }
999 }
1000
Shilpasri G Bhatc89f2682016-02-03 01:11:41 +05301001 chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
Shilpasri G Bhat053819e2015-07-16 13:34:18 +05301002 if (!chips)
Michael Neuling3e5963b2016-03-21 22:24:52 +05301003 return -ENOMEM;
Shilpasri G Bhat053819e2015-07-16 13:34:18 +05301004
1005 for (i = 0; i < nr_chips; i++) {
1006 chips[i].id = chip[i];
Shilpasri G Bhat735366f2015-07-16 13:34:21 +05301007 cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i]));
1008 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
Michael Neuling3e5963b2016-03-21 22:24:52 +05301009 for_each_cpu(cpu, &chips[i].mask)
1010 per_cpu(chip_info, cpu) = &chips[i];
Shilpasri G Bhat053819e2015-07-16 13:34:18 +05301011 }
1012
1013 return 0;
1014}
1015
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301016static inline void clean_chip_info(void)
1017{
1018 kfree(chips);
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301019}
1020
1021static inline void unregister_all_notifiers(void)
1022{
1023 opal_message_notifier_unregister(OPAL_MSG_OCC,
1024 &powernv_cpufreq_opal_nb);
1025 unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
1026}
1027
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301028static int __init powernv_cpufreq_init(void)
1029{
1030 int rc = 0;
1031
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +05301032 /* Don't probe on pseries (guest) platforms */
Stewart Smithe4d54f72015-12-09 17:18:20 +11001033 if (!firmware_has_feature(FW_FEATURE_OPAL))
Vaidyanathan Srinivasan6174bac2014-08-03 14:54:05 +05301034 return -ENODEV;
1035
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301036 /* Discover pstates from device tree and init */
1037 rc = init_powernv_pstates();
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301038 if (rc)
1039 goto out;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301040
Shilpasri G Bhat053819e2015-07-16 13:34:18 +05301041 /* Populate chip info */
1042 rc = init_chip_info();
1043 if (rc)
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301044 goto out;
Shilpasri G Bhat053819e2015-07-16 13:34:18 +05301045
Shilpasri G Bhatcf30af762014-09-29 15:49:11 +02001046 register_reboot_notifier(&powernv_cpufreq_reboot_nb);
Shilpasri G Bhatcb166fa2015-07-16 13:34:20 +05301047 opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301048
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +05301049 if (powernv_pstate_info.wof_enabled)
1050 powernv_cpufreq_driver.boost_enabled = true;
1051 else
1052 powernv_cpu_freq_attr[SCALING_BOOST_FREQS_ATTR_INDEX] = NULL;
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301053
Shilpasri G Bhatb12f7a22017-01-03 16:36:00 +05301054 rc = cpufreq_register_driver(&powernv_cpufreq_driver);
1055 if (rc) {
1056 pr_info("Failed to register the cpufreq driver (%d)\n", rc);
1057 goto cleanup_notifiers;
1058 }
1059
1060 if (powernv_pstate_info.wof_enabled)
1061 cpufreq_enable_boost_support();
1062
1063 return 0;
1064cleanup_notifiers:
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301065 unregister_all_notifiers();
1066 clean_chip_info();
1067out:
1068 pr_info("Platform driver disabled. System does not support PState control\n");
1069 return rc;
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301070}
1071module_init(powernv_cpufreq_init);
1072
1073static void __exit powernv_cpufreq_exit(void)
1074{
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301075 cpufreq_unregister_driver(&powernv_cpufreq_driver);
Shilpasri G Bhatc5e29ea2016-02-26 16:06:51 +05301076 unregister_all_notifiers();
1077 clean_chip_info();
Vaidyanathan Srinivasanb3d627a2014-04-01 12:43:26 +05301078}
1079module_exit(powernv_cpufreq_exit);
1080
1081MODULE_LICENSE("GPL");
1082MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");