blob: 7636a4229f783304ba612550b2b933fbe8b79523 [file] [log] [blame]
/* Copyright (c) 2010-2017, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/hrtimer.h>
#include <linux/devfreq_cooling.h>
#include "kgsl.h"
#include "kgsl_pwrscale.h"
#include "kgsl_device.h"
#include "kgsl_trace.h"
/*
* "SLEEP" is generic counting both NAP & SLUMBER
* PERIODS generally won't exceed 9 for the relavent 150msec
* window, but can be significantly smaller and still POPP
* pushable in cases where SLUMBER is involved. Hence the
* additional reliance on PERCENT to make sure a reasonable
* amount of down-time actually exists.
*/
#define MIN_SLEEP_PERIODS 3
#define MIN_SLEEP_PERCENT 5
static struct kgsl_popp popp_param[POPP_MAX] = {
{0, 0},
{-5, 20},
{-5, 0},
{0, 0},
};
/**
* struct kgsl_midframe_info - midframe power stats sampling info
* @timer - midframe sampling timer
* @timer_check_ws - Updates powerstats on midframe expiry
* @device - pointer to kgsl_device
*/
static struct kgsl_midframe_info {
struct hrtimer timer;
struct work_struct timer_check_ws;
struct kgsl_device *device;
} *kgsl_midframe = NULL;
static void do_devfreq_suspend(struct work_struct *work);
static void do_devfreq_resume(struct work_struct *work);
static void do_devfreq_notify(struct work_struct *work);
/*
* These variables are used to keep the latest data
* returned by kgsl_devfreq_get_dev_status
*/
static struct xstats last_xstats;
static struct devfreq_dev_status last_status = { .private_data = &last_xstats };
/*
* kgsl_pwrscale_sleep - notify governor that device is going off
* @device: The device
*
* Called shortly after all pending work is completed.
*/
void kgsl_pwrscale_sleep(struct kgsl_device *device)
{
struct kgsl_pwrscale *psc = &device->pwrscale;
if (!device->pwrscale.enabled)
return;
device->pwrscale.on_time = 0;
psc->popp_level = 0;
clear_bit(POPP_PUSH, &device->pwrscale.popp_state);
/* to call devfreq_suspend_device() from a kernel thread */
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_suspend_ws);
}
EXPORT_SYMBOL(kgsl_pwrscale_sleep);
/*
* kgsl_pwrscale_wake - notify governor that device is going on
* @device: The device
*
* Called when the device is returning to an active state.
*/
void kgsl_pwrscale_wake(struct kgsl_device *device)
{
struct kgsl_power_stats stats;
struct kgsl_pwrscale *psc = &device->pwrscale;
if (!device->pwrscale.enabled)
return;
/* clear old stats before waking */
memset(&psc->accum_stats, 0, sizeof(psc->accum_stats));
memset(&last_xstats, 0, sizeof(last_xstats));
/* and any hw activity from waking up*/
device->ftbl->power_stats(device, &stats);
psc->time = ktime_get();
psc->next_governor_call = ktime_add_us(psc->time,
KGSL_GOVERNOR_CALL_INTERVAL);
/* to call devfreq_resume_device() from a kernel thread */
queue_work(psc->devfreq_wq, &psc->devfreq_resume_ws);
}
EXPORT_SYMBOL(kgsl_pwrscale_wake);
/*
* kgsl_pwrscale_busy - update pwrscale state for new work
* @device: The device
*
* Called when new work is submitted to the device.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_busy(struct kgsl_device *device)
{
if (!device->pwrscale.enabled)
return;
if (device->pwrscale.on_time == 0)
device->pwrscale.on_time = ktime_to_us(ktime_get());
}
EXPORT_SYMBOL(kgsl_pwrscale_busy);
/**
* kgsl_pwrscale_update_stats() - update device busy statistics
* @device: The device
*
* Read hardware busy counters and accumulate the results.
*/
void kgsl_pwrscale_update_stats(struct kgsl_device *device)
{
struct kgsl_pwrctrl *pwrctrl = &device->pwrctrl;
struct kgsl_pwrscale *psc = &device->pwrscale;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (!psc->enabled)
return;
if (device->state == KGSL_STATE_ACTIVE) {
struct kgsl_power_stats stats;
device->ftbl->power_stats(device, &stats);
if (psc->popp_level) {
u64 x = stats.busy_time;
u64 y = stats.ram_time;
do_div(x, 100);
do_div(y, 100);
x *= popp_param[psc->popp_level].gpu_x;
y *= popp_param[psc->popp_level].ddr_y;
trace_kgsl_popp_mod(device, x, y);
stats.busy_time += x;
stats.ram_time += y;
}
device->pwrscale.accum_stats.busy_time += stats.busy_time;
device->pwrscale.accum_stats.ram_time += stats.ram_time;
device->pwrscale.accum_stats.ram_wait += stats.ram_wait;
pwrctrl->clock_times[pwrctrl->active_pwrlevel] +=
stats.busy_time;
}
}
EXPORT_SYMBOL(kgsl_pwrscale_update_stats);
/**
* kgsl_pwrscale_update() - update device busy statistics
* @device: The device
*
* If enough time has passed schedule the next call to devfreq
* get_dev_status.
*/
void kgsl_pwrscale_update(struct kgsl_device *device)
{
ktime_t t;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (!device->pwrscale.enabled)
return;
t = ktime_get();
if (ktime_compare(t, device->pwrscale.next_governor_call) < 0)
return;
device->pwrscale.next_governor_call = ktime_add_us(t,
KGSL_GOVERNOR_CALL_INTERVAL);
/* to call srcu_notifier_call_chain() from a kernel thread */
if (device->state != KGSL_STATE_SLUMBER)
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_notify_ws);
kgsl_pwrscale_midframe_timer_restart(device);
}
EXPORT_SYMBOL(kgsl_pwrscale_update);
void kgsl_pwrscale_midframe_timer_restart(struct kgsl_device *device)
{
if (kgsl_midframe) {
WARN_ON(!mutex_is_locked(&device->mutex));
/* If the timer is already running, stop it */
if (hrtimer_active(&kgsl_midframe->timer))
hrtimer_cancel(
&kgsl_midframe->timer);
hrtimer_start(&kgsl_midframe->timer,
ns_to_ktime(KGSL_GOVERNOR_CALL_INTERVAL
* NSEC_PER_USEC), HRTIMER_MODE_REL);
}
}
EXPORT_SYMBOL(kgsl_pwrscale_midframe_timer_restart);
void kgsl_pwrscale_midframe_timer_cancel(struct kgsl_device *device)
{
if (kgsl_midframe) {
WARN_ON(!mutex_is_locked(&device->mutex));
hrtimer_cancel(&kgsl_midframe->timer);
}
}
EXPORT_SYMBOL(kgsl_pwrscale_midframe_timer_cancel);
static void kgsl_pwrscale_midframe_timer_check(struct work_struct *work)
{
struct kgsl_device *device = kgsl_midframe->device;
mutex_lock(&device->mutex);
if (device->state == KGSL_STATE_ACTIVE)
kgsl_pwrscale_update(device);
mutex_unlock(&device->mutex);
}
static enum hrtimer_restart kgsl_pwrscale_midframe_timer(struct hrtimer *timer)
{
struct kgsl_device *device = kgsl_midframe->device;
queue_work(device->pwrscale.devfreq_wq,
&kgsl_midframe->timer_check_ws);
return HRTIMER_NORESTART;
}
/*
* kgsl_pwrscale_disable - temporarily disable the governor
* @device: The device
* @turbo: Indicates if pwrlevel should be forced to turbo
*
* Temporarily disable the governor, to prevent interference
* with profiling tools that expect a fixed clock frequency.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_disable(struct kgsl_device *device, bool turbo)
{
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (device->pwrscale.devfreqptr)
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_suspend_ws);
device->pwrscale.enabled = false;
if (turbo)
kgsl_pwrctrl_pwrlevel_change(device, KGSL_PWRLEVEL_TURBO);
}
EXPORT_SYMBOL(kgsl_pwrscale_disable);
/*
* kgsl_pwrscale_enable - re-enable the governor
* @device: The device
*
* Reenable the governor after a kgsl_pwrscale_disable() call.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_enable(struct kgsl_device *device)
{
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (device->pwrscale.devfreqptr) {
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_resume_ws);
device->pwrscale.enabled = true;
} else {
/*
* Don't enable it if devfreq is not set and let the device
* run at default level;
*/
kgsl_pwrctrl_pwrlevel_change(device,
device->pwrctrl.default_pwrlevel);
device->pwrscale.enabled = false;
}
}
EXPORT_SYMBOL(kgsl_pwrscale_enable);
static int _thermal_adjust(struct kgsl_pwrctrl *pwr, int level)
{
if (level < pwr->active_pwrlevel)
return pwr->active_pwrlevel;
/*
* A lower frequency has been recommended! Stop thermal
* cycling (but keep the upper thermal limit) and switch to
* the lower frequency.
*/
pwr->thermal_cycle = CYCLE_ENABLE;
del_timer_sync(&pwr->thermal_timer);
return level;
}
/*
* Use various metrics including level stability, NAP intervals, and
* overall GPU freq / DDR freq combination to decide if POPP should
* be activated.
*/
static bool popp_stable(struct kgsl_device *device)
{
s64 t;
s64 nap_time = 0;
s64 go_time = 0;
int i, index;
int nap = 0;
s64 percent_nap = 0;
struct kgsl_pwr_event *e;
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct kgsl_pwrscale *psc = &device->pwrscale;
if (!test_bit(POPP_ON, &psc->popp_state))
return false;
/* If already pushed or running naturally at min don't push further */
if (test_bit(POPP_PUSH, &psc->popp_state))
return false;
if (!psc->popp_level &&
(pwr->active_pwrlevel == pwr->min_pwrlevel))
return false;
if (psc->history[KGSL_PWREVENT_STATE].events == NULL)
return false;
t = ktime_to_ms(ktime_get());
/* Check for recent NAP statistics: NAPping regularly and well? */
if (pwr->active_pwrlevel == 0) {
index = psc->history[KGSL_PWREVENT_STATE].index;
i = index > 0 ? (index - 1) :
(psc->history[KGSL_PWREVENT_STATE].size - 1);
while (i != index) {
e = &psc->history[KGSL_PWREVENT_STATE].events[i];
if (e->data == KGSL_STATE_NAP ||
e->data == KGSL_STATE_SLUMBER) {
if (ktime_to_ms(e->start) + STABLE_TIME > t) {
nap++;
nap_time += e->duration;
}
} else if (e->data == KGSL_STATE_ACTIVE) {
if (ktime_to_ms(e->start) + STABLE_TIME > t)
go_time += e->duration;
}
if (i == 0)
i = psc->history[KGSL_PWREVENT_STATE].size - 1;
else
i--;
}
if (nap_time && go_time) {
percent_nap = 100 * nap_time;
do_div(percent_nap, nap_time + go_time);
}
trace_kgsl_popp_nap(device, (int)nap_time / 1000, nap,
percent_nap);
/* If running high at turbo, don't push */
if (nap < MIN_SLEEP_PERIODS || percent_nap < MIN_SLEEP_PERCENT)
return false;
}
/* Finally check that there hasn't been a recent change */
if ((device->pwrscale.freq_change_time + STABLE_TIME) < t) {
device->pwrscale.freq_change_time = t;
return true;
}
return false;
}
bool kgsl_popp_check(struct kgsl_device *device)
{
int i;
struct kgsl_pwrscale *psc = &device->pwrscale;
struct kgsl_pwr_event *e;
if (!test_bit(POPP_ON, &psc->popp_state))
return false;
if (!test_bit(POPP_PUSH, &psc->popp_state))
return false;
if (psc->history[KGSL_PWREVENT_STATE].events == NULL) {
clear_bit(POPP_PUSH, &psc->popp_state);
return false;
}
e = &psc->history[KGSL_PWREVENT_STATE].
events[psc->history[KGSL_PWREVENT_STATE].index];
if (e->data == KGSL_STATE_SLUMBER)
e->duration = ktime_us_delta(ktime_get(), e->start);
/* If there's been a long SLUMBER in recent history, clear the _PUSH */
for (i = 0; i < psc->history[KGSL_PWREVENT_STATE].size; i++) {
e = &psc->history[KGSL_PWREVENT_STATE].events[i];
if ((e->data == KGSL_STATE_SLUMBER) &&
(e->duration > POPP_RESET_TIME)) {
clear_bit(POPP_PUSH, &psc->popp_state);
return false;
}
}
return true;
}
/*
* The GPU has been running at the current frequency for a while. Attempt
* to lower the frequency for boarderline cases.
*/
static void popp_trans1(struct kgsl_device *device)
{
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct kgsl_pwrlevel *pl = &pwr->pwrlevels[pwr->active_pwrlevel];
struct kgsl_pwrscale *psc = &device->pwrscale;
int old_level = psc->popp_level;
switch (old_level) {
case 0:
psc->popp_level = 2;
/* If the current level has a high default bus don't push it */
if (pl->bus_freq == pl->bus_max)
pwr->bus_mod = 1;
kgsl_pwrctrl_pwrlevel_change(device, pwr->active_pwrlevel + 1);
break;
case 1:
case 2:
psc->popp_level++;
break;
case 3:
set_bit(POPP_PUSH, &psc->popp_state);
psc->popp_level = 0;
break;
case POPP_MAX:
default:
psc->popp_level = 0;
break;
}
trace_kgsl_popp_level(device, old_level, psc->popp_level);
}
/*
* The GPU DCVS algorithm recommends a level change. Apply any
* POPP restrictions and update the level accordingly
*/
static int popp_trans2(struct kgsl_device *device, int level)
{
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct kgsl_pwrscale *psc = &device->pwrscale;
int old_level = psc->popp_level;
if (!test_bit(POPP_ON, &psc->popp_state))
return level;
clear_bit(POPP_PUSH, &psc->popp_state);
/* If the governor recommends going down, do it! */
if (pwr->active_pwrlevel < level) {
psc->popp_level = 0;
trace_kgsl_popp_level(device, old_level, psc->popp_level);
return level;
}
switch (psc->popp_level) {
case 0:
/* If the feature isn't engaged, go up immediately */
break;
case 1:
/* Turn off mitigation, and go up a level */
psc->popp_level = 0;
break;
case 2:
case 3:
/* Try a more aggressive mitigation */
psc->popp_level--;
level++;
/* Update the stable timestamp */
device->pwrscale.freq_change_time = ktime_to_ms(ktime_get());
break;
case POPP_MAX:
default:
psc->popp_level = 0;
break;
}
trace_kgsl_popp_level(device, old_level, psc->popp_level);
return level;
}
#ifdef DEVFREQ_FLAG_WAKEUP_MAXFREQ
static inline bool _check_maxfreq(u32 flags)
{
return (flags & DEVFREQ_FLAG_WAKEUP_MAXFREQ);
}
#else
static inline bool _check_maxfreq(u32 flags)
{
return false;
}
#endif
/*
* kgsl_devfreq_target - devfreq_dev_profile.target callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_target(struct device *dev, unsigned long *freq, u32 flags)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwr;
struct kgsl_pwrlevel *pwr_level;
int level, i;
unsigned long cur_freq, rec_freq;
struct dev_pm_opp *opp;
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
if (!device->pwrscale.enabled)
return 0;
pwr = &device->pwrctrl;
if (_check_maxfreq(flags)) {
/*
* The GPU is about to get suspended,
* but it needs to be at the max power level when waking up
*/
pwr->wakeup_maxpwrlevel = 1;
return 0;
}
/*
* Thermal framework might have disabled/enabled OPP entries
* for mitigation. So find the recommended frequency matching
* the available opp entries
*/
rcu_read_lock();
rec_freq = *freq;
opp = devfreq_recommended_opp(dev, &rec_freq, flags);
if (IS_ERR(opp)) {
rcu_read_unlock();
return PTR_ERR(opp);
}
rec_freq = dev_pm_opp_get_freq(opp);
rcu_read_unlock();
mutex_lock(&device->mutex);
cur_freq = kgsl_pwrctrl_active_freq(pwr);
level = pwr->active_pwrlevel;
pwr_level = &pwr->pwrlevels[level];
/* If the governor recommends a new frequency, update it here */
if (rec_freq != cur_freq) {
level = pwr->max_pwrlevel;
for (i = pwr->min_pwrlevel; i >= pwr->max_pwrlevel; i--)
if (rec_freq <= pwr->pwrlevels[i].gpu_freq) {
if (pwr->thermal_cycle == CYCLE_ACTIVE)
level = _thermal_adjust(pwr, i);
else
level = popp_trans2(device, i);
break;
}
if (level != pwr->active_pwrlevel)
kgsl_pwrctrl_pwrlevel_change(device, level);
} else if (popp_stable(device)) {
popp_trans1(device);
}
*freq = kgsl_pwrctrl_active_freq(pwr);
mutex_unlock(&device->mutex);
return 0;
}
EXPORT_SYMBOL(kgsl_devfreq_target);
/*
* kgsl_devfreq_get_dev_status - devfreq_dev_profile.get_dev_status callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwrctrl;
struct kgsl_pwrscale *pwrscale;
ktime_t tmp;
if (device == NULL)
return -ENODEV;
if (stat == NULL)
return -EINVAL;
pwrscale = &device->pwrscale;
pwrctrl = &device->pwrctrl;
mutex_lock(&device->mutex);
/*
* If the GPU clock is on grab the latest power counter
* values. Otherwise the most recent ACTIVE values will
* already be stored in accum_stats.
*/
kgsl_pwrscale_update_stats(device);
tmp = ktime_get();
stat->total_time = ktime_us_delta(tmp, pwrscale->time);
pwrscale->time = tmp;
stat->busy_time = pwrscale->accum_stats.busy_time;
stat->current_frequency = kgsl_pwrctrl_active_freq(&device->pwrctrl);
stat->private_data = &device->active_context_count;
/*
* keep the latest devfreq_dev_status values
* and vbif counters data
* to be (re)used by kgsl_busmon_get_dev_status()
*/
if (pwrctrl->bus_control) {
struct xstats *last_b =
(struct xstats *)last_status.private_data;
last_status.total_time = stat->total_time;
last_status.busy_time = stat->busy_time;
last_status.current_frequency = stat->current_frequency;
last_b->ram_time = device->pwrscale.accum_stats.ram_time;
last_b->ram_wait = device->pwrscale.accum_stats.ram_wait;
last_b->mod = device->pwrctrl.bus_mod;
}
kgsl_pwrctrl_busy_time(device, stat->total_time, stat->busy_time);
trace_kgsl_pwrstats(device, stat->total_time,
&pwrscale->accum_stats, device->active_context_count);
memset(&pwrscale->accum_stats, 0, sizeof(pwrscale->accum_stats));
mutex_unlock(&device->mutex);
return 0;
}
EXPORT_SYMBOL(kgsl_devfreq_get_dev_status);
/*
* kgsl_devfreq_get_cur_freq - devfreq_dev_profile.get_cur_freq callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
mutex_lock(&device->mutex);
*freq = kgsl_pwrctrl_active_freq(&device->pwrctrl);
mutex_unlock(&device->mutex);
return 0;
}
EXPORT_SYMBOL(kgsl_devfreq_get_cur_freq);
/*
* kgsl_devfreq_add_notifier - add a fine grained notifier.
* @dev: The device
* @nb: Notifier block that will receive updates.
*
* Add a notifier to receive ADRENO_DEVFREQ_NOTIFY_* events
* from the device.
*/
int kgsl_devfreq_add_notifier(struct device *dev,
struct notifier_block *nb)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (nb == NULL)
return -EINVAL;
return srcu_notifier_chain_register(&device->pwrscale.nh, nb);
}
EXPORT_SYMBOL(kgsl_devfreq_add_notifier);
/*
* kgsl_devfreq_del_notifier - remove a fine grained notifier.
* @dev: The device
* @nb: The notifier block.
*
* Remove a notifier registered with kgsl_devfreq_add_notifier().
*/
int kgsl_devfreq_del_notifier(struct device *dev, struct notifier_block *nb)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (nb == NULL)
return -EINVAL;
return srcu_notifier_chain_unregister(&device->pwrscale.nh, nb);
}
EXPORT_SYMBOL(kgsl_devfreq_del_notifier);
/*
* kgsl_busmon_get_dev_status - devfreq_dev_profile.get_dev_status callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_busmon_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
struct xstats *b;
stat->total_time = last_status.total_time;
stat->busy_time = last_status.busy_time;
stat->current_frequency = last_status.current_frequency;
if (stat->private_data) {
struct xstats *last_b =
(struct xstats *)last_status.private_data;
b = (struct xstats *)stat->private_data;
b->ram_time = last_b->ram_time;
b->ram_wait = last_b->ram_wait;
b->mod = last_b->mod;
}
return 0;
}
#ifdef DEVFREQ_FLAG_FAST_HINT
static inline bool _check_fast_hint(u32 flags)
{
return (flags & DEVFREQ_FLAG_FAST_HINT);
}
#else
static inline bool _check_fast_hint(u32 flags)
{
return false;
}
#endif
#ifdef DEVFREQ_FLAG_SLOW_HINT
static inline bool _check_slow_hint(u32 flags)
{
return (flags & DEVFREQ_FLAG_SLOW_HINT);
}
#else
static inline bool _check_slow_hint(u32 flags)
{
return false;
}
#endif
/*
* kgsl_busmon_target - devfreq_dev_profile.target callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_busmon_target(struct device *dev, unsigned long *freq, u32 flags)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwr;
struct kgsl_pwrlevel *pwr_level;
int level, b;
u32 bus_flag;
unsigned long ab_mbytes;
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
if (!device->pwrscale.enabled)
return 0;
pwr = &device->pwrctrl;
if (!pwr->bus_control)
return 0;
mutex_lock(&device->mutex);
level = pwr->active_pwrlevel;
pwr_level = &pwr->pwrlevels[level];
bus_flag = device->pwrscale.bus_profile.flag;
device->pwrscale.bus_profile.flag = 0;
ab_mbytes = device->pwrscale.bus_profile.ab_mbytes;
/*
* Bus devfreq governor has calculated its recomendations
* when gpu was running with *freq frequency.
* If the gpu frequency is different now it's better to
* ignore the call
*/
if (pwr_level->gpu_freq != *freq) {
mutex_unlock(&device->mutex);
return 0;
}
b = pwr->bus_mod;
if (_check_fast_hint(bus_flag) &&
((pwr_level->bus_freq + pwr->bus_mod) < pwr_level->bus_max))
pwr->bus_mod++;
else if (_check_slow_hint(bus_flag) &&
((pwr_level->bus_freq + pwr->bus_mod) > pwr_level->bus_min))
pwr->bus_mod--;
/* Update bus vote if AB or IB is modified */
if ((pwr->bus_mod != b) || (pwr->bus_ab_mbytes != ab_mbytes)) {
pwr->bus_percent_ab = device->pwrscale.bus_profile.percent_ab;
pwr->bus_ab_mbytes = ab_mbytes;
kgsl_pwrctrl_buslevel_update(device, true);
}
mutex_unlock(&device->mutex);
return 0;
}
int kgsl_busmon_get_cur_freq(struct device *dev, unsigned long *freq)
{
return 0;
}
/*
* kgsl_pwrscale_init - Initialize pwrscale.
* @dev: The device
* @governor: The initial governor to use.
*
* Initialize devfreq and any non-constant profile data.
*/
int kgsl_pwrscale_init(struct device *dev, const char *governor)
{
struct kgsl_device *device;
struct kgsl_pwrscale *pwrscale;
struct kgsl_pwrctrl *pwr;
struct devfreq *devfreq;
struct devfreq *bus_devfreq;
struct msm_adreno_extended_profile *gpu_profile;
struct devfreq_dev_profile *profile;
struct devfreq_msm_adreno_tz_data *data;
int i, out = 0;
int ret;
device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
pwrscale = &device->pwrscale;
pwr = &device->pwrctrl;
gpu_profile = &pwrscale->gpu_profile;
profile = &pwrscale->gpu_profile.profile;
srcu_init_notifier_head(&pwrscale->nh);
profile->initial_freq =
pwr->pwrlevels[pwr->default_pwrlevel].gpu_freq;
/* Let's start with 10 ms and tune in later */
profile->polling_ms = 10;
/* do not include the 'off' level or duplicate freq. levels */
for (i = 0; i < (pwr->num_pwrlevels - 1); i++)
pwrscale->freq_table[out++] = pwr->pwrlevels[i].gpu_freq;
/*
* Max_state is the number of valid power levels.
* The valid power levels range from 0 - (max_state - 1)
*/
profile->max_state = pwr->num_pwrlevels - 1;
/* link storage array to the devfreq profile pointer */
profile->freq_table = pwrscale->freq_table;
/* if there is only 1 freq, no point in running a governor */
if (profile->max_state == 1)
governor = "performance";
/* initialize msm-adreno-tz governor specific data here */
data = gpu_profile->private_data;
data->disable_busy_time_burst = of_property_read_bool(
device->pdev->dev.of_node, "qcom,disable-busy-time-burst");
data->ctxt_aware_enable =
of_property_read_bool(device->pdev->dev.of_node,
"qcom,enable-ca-jump");
if (data->ctxt_aware_enable) {
if (of_property_read_u32(device->pdev->dev.of_node,
"qcom,ca-target-pwrlevel",
&data->bin.ctxt_aware_target_pwrlevel))
data->bin.ctxt_aware_target_pwrlevel = 1;
if ((data->bin.ctxt_aware_target_pwrlevel >
pwr->num_pwrlevels))
data->bin.ctxt_aware_target_pwrlevel = 1;
if (of_property_read_u32(device->pdev->dev.of_node,
"qcom,ca-busy-penalty",
&data->bin.ctxt_aware_busy_penalty))
data->bin.ctxt_aware_busy_penalty = 12000;
}
if (of_property_read_bool(device->pdev->dev.of_node,
"qcom,enable-midframe-timer")) {
kgsl_midframe = kzalloc(
sizeof(struct kgsl_midframe_info), GFP_KERNEL);
if (kgsl_midframe) {
hrtimer_init(&kgsl_midframe->timer,
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
kgsl_midframe->timer.function =
kgsl_pwrscale_midframe_timer;
kgsl_midframe->device = device;
} else
KGSL_PWR_ERR(device,
"Failed to enable-midframe-timer feature\n");
}
/*
* If there is a separate GX power rail, allow
* independent modification to its voltage through
* the bus bandwidth vote.
*/
if (pwr->bus_control) {
out = 0;
while (pwr->bus_ib[out] && out <= pwr->pwrlevels[0].bus_max) {
pwr->bus_ib[out] =
pwr->bus_ib[out] >> 20;
out++;
}
data->bus.num = out;
data->bus.ib = &pwr->bus_ib[0];
data->bus.index = &pwr->bus_index[0];
data->bus.width = pwr->bus_width;
} else
data->bus.num = 0;
devfreq = devfreq_add_device(dev, &pwrscale->gpu_profile.profile,
governor, pwrscale->gpu_profile.private_data);
if (IS_ERR(devfreq)) {
device->pwrscale.enabled = false;
return PTR_ERR(devfreq);
}
pwrscale->devfreqptr = devfreq;
pwrscale->cooling_dev = of_devfreq_cooling_register(
device->pdev->dev.of_node, devfreq);
if (IS_ERR(pwrscale->cooling_dev))
pwrscale->cooling_dev = NULL;
pwrscale->gpu_profile.bus_devfreq = NULL;
if (data->bus.num) {
pwrscale->bus_profile.profile.max_state
= pwr->num_pwrlevels - 1;
pwrscale->bus_profile.profile.freq_table
= pwrscale->freq_table;
bus_devfreq = devfreq_add_device(device->busmondev,
&pwrscale->bus_profile.profile, "gpubw_mon", NULL);
if (!IS_ERR(bus_devfreq))
pwrscale->gpu_profile.bus_devfreq = bus_devfreq;
}
ret = sysfs_create_link(&device->dev->kobj,
&devfreq->dev.kobj, "devfreq");
pwrscale->devfreq_wq = create_freezable_workqueue("kgsl_devfreq_wq");
INIT_WORK(&pwrscale->devfreq_suspend_ws, do_devfreq_suspend);
INIT_WORK(&pwrscale->devfreq_resume_ws, do_devfreq_resume);
INIT_WORK(&pwrscale->devfreq_notify_ws, do_devfreq_notify);
if (kgsl_midframe)
INIT_WORK(&kgsl_midframe->timer_check_ws,
kgsl_pwrscale_midframe_timer_check);
pwrscale->next_governor_call = ktime_add_us(ktime_get(),
KGSL_GOVERNOR_CALL_INTERVAL);
/* history tracking */
for (i = 0; i < KGSL_PWREVENT_MAX; i++) {
pwrscale->history[i].events = kzalloc(
pwrscale->history[i].size *
sizeof(struct kgsl_pwr_event), GFP_KERNEL);
pwrscale->history[i].type = i;
}
/* Add links to the devfreq sysfs nodes */
kgsl_gpu_sysfs_add_link(device->gpu_sysfs_kobj,
&pwrscale->devfreqptr->dev.kobj, "governor",
"gpu_governor");
kgsl_gpu_sysfs_add_link(device->gpu_sysfs_kobj,
&pwrscale->devfreqptr->dev.kobj,
"available_governors", "gpu_available_governor");
return 0;
}
EXPORT_SYMBOL(kgsl_pwrscale_init);
/*
* kgsl_pwrscale_close - clean up pwrscale
* @device: the device
*
* This function should be called with the device mutex locked.
*/
void kgsl_pwrscale_close(struct kgsl_device *device)
{
int i;
struct kgsl_pwrscale *pwrscale;
pwrscale = &device->pwrscale;
if (!pwrscale->devfreqptr)
return;
if (pwrscale->cooling_dev)
devfreq_cooling_unregister(pwrscale->cooling_dev);
kgsl_pwrscale_midframe_timer_cancel(device);
flush_workqueue(pwrscale->devfreq_wq);
destroy_workqueue(pwrscale->devfreq_wq);
devfreq_remove_device(device->pwrscale.devfreqptr);
kfree(kgsl_midframe);
kgsl_midframe = NULL;
device->pwrscale.devfreqptr = NULL;
srcu_cleanup_notifier_head(&device->pwrscale.nh);
for (i = 0; i < KGSL_PWREVENT_MAX; i++)
kfree(pwrscale->history[i].events);
}
EXPORT_SYMBOL(kgsl_pwrscale_close);
static void do_devfreq_suspend(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_suspend_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
devfreq_suspend_device(devfreq);
}
static void do_devfreq_resume(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_resume_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
devfreq_resume_device(devfreq);
}
static void do_devfreq_notify(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_notify_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
srcu_notifier_call_chain(&pwrscale->nh,
ADRENO_DEVFREQ_NOTIFY_RETIRE,
devfreq);
}