From aee92c8a4e1950d8cf9db60288cd9a2816c1c379 Mon Sep 17 00:00:00 2001 From: Urvashi Agrawal Date: Wed, 21 Feb 2018 13:37:56 -0800 Subject: [PATCH] msm: kgsl: Add GPU bus governor from the msm-4.9 to msm-4.14 Adding a snapshot of the devfreq files on 4.9 as of commit cf0cc6e60bc7 (msm: pcie:fix PCIe L0s and L1 ASPM support). Change-Id: I627890ce33d88798da5c9ca58be7eb2420a792f0 Signed-off-by: Urvashi Agrawal --- drivers/devfreq/Kconfig | 53 ++ drivers/devfreq/Makefile | 6 + drivers/devfreq/devfreq_trace.h | 44 ++ drivers/devfreq/governor_bw_vbif.c | 141 +++++ drivers/devfreq/governor_gpubw_mon.c | 266 +++++++++ drivers/devfreq/governor_msm_adreno_tz.c | 664 +++++++++++++++++++++++ 6 files changed, 1174 insertions(+) create mode 100644 drivers/devfreq/devfreq_trace.h create mode 100644 drivers/devfreq/governor_bw_vbif.c create mode 100644 drivers/devfreq/governor_gpubw_mon.c create mode 100644 drivers/devfreq/governor_msm_adreno_tz.c diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig index d2b975bd612e..52c2706df556 100644 --- a/drivers/devfreq/Kconfig +++ b/drivers/devfreq/Kconfig @@ -73,6 +73,14 @@ config DEVFREQ_GOV_PASSIVE through sysfs entries. The passive governor recommends that devfreq device uses the OPP table to get the frequency/voltage. +config DEVFREQ_GOV_CPUFREQ + tristate "CPUfreq" + depends on CPU_FREQ + help + Chooses frequency based on the online CPUs' current frequency and a + CPU frequency to device frequency mapping table(s). This governor + can be useful for controlling devices such as DDR, cache, CCI, etc. + config QCOM_BIMC_BWMON tristate "QCOM BIMC Bandwidth monitor hardware" depends on ARCH_QCOM @@ -128,6 +136,15 @@ config DEVFREQ_GOV_QCOM_CACHE_HWMON it can conflict with existing profiling tools. This governor is unlikely to be useful for other devices. +config DEVFREQ_GOV_SPDM_HYP + bool "QTI SPDM Hypervisor Governor" + depends on ARCH_QCOM + help + Hypervisor based governor for CPU bandwidth voting + for QTI chipsets. + Sets the frequency using a "on-demand" algorithm. + This governor is unlikely to be useful for other devices. + config DEVFREQ_GOV_MEMLAT tristate "HW monitor based governor for device BW" depends on ARM_MEMLAT_MON @@ -140,6 +157,24 @@ config DEVFREQ_GOV_MEMLAT comment "DEVFREQ Drivers" +config DEVFREQ_GOV_QCOM_ADRENO_TZ + tristate "Qualcomm Technologies Inc Adreno Trustzone" + depends on QCOM_KGSL && QCOM_SCM + help + Trustzone based governor for the Adreno GPU. Sets + the frequency using a "on-demand" algorithm. This + governor is unlikely to be useful for other + devices. + +config DEVFREQ_GOV_QCOM_GPUBW_MON + tristate "GPU BW voting governor" + depends on DEVFREQ_GOV_QCOM_ADRENO_TZ + help + This governor works together with Adreno Trustzone governor, + and select bus frequency votes using an "on-demand" alorithm. + This governor is unlikely to be useful for non-Adreno + devices. + config ARM_EXYNOS_BUS_DEVFREQ tristate "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" depends on ARCH_EXYNOS || COMPILE_TEST @@ -202,6 +237,24 @@ config QCOM_DEVFREQ_DEVBW agnostic interface to so that some of the devfreq governors can be shared across SoCs. +config SPDM_SCM + bool "QTI SPDM SCM based call support" + depends on DEVFREQ_SPDM + help + SPDM driver support the dcvs algorithm logic being accessed via + scm or hvc calls. This adds the support for SPDM interaction to + tz via SCM based call. If not selected then Hypervisor interaction + will be activated. + +config DEVFREQ_SPDM + bool "QTI SPDM based bandwidth voting" + depends on ARCH_QCOM + select DEVFREQ_GOV_SPDM_HYP + help + This adds the support for SPDM based bandwidth voting on QTI chipsets. + This driver allows any SPDM based client to vote for bandwidth. + Used with the QTI SPDM Hypervisor Governor. + source "drivers/devfreq/event/Kconfig" endif # PM_DEVFREQ diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile index fc5f8f708822..0f669c91334e 100644 --- a/drivers/devfreq/Makefile +++ b/drivers/devfreq/Makefile @@ -6,12 +6,17 @@ obj-$(CONFIG_DEVFREQ_GOV_PERFORMANCE) += governor_performance.o obj-$(CONFIG_DEVFREQ_GOV_POWERSAVE) += governor_powersave.o obj-$(CONFIG_DEVFREQ_GOV_USERSPACE) += governor_userspace.o obj-$(CONFIG_DEVFREQ_GOV_PASSIVE) += governor_passive.o +#obj-$(CONFIG_DEVFREQ_GOV_CPUFREQ) += governor_cpufreq.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_ADRENO_TZ) += governor_msm_adreno_tz.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON) += governor_bw_vbif.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON) += governor_gpubw_mon.o obj-$(CONFIG_QCOM_BIMC_BWMON) += bimc-bwmon.o obj-$(CONFIG_ARM_MEMLAT_MON) += arm-memlat-mon.o obj-$(CONFIG_QCOMCCI_HWMON) += msmcci-hwmon.o obj-$(CONFIG_QCOM_M4M_HWMON) += m4m-hwmon.o obj-$(CONFIG_DEVFREQ_GOV_QCOM_BW_HWMON) += governor_bw_hwmon.o obj-$(CONFIG_DEVFREQ_GOV_QCOM_CACHE_HWMON) += governor_cache_hwmon.o +obj-$(CONFIG_DEVFREQ_GOV_SPDM_HYP) += governor_spdm_bw_hyp.o obj-$(CONFIG_DEVFREQ_GOV_MEMLAT) += governor_memlat.o # DEVFREQ Drivers @@ -20,6 +25,7 @@ obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ) += rk3399_dmc.o obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra-devfreq.o obj-$(CONFIG_QCOM_DEVFREQ_DEVBW) += devfreq_devbw.o obj-$(CONFIG_DEVFREQ_SIMPLE_DEV) += devfreq_simple_dev.o +obj-$(CONFIG_DEVFREQ_SPDM) += devfreq_spdm.o devfreq_spdm_debugfs.o # DEVFREQ Event Drivers obj-$(CONFIG_PM_DEVFREQ_EVENT) += event/ diff --git a/drivers/devfreq/devfreq_trace.h b/drivers/devfreq/devfreq_trace.h new file mode 100644 index 000000000000..7dacc0edc81c --- /dev/null +++ b/drivers/devfreq/devfreq_trace.h @@ -0,0 +1,44 @@ +/* Copyright (c) 2014-2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#if !defined(_DEVFREQ_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _DEVFREQ_TRACE_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM devfreq +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE devfreq_trace + +#include + +TRACE_EVENT(devfreq_msg, + TP_PROTO(const char *msg), + TP_ARGS(msg), + TP_STRUCT__entry( + __string(msg, msg) + ), + TP_fast_assign( + __assign_str(msg, msg); + ), + TP_printk( + "%s", __get_str(msg) + ) +); + +#endif /* _DEVFREQ_TRACE_H */ + +/* This part must be outside protection */ +#include + diff --git a/drivers/devfreq/governor_bw_vbif.c b/drivers/devfreq/governor_bw_vbif.c new file mode 100644 index 000000000000..a19503e7662d --- /dev/null +++ b/drivers/devfreq/governor_bw_vbif.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2014-2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include "governor.h" + +unsigned long (*extern_get_bw)(void) = NULL; +unsigned long *dev_ab; +static unsigned long dev_ib; + +DEFINE_MUTEX(df_lock); +static struct devfreq *df; + +/* + * This function is 'get_target_freq' API for the governor. + * It just calls an external function that should be registered + * by KGSL driver to get and return a value for frequency. + */ +static int devfreq_vbif_get_freq(struct devfreq *df, + unsigned long *freq) +{ + /* If the IB isn't set yet, check if it should be non-zero. */ + if (!dev_ib && extern_get_bw) { + dev_ib = extern_get_bw(); + if (dev_ab) + *dev_ab = dev_ib / 4; + } + + *freq = dev_ib; + return 0; +} + +/* + * Registers a function to be used to request a frequency + * value from legacy vbif based bus bandwidth governor. + * This function is called by KGSL driver. + */ +void devfreq_vbif_register_callback(void *p) +{ + extern_get_bw = p; +} + +int devfreq_vbif_update_bw(unsigned long ib, unsigned long ab) +{ + int ret = 0; + + mutex_lock(&df_lock); + if (df) { + mutex_lock(&df->lock); + dev_ib = ib; + *dev_ab = ab; + ret = update_devfreq(df); + mutex_unlock(&df->lock); + } + mutex_unlock(&df_lock); + return ret; +} + +static int devfreq_vbif_ev_handler(struct devfreq *devfreq, + unsigned int event, void *data) +{ + int ret; + struct devfreq_dev_status stat; + + switch (event) { + case DEVFREQ_GOV_START: + mutex_lock(&df_lock); + df = devfreq; + if (df->profile->get_dev_status && + !df->profile->get_dev_status(df->dev.parent, &stat) && + stat.private_data) + dev_ab = stat.private_data; + else + pr_warn("Device doesn't take AB votes!\n"); + + mutex_unlock(&df_lock); + + ret = devfreq_vbif_update_bw(0, 0); + if (ret) { + pr_err("Unable to update BW! Gov start failed!\n"); + return ret; + } + /* + * Normally at this point governors start the polling with + * devfreq_monitor_start(df); + * This governor doesn't poll, but expect external calls + * of its devfreq_vbif_update_bw() function + */ + pr_debug("Enabled MSM VBIF governor\n"); + break; + + case DEVFREQ_GOV_STOP: + mutex_lock(&df_lock); + df = NULL; + mutex_unlock(&df_lock); + + pr_debug("Disabled MSM VBIF governor\n"); + break; + } + + return 0; +} + +static struct devfreq_governor devfreq_vbif = { + .name = "bw_vbif", + .get_target_freq = devfreq_vbif_get_freq, + .event_handler = devfreq_vbif_ev_handler, +}; + +static int __init devfreq_vbif_init(void) +{ + return devfreq_add_governor(&devfreq_vbif); +} +subsys_initcall(devfreq_vbif_init); + +static void __exit devfreq_vbif_exit(void) +{ + int ret; + + ret = devfreq_remove_governor(&devfreq_vbif); + if (ret) + pr_err("%s: failed remove governor %d\n", __func__, ret); + +} +module_exit(devfreq_vbif_exit); + +MODULE_DESCRIPTION("VBIF based GPU bus BW voting governor"); +MODULE_LICENSE("GPL v2"); + + diff --git a/drivers/devfreq/governor_gpubw_mon.c b/drivers/devfreq/governor_gpubw_mon.c new file mode 100644 index 000000000000..9c24eef6a497 --- /dev/null +++ b/drivers/devfreq/governor_gpubw_mon.c @@ -0,0 +1,266 @@ +/* Copyright (c) 2014-2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include + +#include "devfreq_trace.h" +#include "governor.h" + +#define MIN_BUSY 1000 +#define LONG_FLOOR 50000 +#define HIST 5 +#define TARGET 80 +#define CAP 75 +/* AB vote is in multiple of BW_STEP Mega bytes */ +#define BW_STEP 160 + +static void _update_cutoff(struct devfreq_msm_adreno_tz_data *priv, + unsigned int norm_max) +{ + int i; + + priv->bus.max = norm_max; + for (i = 0; i < priv->bus.num; i++) { + priv->bus.up[i] = priv->bus.p_up[i] * norm_max / 100; + priv->bus.down[i] = priv->bus.p_down[i] * norm_max / 100; + } +} + +static inline int devfreq_get_freq_level(struct devfreq *devfreq, + unsigned long freq) +{ + int lev; + + for (lev = 0; lev < devfreq->profile->max_state; lev++) + if (freq == devfreq->profile->freq_table[lev]) + return lev; + + return -EINVAL; +} + +static int devfreq_gpubw_get_target(struct devfreq *df, + unsigned long *freq) +{ + + struct devfreq_msm_adreno_tz_data *priv = df->data; + struct msm_busmon_extended_profile *bus_profile = container_of( + (df->profile), + struct msm_busmon_extended_profile, + profile); + struct devfreq_dev_status stats; + struct xstats b; + int result; + int level = 0; + int act_level; + int norm_cycles; + int gpu_percent; + /* + * Normalized AB should at max usage be the gpu_bimc frequency in MHz. + * Start with a reasonable value and let the system push it up to max. + */ + static int norm_ab_max = 300; + int norm_ab; + unsigned long ab_mbytes = 0; + + if (priv == NULL) + return 0; + + stats.private_data = &b; + + result = df->profile->get_dev_status(df->dev.parent, &stats); + + *freq = stats.current_frequency; + + priv->bus.total_time += stats.total_time; + priv->bus.gpu_time += stats.busy_time; + priv->bus.ram_time += b.ram_time; + priv->bus.ram_wait += b.ram_wait; + + level = devfreq_get_freq_level(df, stats.current_frequency); + + if (priv->bus.total_time < LONG_FLOOR) + return result; + + norm_cycles = (unsigned int)(priv->bus.ram_time + priv->bus.ram_wait) / + (unsigned int) priv->bus.total_time; + gpu_percent = (100 * (unsigned int)priv->bus.gpu_time) / + (unsigned int) priv->bus.total_time; + + /* + * If there's a new high watermark, update the cutoffs and send the + * FAST hint. Otherwise check the current value against the current + * cutoffs. + */ + if (norm_cycles > priv->bus.max) { + _update_cutoff(priv, norm_cycles); + bus_profile->flag = DEVFREQ_FLAG_FAST_HINT; + } else { + /* GPU votes for IB not AB so don't under vote the system */ + norm_cycles = (100 * norm_cycles) / TARGET; + act_level = priv->bus.index[level] + b.mod; + act_level = (act_level < 0) ? 0 : act_level; + act_level = (act_level >= priv->bus.num) ? + (priv->bus.num - 1) : act_level; + if (norm_cycles > priv->bus.up[act_level] && + gpu_percent > CAP) + bus_profile->flag = DEVFREQ_FLAG_FAST_HINT; + else if (norm_cycles < priv->bus.down[act_level] && level) + bus_profile->flag = DEVFREQ_FLAG_SLOW_HINT; + } + + /* Calculate the AB vote based on bus width if defined */ + if (priv->bus.width) { + norm_ab = (unsigned int)priv->bus.ram_time / + (unsigned int) priv->bus.total_time; + /* Calculate AB in Mega Bytes and roundup in BW_STEP */ + ab_mbytes = (norm_ab * priv->bus.width * 1000000ULL) >> 20; + bus_profile->ab_mbytes = roundup(ab_mbytes, BW_STEP); + } else if (bus_profile->flag) { + /* Re-calculate the AB percentage for a new IB vote */ + norm_ab = (unsigned int)priv->bus.ram_time / + (unsigned int) priv->bus.total_time; + if (norm_ab > norm_ab_max) + norm_ab_max = norm_ab; + bus_profile->percent_ab = (100 * norm_ab) / norm_ab_max; + } + + priv->bus.total_time = 0; + priv->bus.gpu_time = 0; + priv->bus.ram_time = 0; + priv->bus.ram_wait = 0; + + return result; +} + +static int gpubw_start(struct devfreq *devfreq) +{ + struct devfreq_msm_adreno_tz_data *priv; + + struct msm_busmon_extended_profile *bus_profile = container_of( + (devfreq->profile), + struct msm_busmon_extended_profile, + profile); + unsigned int t1, t2 = 2 * HIST; + int i, bus_size; + + + devfreq->data = bus_profile->private_data; + priv = devfreq->data; + + bus_size = sizeof(u32) * priv->bus.num; + priv->bus.up = kzalloc(bus_size, GFP_KERNEL); + priv->bus.down = kzalloc(bus_size, GFP_KERNEL); + priv->bus.p_up = kzalloc(bus_size, GFP_KERNEL); + priv->bus.p_down = kzalloc(bus_size, GFP_KERNEL); + if (priv->bus.up == NULL || priv->bus.down == NULL || + priv->bus.p_up == NULL || priv->bus.p_down == NULL) + return -ENOMEM; + + /* Set up the cut-over percentages for the bus calculation. */ + for (i = 0; i < priv->bus.num; i++) { + t1 = (u32)(100 * priv->bus.ib[i]) / + (u32)priv->bus.ib[priv->bus.num - 1]; + priv->bus.p_up[i] = t1 - HIST; + priv->bus.p_down[i] = t2 - 2 * HIST; + t2 = t1; + } + /* Set the upper-most and lower-most bounds correctly. */ + priv->bus.p_down[0] = 0; + priv->bus.p_down[1] = (priv->bus.p_down[1] > (2 * HIST)) ? + priv->bus.p_down[1] : (2 * HIST); + if (priv->bus.num >= 1) + priv->bus.p_up[priv->bus.num - 1] = 100; + _update_cutoff(priv, priv->bus.max); + + return 0; +} + +static int gpubw_stop(struct devfreq *devfreq) +{ + struct devfreq_msm_adreno_tz_data *priv = devfreq->data; + + if (priv) { + kfree(priv->bus.up); + kfree(priv->bus.down); + kfree(priv->bus.p_up); + kfree(priv->bus.p_down); + } + devfreq->data = NULL; + return 0; +} + +static int devfreq_gpubw_event_handler(struct devfreq *devfreq, + unsigned int event, void *data) +{ + int result = 0; + unsigned long freq; + + mutex_lock(&devfreq->lock); + freq = devfreq->previous_freq; + switch (event) { + case DEVFREQ_GOV_START: + result = gpubw_start(devfreq); + break; + case DEVFREQ_GOV_STOP: + result = gpubw_stop(devfreq); + break; + case DEVFREQ_GOV_RESUME: + /* TODO ..... */ + /* ret = update_devfreq(devfreq); */ + break; + case DEVFREQ_GOV_SUSPEND: + { + struct devfreq_msm_adreno_tz_data *priv = devfreq->data; + + priv->bus.total_time = 0; + priv->bus.gpu_time = 0; + priv->bus.ram_time = 0; + } + break; + default: + result = 0; + break; + } + mutex_unlock(&devfreq->lock); + return result; +} + +static struct devfreq_governor devfreq_gpubw = { + .name = "gpubw_mon", + .get_target_freq = devfreq_gpubw_get_target, + .event_handler = devfreq_gpubw_event_handler, +}; + +static int __init devfreq_gpubw_init(void) +{ + return devfreq_add_governor(&devfreq_gpubw); +} +subsys_initcall(devfreq_gpubw_init); + +static void __exit devfreq_gpubw_exit(void) +{ + int ret; + + ret = devfreq_remove_governor(&devfreq_gpubw); + if (ret) + pr_err("%s: failed remove governor %d\n", __func__, ret); + +} +module_exit(devfreq_gpubw_exit); + +MODULE_DESCRIPTION("GPU bus bandwidth voting driver. Uses VBIF counters"); +MODULE_LICENSE("GPL v2"); + diff --git a/drivers/devfreq/governor_msm_adreno_tz.c b/drivers/devfreq/governor_msm_adreno_tz.c new file mode 100644 index 000000000000..8a93d8d15db8 --- /dev/null +++ b/drivers/devfreq/governor_msm_adreno_tz.c @@ -0,0 +1,664 @@ +/* Copyright (c) 2010-2018, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "governor.h" + +static DEFINE_SPINLOCK(tz_lock); +static DEFINE_SPINLOCK(sample_lock); +static DEFINE_SPINLOCK(suspend_lock); +/* + * FLOOR is 5msec to capture up to 3 re-draws + * per frame for 60fps content. + */ +#define FLOOR 5000 +/* + * MIN_BUSY is 1 msec for the sample to be sent + */ +#define MIN_BUSY 1000 +#define MAX_TZ_VERSION 0 + +/* + * CEILING is 50msec, larger than any standard + * frame length, but less than the idle timer. + */ +#define CEILING 50000 +#define TZ_RESET_ID 0x3 +#define TZ_UPDATE_ID 0x4 +#define TZ_INIT_ID 0x6 + +#define TZ_RESET_ID_64 0x7 +#define TZ_UPDATE_ID_64 0x8 +#define TZ_INIT_ID_64 0x9 + +#define TZ_V2_UPDATE_ID_64 0xA +#define TZ_V2_INIT_ID_64 0xB +#define TZ_V2_INIT_CA_ID_64 0xC +#define TZ_V2_UPDATE_WITH_CA_ID_64 0xD + +#define TAG "msm_adreno_tz: " + +static u64 suspend_time; +static u64 suspend_start; +static unsigned long acc_total, acc_relative_busy; + +static struct msm_adreno_extended_profile *partner_gpu_profile; +static void do_partner_start_event(struct work_struct *work); +static void do_partner_stop_event(struct work_struct *work); +static void do_partner_suspend_event(struct work_struct *work); +static void do_partner_resume_event(struct work_struct *work); + +static struct workqueue_struct *workqueue; + +/* + * Returns GPU suspend time in millisecond. + */ +u64 suspend_time_ms(void) +{ + u64 suspend_sampling_time; + u64 time_diff = 0; + + if (suspend_start == 0) + return 0; + + suspend_sampling_time = (u64)ktime_to_ms(ktime_get()); + time_diff = suspend_sampling_time - suspend_start; + /* Update the suspend_start sample again */ + suspend_start = suspend_sampling_time; + return time_diff; +} + +static ssize_t gpu_load_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + unsigned long sysfs_busy_perc = 0; + /* + * Average out the samples taken since last read + * This will keep the average value in sync with + * with the client sampling duration. + */ + spin_lock(&sample_lock); + if (acc_total) + sysfs_busy_perc = (acc_relative_busy * 100) / acc_total; + + /* Reset the parameters */ + acc_total = 0; + acc_relative_busy = 0; + spin_unlock(&sample_lock); + return snprintf(buf, PAGE_SIZE, "%lu\n", sysfs_busy_perc); +} + +/* + * Returns the time in ms for which gpu was in suspend state + * since last time the entry is read. + */ +static ssize_t suspend_time_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + u64 time_diff = 0; + + spin_lock(&suspend_lock); + time_diff = suspend_time_ms(); + /* + * Adding the previous suspend time also as the gpu + * can go and come out of suspend states in between + * reads also and we should have the total suspend + * since last read. + */ + time_diff += suspend_time; + suspend_time = 0; + spin_unlock(&suspend_lock); + + return snprintf(buf, PAGE_SIZE, "%llu\n", time_diff); +} + +static DEVICE_ATTR(gpu_load, 0444, gpu_load_show, NULL); + +static DEVICE_ATTR(suspend_time, 0444, + suspend_time_show, + NULL); + +static const struct device_attribute *adreno_tz_attr_list[] = { + &dev_attr_gpu_load, + &dev_attr_suspend_time, + NULL +}; + +void compute_work_load(struct devfreq_dev_status *stats, + struct devfreq_msm_adreno_tz_data *priv, + struct devfreq *devfreq) +{ + spin_lock(&sample_lock); + /* + * Keep collecting the stats till the client + * reads it. Average of all samples and reset + * is done when the entry is read + */ + acc_total += stats->total_time; + acc_relative_busy += (stats->busy_time * stats->current_frequency) / + devfreq->profile->freq_table[0]; + spin_unlock(&sample_lock); +} + +/* Trap into the TrustZone, and call funcs there. */ +static int __secure_tz_reset_entry2(unsigned int *scm_data, u32 size_scm_data, + bool is_64) +{ + int ret; + /* sync memory before sending the commands to tz */ + __iowmb(); + + if (!is_64) { + struct scm_desc desc = { + .args[0] = scm_data[0], + .args[1] = scm_data[1], + .arginfo = SCM_ARGS(2), + }; + spin_lock(&tz_lock); + ret = scm_call2_atomic(SCM_SIP_FNID(SCM_SVC_IO, TZ_RESET_ID), + &desc); + spin_unlock(&tz_lock); + } else { + struct scm_desc desc = {0}; + + desc.arginfo = 0; + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_DCVS, + TZ_RESET_ID_64), &desc); + } + return ret; +} + +static int __secure_tz_update_entry3(unsigned int *scm_data, u32 size_scm_data, + int *val, u32 size_val, struct devfreq_msm_adreno_tz_data *priv) +{ + int ret; + /* sync memory before sending the commands to tz */ + __iowmb(); + + if (!priv->is_64) { + struct scm_desc desc = { + .args[0] = scm_data[0], + .args[1] = scm_data[1], + .args[2] = scm_data[2], + .arginfo = SCM_ARGS(3), + }; + spin_lock(&tz_lock); + ret = scm_call2_atomic(SCM_SIP_FNID(SCM_SVC_IO, TZ_UPDATE_ID), + &desc); + spin_unlock(&tz_lock); + *val = ret; + } else { + unsigned int cmd_id; + struct scm_desc desc = {0}; + + desc.args[0] = scm_data[0]; + desc.args[1] = scm_data[1]; + desc.args[2] = scm_data[2]; + + if (!priv->ctxt_aware_enable) { + desc.arginfo = SCM_ARGS(3); + cmd_id = TZ_V2_UPDATE_ID_64; + } else { + /* Add context count infomration to update*/ + desc.args[3] = scm_data[3]; + desc.arginfo = SCM_ARGS(4); + cmd_id = TZ_V2_UPDATE_WITH_CA_ID_64; + } + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_DCVS, cmd_id), + &desc); + *val = desc.ret[0]; + } + return ret; +} + +static int tz_init_ca(struct devfreq_msm_adreno_tz_data *priv) +{ + unsigned int tz_ca_data[2]; + struct scm_desc desc = {0}; + u8 *tz_buf; + int ret; + + /* Set data for TZ */ + tz_ca_data[0] = priv->bin.ctxt_aware_target_pwrlevel; + tz_ca_data[1] = priv->bin.ctxt_aware_busy_penalty; + + tz_buf = kzalloc(PAGE_ALIGN(sizeof(tz_ca_data)), GFP_KERNEL); + if (!tz_buf) + return -ENOMEM; + + memcpy(tz_buf, tz_ca_data, sizeof(tz_ca_data)); + /* Ensure memcpy completes execution */ + mb(); + dmac_flush_range(tz_buf, + tz_buf + PAGE_ALIGN(sizeof(tz_ca_data))); + + desc.args[0] = virt_to_phys(tz_buf); + desc.args[1] = sizeof(tz_ca_data); + desc.arginfo = SCM_ARGS(2, SCM_RW, SCM_VAL); + + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_DCVS, + TZ_V2_INIT_CA_ID_64), + &desc); + + kzfree(tz_buf); + + return ret; +} + +static int tz_init(struct devfreq_msm_adreno_tz_data *priv, + unsigned int *tz_pwrlevels, u32 size_pwrlevels, + unsigned int *version, u32 size_version) +{ + int ret; + /* Make sure all CMD IDs are avaialble */ + if (scm_is_call_available(SCM_SVC_DCVS, TZ_INIT_ID_64) && + scm_is_call_available(SCM_SVC_DCVS, TZ_UPDATE_ID_64) && + scm_is_call_available(SCM_SVC_DCVS, TZ_RESET_ID_64)) { + struct scm_desc desc = {0}; + u8 *tz_buf; + + tz_buf = kzalloc(PAGE_ALIGN(size_pwrlevels), GFP_KERNEL); + if (!tz_buf) + return -ENOMEM; + memcpy(tz_buf, tz_pwrlevels, size_pwrlevels); + /* Ensure memcpy completes execution */ + mb(); + dmac_flush_range(tz_buf, tz_buf + PAGE_ALIGN(size_pwrlevels)); + + desc.args[0] = virt_to_phys(tz_buf); + desc.args[1] = size_pwrlevels; + desc.arginfo = SCM_ARGS(2, SCM_RW, SCM_VAL); + + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_DCVS, TZ_V2_INIT_ID_64), + &desc); + *version = desc.ret[0]; + if (!ret) + priv->is_64 = true; + kzfree(tz_buf); + } else + ret = -EINVAL; + + /* Initialize context aware feature, if enabled. */ + if (!ret && priv->ctxt_aware_enable) { + if (priv->is_64 && + (scm_is_call_available(SCM_SVC_DCVS, + TZ_V2_INIT_CA_ID_64)) && + (scm_is_call_available(SCM_SVC_DCVS, + TZ_V2_UPDATE_WITH_CA_ID_64))) { + ret = tz_init_ca(priv); + /* + * If context aware feature initialization fails, + * just print an error message and return + * success as normal DCVS will still work. + */ + if (ret) { + pr_err(TAG "tz: context aware DCVS init failed\n"); + priv->ctxt_aware_enable = false; + return 0; + } + } else { + pr_warn(TAG "tz: context aware DCVS not supported\n"); + priv->ctxt_aware_enable = false; + } + } + + return ret; +} + +static inline int devfreq_get_freq_level(struct devfreq *devfreq, + unsigned long freq) +{ + int lev; + + for (lev = 0; lev < devfreq->profile->max_state; lev++) + if (freq == devfreq->profile->freq_table[lev]) + return lev; + + return -EINVAL; +} + +static int tz_get_target_freq(struct devfreq *devfreq, unsigned long *freq) +{ + int result = 0; + struct devfreq_msm_adreno_tz_data *priv = devfreq->data; + struct devfreq_dev_status stats; + int val, level = 0; + unsigned int scm_data[4]; + int context_count = 0; + + /* keeps stats.private_data == NULL */ + result = devfreq->profile->get_dev_status(devfreq->dev.parent, &stats); + if (result) { + pr_err(TAG "get_status failed %d\n", result); + return result; + } + + *freq = stats.current_frequency; + priv->bin.total_time += stats.total_time; + priv->bin.busy_time += stats.busy_time; + + if (stats.private_data) + context_count = *((int *)stats.private_data); + + /* Update the GPU load statistics */ + compute_work_load(&stats, priv, devfreq); + /* + * Do not waste CPU cycles running this algorithm if + * the GPU just started, or if less than FLOOR time + * has passed since the last run or the gpu hasn't been + * busier than MIN_BUSY. + */ + if ((stats.total_time == 0) || + (priv->bin.total_time < FLOOR) || + (unsigned int) priv->bin.busy_time < MIN_BUSY) { + return 0; + } + + level = devfreq_get_freq_level(devfreq, stats.current_frequency); + if (level < 0) { + pr_err(TAG "bad freq %ld\n", stats.current_frequency); + return level; + } + + /* + * If there is an extended block of busy processing, + * increase frequency. Otherwise run the normal algorithm. + */ + if (!priv->disable_busy_time_burst && + priv->bin.busy_time > CEILING) { + val = -1 * level; + } else { + + scm_data[0] = level; + scm_data[1] = priv->bin.total_time; + scm_data[2] = priv->bin.busy_time; + scm_data[3] = context_count; + __secure_tz_update_entry3(scm_data, sizeof(scm_data), + &val, sizeof(val), priv); + } + priv->bin.total_time = 0; + priv->bin.busy_time = 0; + + /* + * If the decision is to move to a different level, make sure the GPU + * frequency changes. + */ + if (val) { + level += val; + level = max(level, 0); + level = min_t(int, level, devfreq->profile->max_state - 1); + } + + *freq = devfreq->profile->freq_table[level]; + return 0; +} + +static int tz_notify(struct notifier_block *nb, unsigned long type, void *devp) +{ + int result = 0; + struct devfreq *devfreq = devp; + + switch (type) { + case ADRENO_DEVFREQ_NOTIFY_IDLE: + case ADRENO_DEVFREQ_NOTIFY_RETIRE: + mutex_lock(&devfreq->lock); + result = update_devfreq(devfreq); + mutex_unlock(&devfreq->lock); + /* Nofifying partner bus governor if any */ + if (partner_gpu_profile && partner_gpu_profile->bus_devfreq) { + mutex_lock(&partner_gpu_profile->bus_devfreq->lock); + update_devfreq(partner_gpu_profile->bus_devfreq); + mutex_unlock(&partner_gpu_profile->bus_devfreq->lock); + } + break; + /* ignored by this governor */ + case ADRENO_DEVFREQ_NOTIFY_SUBMIT: + default: + break; + } + return notifier_from_errno(result); +} + +static int tz_start(struct devfreq *devfreq) +{ + struct devfreq_msm_adreno_tz_data *priv; + unsigned int tz_pwrlevels[MSM_ADRENO_MAX_PWRLEVELS + 1]; + int i, out, ret; + unsigned int version; + + struct msm_adreno_extended_profile *gpu_profile = container_of( + (devfreq->profile), + struct msm_adreno_extended_profile, + profile); + + /* + * Assuming that we have only one instance of the adreno device + * connected to this governor, + * can safely restore the pointer to the governor private data + * from the container of the device profile + */ + devfreq->data = gpu_profile->private_data; + partner_gpu_profile = gpu_profile; + + priv = devfreq->data; + priv->nb.notifier_call = tz_notify; + + out = 1; + if (devfreq->profile->max_state < MSM_ADRENO_MAX_PWRLEVELS) { + for (i = 0; i < devfreq->profile->max_state; i++) + tz_pwrlevels[out++] = devfreq->profile->freq_table[i]; + tz_pwrlevels[0] = i; + } else { + pr_err(TAG "tz_pwrlevels[] is too short\n"); + return -EINVAL; + } + + INIT_WORK(&gpu_profile->partner_start_event_ws, + do_partner_start_event); + INIT_WORK(&gpu_profile->partner_stop_event_ws, + do_partner_stop_event); + INIT_WORK(&gpu_profile->partner_suspend_event_ws, + do_partner_suspend_event); + INIT_WORK(&gpu_profile->partner_resume_event_ws, + do_partner_resume_event); + + ret = tz_init(priv, tz_pwrlevels, sizeof(tz_pwrlevels), &version, + sizeof(version)); + if (ret != 0 || version > MAX_TZ_VERSION) { + pr_err(TAG "tz_init failed\n"); + return ret; + } + + for (i = 0; adreno_tz_attr_list[i] != NULL; i++) + device_create_file(&devfreq->dev, adreno_tz_attr_list[i]); + + return kgsl_devfreq_add_notifier(devfreq->dev.parent, &priv->nb); +} + +static int tz_stop(struct devfreq *devfreq) +{ + int i; + struct devfreq_msm_adreno_tz_data *priv = devfreq->data; + + kgsl_devfreq_del_notifier(devfreq->dev.parent, &priv->nb); + + for (i = 0; adreno_tz_attr_list[i] != NULL; i++) + device_remove_file(&devfreq->dev, adreno_tz_attr_list[i]); + + flush_workqueue(workqueue); + + /* leaving the governor and cleaning the pointer to private data */ + devfreq->data = NULL; + partner_gpu_profile = NULL; + return 0; +} + +static int tz_suspend(struct devfreq *devfreq) +{ + struct devfreq_msm_adreno_tz_data *priv = devfreq->data; + unsigned int scm_data[2] = {0, 0}; + + __secure_tz_reset_entry2(scm_data, sizeof(scm_data), priv->is_64); + + priv->bin.total_time = 0; + priv->bin.busy_time = 0; + return 0; +} + +static int tz_handler(struct devfreq *devfreq, unsigned int event, void *data) +{ + int result; + + struct msm_adreno_extended_profile *gpu_profile = container_of( + (devfreq->profile), + struct msm_adreno_extended_profile, + profile); + + switch (event) { + case DEVFREQ_GOV_START: + result = tz_start(devfreq); + break; + + case DEVFREQ_GOV_STOP: + /* Queue the stop work before the TZ is stopped */ + if (partner_gpu_profile && partner_gpu_profile->bus_devfreq) + queue_work(workqueue, + &gpu_profile->partner_stop_event_ws); + spin_lock(&suspend_lock); + suspend_start = 0; + spin_unlock(&suspend_lock); + result = tz_stop(devfreq); + break; + + case DEVFREQ_GOV_SUSPEND: + result = tz_suspend(devfreq); + if (!result) { + spin_lock(&suspend_lock); + /* Collect the start sample for suspend time */ + suspend_start = (u64)ktime_to_ms(ktime_get()); + spin_unlock(&suspend_lock); + } + break; + + case DEVFREQ_GOV_RESUME: + spin_lock(&suspend_lock); + suspend_time += suspend_time_ms(); + /* Reset the suspend_start when gpu resumes */ + suspend_start = 0; + spin_unlock(&suspend_lock); + /* fallthrough */ + case DEVFREQ_GOV_INTERVAL: + /* fallthrough, this governor doesn't use polling */ + default: + result = 0; + break; + } + + if (partner_gpu_profile && partner_gpu_profile->bus_devfreq) + switch (event) { + case DEVFREQ_GOV_START: + queue_work(workqueue, + &gpu_profile->partner_start_event_ws); + break; + case DEVFREQ_GOV_SUSPEND: + queue_work(workqueue, + &gpu_profile->partner_suspend_event_ws); + break; + case DEVFREQ_GOV_RESUME: + queue_work(workqueue, + &gpu_profile->partner_resume_event_ws); + break; + } + + return result; +} + +static void _do_partner_event(struct work_struct *work, unsigned int event) +{ + struct devfreq *bus_devfreq; + + if (partner_gpu_profile == NULL) + return; + + bus_devfreq = partner_gpu_profile->bus_devfreq; + + if (bus_devfreq != NULL && + bus_devfreq->governor && + bus_devfreq->governor->event_handler) + bus_devfreq->governor->event_handler(bus_devfreq, event, NULL); +} + +static void do_partner_start_event(struct work_struct *work) +{ + _do_partner_event(work, DEVFREQ_GOV_START); +} + +static void do_partner_stop_event(struct work_struct *work) +{ + _do_partner_event(work, DEVFREQ_GOV_STOP); +} + +static void do_partner_suspend_event(struct work_struct *work) +{ + _do_partner_event(work, DEVFREQ_GOV_SUSPEND); +} + +static void do_partner_resume_event(struct work_struct *work) +{ + _do_partner_event(work, DEVFREQ_GOV_RESUME); +} + + +static struct devfreq_governor msm_adreno_tz = { + .name = "msm-adreno-tz", + .get_target_freq = tz_get_target_freq, + .event_handler = tz_handler, +}; + +static int __init msm_adreno_tz_init(void) +{ + workqueue = create_freezable_workqueue("governor_msm_adreno_tz_wq"); + + if (workqueue == NULL) + return -ENOMEM; + + return devfreq_add_governor(&msm_adreno_tz); +} +subsys_initcall(msm_adreno_tz_init); + +static void __exit msm_adreno_tz_exit(void) +{ + int ret = devfreq_remove_governor(&msm_adreno_tz); + + if (ret) + pr_err(TAG "failed to remove governor %d\n", ret); + + if (workqueue != NULL) + destroy_workqueue(workqueue); +} + +module_exit(msm_adreno_tz_exit); + +MODULE_LICENSE("GPL v2");