From 541abe7e4a6dd8ba2a1338c36d916889d2deafc2 Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Wed, 20 Feb 2019 17:11:20 +0530 Subject: [PATCH] cpuidle: lpm-levels: Track and predict next rescheduling IPI Add changes to track and predict next rescheduling IPI based on past history. Add a module param to control enabling it. Change-Id: Ie495d8906288ee410708693ee15ed51643aefb44 Signed-off-by: Maulik Shah --- arch/arm/kernel/smp.c | 3 + arch/arm64/kernel/smp.c | 2 + drivers/cpuidle/lpm-levels-of.c | 9 +- drivers/cpuidle/lpm-levels.c | 147 ++++++++++++++++++++++---------- drivers/cpuidle/lpm-levels.h | 3 + include/soc/qcom/lpm_levels.h | 4 +- 6 files changed, 117 insertions(+), 51 deletions(-) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index ddc7c304f5bb..9db313a3bb1d 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -52,6 +52,8 @@ #include #include +#include + #define CREATE_TRACE_POINTS #include @@ -721,6 +723,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) void smp_send_reschedule(int cpu) { + update_ipi_history(cpu); smp_cross_call_common(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 962e0de9a729..ef121f5572a6 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -61,6 +61,7 @@ #include #include +#include #define CREATE_TRACE_POINTS #include @@ -932,6 +933,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) void smp_send_reschedule(int cpu) { BUG_ON(cpu_is_offline(cpu)); + update_ipi_history(cpu); smp_cross_call_common(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/drivers/cpuidle/lpm-levels-of.c b/drivers/cpuidle/lpm-levels-of.c index 87a8954d84a8..f2f0bd4d0023 100644 --- a/drivers/cpuidle/lpm-levels-of.c +++ b/drivers/cpuidle/lpm-levels-of.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2014-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -589,8 +589,11 @@ static int parse_cpu_levels(struct device_node *node, struct lpm_cluster *c) if (ret) goto failed; - key = "qcom,disable-prediction"; - cpu->lpm_prediction = !(of_property_read_bool(node, key)); + cpu->ipi_prediction = !(of_property_read_bool(node, + "qcom,disable-ipi-prediction")); + + cpu->lpm_prediction = !(of_property_read_bool(node, + "qcom,disable-prediction")); if (cpu->lpm_prediction) { key = "qcom,ref-stddev"; diff --git a/drivers/cpuidle/lpm-levels.c b/drivers/cpuidle/lpm-levels.c index b6e4e6a45035..c75e5a72aa6f 100644 --- a/drivers/cpuidle/lpm-levels.c +++ b/drivers/cpuidle/lpm-levels.c @@ -96,6 +96,8 @@ module_param_named(lpm_prediction, lpm_prediction, bool, 0664); static uint32_t bias_hyst; module_param_named(bias_hyst, bias_hyst, uint, 0664); +static bool lpm_ipi_prediction; +module_param_named(lpm_ipi_prediction, lpm_ipi_prediction, bool, 0664); struct lpm_history { uint32_t resi[MAXSAMPLES]; @@ -107,8 +109,14 @@ struct lpm_history { int64_t stime; }; -static DEFINE_PER_CPU(struct lpm_history, hist); +struct ipi_history { + uint32_t interval[MAXSAMPLES]; + uint32_t current_ptr; + ktime_t cpu_idle_resched_ts; +}; +static DEFINE_PER_CPU(struct lpm_history, hist); +static DEFINE_PER_CPU(struct ipi_history, cpu_ipi_history); static DEFINE_PER_CPU(struct lpm_cpu*, cpu_lpm); static bool suspend_in_progress; static struct hrtimer lpm_hrtimer; @@ -468,14 +476,66 @@ static void biastimer_start(uint32_t time_ns) hrtimer_start(cpu_biastimer, bias_ktime, HRTIMER_MODE_REL_PINNED); } -static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, - struct lpm_cpu *cpu, int *idx_restrict, - uint32_t *idx_restrict_time) +static uint64_t find_deviation(int *interval, uint32_t ref_stddev, + int64_t *stime) { - int i, j, divisor; + int divisor, i; uint64_t max, avg, stddev; int64_t thresh = LLONG_MAX; + + if (lpm_ipi_prediction) + ref_stddev += DEFAULT_IPI_STDDEV; + + do { + max = avg = divisor = stddev = 0; + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = interval[i]; + + if (value <= thresh) { + avg += value; + divisor++; + if (value > max) + max = value; + } + } + do_div(avg, divisor); + + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = interval[i]; + + if (value <= thresh) { + int64_t diff = value - avg; + + stddev += diff * diff; + } + } + do_div(stddev, divisor); + stddev = int_sqrt(stddev); + + /* + * If the deviation is less, return the average, else + * ignore one maximum sample and retry + */ + if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1))) + || stddev <= ref_stddev) { + *stime = ktime_to_us(ktime_get()) + avg; + return avg; + } + thresh = max - 1; + + } while (divisor > (MAXSAMPLES - 1)); + + return 0; +} + +static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, + struct lpm_cpu *cpu, int *idx_restrict, + uint32_t *idx_restrict_time, uint32_t *ipi_predicted) +{ + int i, j; + uint64_t avg; struct lpm_history *history = &per_cpu(hist, dev->cpu); + struct ipi_history *ipi_history = &per_cpu(cpu_ipi_history, dev->cpu); if (!lpm_prediction || !cpu->lpm_prediction) return 0; @@ -506,44 +566,9 @@ static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, * that mode. */ -again: - max = avg = divisor = stddev = 0; - for (i = 0; i < MAXSAMPLES; i++) { - int64_t value = history->resi[i]; - - if (value <= thresh) { - avg += value; - divisor++; - if (value > max) - max = value; - } - } - do_div(avg, divisor); - - for (i = 0; i < MAXSAMPLES; i++) { - int64_t value = history->resi[i]; - - if (value <= thresh) { - int64_t diff = value - avg; - - stddev += diff * diff; - } - } - do_div(stddev, divisor); - stddev = int_sqrt(stddev); - - /* - * If the deviation is less, return the average, else - * ignore one maximum sample and retry - */ - if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1))) - || stddev <= cpu->ref_stddev) { - history->stime = ktime_to_us(ktime_get()) + avg; + avg = find_deviation(history->resi, cpu->ref_stddev, &(history->stime)); + if (avg) return avg; - } else if (divisor > (MAXSAMPLES - 1)) { - thresh = max - 1; - goto again; - } /* * Find the number of premature exits for each of the mode, @@ -586,6 +611,17 @@ again: } } } + + if (*idx_restrict_time || !cpu->ipi_prediction || !lpm_ipi_prediction) + return 0; + + avg = find_deviation(ipi_history->interval, cpu->ref_stddev, + &(history->stime)); + if (avg) { + *ipi_predicted = 1; + return avg; + } + return 0; } @@ -659,7 +695,7 @@ static int cpu_power_select(struct cpuidle_device *dev, int i, idx_restrict; uint32_t lvl_latency_us = 0; uint64_t predicted = 0; - uint32_t htime = 0, idx_restrict_time = 0; + uint32_t htime = 0, idx_restrict_time = 0, ipi_predicted = 0; uint32_t next_wakeup_us = (uint32_t)sleep_us; uint32_t min_residency, max_residency; struct power_params *pwr_params; @@ -710,7 +746,8 @@ static int cpu_power_select(struct cpuidle_device *dev, */ if (next_wakeup_us > max_residency) { predicted = lpm_cpuidle_predict(dev, cpu, - &idx_restrict, &idx_restrict_time); + &idx_restrict, &idx_restrict_time, + &ipi_predicted); if (predicted && (predicted < min_residency)) predicted = min_residency; } else @@ -747,7 +784,9 @@ static int cpu_power_select(struct cpuidle_device *dev, if ((predicted || (idx_restrict != (cpu->nlevels + 1))) && (best_level < (cpu->nlevels-1))) { htime = predicted + cpu->tmr_add; - if (htime == cpu->tmr_add) + if (lpm_ipi_prediction && cpu->ipi_prediction) + htime += DEFAULT_IPI_TIMER_ADD; + if (!predicted) htime = idx_restrict_time; else if (htime > max_residency) htime = max_residency; @@ -760,8 +799,8 @@ static int cpu_power_select(struct cpuidle_device *dev, done_select: trace_cpu_power_select(best_level, sleep_us, latency_us, next_event_us); - trace_cpu_pred_select(idx_restrict_time ? 2 : (predicted ? 1 : 0), - predicted, htime); + trace_cpu_pred_select(idx_restrict_time ? 2 : (ipi_predicted ? + 3 : (predicted ? 1 : 0)), predicted, htime); return best_level; } @@ -1398,6 +1437,20 @@ static int lpm_cpuidle_select(struct cpuidle_driver *drv, return cpu_power_select(dev, cpu); } +void update_ipi_history(int cpu) +{ + struct ipi_history *history = &per_cpu(cpu_ipi_history, cpu); + ktime_t now = ktime_get(); + + history->interval[history->current_ptr] = + ktime_to_us(ktime_sub(now, + history->cpu_idle_resched_ts)); + (history->current_ptr)++; + if (history->current_ptr >= MAXSAMPLES) + history->current_ptr = 0; + history->cpu_idle_resched_ts = now; +} + static void update_history(struct cpuidle_device *dev, int idx) { struct lpm_history *history = &per_cpu(hist, dev->cpu); diff --git a/drivers/cpuidle/lpm-levels.h b/drivers/cpuidle/lpm-levels.h index e6207e5ecd89..a8539e77f3dd 100644 --- a/drivers/cpuidle/lpm-levels.h +++ b/drivers/cpuidle/lpm-levels.h @@ -17,7 +17,9 @@ #define CLUST_SMPL_INVLD_TIME 40000 #define DEFAULT_PREMATURE_CNT 3 #define DEFAULT_STDDEV 100 +#define DEFAULT_IPI_STDDEV 400 #define DEFAULT_TIMER_ADD 100 +#define DEFAULT_IPI_TIMER_ADD 900 #define TIMER_ADD_LOW 100 #define TIMER_ADD_HIGH 1500 #define STDDEV_LOW 100 @@ -52,6 +54,7 @@ struct lpm_cpu { uint32_t ref_premature_cnt; uint32_t tmr_add; bool lpm_prediction; + bool ipi_prediction; uint64_t bias; struct cpuidle_driver *drv; struct lpm_cluster *parent; diff --git a/include/soc/qcom/lpm_levels.h b/include/soc/qcom/lpm_levels.h index f28c30187613..acb730c1fd98 100644 --- a/include/soc/qcom/lpm_levels.h +++ b/include/soc/qcom/lpm_levels.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, 2020 The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -22,9 +22,11 @@ struct system_pm_ops { #if defined(CONFIG_MSM_PM) || defined(CONFIG_MSM_PM_LEGACY) uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops); +void update_ipi_history(int cpu); #else static inline uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops) { return -ENODEV; } +static inline void update_ipi_history(int cpu) {} #endif #endif