cpuidle_governor_latency_req() runs on every idle-state selection and aggregates per-CPU resume latency with the global CPU latency and wakeup latency QoS limits. That path repeatedly walks get_cpu_device() and pm_qos_read_value(), which shows up hot under menu_select().
Cache the aggregated constraint per CPU and refresh it only when the corresponding per-CPU generation changes. The generation is already bumped by the global and per-CPU resume latency QoS notifiers added earlier in this series. On a menu governor profile, cpuidle_governor_latency_req() drops from about 19.9% of menu_select time to about 4.2%, roughly a 6x reduction in cost per call (~1.9 us down to ~0.3 us). Signed-off-by: Yaxiong Tian <[email protected]> --- drivers/cpuidle/governor.c | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index d286ccf19a69..67909470c14b 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -32,11 +32,17 @@ struct cpuidle_governor *cpuidle_prev_governor; */ static DEFINE_PER_CPU(atomic_t, latency_req_gen); +struct cpuidle_latency_req_cache { + unsigned int gen; + s64 latency_ns; +}; + struct cpuidle_cpu_qos_nb { struct notifier_block nb; unsigned int cpu; }; +static DEFINE_PER_CPU(struct cpuidle_latency_req_cache, latency_req_cache); static DEFINE_PER_CPU(struct cpuidle_cpu_qos_nb, cpuidle_cpu_qos_nb); static void cpuidle_latency_req_invalidate_cpu(unsigned int cpu) @@ -212,10 +218,22 @@ int cpuidle_register_governor(struct cpuidle_governor *gov) */ s64 cpuidle_governor_latency_req(unsigned int cpu) { - struct device *device = get_cpu_device(cpu); - int device_req = dev_pm_qos_raw_resume_latency(device); - int global_req = cpu_latency_qos_limit(); - int global_wake_req = cpu_wakeup_latency_qos_limit(); + struct cpuidle_latency_req_cache *cache; + unsigned int gen; + struct device *device; + int device_req, global_req, global_wake_req; + s64 latency_ns; + + cache = per_cpu_ptr(&latency_req_cache, cpu); + gen = atomic_read(per_cpu_ptr(&latency_req_gen, cpu)); + + if (likely(READ_ONCE(cache->gen) == gen)) + return READ_ONCE(cache->latency_ns); + + device = get_cpu_device(cpu); + device_req = dev_pm_qos_raw_resume_latency(device); + global_req = cpu_latency_qos_limit(); + global_wake_req = cpu_wakeup_latency_qos_limit(); if (global_req > global_wake_req) global_req = global_wake_req; @@ -223,5 +241,14 @@ s64 cpuidle_governor_latency_req(unsigned int cpu) if (device_req > global_req) device_req = global_req; - return (s64)device_req * NSEC_PER_USEC; + latency_ns = (s64)device_req * NSEC_PER_USEC; + + WRITE_ONCE(cache->latency_ns, latency_ns); + /* + * Store gen last so a concurrent invalidate cannot leave a stale + * latency_ns marked as current. + */ + WRITE_ONCE(cache->gen, gen); + + return latency_ns; } -- 2.43.0
