On 7/21/26 10:25, Yaxiong Tian wrote:
> cpuidle_governor_latency_req() runs on every idle-state selection and
> aggregates per-CPU resume latency with the global CPU latency and
> wakeup latency QoS limits. That path repeatedly walks get_cpu_device()
> and pm_qos_read_value(), which shows up hot under menu_select().
>
> Cache the aggregated constraint per CPU and refresh it only when the
> corresponding per-CPU generation changes. The generation is already
> bumped by the global and per-CPU resume latency QoS notifiers added
> earlier in this series.
>
> On a menu governor profile, cpuidle_governor_latency_req() drops from
> about 19.9% of menu_select time to about 4.2%, roughly a 6x reduction
> in cost per call (~1.9 us down to ~0.3 us).
>
> Signed-off-by: Yaxiong Tian <[email protected]>
> ---
> drivers/cpuidle/governor.c | 37 ++++++++++++++++++++++++++++++++-----
> 1 file changed, 32 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
> index d286ccf19a69..67909470c14b 100644
> --- a/drivers/cpuidle/governor.c
> +++ b/drivers/cpuidle/governor.c
> @@ -32,11 +32,17 @@ struct cpuidle_governor *cpuidle_prev_governor;
> */
> static DEFINE_PER_CPU(atomic_t, latency_req_gen);
>
> +struct cpuidle_latency_req_cache {
> + unsigned int gen;
> + s64 latency_ns;
> +};
> +
> struct cpuidle_cpu_qos_nb {
> struct notifier_block nb;
> unsigned int cpu;
> };
>
> +static DEFINE_PER_CPU(struct cpuidle_latency_req_cache, latency_req_cache);
So this will initialize with latency_req_cache.gen = 0 and
latency.req_cache.latency_ns = 0...
Come to think of it maybe a no-req run is also good as the first and last
selftest case.
> static DEFINE_PER_CPU(struct cpuidle_cpu_qos_nb, cpuidle_cpu_qos_nb);
>
> static void cpuidle_latency_req_invalidate_cpu(unsigned int cpu)
> @@ -212,10 +218,22 @@ int cpuidle_register_governor(struct cpuidle_governor
> *gov)
> */
> s64 cpuidle_governor_latency_req(unsigned int cpu)
> {
> - struct device *device = get_cpu_device(cpu);
> - int device_req = dev_pm_qos_raw_resume_latency(device);
> - int global_req = cpu_latency_qos_limit();
> - int global_wake_req = cpu_wakeup_latency_qos_limit();
> + struct cpuidle_latency_req_cache *cache;
> + unsigned int gen;
> + struct device *device;
> + int device_req, global_req, global_wake_req;
> + s64 latency_ns;
> +
> + cache = per_cpu_ptr(&latency_req_cache, cpu);
> + gen = atomic_read(per_cpu_ptr(&latency_req_gen, cpu));
> +
> + if (likely(READ_ONCE(cache->gen) == gen))
> + return READ_ONCE(cache->latency_ns);
> +
> + device = get_cpu_device(cpu);
> + device_req = dev_pm_qos_raw_resume_latency(device);
> + global_req = cpu_latency_qos_limit();
> + global_wake_req = cpu_wakeup_latency_qos_limit();
>
> if (global_req > global_wake_req)
> global_req = global_wake_req;
> @@ -223,5 +241,14 @@ s64 cpuidle_governor_latency_req(unsigned int cpu)
> if (device_req > global_req)
> device_req = global_req;
>
> - return (s64)device_req * NSEC_PER_USEC;
> + latency_ns = (s64)device_req * NSEC_PER_USEC;
> +
> + WRITE_ONCE(cache->latency_ns, latency_ns);
> + /*
> + * Store gen last so a concurrent invalidate cannot leave a stale
> + * latency_ns marked as current.
> + */
This doesn't guarantee the ordering.
> + WRITE_ONCE(cache->gen, gen);
> +
> + return latency_ns;
> }
