From: Kunwu Chan <[email protected]> Implement the observe_*() core: per-CPU monotonic counters, the event-lifecycle hooks with lazily-allocated per-event per-CPU state alongside the global per-CPU stage, NMI-safe sample classification, ring enqueue/dequeue/peak, match/miss/update, and framework init. The global per-CPU stage advances monotonically so a late create/bind/ enable cannot regress a CPU that is already RUNNING.
damon_perf_observe_sample() records the execution context (process/softirq/hardirq/NMI) so overflow callbacks can be verified to fire in the expected context. Co-developed-by: Lian Wang <[email protected]> Signed-off-by: Lian Wang <[email protected]> Signed-off-by: Kunwu Chan <[email protected]> --- mm/damon/perf/Makefile | 2 + mm/damon/perf/stats.c | 295 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 mm/damon/perf/stats.c diff --git a/mm/damon/perf/Makefile b/mm/damon/perf/Makefile index cc0d4f1d1d28..5c46d3da7ef8 100644 --- a/mm/damon/perf/Makefile +++ b/mm/damon/perf/Makefile @@ -1,3 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 # Observability: per-CPU counters, tracepoints, debugfs perf_stats +obj-$(CONFIG_DAMON_PERF_OBSERVE) += damon-perf.o +damon-perf-objs := stats.o diff --git a/mm/damon/perf/stats.c b/mm/damon/perf/stats.c new file mode 100644 index 000000000000..a869f115bb26 --- /dev/null +++ b/mm/damon/perf/stats.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Perf Observability — Per-CPU Statistics & Observe API + * + * All hardware-sampling backends funnel through the observe functions + * defined here. Counters always increment when + * CONFIG_DAMON_PERF_OBSERVE=y; tracepoints are guarded by + * trace_*_enabled() for zero overhead when ftrace is not attached. + * + * Per-CPU counters are best-effort: individual u64 writes are atomic + * on 64-bit platforms, but no cross-field consistency is guaranteed. + * Snapshot reads may race with concurrent writers. This is debug data + * — do not build policy on it. + */ + +#include <linux/cpu.h> +#include <linux/percpu.h> + +#include <trace/events/damon.h> + +#include "perf.h" + +/* + * Per-CPU statistics + */ +static DEFINE_PER_CPU(struct damon_perf_stats, damon_perf_stats); + +/* + * Per-CPU event state for the state machine — the maximum lifecycle + * stage ever reached on each CPU (any event). Set by the NMI + * observe_sample() (ENABLED→RUNNING) and by lifecycle functions as a + * monotonic "best CPU so far". Never reset to UNINIT — destroying one + * event must not hide that another event is still running on the same + * CPU. Per-event per-CPU state (event->cpu_state) tracks individual + * event lifecycle for correctness when multiple events exist. + */ +static DEFINE_PER_CPU(int, damon_perf_cpu_state); + +/* + * Event lifecycle — process context (kdamond / cpuhp callbacks). + * + * Each event tracks its own per-CPU lifecycle state via + * event->cpu_state (allocated on first CREATED, freed on DESTROYED). + * The global damon_perf_cpu_state is also advanced (monotonic) so that + * debugfs perf_stats shows the "best" stage any event reached on each + * CPU. Diagnostics are emitted via tracepoints, not dmesg. + */ + +/* + * Advance the global per-CPU lifecycle state monotonically. The global + * reflects the best stage any event ever reached on this CPU; a late + * create/bind/enable must not regress a CPU that is already RUNNING. + */ +static void damon_perf_cpu_state_advance(int cpu, int state) +{ + int cur = READ_ONCE(per_cpu(damon_perf_cpu_state, cpu)); + + if (state > cur) + WRITE_ONCE(per_cpu(damon_perf_cpu_state, cpu), state); +} + +void damon_perf_observe_event_created(struct damon_perf_event *event, int cpu) +{ + if (!event->cpu_state) { + event->cpu_state = alloc_percpu(int); + if (!event->cpu_state) + return; + } + *per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_CREATED; + damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_CREATED); +} + +void damon_perf_observe_event_bound(struct damon_perf_event *event, + int cpu, struct perf_event *perf_event) +{ + if (event->cpu_state) + *per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_BOUND; + damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_BOUND); +} + +void damon_perf_observe_event_enabled(struct damon_perf_event *event, + int cpu, int state, int oncpu) +{ + if (event->cpu_state) + *per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_ENABLED; + damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_ENABLED); +} + +void damon_perf_observe_event_disabled(struct damon_perf_event *event, + int cpu, int state) +{ + /* State unchanged: the event may be re-enabled later. */ +} + +void damon_perf_observe_event_destroyed(struct damon_perf_event *event, int cpu) +{ + /* + * Reset only this CPU's slot. The per-CPU array is owned by the + * event as a whole and must NOT be freed here: destroyed() is + * invoked from the per-CPU CPU-offline callback, so freeing the + * whole array on the first offline CPU would leave every other + * still-online CPU with a dangling pointer. The array is freed + * once by damon_perf_observe_event_free() at event teardown. + */ + if (event->cpu_state) + *per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_UNINIT; +} + +void damon_perf_observe_event_free(struct damon_perf_event *event) +{ + if (event->cpu_state) { + free_percpu(event->cpu_state); + event->cpu_state = NULL; + } +} + +/* + * Sample observed — NMI-safe. + * + * The @cpu argument is the CPU the sample fired on; it is passed + * through to the tracepoint but stats are always written on the + * current CPU via this_cpu_ptr(). + */ + +void damon_perf_observe_sample(unsigned long addr, u64 data_src, + u64 period, int cpu, u8 reason, + u64 sample_flags, u64 sample_type) +{ + this_cpu_inc(damon_perf_stats.callback); + + switch (reason) { + case 0: + this_cpu_inc(damon_perf_stats.sample_valid); + break; + case 1: + this_cpu_inc(damon_perf_stats.sample_null); + break; + case 2: + this_cpu_inc(damon_perf_stats.sample_addr_zero); + break; + case 3: + this_cpu_inc(damon_perf_stats.sample_kernel); + break; + case 4: + this_cpu_inc(damon_perf_stats.sample_invalid_phys); + break; + } + + /* First callback advances state to RUNNING */ + if (this_cpu_read(damon_perf_cpu_state) == DAMON_PERF_STATE_ENABLED) + this_cpu_write(damon_perf_cpu_state, DAMON_PERF_STATE_RUNNING); + + /* + * Record the execution context so callers can verify whether + * overflow callbacks actually fire in the expected context + * (e.g. NMI for IBS, process for SPE AUX drain). + * + * 0 = process, 1 = softirq, 2 = hardirq, 3 = NMI + */ + if (trace_damon_perf_sample_enabled()) + trace_damon_perf_sample(addr, data_src, period, cpu, reason, + sample_flags, sample_type, + in_nmi() ? 3 : in_hardirq() ? 2 : + in_serving_softirq() ? 1 : 0); +} + +/* + * Ring operations — enqueue / overflow are NMI-safe + */ + +void damon_perf_observe_ring_enqueue(void) +{ + this_cpu_inc(damon_perf_stats.enqueue); +} + +void damon_perf_observe_ring_overflow(int cpu) +{ + this_cpu_inc(damon_perf_stats.overflow); + if (trace_damon_perf_ring_overflow_enabled()) + trace_damon_perf_ring_overflow(cpu); +} + +void damon_perf_observe_ring_dequeue(int cpu) +{ + per_cpu_ptr(&damon_perf_stats, cpu)->dequeue++; +} + +void damon_perf_observe_ring_peak(unsigned int occupancy) +{ + struct damon_perf_stats *st = this_cpu_ptr(&damon_perf_stats); + + if (occupancy > READ_ONCE(st->ring_peak)) + WRITE_ONCE(st->ring_peak, occupancy); +} + +/* + * Matching — process context (kdamond drain loop) + */ + +void damon_perf_observe_match(unsigned long addr, int cpu) +{ + per_cpu_ptr(&damon_perf_stats, cpu)->match++; +} + +void damon_perf_observe_miss(unsigned long addr, int cpu, int reason) +{ + struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu); + + switch (reason) { + case DAMON_REPORT_MISS_TGID: + st->miss_tgid++; + break; + case DAMON_REPORT_MISS_NOREGION: + st->miss_region++; + break; + case DAMON_REPORT_MISS_BOUNDARY: + st->miss_boundary++; + break; + } + + if (trace_damon_perf_report_missed_enabled()) + trace_damon_perf_report_missed(addr, cpu, reason); +} + +void damon_perf_observe_update(int cpu) +{ + per_cpu_ptr(&damon_perf_stats, cpu)->update++; +} + +void damon_perf_observe_drain(unsigned int total, unsigned int matched) +{ + if (trace_damon_perf_drain_enabled()) + trace_damon_perf_drain(total, matched); +} + +/* + * Stats accessors (for debugfs) + */ + +/* + * Best-effort per-CPU snapshot. Individual u64 writes are atomic on + * 64-bit platforms; no cross-field consistency is guaranteed. This is + * debug data only — do not build policy on it. + */ +void damon_perf_stats_snapshot(int cpu, struct damon_perf_stats *dst) +{ + struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu); + + *dst = *st; + dst->cpu_state = per_cpu(damon_perf_cpu_state, cpu); +} + +void damon_perf_stats_aggregate(struct damon_perf_stats *dst) +{ + int cpu; + + memset(dst, 0, sizeof(*dst)); + dst->cpu_state = DAMON_PERF_STATE_RUNNING; /* start optimistic, take min */ + cpus_read_lock(); + for_each_online_cpu(cpu) { + struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu); + + dst->callback += st->callback; + dst->sample_valid += st->sample_valid; + dst->sample_null += st->sample_null; + dst->sample_addr_zero += st->sample_addr_zero; + dst->sample_kernel += st->sample_kernel; + dst->sample_invalid_phys += st->sample_invalid_phys; + dst->enqueue += st->enqueue; + dst->dequeue += st->dequeue; + dst->overflow += st->overflow; + dst->ring_peak = max(dst->ring_peak, st->ring_peak); + dst->match += st->match; + dst->miss_tgid += st->miss_tgid; + dst->miss_region += st->miss_region; + dst->miss_boundary += st->miss_boundary; + dst->update += st->update; + /* + * Aggregate per-CPU state: the pipeline hasn't passed a + * stage until ALL CPUs have passed it, so take the min. + */ + dst->cpu_state = min_t(int, dst->cpu_state, + per_cpu(damon_perf_cpu_state, cpu)); + } + cpus_read_unlock(); +} + +/* + * Subsystem init + */ + +int damon_perf_framework_init(void) +{ + return 0; +} -- 2.43.0
