To modify PMU guest counter reservations dynamically, the per-CPU available counter mask must be updated while host perf events are temporarily scheduled out.
Introduce perf_pmu_resched_update() to update PMU state while holding cpuctx->ctx.lock in between scheduling perf events out and scheduling them back in. It accepts a callback invoked between ctx_sched_out() and perf_event_sched_in(), accomplishing atomic counter mask updates with minimal perf API expansion. Refactor ctx_resched() into __ctx_resched() to invoke the callback at the appropriate point. Signed-off-by: Colton Lewis <[email protected]> --- include/linux/perf_event.h | 3 +++ kernel/events/core.c | 31 ++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5842552294c19..32e3c0d365963 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1242,6 +1242,9 @@ extern int perf_event_task_disable(void); extern int perf_event_task_enable(void); extern void perf_pmu_resched(struct pmu *pmu); +extern void perf_pmu_resched_update(struct pmu *pmu, + void (*update)(struct pmu *, void *), + void *data); extern int perf_event_refresh(struct perf_event *event, int refresh); extern void perf_event_update_userpage(struct perf_event *event); diff --git a/kernel/events/core.c b/kernel/events/core.c index db7b76d6b68aa..06d1128d194fa 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2991,9 +2991,10 @@ static void perf_event_sched_in(struct perf_cpu_context *cpuctx, * event_type is a bit mask of the types of events involved. For CPU events, * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. */ -static void ctx_resched(struct perf_cpu_context *cpuctx, - struct perf_event_context *task_ctx, - struct pmu *pmu, enum event_type_t event_type) +static void __ctx_resched(struct perf_cpu_context *cpuctx, + struct perf_event_context *task_ctx, + struct pmu *pmu, enum event_type_t event_type, + void (*update)(struct pmu *, void *), void *data) { bool cpu_event = !!(event_type & EVENT_CPU); struct perf_event_pmu_context *epc; @@ -3029,6 +3030,9 @@ static void ctx_resched(struct perf_cpu_context *cpuctx, else if (event_type & EVENT_PINNED) ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); + if (update) + update(pmu, data); + perf_event_sched_in(cpuctx, task_ctx, pmu, 0); for_each_epc(epc, &cpuctx->ctx, pmu, 0) @@ -3040,6 +3044,27 @@ static void ctx_resched(struct perf_cpu_context *cpuctx, } } +static void ctx_resched(struct perf_cpu_context *cpuctx, + struct perf_event_context *task_ctx, + struct pmu *pmu, enum event_type_t event_type) +{ + __ctx_resched(cpuctx, task_ctx, pmu, event_type, NULL, NULL); +} + +void perf_pmu_resched_update(struct pmu *pmu, void (*update)(struct pmu *, void *), void *data) +{ + struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); + struct perf_event_context *task_ctx = cpuctx->task_ctx; + unsigned long flags; + + local_irq_save(flags); + perf_ctx_lock(cpuctx, task_ctx); + __ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL|EVENT_CPU, update, data); + perf_ctx_unlock(cpuctx, task_ctx); + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(perf_pmu_resched_update); + void perf_pmu_resched(struct pmu *pmu) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); -- 2.56.0.rc1.310.g51773c2048-goog
