Now that the pmonr state machine and pqr_common are in place, add
pmonr_update_sched_rmid to find the adequate rmid to use. With it,
complete the body of PMUs functions that start/stop, add/del events.
A pmonr in Unused state tries to allocate a free rmid the first time one
of its monitored threads is scheduled in a CPU package (lazy allocation
of rmids). If there is no available rmids in that package, the pmonr
enters a Dep_Idle state (it borrows the sched_rmid from its
Lowest Monitored Ancestor (lma) pmonr).
When a event is stopped, no other event runs in that CPU and PQR msr
uses the rmid of the monr_hrchy_root's pmonr for that CPU's package.
Details in pmonr state machine's comments.
Signed-off-by: David Carrillo-Cisneros <davi...@google.com>
---
arch/x86/events/intel/cmt.c | 101 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 101 insertions(+)
diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c
index ce5be74..9421a3e 100644
--- a/arch/x86/events/intel/cmt.c
+++ b/arch/x86/events/intel/cmt.c
@@ -650,6 +650,74 @@ static int monr_append_event(struct monr *monr, struct
perf_event *event)
return err;
}
+/**
+ * pmonr_update_sched_rmid() - Update sched_rmid for @pmonr in current package.
+ *
+ * Always finds valid rmids for non Off pmonr. Safe to call with IRQs disabled.
+ * A lock-free fast path reuses the rmid when the pmonr has been scheduled
+ * before in this package. Otherwise, tries to get a free rmid. On failure,
+ * enters Dep_Idle state and uses the rmid of its lender. There is always a
+ * pmonr to borrow from since monr_hrchy_root has all its pmonrs in Active
+ * state.
+ * Return: new pmonr_rmids for pmonr.
+ */
+static inline union pmonr_rmids pmonr_update_sched_rmid(struct pmonr *pmonr)
+{
+ struct pkg_data *pkgd = pmonr->pkgd;
+ union pmonr_rmids rmids;
+ u32 free_rmid;
+
+ /* Use atomic_rmids to check state in a lock-free fastpath. */
+ rmids.value = atomic64_read(&pmonr->atomic_rmids);
+ if (rmids.sched_rmid != INVALID_RMID)
+ return rmids;
+
+ /* No need to obtain RMID if in Off state. */
+ if (rmids.sched_rmid == rmids.read_rmid)
+ return rmids;
+
+ /*
+ * Lock-free path failed. Now acquire lock and verify that state
+ * and atomic_rmids haven't changed. If still Unused, try to
+ * obtain a free RMID.
+ */
+ raw_spin_lock(&pkgd->lock);
+
+ /* With lock acquired it is ok to read pmonr::state. */
+ if (pmonr->state != PMONR_UNUSED) {
+ /* Update rmids in case they changed before acquiring lock. */
+ rmids.value = atomic64_read(&pmonr->atomic_rmids);
+ raw_spin_unlock(&pkgd->lock);
+ return rmids;
+ }
+
+ free_rmid = find_first_bit(pkgd->free_rmids, CMT_MAX_NR_RMIDS);
+ if (free_rmid == CMT_MAX_NR_RMIDS)
+ pmonr_unused_to_dep_idle(pmonr);
+ else
+ pmonr_unused_to_active(pmonr, free_rmid);
+
+ raw_spin_unlock(&pkgd->lock);
+
+ rmids.value = atomic64_read(&pmonr->atomic_rmids);
+
+ return rmids;
+}
+
+static inline union pmonr_rmids monr_get_sched_in_rmids(struct monr *monr)
+{
+ struct pmonr *pmonr;
+ union pmonr_rmids rmids;
+ u16 pkgid = topology_logical_package_id(smp_processor_id());
+
+ rcu_read_lock();
+ pmonr = rcu_dereference(monr->pmonrs[pkgid]);
+ rmids = pmonr_update_sched_rmid(pmonr);
+ rcu_read_unlock();
+
+ return rmids;
+}
+
static void monr_hrchy_insert_leaf(struct monr *monr, struct monr *parent)
{
unsigned long flags;
@@ -865,16 +933,49 @@ static void intel_cmt_event_read(struct perf_event *event)
{
}
+static inline void __intel_cmt_event_start(struct perf_event *event,
+ union pmonr_rmids rmids)
+{
+ if (!(event->hw.state & PERF_HES_STOPPED))
+ return;
+ event->hw.state &= ~PERF_HES_STOPPED;
+ pqr_cache_update_rmid(rmids.sched_rmid);
+}
+
static void intel_cmt_event_start(struct perf_event *event, int mode)
{
+ union pmonr_rmids rmids;
+
+ rmids = monr_get_sched_in_rmids(monr_from_event(event));
+ __intel_cmt_event_start(event, rmids);
}
static void intel_cmt_event_stop(struct perf_event *event, int mode)
{
+ union pmonr_rmids rmids;
+
+ if (event->hw.state & PERF_HES_STOPPED)
+ return;
+ event->hw.state |= PERF_HES_STOPPED;
+ rmids = monr_get_sched_in_rmids(monr_hrchy_root);
+ /*
+ * HW tracks the rmid even when event is not scheduled and event
+ * reads occur even if event is Inactive. Therefore there is no need to
+ * read when event is stopped.
+ */
+ pqr_cache_update_rmid(rmids.sched_rmid);
}
static int intel_cmt_event_add(struct perf_event *event, int mode)
{
+ union pmonr_rmids rmids;
+
+ event->hw.state = PERF_HES_STOPPED;
+ rmids = monr_get_sched_in_rmids(monr_from_event(event));
+
+ if (mode & PERF_EF_START)
+ __intel_cmt_event_start(event, rmids);
+
return 0;
}
--
2.8.0.rc3.226.g39d4020
