This uflag allows user to specify that a rmid must be allocated at monr's
initialization or fail otherwise.
For this to work we split __pmonr_apply_uflags into reserve and apply
modes. The reserve mode will try to reserve a free rmid, and if successful,
the apply mode can proceed using the rmid previously reserved.
Signed-off-by: David Carrillo-Cisneros <davi...@google.com>
---
arch/x86/events/intel/cmt.c | 116 +++++++++++++++++++++++++++++++++++++++-----
arch/x86/events/intel/cmt.h | 5 +-
2 files changed, 109 insertions(+), 12 deletions(-)
diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c
index 9421a3e..3883cb4 100644
--- a/arch/x86/events/intel/cmt.c
+++ b/arch/x86/events/intel/cmt.c
@@ -33,7 +33,8 @@ static unsigned int __min_max_rmid; /* minimum max_rmid
across all pkgs. */
static struct monr *monr_hrchy_root;
/* Flags for root monr and all its pmonrs while being monitored. */
-static enum cmt_user_flags root_monr_uflags = CMT_UF_HAS_USER;
+static enum cmt_user_flags root_monr_uflags =
+ CMT_UF_HAS_USER | CMT_UF_NOLAZY_RMID;
/* Auxiliar flags */
static enum cmt_user_flags *pkg_uflags_zeroes;
@@ -414,6 +415,7 @@ static void monr_dealloc(struct monr *monr)
u16 p, nr_pkgs = topology_max_packages();
if (WARN_ON_ONCE(monr->nr_has_user) ||
+ WARN_ON_ONCE(monr->nr_nolazy_rmid) ||
WARN_ON_ONCE(monr->mon_events))
return;
@@ -478,11 +480,28 @@ static enum cmt_user_flags pmonr_uflags(struct pmonr
*pmonr)
return monr->uflags | monr->pkg_uflags[pmonr->pkgd->pkgid];
}
+/*
+ * Callable in two modes:
+ * 1) @reserve == true: will check if uflags are applicable and store in
+ * @res_rmid the "reserved" rmid.
+ * 2) @reserve == false: will apply pmonr_uflags using the rmid stored in
+ * @res_rmid rmid (if any). Cannot fail.
+ */
static int __pmonr_apply_uflags(struct pmonr *pmonr,
- enum cmt_user_flags pmonr_uflags)
+ enum cmt_user_flags pmonr_uflags, bool reserve, u32 *res_rmid)
{
+ struct pkg_data *pkgd = pmonr->pkgd;
+ u32 free_rmid;
+
+ if (WARN_ON_ONCE(!res_rmid))
+ return -EINVAL;
+ if (WARN_ON_ONCE(reserve && *res_rmid != INVALID_RMID))
+ return -EINVAL;
+
if (!(pmonr_uflags & CMT_UF_HAS_USER)) {
if (pmonr->state != PMONR_OFF) {
+ if (reserve)
+ return 0;
pmonr_to_unused(pmonr);
pmonr_unused_to_off(pmonr);
}
@@ -492,8 +511,40 @@ static int __pmonr_apply_uflags(struct pmonr *pmonr,
if (monr_is_root(pmonr->monr) && (~pmonr_uflags & root_monr_uflags))
return -EINVAL;
- if (pmonr->state == PMONR_OFF)
- pmonr_to_unused(pmonr);
+ if (pmonr->state == PMONR_OFF) {
+ if (!reserve)
+ pmonr_to_unused(pmonr);
+ }
+ if (pmonr->state == PMONR_ACTIVE)
+ return 0;
+ if (!(pmonr_uflags & CMT_UF_NOLAZY_RMID))
+ return 0;
+ if (pmonr->state == PMONR_DEP_DIRTY) {
+ if (!reserve)
+ pmonr_dep_dirty_to_active(pmonr);
+ return 0;
+ }
+
+ /*
+ * At this point pmonr is in either Unused or Dep_Idle state and
+ * needs a rmid to transition to Active.
+ */
+ if (reserve) {
+ free_rmid = find_first_bit(pkgd->free_rmids, CMT_MAX_NR_RMIDS);
+ if (free_rmid == CMT_MAX_NR_RMIDS)
+ return -ENOSPC;
+ *res_rmid = free_rmid;
+ __clear_bit(*res_rmid, pkgd->free_rmids);
+ return 0;
+ }
+
+ /* both cases use the reserved rmid. */
+ if (pmonr->state == PMONR_UNUSED) {
+ pmonr_unused_to_active(pmonr, *res_rmid);
+ } else {
+ WARN_ON_ONCE(pmonr->state != PMONR_DEP_IDLE);
+ pmonr_dep_idle_to_active(pmonr, *res_rmid);
+ }
return 0;
}
@@ -514,7 +565,10 @@ static bool monr_has_user(struct monr *monr)
pkg_uflags_has_user(monr->pkg_uflags);
}
-static int __monr_apply_uflags(struct monr *monr, enum cmt_user_flags *puflags)
+static int __monr_apply_uflags(struct monr *monr,
+ enum cmt_user_flags *puflags,
+ bool reserve,
+ u32 *res_rmids)
{
enum cmt_user_flags pmonr_uflags;
struct pkg_data *pkgd = NULL;
@@ -526,7 +580,10 @@ static int __monr_apply_uflags(struct monr *monr, enum
cmt_user_flags *puflags)
pmonr_uflags = monr->uflags |
(puflags ? puflags[p] : monr->pkg_uflags[p]);
pmonr = pkgd_pmonr(pkgd, monr);
- err = __pmonr_apply_uflags(pmonr, pmonr_uflags);
+ err = __pmonr_apply_uflags(pmonr, pmonr_uflags,
+ reserve, &res_rmids[p]);
+ /* Should not fail if reserve is set. */
+ WARN_ON_ONCE(!reserve && err);
if (err)
return err;
}
@@ -537,17 +594,26 @@ static int __monr_apply_uflags(struct monr *monr, enum
cmt_user_flags *puflags)
/* Apply puflags for all packages or rollback and fail. */
static int monr_apply_uflags(struct monr *monr, enum cmt_user_flags *puflags)
{
+ struct pkg_data *pkgd = NULL;
+ u32 res_rmids[CMT_MAX_NR_PKGS];
int p, err;
unsigned long flags;
monr_hrchy_assert_held_mutexes();
monr_hrchy_acquire_locks(&flags);
- err = __monr_apply_uflags(monr, puflags);
+ for (p = 0; p < CMT_MAX_NR_PKGS; p++)
+ res_rmids[p] = INVALID_RMID;
+
+ /* first call of __monr_apply_uflags will "reserve" rmid only.*/
+ err = __monr_apply_uflags(monr, puflags, true, res_rmids);
if (err)
- goto exit;
+ goto error;
+
+ /* second call actually applies the flags. */
+ err = __monr_apply_uflags(monr, puflags, false, res_rmids);
+ WARN_ON_ONCE(err);
- /* Proceed to exit if no uflags to update to pkg_uflags. */
if (!puflags)
goto exit;
@@ -563,6 +629,14 @@ static int monr_apply_uflags(struct monr *monr, enum
cmt_user_flags *puflags)
monr_hrchy_release_locks(&flags);
return err;
+
+error:
+ while ((pkgd = cmt_pkgs_data_next_rcu(pkgd))) {
+ p = pkgd->pkgid;
+ if (res_rmids[p] != INVALID_RMID)
+ __set_bit(res_rmids[p], pkgd->free_rmids);
+ }
+ goto exit;
}
static inline struct monr *monr_from_event(struct perf_event *event)
@@ -583,8 +657,11 @@ static bool monr_account_uflags(struct monr *monr,
if (uflags & CMT_UF_HAS_USER)
monr->nr_has_user += account ? 1 : -1;
+ if (uflags & CMT_UF_NOLAZY_RMID)
+ monr->nr_nolazy_rmid += account ? 1 : -1;
- monr->uflags = (monr->nr_has_user ? CMT_UF_HAS_USER : 0);
+ monr->uflags = (monr->nr_has_user ? CMT_UF_HAS_USER : 0) |
+ (monr->nr_nolazy_rmid ? CMT_UF_NOLAZY_RMID : 0);
return old_flags != monr->uflags;
}
@@ -1165,6 +1242,7 @@ static int init_pkg_data(int cpu)
struct pmonr *pmonr;
unsigned long flags;
int err = 0;
+ u32 res_rmid;
u16 pkgid = topology_logical_package_id(cpu);
lockdep_assert_held(&cmt_mutex);
@@ -1190,9 +1268,25 @@ static int init_pkg_data(int cpu)
*/
RCU_INIT_POINTER(pos->pmonrs[pkgid], pmonr);
+ res_rmid = INVALID_RMID;
raw_spin_lock_irqsave(&pkgd->lock, flags);
- err = __pmonr_apply_uflags(pmonr, pmonr_uflags(pmonr));
+ err = __pmonr_apply_uflags(pmonr, pmonr_uflags(pmonr),
+ true, &res_rmid);
+ if (!err)
+ __pmonr_apply_uflags(pmonr, pmonr_uflags(pmonr),
+ false, &res_rmid);
raw_spin_unlock_irqrestore(&pkgd->lock, flags);
+
+ /*
+ * Do not fail the whole package initialization because a pmonr
+ * failed to apply its uflags, just report the error.
+ */
+ if (err) {
+ pr_err("Not enough free RMIDs in package %d for Intel
CMT.\n",
+ pkgid);
+ pos->pkg_uflags[pkgid] |= CMT_UF_ERROR;
+ err = 0;
+ }
}
if (err) {
diff --git a/arch/x86/events/intel/cmt.h b/arch/x86/events/intel/cmt.h
index bf90c26..754a9c8 100644
--- a/arch/x86/events/intel/cmt.h
+++ b/arch/x86/events/intel/cmt.h
@@ -245,7 +245,8 @@ struct pkg_data {
enum cmt_user_flags {
/* if no has_user other flags are meaningless. */
CMT_UF_HAS_USER = BIT(0), /* has cgroup or event users */
- CMT_UF_MAX = BIT(1) - 1,
+ CMT_UF_NOLAZY_RMID = BIT(1), /* try to obtain rmid on creation */
+ CMT_UF_MAX = BIT(2) - 1,
CMT_UF_ERROR = CMT_UF_MAX + 1,
};
@@ -258,6 +259,7 @@ enum cmt_user_flags {
* @children: List of children in monr hierarchy.
* @parent_entry: Entry in parent's children list.
* @nr_has_user: nr of CMT_UF_HAS_USER set in events in mon_events.
+ * @nr_nolazy_user: nr of CMT_UF_NOLAZY_RMID set in events in mon_events.
* @uflags: monr level cmt_user_flags, or'ed with pkg_uflags.
* @pkg_uflags: package level cmt_user_flags, each entry is
used as
* pmonr uflags if that package is online.
@@ -278,6 +280,7 @@ struct monr {
struct list_head parent_entry;
int nr_has_user;
+ int nr_nolazy_rmid;
enum cmt_user_flags uflags;
enum cmt_user_flags pkg_uflags[];
};
--
2.8.0.rc3.226.g39d4020
