The branch main has been updated by mhorne: URL: https://cgit.FreeBSD.org/src/commit/?id=f19f44634afd859e445c7b13bbdf715a493b499b
commit f19f44634afd859e445c7b13bbdf715a493b499b Author: Ali Mashtizadeh <[email protected]> AuthorDate: 2026-08-13 23:08:20 +0000 Commit: Mitchell Horne <[email protected]> CommitDate: 2026-09-03 16:04:06 +0000 pmc: Add support for AMD UMC counters This change adds support for AMD's UMC performance counters. It is a bit more complicated than existing counters because the enable bit has moved. This supports Zen 4 through most Zen 6 chips as UMC counters are per-node, where a node does not necessarily translate to a NUMA domain. A few follow up changes to PMC will address this limitation. Reviewed by: mhorne Sponsored by: Netflix Pull Request: https://github.com/freebsd/freebsd-src/pull/2368 --- lib/libpmc/libpmc_pmu_util.c | 7 ++++ lib/libpmc/pmu-events/jevents.c | 4 +- sys/dev/hwpmc/hwpmc_amd.c | 92 ++++++++++++++++++++++++++++++++--------- sys/dev/hwpmc/hwpmc_amd.h | 31 +++++++++++++- 4 files changed, 112 insertions(+), 22 deletions(-) diff --git a/lib/libpmc/libpmc_pmu_util.c b/lib/libpmc/libpmc_pmu_util.c index 050cbc120cd8..647fceabb05d 100644 --- a/lib/libpmc/libpmc_pmu_util.c +++ b/lib/libpmc/libpmc_pmu_util.c @@ -230,6 +230,7 @@ struct pmu_event_desc { uint32_t ped_coreid; uint32_t ped_allsources; uint32_t ped_allcores; + uint32_t ped_rdwrmask; uint32_t ped_event; uint32_t ped_frontend; uint32_t ped_ldlat; @@ -378,6 +379,8 @@ pmu_parse_event(struct pmu_event_desc *ped, const char *eventin) ped->ped_allcores = strtol(value, NULL, 0); else if (strcmp(key, "allsources") == 0) ped->ped_allsources = strtol(value, NULL, 0); + else if (strcmp(key, "rdwrmask") == 0) + ped->ped_rdwrmask = strtol(value, NULL, 0); else if (strcmp(key, "pebs") == 0) ped->ped_pebs = strtol(value, NULL, 10); else { @@ -589,6 +592,10 @@ pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, amd->pm_amd_config |= AMD_PMC_DF2_TO_UNITMASK(ped->ped_umask); } + } else if (strcmp("amd_umc", pe->pmu) == 0) { + amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_UMC; + amd->pm_amd_config |= AMD_PMC_UMC_TO_EVENTMASK(ped->ped_event); + amd->pm_amd_config |= AMD_PMC_UMC_TO_RDWRMASK(ped->ped_rdwrmask); } else { printf("PMC pmu '%s' is not supported!\n", pe->pmu); return (EOPNOTSUPP); diff --git a/lib/libpmc/pmu-events/jevents.c b/lib/libpmc/pmu-events/jevents.c index b4d073f13708..7e6b19c1b30b 100644 --- a/lib/libpmc/pmu-events/jevents.c +++ b/lib/libpmc/pmu-events/jevents.c @@ -265,6 +265,7 @@ static struct map { /* AMD */ { "L3PMC", "amd_l3" }, { "DFPMC", "amd_df" }, + { "UMCPMC", "amd_umc" }, /* ARM HiSilicon */ { "hisi_sicl,cpa", "hisi_sicl,cpa"}, { "hisi_sccl,ddrc", "hisi_sccl,ddrc" }, @@ -648,7 +649,8 @@ static int json_events(const char *fn, if (nz) addfield(map, &cmask, "", "cmask=", val); } else if (json_streq(map, field, "RdWrMask")) { - /* AMD UMC */ + if (nz) + addfield(map, &cmask, "", "rdwrmask=", val); } else if (json_streq(map, field, "Invert")) { if (nz) addfield(map, &inv, "", "inv=", val); diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 1afdb59ed745..5c4b14ad1003 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -64,7 +64,7 @@ struct amd_descr { }; static int amd_npmcs; -static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; +static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs, amd_umc_npmcs; static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ @@ -186,10 +186,12 @@ static struct amd_cpu **amd_pcpu; static uint64_t amd_core_allowed_mask; static uint64_t amd_l3_allowed_mask; static uint64_t amd_df_allowed_mask; +static uint64_t amd_umc_allowed_mask; static uint64_t amd_core_extra_mask; static uint64_t amd_l3_extra_mask; static uint64_t amd_df_extra_mask; +static uint64_t amd_umc_extra_mask; SYSCTL_DECL(_kern_hwpmc); @@ -205,6 +207,10 @@ SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_df_extra_mask, CTLFLAG_RDTUN, &amd_df_extra_mask, 0, "Extra allowed bits in AMD DF PMU control (override; default 0)"); +SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_umc_extra_mask, CTLFLAG_RDTUN, + &amd_umc_extra_mask, 0, + "Extra allowed bits in AMD UMC PMU control (override; default 0)"); + static void amd_init_policy(void) { @@ -219,6 +225,8 @@ amd_init_policy(void) amd_df_allowed_mask = (family <= 0x19) ? AMD_PMC_DF_FAMILY17_MASK : AMD_PMC_DF_FAMILY1A_MASK; + + amd_umc_allowed_mask = AMD_PMC_UMC_MASK; } static uint64_t @@ -234,6 +242,8 @@ amd_config_mask(enum sub_class subclass, uint64_t caps) return (amd_l3_allowed_mask | amd_l3_extra_mask); case PMC_AMD_SUB_CLASS_DATA_FABRIC: return (amd_df_allowed_mask | amd_df_extra_mask); + case PMC_AMD_SUB_CLASS_UMC: + return (amd_umc_allowed_mask | amd_umc_extra_mask); default: return (0); } @@ -526,16 +536,24 @@ amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm) PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri); /* - * Triggered by DF counters because all DF MSRs are shared. We need to - * change the code to honor the per-package flag in the JSON event - * definitions. + * Asserts triggered by DF/UMC counters because all DF/UMC MSRs are + * shared. While userspace now honors the per-node flags, we should + * enforce this in the kernel. */ - KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), - ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, - ri, cpu, pd->pm_descr.pd_name)); - /* turn on the PMC ENABLE bit */ - config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { + KASSERT(AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), + ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", + __LINE__, ri, cpu, pd->pm_descr.pd_name)); + + config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_UMC_ENABLE; + } else { + KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), + ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", + __LINE__, ri, cpu, pd->pm_descr.pd_name)); + + config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; + } PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config); @@ -560,14 +578,22 @@ amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) pd = &amd_pmcdesc[ri]; - KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), - ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", - __LINE__, ri, cpu, pd->pm_descr.pd_name)); - PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri); /* turn off the PMC ENABLE bit */ - config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC) { + KASSERT(!AMD_PMC_UMC_IS_STOPPED(pd->pm_evsel), + ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", + __LINE__, ri, cpu, pd->pm_descr.pd_name)); + + config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE; + } else { + KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), + ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", + __LINE__, ri, cpu, pd->pm_descr.pd_name)); + + config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; + } wrmsr(pd->pm_evsel, config); /* @@ -654,12 +680,22 @@ amd_intr(struct trapframe *tf) v = pm->pm_sc.pm_reloadcount; config = rdmsr(evsel); - KASSERT((config & ~AMD_PMC_ENABLE) == - (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), - ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, - (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); - wrmsr(evsel, config & ~AMD_PMC_ENABLE); + if (amd_pmcdesc[i].pm_subclass == PMC_AMD_SUB_CLASS_UMC) { + KASSERT((config & ~AMD_PMC_UMC_ENABLE) == + (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE), + ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, + (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); + + wrmsr(evsel, config & ~AMD_PMC_UMC_ENABLE); + } else { + KASSERT((config & ~AMD_PMC_ENABLE) == + (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), + ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, + (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); + + wrmsr(evsel, config & ~AMD_PMC_ENABLE); + } wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); /* Restart the counter if logging succeeded. */ @@ -953,6 +989,7 @@ pmc_amd_initialize(void) int ncpus, nclasses, i; int family, model, stepping; int error; + int pmcs_per_umc; /* * The presence of hardware performance counters on the AMD @@ -996,12 +1033,16 @@ pmc_amd_initialize(void) } amd_l3_npmcs = AMD_PMC_L3_DEFAULT; amd_df_npmcs = AMD_PMC_DF_DEFAULT; + amd_umc_npmcs = 0; + pmcs_per_umc = 0; if (cpu_exthigh >= CPUID_EXTPERFMON) { do_cpuid(CPUID_EXTPERFMON, regs); if (regs[1] != 0) { amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); + amd_umc_npmcs = EXTPERFMON_UMC_PMCS(regs[1]); + pmcs_per_umc = amd_umc_npmcs / popcntq(regs[2]); } } @@ -1064,6 +1105,19 @@ pmc_amd_initialize(void) amd_npmcs += amd_df_npmcs; } + for (i = 0; i < amd_umc_npmcs; i++) { + d = &amd_pmcdesc[amd_npmcs + i]; + snprintf(d->pm_descr.pd_name, PMC_NAME_MAX, + "K8-UMC%d-%d", i / pmcs_per_umc, i); + d->pm_descr.pd_class = PMC_CLASS_K8; + d->pm_descr.pd_caps = AMD_PMC_UMC_CAPS; + d->pm_descr.pd_width = 48; + d->pm_evsel = AMD_PMC_UMC_BASE + 2 * i; + d->pm_perfctr = AMD_PMC_UMC_BASE + 2 * i + 1; + d->pm_subclass = PMC_AMD_SUB_CLASS_UMC; + } + amd_npmcs += amd_umc_npmcs; + /* * Sanity check that the hardware is safe to use. Do not read or write * any of the PMC MSRs until after this check passes. diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h index 616d115ecc25..4a8a5f6b6433 100644 --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -35,6 +35,7 @@ #define CPUID_EXTPERFMON 0x80000022 #define EXTPERFMON_CORE_PMCS(x) ((x) & 0x0F) #define EXTPERFMON_DF_PMCS(x) (((x) >> 10) & 0x3F) +#define EXTPERFMON_UMC_PMCS(x) (((x) >> 16) & 0xFF) /* AMD K8 PMCs */ #define AMD_PMC_EVSEL_0 0xC0010000 @@ -170,9 +171,34 @@ AMD_PMC_DF2_TO_EVENTMASK(0x7fff) | \ AMD_PMC_DF2_TO_UNITMASK(0xfff)) +/* + * UMC counters + * + * Refer to the following documents: + * PPR for AMD Family 1Ah Model 02h C1 57238 Rev. 0.49 March 6, 2026 + */ + +#define AMD_PMC_UMC_BASE 0xC0010800 +#define AMD_PMC_UMC_MAX 256 + +#define AMD_PMC_UMC_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | \ + PMC_CAP_QUALIFIER | PMC_CAP_DOMWIDE) + +#define AMD_PMC_UMC_ENABLE 0x80000000 +#define AMD_PMC_UMC_RDWRMASK 0x00000300 +#define AMD_PMC_UMC_EVENTMASK 0x000000FF + +#define AMD_PMC_UMC_MASK (AMD_PMC_UMC_ENABLE | AMD_PMC_UMC_RDWRMASK | \ + AMD_PMC_UMC_EVENTMASK) + +#define AMD_PMC_UMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_UMC_ENABLE) == 0) + +#define AMD_PMC_UMC_TO_EVENTMASK(x) ((x) & AMD_PMC_UMC_EVENTMASK) +#define AMD_PMC_UMC_TO_RDWRMASK(x) (((x) << 8) & AMD_PMC_UMC_RDWRMASK) + #define AMD_NPMCS_K8 4 #define AMD_NPMCS_MAX (AMD_PMC_CORE_MAX + AMD_PMC_L3_MAX + \ - AMD_PMC_DF_MAX) + AMD_PMC_DF_MAX + AMD_PMC_UMC_MAX) #define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) #define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) @@ -183,7 +209,8 @@ enum sub_class { PMC_AMD_SUB_CLASS_CORE, PMC_AMD_SUB_CLASS_L3_CACHE, - PMC_AMD_SUB_CLASS_DATA_FABRIC + PMC_AMD_SUB_CLASS_DATA_FABRIC, + PMC_AMD_SUB_CLASS_UMC }; struct pmc_md_amd_op_pmcallocate {
