Module Name: src Committed By: ad Date: Mon Dec 2 23:22:44 UTC 2019
Modified Files: src/sys/arch/aarch64/aarch64: cpufunc.c src/sys/arch/arm/arm32: arm32_boot.c cpu.c src/sys/arch/mips/mips: cpu_subr.c src/sys/arch/x86/x86: cpu_topology.c src/sys/kern: init_main.c kern_cpu.c src/sys/sys: cpu.h cpu_data.h Log Message: Take the basic CPU topology information we already collect, and use it to make circular lists of CPU siblings in the same core, and in the same package. Nothing fancy, just enough to have a bit of fun in the scheduler trying out different tactics. To generate a diff of this commit: cvs rdiff -u -r1.8 -r1.9 src/sys/arch/aarch64/aarch64/cpufunc.c cvs rdiff -u -r1.33 -r1.34 src/sys/arch/arm/arm32/arm32_boot.c cvs rdiff -u -r1.134 -r1.135 src/sys/arch/arm/arm32/cpu.c cvs rdiff -u -r1.38 -r1.39 src/sys/arch/mips/mips/cpu_subr.c cvs rdiff -u -r1.14 -r1.15 src/sys/arch/x86/x86/cpu_topology.c cvs rdiff -u -r1.507 -r1.508 src/sys/kern/init_main.c cvs rdiff -u -r1.78 -r1.79 src/sys/kern/kern_cpu.c cvs rdiff -u -r1.44 -r1.45 src/sys/sys/cpu.h cvs rdiff -u -r1.40 -r1.41 src/sys/sys/cpu_data.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/aarch64/aarch64/cpufunc.c diff -u src/sys/arch/aarch64/aarch64/cpufunc.c:1.8 src/sys/arch/aarch64/aarch64/cpufunc.c:1.9 --- src/sys/arch/aarch64/aarch64/cpufunc.c:1.8 Fri Nov 22 05:21:19 2019 +++ src/sys/arch/aarch64/aarch64/cpufunc.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $ */ +/* $NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu <r...@nerv.org> @@ -29,13 +29,13 @@ #include "opt_multiprocessor.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.8 2019/11/22 05:21:19 mlelstv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.9 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/types.h> #include <sys/kmem.h> +#include <sys/cpu.h> -#include <aarch64/cpu.h> #include <aarch64/cpufunc.h> u_int cputype; /* compat arm */ @@ -93,12 +93,15 @@ aarch64_gettopology(struct cpu_info * co { if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } } Index: src/sys/arch/arm/arm32/arm32_boot.c diff -u src/sys/arch/arm/arm32/arm32_boot.c:1.33 src/sys/arch/arm/arm32/arm32_boot.c:1.34 --- src/sys/arch/arm/arm32/arm32_boot.c:1.33 Sat Mar 16 10:05:40 2019 +++ src/sys/arch/arm/arm32/arm32_boot.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $ */ +/* $NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. @@ -122,7 +122,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.33 2019/03/16 10:05:40 skrll Exp $"); +__KERNEL_RCSID(1, "$NetBSD: arm32_boot.c,v 1.34 2019/12/02 23:22:43 ad Exp $"); #include "opt_arm_debug.h" #include "opt_cputypes.h" @@ -358,12 +358,15 @@ cpu_hatch(struct cpu_info *ci, u_int cpu uint32_t mpidr = armreg_mpidr_read(); ci->ci_mpidr = mpidr; if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } ci->ci_arm_cpuid = cpu_idnum(); Index: src/sys/arch/arm/arm32/cpu.c diff -u src/sys/arch/arm/arm32/cpu.c:1.134 src/sys/arch/arm/arm32/cpu.c:1.135 --- src/sys/arch/arm/arm32/cpu.c:1.134 Sun Oct 20 14:25:14 2019 +++ src/sys/arch/arm/arm32/cpu.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $ */ +/* $NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $ */ /* * Copyright (c) 1995 Mark Brinicombe. @@ -46,7 +46,7 @@ #include "opt_multiprocessor.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.134 2019/10/20 14:25:14 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.135 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/conf.h> @@ -111,12 +111,15 @@ cpu_attach(device_t dv, cpuid_t id) ci->ci_mpidr = mpidr; if (mpidr & MPIDR_MT) { - ci->ci_smt_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF1); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF2); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF2), + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0)); } else { - ci->ci_core_id = __SHIFTOUT(mpidr, MPIDR_AFF0); - ci->ci_package_id = __SHIFTOUT(mpidr, MPIDR_AFF1); + cpu_topology_set(ci, + __SHIFTOUT(mpidr, MPIDR_AFF1), + __SHIFTOUT(mpidr, MPIDR_AFF0), + 0); } #endif } else { Index: src/sys/arch/mips/mips/cpu_subr.c diff -u src/sys/arch/mips/mips/cpu_subr.c:1.38 src/sys/arch/mips/mips/cpu_subr.c:1.39 --- src/sys/arch/mips/mips/cpu_subr.c:1.38 Sun Dec 1 14:52:13 2019 +++ src/sys/arch/mips/mips/cpu_subr.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $ */ +/* $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2010, 2019 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.38 2019/12/01 14:52:13 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $"); #include "opt_cputype.h" #include "opt_ddb.h" @@ -182,9 +182,6 @@ cpu_info_alloc(struct pmap_tlb_info *ti, KASSERT(cpu_id != 0); ci->ci_cpuid = cpu_id; ci->ci_pmap_kern_segtab = &pmap_kern_segtab, - ci->ci_package_id = cpu_package_id; - ci->ci_core_id = cpu_core_id; - ci->ci_smt_id = cpu_smt_id; ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq; ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq; ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz; @@ -192,6 +189,8 @@ cpu_info_alloc(struct pmap_tlb_info *ti, ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip; ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count; + cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id); + pmap_md_alloc_ephemeral_address_space(ci); mi_cpu_attach(ci); Index: src/sys/arch/x86/x86/cpu_topology.c diff -u src/sys/arch/x86/x86/cpu_topology.c:1.14 src/sys/arch/x86/x86/cpu_topology.c:1.15 --- src/sys/arch/x86/x86/cpu_topology.c:1.14 Wed Nov 21 10:34:53 2018 +++ src/sys/arch/x86/x86/cpu_topology.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $ */ +/* $NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2009 Mindaugas Rasiukevicius <rmind at NetBSD org>, @@ -36,13 +36,13 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.14 2018/11/21 10:34:53 msaitoh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.15 2019/12/02 23:22:43 ad Exp $"); #include <sys/param.h> #include <sys/bitops.h> +#include <sys/cpu.h> #include <machine/specialreg.h> -#include <machine/cpu.h> #include <x86/cpufunc.h> #include <x86/cputypes.h> @@ -55,23 +55,28 @@ x86_cpu_topology(struct cpu_info *ci) u_int core_max; /* Core per package */ int n, cpu_family, apic_id, smt_bits, core_bits = 0; uint32_t descs[4]; + int package_id, core_id, smt_id; apic_id = ci->ci_initapicid; cpu_family = CPUID_TO_FAMILY(ci->ci_signature); /* Initial values. */ - ci->ci_package_id = apic_id; - ci->ci_core_id = 0; - ci->ci_smt_id = 0; + package_id = apic_id; + core_id = 0; + smt_id = 0; switch (cpu_vendor) { case CPUVENDOR_INTEL: - if (cpu_family < 6) + if (cpu_family < 6) { + cpu_topology_set(ci, package_id, core_id, smt_id); return; + } break; case CPUVENDOR_AMD: - if (cpu_family < 0xf) + if (cpu_family < 0xf) { + cpu_topology_set(ci, package_id, core_id, smt_id); return; + } break; default: return; @@ -164,16 +169,18 @@ x86_cpu_topology(struct cpu_info *ci) if (smt_bits + core_bits) { if (smt_bits + core_bits < sizeof(apic_id) * NBBY) - ci->ci_package_id = apic_id >> (smt_bits + core_bits); + package_id = apic_id >> (smt_bits + core_bits); else - ci->ci_package_id = 0; + package_id = 0; } if (core_bits) { u_int core_mask = __BITS(smt_bits, smt_bits + core_bits - 1); - ci->ci_core_id = __SHIFTOUT(apic_id, core_mask); + core_id = __SHIFTOUT(apic_id, core_mask); } if (smt_bits) { u_int smt_mask = __BITS(0, smt_bits - 1); - ci->ci_smt_id = __SHIFTOUT(apic_id, smt_mask); + smt_id = __SHIFTOUT(apic_id, smt_mask); } + + cpu_topology_set(ci, package_id, core_id, smt_id); } Index: src/sys/kern/init_main.c diff -u src/sys/kern/init_main.c:1.507 src/sys/kern/init_main.c:1.508 --- src/sys/kern/init_main.c:1.507 Sun Dec 1 17:08:31 2019 +++ src/sys/kern/init_main.c Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $ */ +/* $NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc. @@ -97,7 +97,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.507 2019/12/01 17:08:31 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.508 2019/12/02 23:22:43 ad Exp $"); #include "opt_ddb.h" #include "opt_inet.h" @@ -788,6 +788,9 @@ configure2(void) struct cpu_info *ci; int s; + /* Fix up CPU topology info, which has all been collected by now. */ + cpu_topology_init(); + /* * Now that we've found all the hardware, start the real time * and statistics clocks. Index: src/sys/kern/kern_cpu.c diff -u src/sys/kern/kern_cpu.c:1.78 src/sys/kern/kern_cpu.c:1.79 --- src/sys/kern/kern_cpu.c:1.78 Sun Dec 1 15:34:46 2019 +++ src/sys/kern/kern_cpu.c Mon Dec 2 23:22:43 2019 @@ -1,7 +1,7 @@ -/* $NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $ */ +/* $NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $ */ /*- - * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. + * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -56,7 +56,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.78 2019/12/01 15:34:46 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.79 2019/12/02 23:22:43 ad Exp $"); #include "opt_cpu_ucode.h" @@ -119,6 +119,7 @@ kmutex_t cpu_lock __cacheline_aligned; int ncpu __read_mostly; int ncpuonline __read_mostly; bool mp_online __read_mostly; +static bool cpu_topology_present __read_mostly; /* An array of CPUs. There are ncpu entries. */ struct cpu_info **cpu_infos __read_mostly; @@ -587,6 +588,117 @@ cpu_softintr_p(void) return (curlwp->l_pflag & LP_INTR) != 0; } +/* + * Collect CPU topology information as each CPU is attached. This can be + * called early during boot, so we need to be careful what we do. + */ +void +cpu_topology_set(struct cpu_info *ci, int package_id, int core_id, int smt_id) +{ + + cpu_topology_present = true; + ci->ci_package_id = package_id; + ci->ci_core_id = core_id; + ci->ci_smt_id = smt_id; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_ncore_cpus = 1; +} + +/* + * Fake up toplogy info if we have none, or if what we got was bogus. + */ +static void +cpu_topology_fake(void) +{ + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + for (CPU_INFO_FOREACH(cii, ci)) { + ci->ci_package_id = cpu_index(ci); + ci->ci_core_id = 0; + ci->ci_smt_id = 0; + ci->ci_ncore_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + } +} + +/* + * Fix up basic CPU topology info. Right now that means attach each CPU to + * circular lists of its siblings in the same core, and in the same package. + */ +void +cpu_topology_init(void) +{ + CPU_INFO_ITERATOR cii, cii2; + struct cpu_info *ci, *ci2, *ci3; + + if (!cpu_topology_present) { + cpu_topology_fake(); + return; + } + + for (CPU_INFO_FOREACH(cii, ci)) { + ci->ci_ncore_cpus = 1; + ci->ci_core_cpus = ci; + ci->ci_package_cpus = ci; + ci->ci_npackage_cpus = 1; + } + + for (CPU_INFO_FOREACH(cii, ci)) { + for (CPU_INFO_FOREACH(cii2, ci2)) { + /* Avoid bad things happening. */ + if (ci2->ci_package_id == ci->ci_package_id && + ci2->ci_core_id == ci->ci_core_id && + ci2->ci_smt_id == ci->ci_smt_id && + ci2 != ci) { + printf("cpu_topology_init: info bogus, " + "faking it\n"); + cpu_topology_fake(); + return; + } + if (ci2 == ci || + ci2->ci_package_id != ci->ci_package_id) { + continue; + } + /* + * Find CPUs in the same core. Walk to the end of + * the existing circular list and append. + */ + if (ci->ci_ncore_cpus == 1 && + ci->ci_core_id == ci2->ci_core_id) { + for (ci3 = ci2;; ci3 = ci3->ci_core_cpus) { + ci3->ci_ncore_cpus++; + if (ci3->ci_core_cpus == ci2) { + break; + } + } + ci->ci_core_cpus = ci2; + ci3->ci_core_cpus = ci; + ci->ci_ncore_cpus = ci3->ci_ncore_cpus; + } + /* Same, but for package. */ + if (ci->ci_npackage_cpus == 1) { + for (ci3 = ci2;; ci3 = ci3->ci_package_cpus) { + ci3->ci_npackage_cpus++; + if (ci3->ci_package_cpus == ci2) { + break; + } + } + ci->ci_package_cpus = ci2; + ci3->ci_package_cpus = ci; + ci->ci_npackage_cpus = ci3->ci_npackage_cpus; + } + if (ci->ci_ncore_cpus > 1 && ci->ci_npackage_cpus > 1) { + break; + } + } + } +} + #ifdef CPU_UCODE int cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) Index: src/sys/sys/cpu.h diff -u src/sys/sys/cpu.h:1.44 src/sys/sys/cpu.h:1.45 --- src/sys/sys/cpu.h:1.44 Sat Nov 23 19:42:52 2019 +++ src/sys/sys/cpu.h Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.h,v 1.44 2019/11/23 19:42:52 ad Exp $ */ +/* $NetBSD: cpu.h,v 1.45 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2007 YAMAMOTO Takashi, @@ -90,6 +90,8 @@ bool cpu_kpreempt_disabled(void); int cpu_lwp_setprivate(struct lwp *, void *); void cpu_intr_redistribute(void); u_int cpu_intr_count(struct cpu_info *); +void cpu_topology_set(struct cpu_info *, int, int, int); +void cpu_topology_init(void); #endif #ifdef _KERNEL Index: src/sys/sys/cpu_data.h diff -u src/sys/sys/cpu_data.h:1.40 src/sys/sys/cpu_data.h:1.41 --- src/sys/sys/cpu_data.h:1.40 Sun Dec 1 15:34:47 2019 +++ src/sys/sys/cpu_data.h Mon Dec 2 23:22:43 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_data.h,v 1.40 2019/12/01 15:34:47 ad Exp $ */ +/* $NetBSD: cpu_data.h,v 1.41 2019/12/02 23:22:43 ad Exp $ */ /*- * Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc. @@ -71,6 +71,15 @@ struct cpu_data { uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */ struct schedstate_percpu cpu_schedstate; /* scheduler state */ + /* Basic topology information. May be fake. */ + cpuid_t cpu_package_id; + cpuid_t cpu_core_id; + cpuid_t cpu_smt_id; + u_int cpu_npackage_cpus; + u_int cpu_ncore_cpus; + struct cpu_info *cpu_package_cpus; /* sibling CPUs in package */ + struct cpu_info *cpu_core_cpus; /* sibling CPUs in core */ + /* * This section is mostly CPU-private. */ @@ -101,9 +110,7 @@ struct cpu_data { int64_t cpu_cc_skew; /* counter skew vs cpu0 */ char cpu_name[8]; /* eg, "cpu4" */ kcpuset_t *cpu_kcpuset; /* kcpuset_t of this cpu only */ - cpuid_t cpu_package_id; - cpuid_t cpu_core_id; - cpuid_t cpu_smt_id; + struct lwp * volatile cpu_pcu_curlwp[PCU_UNIT_COUNT]; }; @@ -124,6 +131,10 @@ struct cpu_data { #define ci_package_id ci_data.cpu_package_id #define ci_core_id ci_data.cpu_core_id #define ci_smt_id ci_data.cpu_smt_id +#define ci_npackage_cpus ci_data.cpu_npackage_cpus +#define ci_ncore_cpus ci_data.cpu_ncore_cpus +#define ci_package_cpus ci_data.cpu_package_cpus +#define ci_core_cpus ci_data.cpu_core_cpus void mi_cpu_init(void); int mi_cpu_attach(struct cpu_info *);