Author: markj Date: Mon Feb 3 18:23:50 2020 New Revision: 357460 URL: https://svnweb.freebsd.org/changeset/base/357460
Log: Dynamically select LSE-based atomic(9)s on arm64. Once all CPUs are online, determine if they all support LSE atomics and set lse_supported to indicate this. For now the atomic(9) implementations are still always inlined, though it would be preferable to create out-of-line functions to avoid text bloat. This was not done here since big.little systems exist in which some CPUs implement LSE while others do not, and ifunc resolution must occur well before this scenario can be detected. It does seem unlikely that FreeBSD will ever run on such platforms, however, so converting atomic(9) to use ifuncs is probably a good next step. Add a LSE_ATOMICS arm64 kernel configuration option to unconditionally select LSE-based atomic(9) implementations when the target system is known. Reviewed by: andrew, kib MFC after: 1 month Sponsored by: The FreeBSD Foundation, Amazon (hardware) Differential Revision: https://reviews.freebsd.org/D23325 Modified: head/sys/arm64/arm64/identcpu.c head/sys/arm64/include/atomic.h head/sys/conf/options.arm64 Modified: head/sys/arm64/arm64/identcpu.c ============================================================================== --- head/sys/arm64/arm64/identcpu.c Mon Feb 3 18:23:35 2020 (r357459) +++ head/sys/arm64/arm64/identcpu.c Mon Feb 3 18:23:50 2020 (r357460) @@ -114,8 +114,8 @@ struct cpu_desc { uint64_t id_aa64pfr1; }; -struct cpu_desc cpu_desc[MAXCPU]; -struct cpu_desc user_cpu_desc; +static struct cpu_desc cpu_desc[MAXCPU]; +static struct cpu_desc user_cpu_desc; static u_int cpu_print_regs; #define PRINT_ID_AA64_AFR0 0x00000001 #define PRINT_ID_AA64_AFR1 0x00000002 @@ -961,6 +961,7 @@ update_user_regs(u_int cpu) /* HWCAP */ extern u_long elf_hwcap; +bool __read_frequently lse_supported = false; static void identify_cpu_sysinit(void *dummy __unused) @@ -975,7 +976,6 @@ identify_cpu_sysinit(void *dummy __unused) ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64; user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8; - CPU_FOREACH(cpu) { print_cpu_features(cpu); hwcap = parse_cpu_features_hwcap(cpu); @@ -985,6 +985,16 @@ identify_cpu_sysinit(void *dummy __unused) elf_hwcap &= hwcap; update_user_regs(cpu); } + + if ((elf_hwcap & HWCAP_ATOMICS) != 0) { + lse_supported = true; + if (bootverbose) + printf("Enabling LSE atomics in the kernel\n"); + } +#ifdef LSE_ATOMICS + if (!lse_supported) + panic("CPU does not support LSE atomic instructions"); +#endif install_undef_handler(true, user_mrs_handler); } Modified: head/sys/arm64/include/atomic.h ============================================================================== --- head/sys/arm64/include/atomic.h Mon Feb 3 18:23:35 2020 (r357459) +++ head/sys/arm64/include/atomic.h Mon Feb 3 18:23:50 2020 (r357460) @@ -59,6 +59,18 @@ #include <sys/atomic_common.h> +#ifdef _KERNEL +extern bool lse_supported; + +#ifdef LSE_ATOMICS +#define _ATOMIC_LSE_SUPPORTED 1 +#else +#define _ATOMIC_LSE_SUPPORTED lse_supported +#endif +#else +#define _ATOMIC_LSE_SUPPORTED 0 +#endif + #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ static __inline void \ atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) @@ -98,7 +110,10 @@ _ATOMIC_OP_PROTO(t, op, bar, _lse) \ \ _ATOMIC_OP_PROTO(t, op, bar, ) \ { \ - atomic_##op##_##bar##t##_llsc(p, val); \ + if (_ATOMIC_LSE_SUPPORTED) \ + atomic_##op##_##bar##t##_lse(p, val); \ + else \ + atomic_##op##_##bar##t##_llsc(p, val); \ } #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ @@ -175,7 +190,12 @@ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ \ _ATOMIC_CMPSET_PROTO(t, bar, ) \ { \ - return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ + newval)); \ + else \ + return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ + newval)); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ @@ -223,7 +243,12 @@ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ \ _ATOMIC_FCMPSET_PROTO(t, bar, ) \ { \ - return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ + newval)); \ + else \ + return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ + newval)); \ } #define _ATOMIC_CMPSET(bar, a, l) \ @@ -277,7 +302,10 @@ _ATOMIC_FETCHADD_PROTO(t, _lse) \ \ _ATOMIC_FETCHADD_PROTO(t, ) \ { \ - return (atomic_fetchadd_##t##_llsc(p, val)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_fetchadd_##t##_lse(p, val)); \ + else \ + return (atomic_fetchadd_##t##_llsc(p, val)); \ } _ATOMIC_FETCHADD_IMPL(32, w) @@ -327,7 +355,10 @@ _ATOMIC_SWAP_PROTO(t, _lse) \ \ _ATOMIC_SWAP_PROTO(t, ) \ { \ - return (atomic_swap_##t##_llsc(p, val)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_swap_##t##_lse(p, val)); \ + else \ + return (atomic_swap_##t##_llsc(p, val)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ @@ -354,7 +385,10 @@ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ \ _ATOMIC_READANDCLEAR_PROTO(t, ) \ { \ - return (atomic_readandclear_##t##_llsc(p)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_readandclear_##t##_lse(p)); \ + else \ + return (atomic_readandclear_##t##_llsc(p)); \ } _ATOMIC_SWAP_IMPL(32, w, wzr) @@ -403,7 +437,10 @@ _ATOMIC_TEST_OP_PROTO(t, op, _lse) \ \ _ATOMIC_TEST_OP_PROTO(t, op, ) \ { \ - return (atomic_testand##op##_##t##_llsc(p, val)); \ + if (_ATOMIC_LSE_SUPPORTED) \ + return (atomic_testand##op##_##t##_lse(p, val)); \ + else \ + return (atomic_testand##op##_##t##_llsc(p, val)); \ } #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ Modified: head/sys/conf/options.arm64 ============================================================================== --- head/sys/conf/options.arm64 Mon Feb 3 18:23:35 2020 (r357459) +++ head/sys/conf/options.arm64 Mon Feb 3 18:23:50 2020 (r357460) @@ -7,6 +7,7 @@ SOCDEV_VA opt_global.h THUNDERX_PASS_1_1_ERRATA opt_global.h VFP opt_global.h LINUX_BOOT_ABI opt_global.h +LSE_ATOMICS opt_global.h # Binary compatibility COMPAT_FREEBSD32 opt_global.h _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "[email protected]"
