The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=88ac318759f28f34aa991ecc6f336326ec00c501
commit 88ac318759f28f34aa991ecc6f336326ec00c501 Author: Andrew Turner <[email protected]> AuthorDate: 2022-05-17 13:52:30 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2022-05-19 10:30:21 +0000 Support LSE atomics in the arm64 casue* functions As with atomic(9) use the ARMv8.1 Large System Extension atomic instructions to implement the userspace compare and swap functions. Reviewed by: kib Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D35234 --- sys/arm64/arm64/support.S | 52 +++++++++++++++++++++++++++++++----- sys/arm64/arm64/support_ifunc.c | 58 +++++++++++++++++++++++++++++++++++++++++ sys/conf/files.arm64 | 1 + 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S index 4771814a9b45..885af5676159 100644 --- a/sys/arm64/arm64/support.S +++ b/sys/arm64/arm64/support.S @@ -56,9 +56,9 @@ fsu_fault_nopcb: END(fsu_fault) /* - * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) + * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ -ENTRY(casueword32) +ENTRY(casueword32_llsc) check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ mov w5, #1 @@ -73,12 +73,32 @@ ENTRY(casueword32) str w4, [x2] /* Store the read data */ mov w0, w5 /* Result same as store status */ ret /* Return */ -END(casueword32) +END(casueword32_llsc) /* - * int casueword(volatile u_long *, u_long, u_long *, u_long) + * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ -ENTRY(casueword) +ENTRY(casueword32_lse) + check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb + adr x6, fsu_fault /* Load the fault handler */ + SET_FAULT_HANDLER(x6, x4) /* And set it */ + ENTER_USER_ACCESS(w6, x4) + mov w7, w1 /* Back up the compare value */ + .arch_extension lse + cas w1, w3, [x0] /* Compare and Swap */ + .arch_extension nolse + cmp w1, w7 /* Check if successful */ + cset w0, ne /* Return 0 on success, 1 on failure */ + EXIT_USER_ACCESS(w6) + SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ + str w1, [x2] /* Store the read data */ + ret /* Return */ +END(casueword32_lse) + +/* + * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long) + */ +ENTRY(casueword_llsc) check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ mov w5, #1 @@ -93,7 +113,27 @@ ENTRY(casueword) str x4, [x2] /* Store the read data */ mov w0, w5 /* Result same as store status */ ret /* Return */ -END(casueword) +END(casueword_llsc) + +/* + * int casueword_lse(volatile u_long *, u_long, u_long *, u_long) + */ +ENTRY(casueword_lse) + check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb + adr x6, fsu_fault /* Load the fault handler */ + SET_FAULT_HANDLER(x6, x4) /* And set it */ + ENTER_USER_ACCESS(w6, x4) + mov x7, x1 /* Back up the compare value */ + .arch_extension lse + cas x1, x3, [x0] /* Compare and Swap */ + .arch_extension nolse + cmp x1, x7 /* Check if successful */ + cset w0, ne /* Return 0 on success, 1 on failure */ + EXIT_USER_ACCESS(w6) + SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ + str x1, [x2] /* Store the read data */ + ret /* Return */ +END(casueword_lse) .macro fsudata insn, ret_reg, user_arg adr x7, fsu_fault /* Load the fault handler */ diff --git a/sys/arm64/arm64/support_ifunc.c b/sys/arm64/arm64/support_ifunc.c new file mode 100644 index 000000000000..8abf3f3c75af --- /dev/null +++ b/sys/arm64/arm64/support_ifunc.c @@ -0,0 +1,58 @@ +/*- + * Copyright (c) 2022 The FreeBSD Foundation + * + * This software was developed by Andrew Turner under sponsorship from + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> + +#include <sys/param.h> + +#include <machine/atomic.h> +#include <machine/ifunc.h> + +int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t); +int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t); + +int casueword_llsc(volatile u_long *, u_long, u_long *, u_long); +int casueword_lse(volatile u_long *, u_long, u_long *, u_long); + +DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *base, uint32_t oldval, + uint32_t *oldvalp, uint32_t newval)) +{ + if (lse_supported) + return (casueword32_lse); + + return (casueword32_llsc); +} + +DEFINE_IFUNC(, int, casueword, (volatile u_long *base, u_long oldval, + u_long *oldvalp, u_long newval)) +{ + if (lse_supported) + return (casueword_lse); + + return (casueword_llsc); +} diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 30358a5d67d1..e0f529506d3d 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -72,6 +72,7 @@ arm64/arm64/pmap.c standard arm64/arm64/ptrace_machdep.c standard arm64/arm64/sigtramp.S standard arm64/arm64/stack_machdep.c optional ddb | stack +arm64/arm64/support_ifunc.c standard arm64/arm64/support.S standard arm64/arm64/swtch.S standard arm64/arm64/sys_machdep.c standard
