Author: avg
Date: Fri Nov 8 12:06:03 2019
New Revision: 354534
URL: https://svnweb.freebsd.org/changeset/base/354534
Log:
MFC r353340,r353340: cleanup of illumos compatibility atomics
Modified:
stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
stable/12/sys/cddl/compat/opensolaris/sys/atomic.h
stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
==============================================================================
--- stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c Fri Nov
8 11:09:50 2019 (r354533)
+++ stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c Fri Nov
8 12:06:03 2019 (r354534)
@@ -32,6 +32,9 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/atomic.h>
+#if !defined(__LP64__) && !defined(__mips_n32) && \
+ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
+
#ifdef _KERNEL
#include <sys/kernel.h>
@@ -52,8 +55,6 @@ atomic_init(void)
}
#endif
-#if !defined(__LP64__) && !defined(__mips_n32) && \
- !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
void
atomic_add_64(volatile uint64_t *target, int64_t delta)
{
@@ -94,7 +95,6 @@ atomic_load_64(volatile uint64_t *a)
mtx_unlock(&atomic_mtx);
return (ret);
}
-#endif
uint64_t
atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
@@ -107,27 +107,6 @@ atomic_add_64_nv(volatile uint64_t *target, int64_t de
return (newval);
}
-#if defined(__powerpc__) || defined(__arm__) || defined(__mips__)
-void
-atomic_or_8(volatile uint8_t *target, uint8_t value)
-{
- mtx_lock(&atomic_mtx);
- *target |= value;
- mtx_unlock(&atomic_mtx);
-}
-#endif
-
-uint8_t
-atomic_or_8_nv(volatile uint8_t *target, uint8_t value)
-{
- uint8_t newval;
-
- mtx_lock(&atomic_mtx);
- newval = (*target |= value);
- mtx_unlock(&atomic_mtx);
- return (newval);
-}
-
uint64_t
atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
{
@@ -140,19 +119,7 @@ atomic_cas_64(volatile uint64_t *target, uint64_t cmp,
mtx_unlock(&atomic_mtx);
return (oldval);
}
-
-uint32_t
-atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
-{
- uint32_t oldval;
-
- mtx_lock(&atomic_mtx);
- oldval = *target;
- if (oldval == cmp)
- *target = newval;
- mtx_unlock(&atomic_mtx);
- return (oldval);
-}
+#endif
void
membar_producer(void)
Modified: stable/12/sys/cddl/compat/opensolaris/sys/atomic.h
==============================================================================
--- stable/12/sys/cddl/compat/opensolaris/sys/atomic.h Fri Nov 8 11:09:50
2019 (r354533)
+++ stable/12/sys/cddl/compat/opensolaris/sys/atomic.h Fri Nov 8 12:06:03
2019 (r354534)
@@ -32,42 +32,28 @@
#include <sys/types.h>
#include <machine/atomic.h>
-#define casptr(_a, _b, _c) \
- atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b),
(uintptr_t) (_c))
-#define cas32 atomic_cmpset_32
-
#if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE))
#define I386_HAVE_ATOMIC64
#endif
+#if defined(__i386__) || defined(__amd64__) || defined(__arm__)
+/* No spurious failures from fcmpset. */
+#define STRONG_FCMPSET
+#endif
+
#if !defined(__LP64__) && !defined(__mips_n32) && \
!defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
extern void atomic_add_64(volatile uint64_t *target, int64_t delta);
extern void atomic_dec_64(volatile uint64_t *target);
extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value);
extern uint64_t atomic_load_64(volatile uint64_t *a);
-#endif
-#ifndef __sparc64__
-extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp,
- uint32_t newval);
+extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta);
extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp,
uint64_t newval);
#endif
-extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta);
-extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value);
+
extern void membar_producer(void);
-#if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \
- defined(__mips__) || defined(__aarch64__) || defined(__riscv)
-extern void atomic_or_8(volatile uint8_t *target, uint8_t value);
-#else
-static __inline void
-atomic_or_8(volatile uint8_t *target, uint8_t value)
-{
- atomic_set_8(target, value);
-}
-#endif
-
static __inline uint32_t
atomic_add_32_nv(volatile uint32_t *target, int32_t delta)
{
@@ -81,6 +67,18 @@ atomic_add_int_nv(volatile u_int *target, int delta)
}
static __inline void
+atomic_inc_32(volatile uint32_t *target)
+{
+ atomic_add_32(target, 1);
+}
+
+static __inline uint32_t
+atomic_inc_32_nv(volatile uint32_t *target)
+{
+ return (atomic_add_32_nv(target, 1));
+}
+
+static __inline void
atomic_dec_32(volatile uint32_t *target)
{
atomic_subtract_32(target, 1);
@@ -89,9 +87,27 @@ atomic_dec_32(volatile uint32_t *target)
static __inline uint32_t
atomic_dec_32_nv(volatile uint32_t *target)
{
- return (atomic_fetchadd_32(target, -1) - 1);
+ return (atomic_add_32_nv(target, -1));
}
+#ifndef __sparc64__
+static inline uint32_t
+atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
+{
+#ifdef STRONG_FCMPSET
+ (void)atomic_fcmpset_32(target, &cmp, newval);
+#else
+ uint32_t expected = cmp;
+
+ do {
+ if (atomic_fcmpset_32(target, &cmp, newval))
+ break;
+ } while (cmp == expected);
+#endif
+ return (cmp);
+}
+#endif
+
#if defined(__LP64__) || defined(__mips_n32) || \
defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64)
static __inline void
@@ -99,19 +115,31 @@ atomic_dec_64(volatile uint64_t *target)
{
atomic_subtract_64(target, 1);
}
-#endif
-static __inline void
-atomic_inc_32(volatile uint32_t *target)
+static inline uint64_t
+atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
{
- atomic_add_32(target, 1);
+ return (atomic_fetchadd_64(target, delta) + delta);
}
-static __inline uint32_t
-atomic_inc_32_nv(volatile uint32_t *target)
+#ifndef __sparc64__
+static inline uint64_t
+atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
{
- return (atomic_add_32_nv(target, 1));
+#ifdef STRONG_FCMPSET
+ (void)atomic_fcmpset_64(target, &cmp, newval);
+#else
+ uint64_t expected = cmp;
+
+ do {
+ if (atomic_fcmpset_64(target, &cmp, newval))
+ break;
+ } while (cmp == expected);
+#endif
+ return (cmp);
}
+#endif
+#endif
static __inline void
atomic_inc_64(volatile uint64_t *target)
Modified:
stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
==============================================================================
---
stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
Fri Nov 8 11:09:50 2019 (r354533)
+++
stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
Fri Nov 8 12:06:03 2019 (r354534)
@@ -28,58 +28,6 @@
#include <machine/asm.h>
-/*
- * uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
- */
-ENTRY(atomic_add_64_nv)
-1: ldxr x2, [x0] /* Load *target */
- add x2, x2, x1 /* x2 = x2 + delta */
- stxr w3, x2, [x0] /* Store *target */
- cbnz w3, 1b /* Check if the store succeeded */
- mov x0, x2 /* Return the new value */
- ret
-END(atomic_add_64_nv)
-
-/*
- * uint32_t
- * atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
- */
-ENTRY(atomic_cas_32)
-1: ldxr w3, [x0] /* Load *target */
- cmp w3, w1 /* Does *targe == cmp? */
- b.ne 2f /* If not exit */
- stxr w4, w2, [x0] /* Store newval to *target */
- cbnz w4, 1b /* Check if the store succeeded */
-2: mov w0, w3 /* Return the old value */
- ret
-END(atomic_cas_32)
-
-/*
- * uint64_t
- * atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
- */
-ENTRY(atomic_cas_64)
-1: ldxr x3, [x0] /* Load *target */
- cmp x3, x1 /* Does *targe == cmp? */
- b.ne 2f /* If not exit */
- stxr w4, x2, [x0] /* Store newval to *target */
- cbnz w4, 1b /* Check if the store succeeded */
-2: mov x0, x3 /* Return the old value */
- ret
-END(atomic_cas_64)
-
-/*
- * uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value)
- */
-ENTRY(atomic_or_8_nv)
-1: ldxrb w2, [x0] /* Load *target */
- orr w2, w2, w1 /* x2 = x2 | delta */
- stxrb w3, w2, [x0] /* Store *target */
- cbnz w3, 1b /* Check if the store succeeded */
- mov w0, w2 /* Return the new value */
- ret
-END(atomic_or_8_nv)
-
ENTRY(membar_producer)
dmb ish
ret
Modified:
stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
==============================================================================
---
stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
Fri Nov 8 11:09:50 2019 (r354533)
+++
stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
Fri Nov 8 12:06:03 2019 (r354534)
@@ -28,40 +28,6 @@
#define _ASM
#include <sys/asm_linkage.h>
- ENTRY(atomic_add_64_nv)
- mov %rsi, %rax // %rax = delta addend
- lock
- xaddq %rsi, (%rdi) // %rsi = old value, (%rdi) = sum
- addq %rsi, %rax // new value = original value + delta
- ret
- SET_SIZE(atomic_add_64_nv)
-
- ENTRY(atomic_or_8_nv)
- movb (%rdi), %al // %al = old value
-1:
- movb %sil, %cl
- orb %al, %cl // %cl = new value
- lock
- cmpxchgb %cl, (%rdi) // try to stick it in
- jne 1b
- movzbl %cl, %eax // return new value
- ret
- SET_SIZE(atomic_or_8_nv)
-
- ENTRY(atomic_cas_32)
- movl %esi, %eax
- lock
- cmpxchgl %edx, (%rdi)
- ret
- SET_SIZE(atomic_cas_32)
-
- ENTRY(atomic_cas_64)
- movq %rsi, %rax
- lock
- cmpxchgq %rdx, (%rdi)
- ret
- SET_SIZE(atomic_cas_64)
-
ENTRY(membar_producer)
sfence
ret
Modified:
stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
==============================================================================
---
stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
Fri Nov 8 11:09:50 2019 (r354533)
+++
stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
Fri Nov 8 12:06:03 2019 (r354534)
@@ -89,28 +89,6 @@
SET_SIZE(atomic_add_64_nv)
SET_SIZE(atomic_add_64)
- ENTRY(atomic_or_8_nv)
- movl 4(%esp), %edx // %edx = target address
- movb (%edx), %al // %al = old value
-1:
- movl 8(%esp), %ecx // %ecx = delta
- orb %al, %cl // %cl = new value
- lock
- cmpxchgb %cl, (%edx) // try to stick it in
- jne 1b
- movzbl %cl, %eax // return new value
- ret
- SET_SIZE(atomic_or_8_nv)
-
- ENTRY(atomic_cas_32)
- movl 4(%esp), %edx
- movl 8(%esp), %eax
- movl 12(%esp), %ecx
- lock
- cmpxchgl %ecx, (%edx)
- ret
- SET_SIZE(atomic_cas_32)
-
ENTRY(atomic_cas_64)
pushl %ebx
pushl %esi
Modified:
stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
==============================================================================
---
stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
Fri Nov 8 11:09:50 2019 (r354533)
+++
stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
Fri Nov 8 12:06:03 2019 (r354534)
@@ -27,61 +27,6 @@
#include <machine/asm.h>
-ENTRY(atomic_add_64_nv)
- 1: ldarx %r5,0,%r3
- add %r5,%r4,%r5
- stdcx. %r5,0,%r3
- bne- 1b
-
- mr %r3,%r5
- blr
-
-ENTRY(atomic_cas_32)
- 1: lwarx %r6,0,%r3
- cmplw %r6,%r4
- bne 2f
- stwcx. %r5,0,%r3
- bne- 1b
- b 3f
-
- 2: stwcx. %r6,0,%r3 /* clear reservation */
-
- 3: mr %r3,%r6
- blr
-
-ENTRY(atomic_cas_64)
- 1: ldarx %r6,0,%r3
- cmpld %r6,%r4
- bne 2f
- stdcx. %r5,0,%r3
- bne- 1b
- b 3f
-
- 2: stdcx. %r6,0,%r3 /* clear reservation */
-
- 3: mr %r3,%r6
- blr
-
-ENTRY(atomic_or_8_nv)
- li %r6,3
- andc. %r6,%r3,%r6 /* r6 = r3 & ~4 */
- addi %r7,%r6,3
- sub %r7,%r7,%r3 /* offset in r7 */
- sldi %r7,%r7,3 /* bits to shift in r7 */
-
- rlwinm %r4,%r4,0,24,31 /* mask and rotate the argument */
- slw %r4,%r4,%r7
-
- 1: lwarx %r5,0,%r6
- or %r5,%r4,%r5
- stwcx. %r5,0,%r6
- bne- 1b
-
- srw %r3,%r5,%r7
- rlwinm %r3,%r3,0,24,31 /* mask return value */
-
- blr
-
ENTRY(membar_producer)
eieio
blr
Modified:
stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
==============================================================================
---
stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
Fri Nov 8 11:09:50 2019 (r354533)
+++
stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
Fri Nov 8 12:06:03 2019 (r354534)
@@ -40,67 +40,6 @@
#endif
/*
- * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
- * separated, you need to also edit the libc sparcv9 platform
- * specific mapfile and remove the NODYNSORT attribute
- * from atomic_add_64_nv.
- */
- ENTRY(atomic_add_64)
- ALTENTRY(atomic_add_64_nv)
- ALTENTRY(atomic_add_ptr)
- ALTENTRY(atomic_add_ptr_nv)
- ALTENTRY(atomic_add_long)
- ALTENTRY(atomic_add_long_nv)
-add_64:
- ldx [%o0], %o2
-1:
- add %o2, %o1, %o3
- casxa [%o0] __ASI_ATOMIC, %o2, %o3
- cmp %o2, %o3
- bne,a,pn %xcc, 1b
- mov %o3, %o2
- retl
- add %o2, %o1, %o0 ! return new value
- SET_SIZE(atomic_add_long_nv)
- SET_SIZE(atomic_add_long)
- SET_SIZE(atomic_add_ptr_nv)
- SET_SIZE(atomic_add_ptr)
- SET_SIZE(atomic_add_64_nv)
- SET_SIZE(atomic_add_64)
-
- /*
- * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
- * separated, you need to also edit the libc sparcv9 platform
- * specific mapfile and remove the NODYNSORT attribute
- * from atomic_or_8_nv.
- */
- ENTRY(atomic_or_8)
- ALTENTRY(atomic_or_8_nv)
- ALTENTRY(atomic_or_uchar)
- and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right
- xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left
- sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left
- set 0xff, %o3 ! %o3 = mask
- sll %o3, %g1, %o3 ! %o3 = shifted to bit offset
- sll %o1, %g1, %o1 ! %o1 = shifted to bit offset
- and %o1, %o3, %o1 ! %o1 = single byte value
- andn %o0, 0x3, %o0 ! %o0 = word address
- ld [%o0], %o2 ! read old value
-1:
- or %o2, %o1, %o5 ! or in the new value
- casa [%o0] __ASI_ATOMIC, %o2, %o5
- cmp %o2, %o5
- bne,a,pn %icc, 1b
- mov %o5, %o2 ! %o2 = old value
- or %o2, %o1, %o5
- and %o5, %o3, %o5
- retl
- srl %o5, %g1, %o0 ! %o0 = new value
- SET_SIZE(atomic_or_uchar)
- SET_SIZE(atomic_or_8_nv)
- SET_SIZE(atomic_or_8)
-
- /*
* Spitfires and Blackbirds have a problem with membars in the
* delay slot (SF_ERRATA_51). For safety's sake, we assume
* that the whole world needs the workaround.
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
