Author: cognet
Date: Sat Jan 28 16:24:06 2017
New Revision: 312925
URL: https://svnweb.freebsd.org/changeset/base/312925
Log:
Implement atomic_fcmpset_* for arm and arm64.
Modified:
head/sys/arm/include/atomic-v4.h
head/sys/arm/include/atomic-v6.h
head/sys/arm/include/atomic.h
head/sys/arm64/include/atomic.h
Modified: head/sys/arm/include/atomic-v4.h
==============================================================================
--- head/sys/arm/include/atomic-v4.h Sat Jan 28 15:44:14 2017
(r312924)
+++ head/sys/arm/include/atomic-v4.h Sat Jan 28 16:24:06 2017
(r312925)
@@ -112,6 +112,43 @@ atomic_clear_64(volatile uint64_t *addre
__with_interrupts_disabled(*address &= ~clearmask);
}
+static __inline int
+atomic_fcmpset_32(volatile u_int32_t *p, volatile u_int32_t *cmpval, volatile
u_int32_t newval)
+{
+ u_int32_t ret;
+
+ __with_interrupts_disabled(
+ {
+ ret = *p;
+ if (*p == *cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ *cmpval = *p;
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+
+static __inline int
+atomic_fcmpset_64(volatile u_int64_t *p, volatile u_int64_t *cmpval, volatile
u_int64_t newval)
+{
+ u_int64_t ret;
+
+ __with_interrupts_disabled(
+ {
+ if (*p == *cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ *cmpval = *p;
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+
static __inline u_int32_t
atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile
u_int32_t newval)
{
@@ -370,6 +407,12 @@ atomic_swap_32(volatile u_int32_t *p, u_
return (__swp(v, p));
}
+#define atomic_fcmpset_rel_32 atomic_fcmpset_32
+#define atomic_fcmpset_acq_32 atomic_fcmpset_32
+#define atomic_fcmpset_rel_64 atomic_fcmpset_64
+#define atomic_fcmpset_acq_64 atomic_fcmpset_64
+#define atomic_fcmpset_acq_long atomic_fcmpset_long
+#define atomic_fcmpset_rel_long atomic_fcmpset_long
#define atomic_cmpset_rel_32 atomic_cmpset_32
#define atomic_cmpset_acq_32 atomic_cmpset_32
#define atomic_cmpset_rel_64 atomic_cmpset_64
@@ -421,6 +464,14 @@ atomic_cmpset_long(volatile u_long *dst,
}
static __inline u_long
+atomic_fcmpset_long(volatile u_long *dst, u_long *old, u_long newe)
+{
+
+ return (atomic_fcmpset_32((volatile uint32_t *)dst,
+ (uint32_t *)old, newe));
+}
+
+static __inline u_long
atomic_fetchadd_long(volatile u_long *p, u_long v)
{
Modified: head/sys/arm/include/atomic-v6.h
==============================================================================
--- head/sys/arm/include/atomic-v6.h Sat Jan 28 15:44:14 2017
(r312924)
+++ head/sys/arm/include/atomic-v6.h Sat Jan 28 16:24:06 2017
(r312925)
@@ -190,6 +190,116 @@ ATOMIC_ACQ_REL(clear, 32)
ATOMIC_ACQ_REL(clear, 64)
ATOMIC_ACQ_REL_LONG(clear)
+static __inline int
+atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+{
+ uint32_t tmp;
+ uint32_t _cmpval = *cmpval;
+ int ret;
+
+ __asm __volatile(
+ "1: mov %0, #1 \n"
+ " ldrex %1, [%2] \n"
+ " cmp %1, %3 \n"
+ " it ne \n"
+ " bne 2f \n"
+ " strex %0, %4, [%2] \n"
+ "2:"
+ : "=&r" (ret), "=&r" (tmp), "+r" (p), "+r" (_cmpval), "+r" (newval)
+ : : "cc", "memory");
+ *cmpval = tmp;
+ return (!ret);
+}
+
+static __inline uint64_t
+atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+{
+ uint64_t tmp;
+ uint64_t _cmpval = *cmpval;
+ int ret;
+
+ __asm __volatile(
+ "1: mov %[ret], #1 \n"
+ " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n"
+ " teq %Q[tmp], %Q[_cmpval] \n"
+ " itee eq \n"
+ " teqeq %R[tmp], %R[_cmpval] \n"
+ " bne 2f \n"
+ " strexd %[ret], %Q[newval], %R[newval], [%[ptr]]\n"
+ "2: \n"
+ : [ret] "=&r" (ret),
+ [tmp] "=&r" (tmp)
+ : [ptr] "r" (p),
+ [_cmpval] "r" (_cmpval),
+ [newval] "r" (newval)
+ : "cc", "memory");
+ *cmpval = tmp;
+ return (!ret);
+}
+
+static __inline u_long
+atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
+{
+
+ return (atomic_fcmpset_32((volatile uint32_t *)p,
+ (uint32_t *)cmpval, newval));
+}
+
+static __inline uint64_t
+atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+{
+ uint64_t ret;
+
+ ret = atomic_fcmpset_64(p, cmpval, newval);
+ dmb();
+ return (ret);
+}
+
+static __inline u_long
+atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
+{
+ u_long ret;
+
+ ret = atomic_fcmpset_long(p, cmpval, newval);
+ dmb();
+ return (ret);
+}
+
+static __inline uint32_t
+atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+{
+
+ uint32_t ret;
+
+ ret = atomic_fcmpset_32(p, cmpval, newval);
+ dmb();
+ return (ret);
+}
+
+static __inline uint32_t
+atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+{
+
+ dmb();
+ return (atomic_fcmpset_32(p, cmpval, newval));
+}
+
+static __inline uint64_t
+atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+{
+
+ dmb();
+ return (atomic_fcmpset_64(p, cmpval, newval));
+}
+
+static __inline u_long
+atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
+{
+
+ dmb();
+ return (atomic_fcmpset_long(p, cmpval, newval));
+}
+
static __inline uint32_t
atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
{
Modified: head/sys/arm/include/atomic.h
==============================================================================
--- head/sys/arm/include/atomic.h Sat Jan 28 15:44:14 2017
(r312924)
+++ head/sys/arm/include/atomic.h Sat Jan 28 16:24:06 2017
(r312925)
@@ -84,6 +84,9 @@ atomic_store_long(volatile u_long *dst,
#define atomic_set_ptr atomic_set_32
#define atomic_set_acq_ptr atomic_set_acq_32
#define atomic_set_rel_ptr atomic_set_rel_32
+#define atomic_fcmpset_ptr atomic_fcmpset_32
+#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_32
+#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_32
#define atomic_cmpset_ptr atomic_cmpset_32
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32
@@ -105,6 +108,9 @@ atomic_store_long(volatile u_long *dst,
#define atomic_set_int atomic_set_32
#define atomic_set_acq_int atomic_set_acq_32
#define atomic_set_rel_int atomic_set_rel_32
+#define atomic_fcmpset_int atomic_fcmpset_32
+#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
+#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
#define atomic_cmpset_int atomic_cmpset_32
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
Modified: head/sys/arm64/include/atomic.h
==============================================================================
--- head/sys/arm64/include/atomic.h Sat Jan 28 15:44:14 2017
(r312924)
+++ head/sys/arm64/include/atomic.h Sat Jan 28 16:24:06 2017
(r312925)
@@ -98,6 +98,61 @@ ATOMIC(clear, bic)
ATOMIC(set, orr)
ATOMIC(subtract, sub)
+#define ATOMIC_FCMPSET(bar, a, l)
\
+static __inline int \
+atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \
+ uint32_t newval) \
+{ \
+ uint32_t tmp; \
+ uint32_t _cmpval = *cmpval; \
+ int res; \
+ \
+ __asm __volatile( \
+ "1: mov %w1, #1 \n" \
+ " ld"#a"xr %w0, [%2] \n" \
+ " cmp %w0, %w3 \n" \
+ " b.ne 2f \n" \
+ " st"#l"xr %w1, %w4, [%2] \n" \
+ "2:" \
+ : "=&r"(tmp), "=&r"(res) \
+ : "r" (p), "r" (_cmpval), "r" (newval) \
+ : "cc", "memory" \
+ ); \
+ *cmpval = tmp; \
+ \
+ return (!res); \
+} \
+ \
+static __inline int \
+atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \
+ uint64_t newval) \
+{ \
+ uint64_t tmp; \
+ uint64_t _cmpval = *cmpval; \
+ int res; \
+ \
+ __asm __volatile( \
+ "1: mov %w1, #1 \n" \
+ " ld"#a"xr %0, [%2] \n" \
+ " cmp %0, %3 \n" \
+ " b.ne 2f \n" \
+ " st"#l"xr %w1, %4, [%2] \n" \
+ "2:" \
+ : "=&r"(tmp), "=&r"(res) \
+ : "r" (p), "r" (_cmpval), "r" (newval) \
+ : "cc", "memory" \
+ ); \
+ *cmpval = tmp; \
+ \
+ return (!res); \
+}
+
+ATOMIC_FCMPSET( , , )
+ATOMIC_FCMPSET(acq_, a, )
+ATOMIC_FCMPSET(rel_, ,l)
+
+#undef ATOMIC_FCMPSET
+
#define ATOMIC_CMPSET(bar, a, l)
\
static __inline int \
atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \
@@ -311,6 +366,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_add_int atomic_add_32
+#define atomic_fcmpset_int atomic_fcmpset_32
#define atomic_clear_int atomic_clear_32
#define atomic_cmpset_int atomic_cmpset_32
#define atomic_fetchadd_int atomic_fetchadd_32
@@ -320,6 +376,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_int atomic_subtract_32
#define atomic_add_acq_int atomic_add_acq_32
+#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
#define atomic_clear_acq_int atomic_clear_acq_32
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
#define atomic_load_acq_int atomic_load_acq_32
@@ -327,6 +384,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_acq_int atomic_subtract_acq_32
#define atomic_add_rel_int atomic_add_rel_32
+#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
#define atomic_clear_rel_int atomic_add_rel_32
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
#define atomic_set_rel_int atomic_set_rel_32
@@ -334,6 +392,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_store_rel_int atomic_store_rel_32
#define atomic_add_long atomic_add_64
+#define atomic_fcmpset_long atomic_fcmpset_64
#define atomic_clear_long atomic_clear_64
#define atomic_cmpset_long atomic_cmpset_64
#define atomic_fetchadd_long atomic_fetchadd_64
@@ -343,6 +402,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_long atomic_subtract_64
#define atomic_add_ptr atomic_add_64
+#define atomic_fcmpset_ptr atomic_fcmpset_64
#define atomic_clear_ptr atomic_clear_64
#define atomic_cmpset_ptr atomic_cmpset_64
#define atomic_fetchadd_ptr atomic_fetchadd_64
@@ -352,6 +412,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_ptr atomic_subtract_64
#define atomic_add_acq_long atomic_add_acq_64
+#define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
#define atomic_clear_acq_long atomic_add_acq_64
#define atomic_cmpset_acq_long atomic_cmpset_acq_64
#define atomic_load_acq_long atomic_load_acq_64
@@ -359,6 +420,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_acq_long atomic_subtract_acq_64
#define atomic_add_acq_ptr atomic_add_acq_64
+#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
#define atomic_clear_acq_ptr atomic_add_acq_64
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
#define atomic_load_acq_ptr atomic_load_acq_64
@@ -366,6 +428,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_subtract_acq_ptr atomic_subtract_acq_64
#define atomic_add_rel_long atomic_add_rel_64
+#define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
#define atomic_clear_rel_long atomic_clear_rel_64
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
#define atomic_set_rel_long atomic_set_rel_64
@@ -373,6 +436,7 @@ atomic_store_rel_64(volatile uint64_t *p
#define atomic_store_rel_long atomic_store_rel_64
#define atomic_add_rel_ptr atomic_add_rel_64
+#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
#define atomic_clear_rel_ptr atomic_clear_rel_64
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
#define atomic_set_rel_ptr atomic_set_rel_64
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
