Signed-off-by: Ola Liljedahl <[email protected]>
Reviewed-by: Brian Brooks <[email protected]>
---
platform/linux-generic/include/odp_atomic16.h | 214 ++++++++++++++++++++++++++
1 file changed, 214 insertions(+)
create mode 100644 platform/linux-generic/include/odp_atomic16.h
diff --git a/platform/linux-generic/include/odp_atomic16.h
b/platform/linux-generic/include/odp_atomic16.h
new file mode 100644
index 00000000..bbebb803
--- /dev/null
+++ b/platform/linux-generic/include/odp_atomic16.h
@@ -0,0 +1,214 @@
+/* Copyright (c) 2017, ARM Limited.
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef _ODP_ATOMIC16_H_
+#define _ODP_ATOMIC16_H_
+
+#include "odp_llsc.h"
+
+#if defined __ARM_ARCH && __ARM_ARCH == 8
+
+static inline __int128 __atomic_load_16(__int128 *var, int mo)
+{
+ __int128 old = *var; /* Possibly torn read */
+
+ /* Do CAS to ensure atomicity
+ * Either CAS succeeds (writing back the same value)
+ * Or CAS fails and returns the old value (atomic read)
+ */
+ (void)__atomic_compare_exchange_n(var,
+ &old,
+ old,
+ false, /* weak= */
+ mo,
+ mo);
+ return old;
+}
+
+#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */
+static inline bool
+__atomic_compare_exchange_16(register __int128 *var, __int128 *exp,
+ register __int128 neu, bool weak, int mo_success,
+ int mo_failure)
+{
+ (void)weak; /* Always do strong CAS or we can't perform atomic read */
+ /* Ignore memory ordering for failure, memory order for
+ * success must be stronger or equal
+ */
+ (void)mo_failure;
+ int ll_mo;
+ int sc_mo;
+
+ register __int128 old;
+ register __int128 expected;
+
+ ll_mo = mo_success == __ATOMIC_ACQUIRE ||
+ mo_success == __ATOMIC_ACQ_REL ?
+ __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
+
+ sc_mo = mo_success == __ATOMIC_RELEASE ||
+ mo_success == __ATOMIC_ACQ_REL ?
+ __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+
+ expected = *exp;
+ __asm__ volatile("" ::: "memory");
+ do {
+ /* Atomicity of LLD is not guaranteed */
+ old = lld(var, ll_mo);
+ /* Must write back neu or old to verify atomicity of LLD */
+ } while (odp_unlikely(scd(var, old == expected ? neu : old, sc_mo)));
+ *exp = old; /* Always update, atomically read value */
+ return old == expected;
+}
+
+static inline __int128 __atomic_exchange_16(__int128 *var,
+ __int128 neu, int mo)
+{
+ int ll_mo;
+ int sc_mo;
+ register __int128 old;
+
+ ll_mo = mo == __ATOMIC_ACQUIRE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
+ sc_mo = mo == __ATOMIC_RELEASE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+ do {
+ /* Atomicity of LLD is not guaranteed */
+ old = lld(var, ll_mo);
+ /* Must successfully write back to verify atomicity of LLD */
+ } while (odp_unlikely(scd(var, neu, sc_mo)));
+ return old;
+}
+
+static inline __int128 __atomic_fetch_and_16(__int128 *var,
+ __int128 mask, int mo)
+{
+ int ll_mo;
+ int sc_mo;
+ register __int128 old;
+
+ ll_mo = mo == __ATOMIC_ACQUIRE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
+ sc_mo = mo == __ATOMIC_RELEASE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+ do {
+ /* Atomicity of LLD is not guaranteed */
+ old = lld(var, ll_mo);
+ /* Must successfully write back to verify atomicity of LLD */
+ } while (odp_unlikely(scd(var, old & mask, sc_mo)));
+ return old;
+}
+
+static inline __int128 __atomic_fetch_or_16(__int128 *var,
+ __int128 mask,
+ int mo)
+{
+ int ll_mo;
+ int sc_mo;
+ register __int128 old;
+
+ ll_mo = mo == __ATOMIC_ACQUIRE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
+ sc_mo = mo == __ATOMIC_RELEASE || mo == __ATOMIC_ACQ_REL ?
+ __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+ do {
+ /* Atomicity of LLD is not guaranteed */
+ old = lld(var, ll_mo);
+ /* Must successfully write back to verify atomicity of LLD */
+ } while (odp_unlikely(scd(var, old | mask, sc_mo)));
+ return old;
+}
+
+#else
+
+static inline __int128 casp(__int128 *var, __int128 old, __int128 neu, int mo)
+{
+ if (mo == __ATOMIC_RELAXED) {
+ __asm__ volatile("casp %0, %H0, %1, %H1, [%2]"
+ : "+r" (old)
+ : "r" (neu), "r" (var)
+ : "memory");
+ } else if (mo == __ATOMIC_ACQUIRE) {
+ __asm__ volatile("caspa %0, %H0, %1, %H1, [%2]"
+ : "+r" (old)
+ : "r" (neu), "r" (var)
+ : "memory");
+ } else if (mo == __ATOMIC_ACQ_REL) {
+ __asm__ volatile("caspal %0, %H0, %1, %H1, [%2]"
+ : "+r" (old)
+ : "r" (neu), "r" (var)
+ : "memory");
+ } else if (mo == __ATOMIC_RELEASE) {
+ __asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
+ : "+r" (old)
+ : "r" (neu), "r" (var)
+ : "memory");
+ } else {
+ abort();
+ }
+ return old;
+}
+
+static inline bool
+__atomic_compare_exchange_16(register __int128 *var, __int128 *exp,
+ register __int128 neu, bool weak, int mo_success,
+ int mo_failure)
+{
+ (void)weak;
+ (void)mo_failure;
+ __int128 old;
+ __int128 expected;
+
+ expected = *exp;
+ old = casp(var, expected, neu, mo_success);
+ *exp = old; /* Always update, atomically read value */
+ return old == expected;
+}
+
+static inline __int128 __atomic_exchange_16(__int128 *var,
+ __int128 neu, int mo)
+{
+ __int128 old;
+ __int128 expected;
+
+ do {
+ expected = *var;
+ old = casp(var, expected, neu, mo);
+ } while (old != expected);
+ return old;
+}
+
+static inline __int128 __atomic_fetch_and_16(__int128 *var,
+ __int128 mask, int mo)
+{
+ __int128 old;
+ __int128 expected;
+
+ do {
+ expected = *var;
+ old = casp(var, expected, expected & mask, mo);
+ } while (old != expected);
+ return old;
+}
+
+static inline __int128 __atomic_fetch_or_16(__int128 *var,
+ __int128 mask,
+ int mo)
+{
+ __int128 old;
+ __int128 expected;
+
+ do {
+ expected = *var;
+ old = casp(var, expected, expected | mask, mo);
+ } while (old != expected);
+ return old;
+}
+
+#endif
+
+#endif
+#endif
--
2.12.2
