Signed-off-by: Ola Liljedahl <[email protected]>
Reviewed-by: Brian Brooks <[email protected]>
Reviewed-by: Honnappa Nagarahalli <[email protected]>
---
platform/linux-generic/include/odp_llsc.h | 332 ++++++++++++++++++++++++++++++
1 file changed, 332 insertions(+)
create mode 100644 platform/linux-generic/include/odp_llsc.h
diff --git a/platform/linux-generic/include/odp_llsc.h
b/platform/linux-generic/include/odp_llsc.h
new file mode 100644
index 00000000..ea60c54b
--- /dev/null
+++ b/platform/linux-generic/include/odp_llsc.h
@@ -0,0 +1,332 @@
+/* Copyright (c) 2017, ARM Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_LLSC_H_
+#define ODP_LLSC_H_
+
+#include <odp_config_internal.h>
+
+/******************************************************************************
+ * LL/SC primitives
+ *****************************************************************************/
+
+#if defined __ARM_ARCH
+#if __ARM_ARCH == 7 || (__ARM_ARCH == 8 && __ARM_64BIT_STATE == 0)
+static inline void dmb(void)
+{
+ __asm__ volatile("dmb" : : : "memory");
+}
+
+static inline uint32_t ll8(uint8_t *var, int mm)
+{
+ uint8_t old;
+
+ __asm__ volatile("ldrexb %0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ /* Barrier after an acquiring load */
+ if (mm == __ATOMIC_ACQUIRE)
+ dmb();
+ return old;
+}
+
+static inline uint32_t ll(uint32_t *var, int mm)
+{
+ uint32_t old;
+
+ __asm__ volatile("ldrex %0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ /* Barrier after an acquiring load */
+ if (mm == __ATOMIC_ACQUIRE)
+ dmb();
+ return old;
+}
+
+#define ll32(a, b) ll((a), (b))
+
+/* Return 0 on success, 1 on failure */
+static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm)
+{
+ uint32_t ret;
+
+ /* Barrier before a releasing store */
+ if (mm == __ATOMIC_RELEASE)
+ dmb();
+ __asm__ volatile("strex %0, %1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : );
+ return ret;
+}
+
+#define sc32(a, b, c) sc((a), (b), (c))
+
+static inline uint64_t lld(uint64_t *var, int mm)
+{
+ uint64_t old;
+
+ __asm__ volatile("ldrexd %0, %H0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ /* Barrier after an acquiring load */
+ if (mm == __ATOMIC_ACQUIRE)
+ dmb();
+ return old;
+}
+
+#define ll64(a, b) lld((a), (b))
+
+/* Return 0 on success, 1 on failure */
+static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm)
+{
+ uint32_t ret;
+
+ /* Barrier before a releasing store */
+ if (mm == __ATOMIC_RELEASE)
+ dmb();
+ __asm__ volatile("strexd %0, %1, %H1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : );
+ return ret;
+}
+
+#define sc64(a, b, c) scd((a), (b), (c))
+
+#endif
+
+#if __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1
+static inline uint16_t ll8(uint8_t *var, int mm)
+{
+ uint16_t old;
+
+ if (mm == __ATOMIC_ACQUIRE)
+ __asm__ volatile("ldaxrb %w0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("ldxrb %w0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return old;
+}
+
+static inline uint32_t ll32(uint32_t *var, int mm)
+{
+ uint32_t old;
+
+ if (mm == __ATOMIC_ACQUIRE)
+ __asm__ volatile("ldaxr %w0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("ldxr %w0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return old;
+}
+
+/* Return 0 on success, 1 on failure */
+static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm)
+{
+ uint32_t ret;
+
+ if (mm == __ATOMIC_RELEASE)
+ __asm__ volatile("stlxr %w0, %w1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("stxr %w0, %w1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return ret;
+}
+
+static inline uint64_t ll(uint64_t *var, int mm)
+{
+ uint64_t old;
+
+ if (mm == __ATOMIC_ACQUIRE)
+ __asm__ volatile("ldaxr %0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("ldxr %0, [%1]"
+ : "=&r" (old)
+ : "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return old;
+}
+
+#define ll64(a, b) ll((a), (b))
+
+/* Return 0 on success, 1 on failure */
+static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm)
+{
+ uint32_t ret;
+
+ if (mm == __ATOMIC_RELEASE)
+ __asm__ volatile("stlxr %w0, %1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("stxr %w0, %1, [%2]"
+ : "=&r" (ret)
+ : "r" (neu), "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return ret;
+}
+
+#define sc64(a, b, c) sc((a), (b), (c))
+
+union i128 {
+ __int128 i128;
+ int64_t i64[2];
+};
+
+static inline __int128 lld(__int128 *var, int mm)
+{
+ union i128 old;
+
+ if (mm == __ATOMIC_ACQUIRE)
+ __asm__ volatile("ldaxp %0, %1, [%2]"
+ : "=&r" (old.i64[0]), "=&r" (old.i64[1])
+ : "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("ldxp %0, %1, [%2]"
+ : "=&r" (old.i64[0]), "=&r" (old.i64[1])
+ : "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return old.i128;
+}
+
+/* Return 0 on success, 1 on failure */
+static inline uint32_t scd(__int128 *var, __int128 neu, int mm)
+{
+ uint32_t ret;
+
+ if (mm == __ATOMIC_RELEASE)
+ __asm__ volatile("stlxp %w0, %1, %2, [%3]"
+ : "=&r" (ret)
+ : "r" (((union i128)neu).i64[0]),
+ "r" (((union i128)neu).i64[1]),
+ "r" (var)
+ : "memory");
+ else if (mm == __ATOMIC_RELAXED)
+ __asm__ volatile("stxp %w0, %1, %2, [%3]"
+ : "=&r" (ret)
+ : "r" (((union i128)neu).i64[0]),
+ "r" (((union i128)neu).i64[1]),
+ "r" (var)
+ : );
+ else
+ ODP_ABORT();
+ return ret;
+}
+#endif
+#endif
+
+static inline void sevl(void)
+{
+#if defined __ARM_ARCH
+ __asm__ volatile("sevl" : : : );
+#endif
+}
+
+static inline void sev(void)
+{
+#if defined __ARM_ARCH
+ __asm__ volatile("sev" : : : "memory");
+#endif
+}
+
+static inline int wfe(void)
+{
+#if defined __ARM_ARCH
+ __asm__ volatile("wfe" : : : "memory");
+#endif
+ return 1;
+}
+
+#ifdef ODP_CONFIG_DMBSTR
+
+#if defined __ARM_ARCH && __ARM_ARCH == 8
+/* Only ARMv8 supports DMB ISHLD */
+/* A load only barrier is much cheaper than full barrier */
+#define _odp_release_barrier(ro) \
+do { \
+ if (ro) \
+ __asm__ volatile("dmb ishld" ::: "memory"); \
+ else \
+ __asm__ volatile("dmb ish" ::: "memory"); \
+} while (0)
+#else
+#define _odp_release_barrier(ro) \
+ __atomic_thread_fence(__ATOMIC_RELEASE)
+#endif
+
+#define atomic_store_release(loc, val, ro) \
+do { \
+ _odp_release_barrier(ro); \
+ __atomic_store_n(loc, val, __ATOMIC_RELAXED); \
+} while (0)
+
+#else
+
+#define atomic_store_release(loc, val, ro) \
+ __atomic_store_n(loc, val, __ATOMIC_RELEASE)
+
+#endif
+
+#ifdef ODP_CONFIG_USE_WFE
+#define SEVL() sevl()
+#define WFE() wfe()
+#define SEV() do { __asm__ volatile("dsb ish" ::: "memory"); sev(); } while (0)
+#if defined __ARM_ARCH && __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1
+#define LDXR128(addr, mo) lld((addr), (mo))
+#endif
+#define LDXR64(addr, mo) ll64((addr), (mo))
+#define LDXR32(addr, mo) ll32((addr), (mo))
+#define LDXR8(addr, mo) ll8((addr), (mo))
+/* When using WFE do not stall the pipeline using other means */
+#define DOZE() (void)0
+#else
+#define SEVL() (void)0
+#define WFE() 1
+#define SEV() (void)0
+#define LDXR128(addr, mo) __atomic_load_n((addr), (mo))
+#define LDXR64(addr, mo) __atomic_load_n((addr), (mo))
+#define LDXR32(addr, mo) __atomic_load_n((addr), (mo))
+#define LDXR8(addr, mo) __atomic_load_n((addr), (mo))
+#define DOZE() odp_cpu_pause()
+#endif
+
+#endif
--
2.12.2
