On 04/04/17 21:48, Brian Brooks wrote:
> Signed-off-by: Ola Liljedahl <[email protected]>
> Reviewed-by: Brian Brooks <[email protected]>
> Reviewed-by: Honnappa Nagarahalli <[email protected]>
> ---
> platform/linux-generic/include/odp_llsc.h | 332
> ++++++++++++++++++++++++++++++
> 1 file changed, 332 insertions(+)
> create mode 100644 platform/linux-generic/include/odp_llsc.h
>
> diff --git a/platform/linux-generic/include/odp_llsc.h
> b/platform/linux-generic/include/odp_llsc.h
> new file mode 100644
> index 00000000..ea60c54b
> --- /dev/null
> +++ b/platform/linux-generic/include/odp_llsc.h
file has to be in:
platform/linux-generic/arch/arm/
> @@ -0,0 +1,332 @@
> +/* Copyright (c) 2017, ARM Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier: BSD-3-Clause
> + */
> +
> +#ifndef ODP_LLSC_H_
> +#define ODP_LLSC_H_
> +
> +#include <odp_config_internal.h>
> +
> +/******************************************************************************
> + * LL/SC primitives
> +
> *****************************************************************************/
> +
> +#if defined __ARM_ARCH
> +#if __ARM_ARCH == 7 || (__ARM_ARCH == 8 && __ARM_64BIT_STATE == 0)
> +static inline void dmb(void)
> +{
> + __asm__ volatile("dmb" : : : "memory");
> +}
> +
> +static inline uint32_t ll8(uint8_t *var, int mm)
> +{
> + uint8_t old;
> +
> + __asm__ volatile("ldrexb %0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + /* Barrier after an acquiring load */
> + if (mm == __ATOMIC_ACQUIRE)
> + dmb();
> + return old;
> +}
> +
> +static inline uint32_t ll(uint32_t *var, int mm)
> +{
> + uint32_t old;
> +
> + __asm__ volatile("ldrex %0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + /* Barrier after an acquiring load */
> + if (mm == __ATOMIC_ACQUIRE)
> + dmb();
> + return old;
> +}
> +
> +#define ll32(a, b) ll((a), (b))
> +
> +/* Return 0 on success, 1 on failure */
> +static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm)
> +{
> + uint32_t ret;
> +
> + /* Barrier before a releasing store */
> + if (mm == __ATOMIC_RELEASE)
> + dmb();
> + __asm__ volatile("strex %0, %1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : );
> + return ret;
> +}
> +
> +#define sc32(a, b, c) sc((a), (b), (c))
> +
> +static inline uint64_t lld(uint64_t *var, int mm)
> +{
> + uint64_t old;
> +
> + __asm__ volatile("ldrexd %0, %H0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + /* Barrier after an acquiring load */
> + if (mm == __ATOMIC_ACQUIRE)
> + dmb();
> + return old;
> +}
> +
> +#define ll64(a, b) lld((a), (b))
> +
> +/* Return 0 on success, 1 on failure */
> +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm)
> +{
> + uint32_t ret;
> +
> + /* Barrier before a releasing store */
> + if (mm == __ATOMIC_RELEASE)
> + dmb();
> + __asm__ volatile("strexd %0, %1, %H1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : );
> + return ret;
> +}
> +
> +#define sc64(a, b, c) scd((a), (b), (c))
> +
> +#endif
> +
> +#if __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1
> +static inline uint16_t ll8(uint8_t *var, int mm)
> +{
> + uint16_t old;
> +
> + if (mm == __ATOMIC_ACQUIRE)
> + __asm__ volatile("ldaxrb %w0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("ldxrb %w0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return old;
> +}
> +
> +static inline uint32_t ll32(uint32_t *var, int mm)
> +{
> + uint32_t old;
> +
> + if (mm == __ATOMIC_ACQUIRE)
> + __asm__ volatile("ldaxr %w0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("ldxr %w0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return old;
> +}
> +
> +/* Return 0 on success, 1 on failure */
> +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm)
> +{
> + uint32_t ret;
> +
> + if (mm == __ATOMIC_RELEASE)
> + __asm__ volatile("stlxr %w0, %w1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("stxr %w0, %w1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return ret;
> +}
> +
> +static inline uint64_t ll(uint64_t *var, int mm)
> +{
> + uint64_t old;
> +
> + if (mm == __ATOMIC_ACQUIRE)
> + __asm__ volatile("ldaxr %0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("ldxr %0, [%1]"
> + : "=&r" (old)
> + : "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return old;
> +}
> +
> +#define ll64(a, b) ll((a), (b))
> +
> +/* Return 0 on success, 1 on failure */
> +static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm)
> +{
> + uint32_t ret;
> +
> + if (mm == __ATOMIC_RELEASE)
> + __asm__ volatile("stlxr %w0, %1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("stxr %w0, %1, [%2]"
> + : "=&r" (ret)
> + : "r" (neu), "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return ret;
> +}
> +
> +#define sc64(a, b, c) sc((a), (b), (c))
> +
> +union i128 {
> + __int128 i128;
> + int64_t i64[2];
> +};
> +
> +static inline __int128 lld(__int128 *var, int mm)
> +{
> + union i128 old;
> +
> + if (mm == __ATOMIC_ACQUIRE)
> + __asm__ volatile("ldaxp %0, %1, [%2]"
> + : "=&r" (old.i64[0]), "=&r" (old.i64[1])
> + : "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("ldxp %0, %1, [%2]"
> + : "=&r" (old.i64[0]), "=&r" (old.i64[1])
> + : "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return old.i128;
> +}
> +
> +/* Return 0 on success, 1 on failure */
> +static inline uint32_t scd(__int128 *var, __int128 neu, int mm)
> +{
> + uint32_t ret;
> +
> + if (mm == __ATOMIC_RELEASE)
> + __asm__ volatile("stlxp %w0, %1, %2, [%3]"
> + : "=&r" (ret)
> + : "r" (((union i128)neu).i64[0]),
> + "r" (((union i128)neu).i64[1]),
> + "r" (var)
> + : "memory");
> + else if (mm == __ATOMIC_RELAXED)
> + __asm__ volatile("stxp %w0, %1, %2, [%3]"
> + : "=&r" (ret)
> + : "r" (((union i128)neu).i64[0]),
> + "r" (((union i128)neu).i64[1]),
> + "r" (var)
> + : );
> + else
> + ODP_ABORT();
> + return ret;
> +}
> +#endif
> +#endif
> +
> +static inline void sevl(void)
> +{
> +#if defined __ARM_ARCH
> + __asm__ volatile("sevl" : : : );
> +#endif
> +}
> +
> +static inline void sev(void)
> +{
> +#if defined __ARM_ARCH
> + __asm__ volatile("sev" : : : "memory");
> +#endif
> +}
> +
> +static inline int wfe(void)
> +{
> +#if defined __ARM_ARCH
> + __asm__ volatile("wfe" : : : "memory");
> +#endif
> + return 1;
> +}
> +
> +#ifdef ODP_CONFIG_DMBSTR
> +
> +#if defined __ARM_ARCH && __ARM_ARCH == 8
> +/* Only ARMv8 supports DMB ISHLD */
> +/* A load only barrier is much cheaper than full barrier */
> +#define _odp_release_barrier(ro) \
> +do { \
> + if (ro) \
> + __asm__ volatile("dmb ishld" ::: "memory"); \
> + else \
> + __asm__ volatile("dmb ish" ::: "memory"); \
> +} while (0)
> +#else
> +#define _odp_release_barrier(ro) \
> + __atomic_thread_fence(__ATOMIC_RELEASE)
> +#endif
> +
> +#define atomic_store_release(loc, val, ro) \
> +do { \
> + _odp_release_barrier(ro); \
> + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \
> +} while (0)
> +
> +#else
> +
> +#define atomic_store_release(loc, val, ro) \
> + __atomic_store_n(loc, val, __ATOMIC_RELEASE)
> +
> +#endif
> +
> +#ifdef ODP_CONFIG_USE_WFE
> +#define SEVL() sevl()
> +#define WFE() wfe()
> +#define SEV() do { __asm__ volatile("dsb ish" ::: "memory"); sev(); } while
> (0)
> +#if defined __ARM_ARCH && __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1
> +#define LDXR128(addr, mo) lld((addr), (mo))
> +#endif
> +#define LDXR64(addr, mo) ll64((addr), (mo))
> +#define LDXR32(addr, mo) ll32((addr), (mo))
> +#define LDXR8(addr, mo) ll8((addr), (mo))
> +/* When using WFE do not stall the pipeline using other means */
> +#define DOZE() (void)0
> +#else
> +#define SEVL() (void)0
> +#define WFE() 1
> +#define SEV() (void)0
> +#define LDXR128(addr, mo) __atomic_load_n((addr), (mo))
> +#define LDXR64(addr, mo) __atomic_load_n((addr), (mo))
> +#define LDXR32(addr, mo) __atomic_load_n((addr), (mo))
> +#define LDXR8(addr, mo) __atomic_load_n((addr), (mo))
> +#define DOZE() odp_cpu_pause()
> +#endif
> +
> +#endif
>
