Signed-off-by: Ola Liljedahl <ola.liljed...@arm.com> Reviewed-by: Brian Brooks <brian.bro...@arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com> --- platform/linux-generic/include/odp_llsc.h | 332 ++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 platform/linux-generic/include/odp_llsc.h
diff --git a/platform/linux-generic/include/odp_llsc.h b/platform/linux-generic/include/odp_llsc.h new file mode 100644 index 00000000..ea60c54b --- /dev/null +++ b/platform/linux-generic/include/odp_llsc.h @@ -0,0 +1,332 @@ +/* Copyright (c) 2017, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_LLSC_H_ +#define ODP_LLSC_H_ + +#include <odp_config_internal.h> + +/****************************************************************************** + * LL/SC primitives + *****************************************************************************/ + +#if defined __ARM_ARCH +#if __ARM_ARCH == 7 || (__ARM_ARCH == 8 && __ARM_64BIT_STATE == 0) +static inline void dmb(void) +{ + __asm__ volatile("dmb" : : : "memory"); +} + +static inline uint32_t ll8(uint8_t *var, int mm) +{ + uint8_t old; + + __asm__ volatile("ldrexb %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +static inline uint32_t ll(uint32_t *var, int mm) +{ + uint32_t old; + + __asm__ volatile("ldrex %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +#define ll32(a, b) ll((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + dmb(); + __asm__ volatile("strex %0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +#define sc32(a, b, c) sc((a), (b), (c)) + +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; + + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +#define ll64(a, b) lld((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + dmb(); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +#define sc64(a, b, c) scd((a), (b), (c)) + +#endif + +#if __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1 +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} + +static inline uint64_t ll(uint64_t *var, int mm) +{ + uint64_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +#define ll64(a, b) ll((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} + +#define sc64(a, b, c) sc((a), (b), (c)) + +union i128 { + __int128 i128; + int64_t i64[2]; +}; + +static inline __int128 lld(__int128 *var, int mm) +{ + union i128 old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + else + ODP_ABORT(); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((union i128)neu).i64[0]), + "r" (((union i128)neu).i64[1]), + "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((union i128)neu).i64[0]), + "r" (((union i128)neu).i64[1]), + "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} +#endif +#endif + +static inline void sevl(void) +{ +#if defined __ARM_ARCH + __asm__ volatile("sevl" : : : ); +#endif +} + +static inline void sev(void) +{ +#if defined __ARM_ARCH + __asm__ volatile("sev" : : : "memory"); +#endif +} + +static inline int wfe(void) +{ +#if defined __ARM_ARCH + __asm__ volatile("wfe" : : : "memory"); +#endif + return 1; +} + +#ifdef ODP_CONFIG_DMBSTR + +#if defined __ARM_ARCH && __ARM_ARCH == 8 +/* Only ARMv8 supports DMB ISHLD */ +/* A load only barrier is much cheaper than full barrier */ +#define _odp_release_barrier(ro) \ +do { \ + if (ro) \ + __asm__ volatile("dmb ishld" ::: "memory"); \ + else \ + __asm__ volatile("dmb ish" ::: "memory"); \ +} while (0) +#else +#define _odp_release_barrier(ro) \ + __atomic_thread_fence(__ATOMIC_RELEASE) +#endif + +#define atomic_store_release(loc, val, ro) \ +do { \ + _odp_release_barrier(ro); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif + +#ifdef ODP_CONFIG_USE_WFE +#define SEVL() sevl() +#define WFE() wfe() +#define SEV() do { __asm__ volatile("dsb ish" ::: "memory"); sev(); } while (0) +#if defined __ARM_ARCH && __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1 +#define LDXR128(addr, mo) lld((addr), (mo)) +#endif +#define LDXR64(addr, mo) ll64((addr), (mo)) +#define LDXR32(addr, mo) ll32((addr), (mo)) +#define LDXR8(addr, mo) ll8((addr), (mo)) +/* When using WFE do not stall the pipeline using other means */ +#define DOZE() (void)0 +#else +#define SEVL() (void)0 +#define WFE() 1 +#define SEV() (void)0 +#define LDXR128(addr, mo) __atomic_load_n((addr), (mo)) +#define LDXR64(addr, mo) __atomic_load_n((addr), (mo)) +#define LDXR32(addr, mo) __atomic_load_n((addr), (mo)) +#define LDXR8(addr, mo) __atomic_load_n((addr), (mo)) +#define DOZE() odp_cpu_pause() +#endif + +#endif -- 2.12.2