On 04/04/17 21:48, Brian Brooks wrote: > Signed-off-by: Ola Liljedahl <ola.liljed...@arm.com> > Reviewed-by: Brian Brooks <brian.bro...@arm.com> > Reviewed-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com> > --- > platform/linux-generic/include/odp_llsc.h | 332 > ++++++++++++++++++++++++++++++ > 1 file changed, 332 insertions(+) > create mode 100644 platform/linux-generic/include/odp_llsc.h > > diff --git a/platform/linux-generic/include/odp_llsc.h > b/platform/linux-generic/include/odp_llsc.h > new file mode 100644 > index 00000000..ea60c54b > --- /dev/null > +++ b/platform/linux-generic/include/odp_llsc.h
file has to be in: platform/linux-generic/arch/arm/ > @@ -0,0 +1,332 @@ > +/* Copyright (c) 2017, ARM Limited > + * All rights reserved. > + * > + * SPDX-License-Identifier: BSD-3-Clause > + */ > + > +#ifndef ODP_LLSC_H_ > +#define ODP_LLSC_H_ > + > +#include <odp_config_internal.h> > + > +/****************************************************************************** > + * LL/SC primitives > + > *****************************************************************************/ > + > +#if defined __ARM_ARCH > +#if __ARM_ARCH == 7 || (__ARM_ARCH == 8 && __ARM_64BIT_STATE == 0) > +static inline void dmb(void) > +{ > + __asm__ volatile("dmb" : : : "memory"); > +} > + > +static inline uint32_t ll8(uint8_t *var, int mm) > +{ > + uint8_t old; > + > + __asm__ volatile("ldrexb %0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + /* Barrier after an acquiring load */ > + if (mm == __ATOMIC_ACQUIRE) > + dmb(); > + return old; > +} > + > +static inline uint32_t ll(uint32_t *var, int mm) > +{ > + uint32_t old; > + > + __asm__ volatile("ldrex %0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + /* Barrier after an acquiring load */ > + if (mm == __ATOMIC_ACQUIRE) > + dmb(); > + return old; > +} > + > +#define ll32(a, b) ll((a), (b)) > + > +/* Return 0 on success, 1 on failure */ > +static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) > +{ > + uint32_t ret; > + > + /* Barrier before a releasing store */ > + if (mm == __ATOMIC_RELEASE) > + dmb(); > + __asm__ volatile("strex %0, %1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : ); > + return ret; > +} > + > +#define sc32(a, b, c) sc((a), (b), (c)) > + > +static inline uint64_t lld(uint64_t *var, int mm) > +{ > + uint64_t old; > + > + __asm__ volatile("ldrexd %0, %H0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + /* Barrier after an acquiring load */ > + if (mm == __ATOMIC_ACQUIRE) > + dmb(); > + return old; > +} > + > +#define ll64(a, b) lld((a), (b)) > + > +/* Return 0 on success, 1 on failure */ > +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) > +{ > + uint32_t ret; > + > + /* Barrier before a releasing store */ > + if (mm == __ATOMIC_RELEASE) > + dmb(); > + __asm__ volatile("strexd %0, %1, %H1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : ); > + return ret; > +} > + > +#define sc64(a, b, c) scd((a), (b), (c)) > + > +#endif > + > +#if __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1 > +static inline uint16_t ll8(uint8_t *var, int mm) > +{ > + uint16_t old; > + > + if (mm == __ATOMIC_ACQUIRE) > + __asm__ volatile("ldaxrb %w0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("ldxrb %w0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + else > + ODP_ABORT(); > + return old; > +} > + > +static inline uint32_t ll32(uint32_t *var, int mm) > +{ > + uint32_t old; > + > + if (mm == __ATOMIC_ACQUIRE) > + __asm__ volatile("ldaxr %w0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("ldxr %w0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + else > + ODP_ABORT(); > + return old; > +} > + > +/* Return 0 on success, 1 on failure */ > +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) > +{ > + uint32_t ret; > + > + if (mm == __ATOMIC_RELEASE) > + __asm__ volatile("stlxr %w0, %w1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("stxr %w0, %w1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : ); > + else > + ODP_ABORT(); > + return ret; > +} > + > +static inline uint64_t ll(uint64_t *var, int mm) > +{ > + uint64_t old; > + > + if (mm == __ATOMIC_ACQUIRE) > + __asm__ volatile("ldaxr %0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("ldxr %0, [%1]" > + : "=&r" (old) > + : "r" (var) > + : ); > + else > + ODP_ABORT(); > + return old; > +} > + > +#define ll64(a, b) ll((a), (b)) > + > +/* Return 0 on success, 1 on failure */ > +static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) > +{ > + uint32_t ret; > + > + if (mm == __ATOMIC_RELEASE) > + __asm__ volatile("stlxr %w0, %1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("stxr %w0, %1, [%2]" > + : "=&r" (ret) > + : "r" (neu), "r" (var) > + : ); > + else > + ODP_ABORT(); > + return ret; > +} > + > +#define sc64(a, b, c) sc((a), (b), (c)) > + > +union i128 { > + __int128 i128; > + int64_t i64[2]; > +}; > + > +static inline __int128 lld(__int128 *var, int mm) > +{ > + union i128 old; > + > + if (mm == __ATOMIC_ACQUIRE) > + __asm__ volatile("ldaxp %0, %1, [%2]" > + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) > + : "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("ldxp %0, %1, [%2]" > + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) > + : "r" (var) > + : ); > + else > + ODP_ABORT(); > + return old.i128; > +} > + > +/* Return 0 on success, 1 on failure */ > +static inline uint32_t scd(__int128 *var, __int128 neu, int mm) > +{ > + uint32_t ret; > + > + if (mm == __ATOMIC_RELEASE) > + __asm__ volatile("stlxp %w0, %1, %2, [%3]" > + : "=&r" (ret) > + : "r" (((union i128)neu).i64[0]), > + "r" (((union i128)neu).i64[1]), > + "r" (var) > + : "memory"); > + else if (mm == __ATOMIC_RELAXED) > + __asm__ volatile("stxp %w0, %1, %2, [%3]" > + : "=&r" (ret) > + : "r" (((union i128)neu).i64[0]), > + "r" (((union i128)neu).i64[1]), > + "r" (var) > + : ); > + else > + ODP_ABORT(); > + return ret; > +} > +#endif > +#endif > + > +static inline void sevl(void) > +{ > +#if defined __ARM_ARCH > + __asm__ volatile("sevl" : : : ); > +#endif > +} > + > +static inline void sev(void) > +{ > +#if defined __ARM_ARCH > + __asm__ volatile("sev" : : : "memory"); > +#endif > +} > + > +static inline int wfe(void) > +{ > +#if defined __ARM_ARCH > + __asm__ volatile("wfe" : : : "memory"); > +#endif > + return 1; > +} > + > +#ifdef ODP_CONFIG_DMBSTR > + > +#if defined __ARM_ARCH && __ARM_ARCH == 8 > +/* Only ARMv8 supports DMB ISHLD */ > +/* A load only barrier is much cheaper than full barrier */ > +#define _odp_release_barrier(ro) \ > +do { \ > + if (ro) \ > + __asm__ volatile("dmb ishld" ::: "memory"); \ > + else \ > + __asm__ volatile("dmb ish" ::: "memory"); \ > +} while (0) > +#else > +#define _odp_release_barrier(ro) \ > + __atomic_thread_fence(__ATOMIC_RELEASE) > +#endif > + > +#define atomic_store_release(loc, val, ro) \ > +do { \ > + _odp_release_barrier(ro); \ > + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ > +} while (0) > + > +#else > + > +#define atomic_store_release(loc, val, ro) \ > + __atomic_store_n(loc, val, __ATOMIC_RELEASE) > + > +#endif > + > +#ifdef ODP_CONFIG_USE_WFE > +#define SEVL() sevl() > +#define WFE() wfe() > +#define SEV() do { __asm__ volatile("dsb ish" ::: "memory"); sev(); } while > (0) > +#if defined __ARM_ARCH && __ARM_ARCH == 8 && __ARM_64BIT_STATE == 1 > +#define LDXR128(addr, mo) lld((addr), (mo)) > +#endif > +#define LDXR64(addr, mo) ll64((addr), (mo)) > +#define LDXR32(addr, mo) ll32((addr), (mo)) > +#define LDXR8(addr, mo) ll8((addr), (mo)) > +/* When using WFE do not stall the pipeline using other means */ > +#define DOZE() (void)0 > +#else > +#define SEVL() (void)0 > +#define WFE() 1 > +#define SEV() (void)0 > +#define LDXR128(addr, mo) __atomic_load_n((addr), (mo)) > +#define LDXR64(addr, mo) __atomic_load_n((addr), (mo)) > +#define LDXR32(addr, mo) __atomic_load_n((addr), (mo)) > +#define LDXR8(addr, mo) __atomic_load_n((addr), (mo)) > +#define DOZE() odp_cpu_pause() > +#endif > + > +#endif >