This is an automated email from the ASF dual-hosted git repository. GUIDINGLI pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit 725b1d3d677e7251fe8ede21f00d8b88d1e2b007 Author: Bowen Wang <[email protected]> AuthorDate: Thu Apr 2 22:32:17 2026 +0800 libc/atomic: support inline arch atomic for generic IRQ path When CONFIG_LIBC_ATOMIC_IRQ or CONFIG_LIBC_ATOMIC_HWSPINLOCK is selected, atomic.h now includes nuttx/lib/arch_atomic.h which provides static inline atomic_*_4 and atomic_*_8 operations using IRQ disable/enable (or hwspinlock). This eliminates function call overhead for NuttX atomic API, following the same ARCH_ATOMIC_SPECIFIER/ARCH_ATOMIC_DEFINE pattern established by the tricore inline atomic commit. The arch_atomic.c retains __atomic_*_{1,2,4,8} symbols with external linkage for GCC libatomic ABI compatibility, and all __sync_* builtins remain unchanged. Also fix a pre-existing race condition in SYNC_*_FETCH macros where the return value was read from shared memory after releasing the lock. Now the result is captured in a local variable before unlocking. Signed-off-by: Bowen Wang <[email protected]> --- include/nuttx/atomic.h | 55 +---- include/nuttx/lib/arch_atomic.h | 191 +++++++++++++++++ include/nuttx/semaphore.h | 9 +- libs/libc/machine/CMakeLists.txt | 2 - libs/libc/machine/Make.defs | 2 - libs/libc/machine/arch_atomic.c | 429 +++++++++++++++++--------------------- libs/libc/machine/arch_atomic.h | 320 ---------------------------- libs/libc/machine/arch_atomic64.c | 227 -------------------- 8 files changed, 392 insertions(+), 843 deletions(-) diff --git a/include/nuttx/atomic.h b/include/nuttx/atomic.h index e6e90d991a1..6477340f3c3 100644 --- a/include/nuttx/atomic.h +++ b/include/nuttx/atomic.h @@ -34,6 +34,9 @@ #ifdef CONFIG_LIBC_ATOMIC_ARCH # include <arch/atomic.h> +#elif defined(CONFIG_LIBC_ATOMIC_IRQ) || \ + defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK) +# include <nuttx/lib/arch_atomic.h> #endif /**************************************************************************** @@ -287,56 +290,4 @@ typedef atomic64_t atomic_ptr_t; typedef atomic_t atomic_ptr_t; #endif -/**************************************************************************** - * Public Function Prototypes - ****************************************************************************/ - -#if defined(__cplusplus) -extern "C" -{ -#endif - -#if !defined(CONFIG_LIBC_ATOMIC_TOOLCHAIN) && !defined(CONFIG_LIBC_ATOMIC_ARCH) -void atomic_store_4(FAR volatile void *ptr, int32_t value, int memorder); -void atomic_store_8(FAR volatile void *ptr, int64_t value, int memorder); -int32_t atomic_load_4(FAR const volatile void *ptr, int memorder); -int64_t atomic_load_8(FAR const volatile void *ptr, int memorder); -int32_t atomic_exchange_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_exchange_8(FAR volatile void *ptr, int64_t value, - int memorder); -bool atomic_compare_exchange_4(FAR volatile void *ptr, - FAR volatile void *expect, - int32_t desired, bool weak, - int success, int failure); -bool atomic_compare_exchange_8(FAR volatile void *ptr, - FAR volatile void *expect, - int64_t desired, bool weak, - int success, int failure); -int32_t atomic_fetch_add_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_fetch_add_8(FAR volatile void *ptr, int64_t value, - int memorder); -int32_t atomic_fetch_sub_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_fetch_sub_8(FAR volatile void *ptr, int64_t value, - int memorder); -int32_t atomic_fetch_and_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_fetch_and_8(FAR volatile void *ptr, int64_t value, - int memorder); -int32_t atomic_fetch_or_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_fetch_or_8(FAR volatile void *ptr, int64_t value, - int memorder); -int32_t atomic_fetch_xor_4(FAR volatile void *ptr, int32_t value, - int memorder); -int64_t atomic_fetch_xor_8(FAR volatile void *ptr, int64_t value, - int memorder); -#endif - -#if defined(__cplusplus) -} -#endif - #endif /* __INCLUDE_NUTTX_ATOMIC_H */ diff --git a/include/nuttx/lib/arch_atomic.h b/include/nuttx/lib/arch_atomic.h new file mode 100644 index 00000000000..bdea1a3a8d7 --- /dev/null +++ b/include/nuttx/lib/arch_atomic.h @@ -0,0 +1,191 @@ +/**************************************************************************** + * include/nuttx/lib/arch_atomic.h + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_NUTTX_LIB_ARCH_ATOMIC_H +#define __INCLUDE_NUTTX_LIB_ARCH_ATOMIC_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include <nuttx/config.h> +#include <nuttx/compiler.h> +#include <nuttx/irq.h> + +#include <stdbool.h> +#include <stdint.h> + +#if !defined(CONFIG_BUILD_FLAT) && !defined(__KERNEL__) +# include <sched.h> +#elif defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK) +# include <nuttx/hwspinlock/hwspinlock.h> +#endif + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifndef ARCH_ATOMIC_SPECIFIER +# define ARCH_ATOMIC_SPECIFIER static always_inline_function +#endif + +#if !defined(CONFIG_BUILD_FLAT) && !defined(__KERNEL__) +static inline irqstate_t atomic_lock(void) +{ + sched_lock(); + return 0; +} + +static inline void atomic_unlock(irqstate_t flags) +{ + sched_unlock(); +} +#elif defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK) +extern struct hwspinlock_dev_s g_atomic_hwspinlock; +static inline irqstate_t atomic_lock(void) +{ + return hwspin_lock_irqsave(&g_atomic_hwspinlock); +} + +static inline void atomic_unlock(irqstate_t flags) +{ + hwspin_unlock_restore(&g_atomic_hwspinlock, flags); +} +#else +static inline irqstate_t atomic_lock(void) +{ + return up_irq_save(); +} + +static inline void atomic_unlock(irqstate_t flags) +{ + up_irq_restore(flags); +} +#endif + +#define ARCH_ATOMIC_STORE(func, t) \ + ARCH_ATOMIC_SPECIFIER \ + void func(FAR volatile void *ptr, t value, int memorder) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + \ + *(FAR t *)ptr = value; \ + \ + atomic_unlock(irqstate); \ + } + +#define ARCH_ATOMIC_LOAD(func, t) \ + ARCH_ATOMIC_SPECIFIER \ + t func(FAR const volatile void *ptr, int memorder) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + \ + t ret = *(FAR t *)ptr; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define ARCH_ATOMIC_EXCHANGE(func, t) \ + ARCH_ATOMIC_SPECIFIER \ + t func(FAR volatile void *ptr, t value, int memorder) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR t *tmp = (FAR t *)ptr; \ + \ + t ret = *tmp; \ + *tmp = value; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define ARCH_ATOMIC_COMPARE_EXCHANGE(func, t) \ + ARCH_ATOMIC_SPECIFIER \ + bool func(FAR volatile void *mem, FAR volatile void *expect, \ + t desired, bool weak, int success, int failure) \ + { \ + bool ret = false; \ + irqstate_t irqstate = atomic_lock(); \ + FAR t *tmpmem = (FAR t *)mem; \ + FAR t *tmpexp = (FAR t *)expect; \ + \ + if (*tmpmem == *tmpexp) \ + { \ + ret = true; \ + *tmpmem = desired; \ + } \ + else \ + { \ + *tmpexp = *tmpmem; \ + } \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define ARCH_ATOMIC_FLAGS_TEST_AND_SET(func, t) \ + ARCH_ATOMIC_SPECIFIER \ + t func(FAR volatile void *ptr, int memorder) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR t *tmp = (FAR t *)ptr; \ + t ret = *tmp; \ + \ + *(FAR t *)ptr = 1; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define ARCH_ATOMIC_FETCH_OP(func, t, op) \ + ARCH_ATOMIC_SPECIFIER \ + t func(FAR volatile void *ptr, t value, int memorder) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR t *tmp = (FAR t *)ptr; \ + t ret = *tmp; \ + \ + *tmp = *tmp op value; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define ARCH_ATOMIC_DEFINE(prefix, t, n) \ + ARCH_ATOMIC_STORE(prefix ## _store_ ## n, t) \ + ARCH_ATOMIC_LOAD(prefix ## _load_ ## n, t) \ + ARCH_ATOMIC_EXCHANGE(prefix ## _exchange_ ## n, t) \ + ARCH_ATOMIC_COMPARE_EXCHANGE(prefix ## _compare_exchange_ ## n, t) \ + ARCH_ATOMIC_FLAGS_TEST_AND_SET(prefix ## _flags_test_and_set_ ## n, t) \ + ARCH_ATOMIC_FETCH_OP(prefix ## _fetch_add_ ## n, t, +) \ + ARCH_ATOMIC_FETCH_OP(prefix ## _fetch_sub_ ## n, t, -) \ + ARCH_ATOMIC_FETCH_OP(prefix ## _fetch_and_ ## n, t, &) \ + ARCH_ATOMIC_FETCH_OP(prefix ## _fetch_or_ ## n, t, |) \ + ARCH_ATOMIC_FETCH_OP(prefix ## _fetch_xor_ ## n, t, ^) + +ARCH_ATOMIC_DEFINE(atomic, uint8_t, 1) +ARCH_ATOMIC_DEFINE(atomic, uint16_t, 2) +ARCH_ATOMIC_DEFINE(atomic, int32_t, 4) +ARCH_ATOMIC_DEFINE(atomic, int64_t, 8) + +#endif /* __INCLUDE_NUTTX_LIB_ARCH_ATOMIC_H */ diff --git a/include/nuttx/semaphore.h b/include/nuttx/semaphore.h index 2b767caa2b8..d516173db25 100644 --- a/include/nuttx/semaphore.h +++ b/include/nuttx/semaphore.h @@ -103,13 +103,14 @@ # define NXSEM_RESTORE_UNINTERRUPTIBLE(tcb, flags) do { } while (0) #endif -/* In user space, the atomics of LIBC_ATOMIC_IRQ are system calls. There, - * the semaphore operations skip the fast path: the slow path is one system - * call. +/* In user space, the atomics of LIBC_ATOMIC_IRQ and LIBC_ATOMIC_HWSPINLOCK + * are system calls. There, the semaphore operations skip the fast path: + * the slow path is one system call. */ #if !defined(CONFIG_BUILD_FLAT) && !defined(__KERNEL__) && \ - defined(CONFIG_LIBC_ATOMIC_IRQ) + (defined(CONFIG_LIBC_ATOMIC_IRQ) || \ + defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK)) # define NXSEM_FASTPATH false #else # define NXSEM_FASTPATH true diff --git a/libs/libc/machine/CMakeLists.txt b/libs/libc/machine/CMakeLists.txt index 35bbf1cbaff..e2dfd540857 100644 --- a/libs/libc/machine/CMakeLists.txt +++ b/libs/libc/machine/CMakeLists.txt @@ -26,8 +26,6 @@ if(CONFIG_LIBC_ATOMIC_IRQ OR CONFIG_LIBC_ATOMIC_HWSPINLOCK) target_sources(c PRIVATE arch_atomic.c) endif() -target_sources(c PRIVATE arch_atomic64.c) - if(CONFIG_MM_KASAN) target_sources(c PRIVATE arch_libc.c) endif() diff --git a/libs/libc/machine/Make.defs b/libs/libc/machine/Make.defs index e8e03b6543f..ef340bd54ca 100644 --- a/libs/libc/machine/Make.defs +++ b/libs/libc/machine/Make.defs @@ -24,8 +24,6 @@ ifneq ($(filter y,$(CONFIG_LIBC_ATOMIC_IRQ)$(CONFIG_LIBC_ATOMIC_HWSPINLOCK)),) CSRCS += arch_atomic.c endif -CSRCS += arch_atomic64.c - ifeq ($(CONFIG_MM_KASAN),y) CSRCS += arch_libc.c endif diff --git a/libs/libc/machine/arch_atomic.c b/libs/libc/machine/arch_atomic.c index e53a857c52f..4b811e4f565 100644 --- a/libs/libc/machine/arch_atomic.c +++ b/libs/libc/machine/arch_atomic.c @@ -26,253 +26,162 @@ #include <nuttx/config.h> -#include <sched.h> -#include <stdbool.h> -#include <stdint.h> -#include <nuttx/irq.h> -#include <nuttx/macro.h> -#if defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK) -# include <nuttx/hwspinlock/hwspinlock.h> -#endif +#define ARCH_ATOMIC_SPECIFIER -#include "arch_atomic.h" +#include <nuttx/atomic.h> +#include <nuttx/macro.h> /**************************************************************************** - * Private Functions - ****************************************************************************/ - -#if defined(CONFIG_LIBC_ATOMIC_HWSPINLOCK) -extern struct hwspinlock_dev_s g_atomic_hwspinlock; - -static inline irqstate_t atomic_lock(void) -{ - return hwspin_lock_irqsave(&g_atomic_hwspinlock); -} - -static inline void atomic_unlock(irqstate_t flags) -{ - hwspin_unlock_restore(&g_atomic_hwspinlock, flags); -} -#elif !defined(CONFIG_BUILD_FLAT) && !defined(__KERNEL__) -/* User code cannot disable the interrupts. Lock the scheduler instead: - * as with up_irq_save(), no other thread runs on this CPU meanwhile. - */ - -static inline irqstate_t atomic_lock(void) -{ - sched_lock(); - return 0; -} - -static inline void atomic_unlock(irqstate_t flags) -{ - sched_unlock(); -} -#elif defined(CONFIG_LIBC_ATOMIC_IRQ) -static inline irqstate_t atomic_lock(void) -{ - return up_irq_save(); -} - -static inline void atomic_unlock(irqstate_t flags) -{ - up_irq_restore(flags); -} -#endif + * Pre-processor Definitions + ****************************************************************************/ + +#define SYNC_ADD_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = *tmp + value; \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_SUB_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = *tmp - value; \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_OR_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = *tmp | value; \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_AND_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = *tmp & value; \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_XOR_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = *tmp ^ value; \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_NAND_FETCH(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type value) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret; \ + \ + *tmp = ~(*tmp & value); \ + ret = *tmp; \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_BOOL_CMP_SWAP(fn, n, type) \ + \ + bool weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type oldvalue, \ + type newvalue) \ + { \ + bool ret = false; \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + \ + if (*tmp == oldvalue) \ + { \ + ret = true; \ + *tmp = newvalue; \ + } \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } + +#define SYNC_VAL_CMP_SWAP(fn, n, type) \ + \ + type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ + type oldvalue, \ + type newvalue) \ + { \ + irqstate_t irqstate = atomic_lock(); \ + FAR type *tmp = (FAR type *)ptr; \ + type ret = *tmp; \ + \ + if (*tmp == oldvalue) \ + { \ + *tmp = newvalue; \ + } \ + \ + atomic_unlock(irqstate); \ + return ret; \ + } /**************************************************************************** * Public Functions ****************************************************************************/ /**************************************************************************** - * Name: __atomic_store_1 - ****************************************************************************/ - -STORE(__atomic_store_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_store_2 - ****************************************************************************/ - -STORE(__atomic_store_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_store_4 - ****************************************************************************/ - -STORE(__atomic_store_, 4, uint32_t) -STORE(atomic_store_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_load_1 - ****************************************************************************/ - -LOAD(__atomic_load_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_load_2 - ****************************************************************************/ - -LOAD(__atomic_load_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_load_4 - ****************************************************************************/ - -LOAD(__atomic_load_, 4, uint32_t) -LOAD(atomic_load_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_exchange_1 - ****************************************************************************/ - -EXCHANGE(__atomic_exchange_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_exchange_2 - ****************************************************************************/ - -EXCHANGE(__atomic_exchange_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_exchange_4 - ****************************************************************************/ - -EXCHANGE(__atomic_exchange_, 4, uint32_t) -EXCHANGE(atomic_exchange_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_compare_exchange_1 - ****************************************************************************/ - -CMP_EXCHANGE(__atomic_compare_exchange_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_compare_exchange_2 - ****************************************************************************/ - -CMP_EXCHANGE(__atomic_compare_exchange_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_compare_exchange_4 - ****************************************************************************/ - -CMP_EXCHANGE(__atomic_compare_exchange_, 4, uint32_t) -CMP_EXCHANGE(atomic_compare_exchange_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_flag_test_and_set_1 - ****************************************************************************/ - -FLAG_TEST_AND_SET(__atomic_flags_test_and_set_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_flag_test_and_set_2 - ****************************************************************************/ - -FLAG_TEST_AND_SET(__atomic_flags_test_and_set_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_flag_test_and_set_4 - ****************************************************************************/ - -FLAG_TEST_AND_SET(__atomic_flags_test_and_set_, 4, uint32_t) -FLAG_TEST_AND_SET(atomic_flags_test_and_set_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_fetch_add_1 - ****************************************************************************/ - -FETCH_ADD(__atomic_fetch_add_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_fetch_add_2 - ****************************************************************************/ - -FETCH_ADD(__atomic_fetch_add_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_fetch_add_4 - ****************************************************************************/ - -FETCH_ADD(__atomic_fetch_add_, 4, uint32_t) -FETCH_ADD(atomic_fetch_add_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_fetch_sub_1 - ****************************************************************************/ - -FETCH_SUB(__atomic_fetch_sub_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_fetch_sub_2 - ****************************************************************************/ - -FETCH_SUB(__atomic_fetch_sub_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_fetch_sub_4 - ****************************************************************************/ - -FETCH_SUB(__atomic_fetch_sub_, 4, uint32_t) -FETCH_SUB(atomic_fetch_sub_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_fetch_and_1 - ****************************************************************************/ - -FETCH_AND(__atomic_fetch_and_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_fetch_and_2 - ****************************************************************************/ - -FETCH_AND(__atomic_fetch_and_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_fetch_and_4 - ****************************************************************************/ - -FETCH_AND(__atomic_fetch_and_, 4, uint32_t) -FETCH_AND(atomic_fetch_and_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_fetch_or_1 - ****************************************************************************/ - -FETCH_OR(__atomic_fetch_or_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_fetch_or_2 - ****************************************************************************/ - -FETCH_OR(__atomic_fetch_or_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_fetch_or_4 - ****************************************************************************/ - -FETCH_OR(__atomic_fetch_or_, 4, uint32_t) -FETCH_OR(atomic_fetch_or_, 4, int32_t) - -/**************************************************************************** - * Name: __atomic_fetch_xor_1 - ****************************************************************************/ - -FETCH_XOR(__atomic_fetch_xor_, 1, uint8_t) - -/**************************************************************************** - * Name: __atomic_fetch_xor_2 - ****************************************************************************/ - -FETCH_XOR(__atomic_fetch_xor_, 2, uint16_t) - -/**************************************************************************** - * Name: __atomic_fetch_xor_4 + * Name: __atomic_*_{1,2,4,8} ****************************************************************************/ -FETCH_XOR(__atomic_fetch_xor_, 4, uint32_t) -FETCH_XOR(atomic_fetch_xor_, 4, int32_t) +ARCH_ATOMIC_DEFINE(__atomic, uint8_t, 1) +ARCH_ATOMIC_DEFINE(__atomic, uint16_t, 2) +ARCH_ATOMIC_DEFINE(__atomic, uint32_t, 4) +ARCH_ATOMIC_DEFINE(__atomic, uint64_t, 8) /* Clang define the __sync builtins, add #ifndef to avoid * redefined/redeclared problem. @@ -298,6 +207,12 @@ SYNC_ADD_FETCH(__sync_add_and_fetch_, 2, uint16_t) SYNC_ADD_FETCH(__sync_add_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_add_and_fetch_8 + ****************************************************************************/ + +SYNC_ADD_FETCH(__sync_add_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_sub_and_fetch_1 ****************************************************************************/ @@ -316,6 +231,12 @@ SYNC_SUB_FETCH(__sync_sub_and_fetch_, 2, uint16_t) SYNC_SUB_FETCH(__sync_sub_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_sub_and_fetch_8 + ****************************************************************************/ + +SYNC_SUB_FETCH(__sync_sub_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_or_and_fetch_1 ****************************************************************************/ @@ -334,6 +255,12 @@ SYNC_OR_FETCH(__sync_or_and_fetch_, 2, uint16_t) SYNC_OR_FETCH(__sync_or_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_or_and_fetch_8 + ****************************************************************************/ + +SYNC_OR_FETCH(__sync_or_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_and_and_fetch_1 ****************************************************************************/ @@ -352,6 +279,12 @@ SYNC_AND_FETCH(__sync_and_and_fetch_, 2, uint16_t) SYNC_AND_FETCH(__sync_and_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_and_and_fetch_8 + ****************************************************************************/ + +SYNC_AND_FETCH(__sync_and_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_xor_and_fetch_1 ****************************************************************************/ @@ -370,6 +303,12 @@ SYNC_XOR_FETCH(__sync_xor_and_fetch_, 2, uint16_t) SYNC_XOR_FETCH(__sync_xor_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_xor_and_fetch_8 + ****************************************************************************/ + +SYNC_XOR_FETCH(__sync_xor_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_nand_and_fetch_1 ****************************************************************************/ @@ -388,6 +327,12 @@ SYNC_NAND_FETCH(__sync_nand_and_fetch_, 2, uint16_t) SYNC_NAND_FETCH(__sync_nand_and_fetch_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_nand_and_fetch_8 + ****************************************************************************/ + +SYNC_NAND_FETCH(__sync_nand_and_fetch_, 8, uint64_t) + /**************************************************************************** * Name: __sync_bool_compare_and_swap_1 ****************************************************************************/ @@ -406,6 +351,12 @@ SYNC_BOOL_CMP_SWAP(__sync_bool_compare_and_swap_, 2, uint16_t) SYNC_BOOL_CMP_SWAP(__sync_bool_compare_and_swap_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_bool_compare_and_swap_8 + ****************************************************************************/ + +SYNC_BOOL_CMP_SWAP(__sync_bool_compare_and_swap_, 8, uint64_t) + /**************************************************************************** * Name: __sync_val_compare_and_swap_1 ****************************************************************************/ @@ -424,6 +375,12 @@ SYNC_VAL_CMP_SWAP(__sync_val_compare_and_swap_, 2, uint16_t) SYNC_VAL_CMP_SWAP(__sync_val_compare_and_swap_, 4, uint32_t) +/**************************************************************************** + * Name: __sync_val_compare_and_swap_8 + ****************************************************************************/ + +SYNC_VAL_CMP_SWAP(__sync_val_compare_and_swap_, 8, uint64_t) + /**************************************************************************** * Name: __sync_synchronize ****************************************************************************/ diff --git a/libs/libc/machine/arch_atomic.h b/libs/libc/machine/arch_atomic.h deleted file mode 100644 index 747271912cb..00000000000 --- a/libs/libc/machine/arch_atomic.h +++ /dev/null @@ -1,320 +0,0 @@ -/**************************************************************************** - * libs/libc/machine/arch_atomic.h - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - ****************************************************************************/ - -#ifndef __LIBS_LIBC_MACHINE_ARCH_ATOMIC_H -#define __LIBS_LIBC_MACHINE_ARCH_ATOMIC_H - -/**************************************************************************** - * Included Files - ****************************************************************************/ - -#include <nuttx/config.h> - -#include <stdbool.h> -#include <stdint.h> -#include <nuttx/irq.h> -#include <nuttx/macro.h> - -/**************************************************************************** - * Pre-processor Definitions - ****************************************************************************/ - -#define STORE(fn, n, type) \ - \ - void weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - \ - *(FAR type *)ptr = value; \ - \ - atomic_unlock(irqstate); \ - } - -#define LOAD(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR const volatile void *ptr, \ - int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - \ - type ret = *(FAR type *)ptr; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define EXCHANGE(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - type ret = *tmp; \ - *tmp = value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define CMP_EXCHANGE(fn, n, type) \ - \ - bool weak_function CONCATENATE(fn, n)(FAR volatile void *mem, \ - FAR volatile void *expect, \ - type desired, bool weak, \ - int success, int failure) \ - { \ - bool ret = false; \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmpmem = (FAR type *)mem; \ - FAR type *tmpexp = (FAR type *)expect; \ - \ - if (*tmpmem == *tmpexp) \ - { \ - ret = true; \ - *tmpmem = desired; \ - } \ - else \ - { \ - *tmpexp = *tmpmem; \ - } \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FLAG_TEST_AND_SET(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *(FAR type *)ptr = 1; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FETCH_ADD(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *tmp = *tmp + value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FETCH_SUB(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *tmp = *tmp - value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FETCH_AND(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *tmp = *tmp & value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FETCH_OR(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *tmp = *tmp | value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define FETCH_XOR(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value, int memorder) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - *tmp = *tmp ^ value; \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define SYNC_ADD_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = *tmp + value; \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_SUB_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = *tmp - value; \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_OR_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = *tmp | value; \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_AND_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = *tmp & value; \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_XOR_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = *tmp ^ value; \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_NAND_FETCH(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type value) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - *tmp = ~(*tmp & value); \ - \ - atomic_unlock(irqstate); \ - return *tmp; \ - } - -#define SYNC_BOOL_CMP_SWAP(fn, n, type) \ - \ - bool weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type oldvalue, \ - type newvalue) \ - { \ - bool ret = false; \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - \ - if (*tmp == oldvalue) \ - { \ - ret = true; \ - *tmp = newvalue; \ - } \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#define SYNC_VAL_CMP_SWAP(fn, n, type) \ - \ - type weak_function CONCATENATE(fn, n)(FAR volatile void *ptr, \ - type oldvalue, \ - type newvalue) \ - { \ - irqstate_t irqstate = atomic_lock(); \ - FAR type *tmp = (FAR type *)ptr; \ - type ret = *tmp; \ - \ - if (*tmp == oldvalue) \ - { \ - *tmp = newvalue; \ - } \ - \ - atomic_unlock(irqstate); \ - return ret; \ - } - -#endif /* __LIBS_LIBC_MACHINE_ARCH_ATOMIC_H */ diff --git a/libs/libc/machine/arch_atomic64.c b/libs/libc/machine/arch_atomic64.c deleted file mode 100644 index 0b90fd8cb9f..00000000000 --- a/libs/libc/machine/arch_atomic64.c +++ /dev/null @@ -1,227 +0,0 @@ -/**************************************************************************** - * libs/libc/machine/arch_atomic64.c - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - ****************************************************************************/ - -/**************************************************************************** - * Included Files - ****************************************************************************/ - -#include <nuttx/config.h> - -#include <sched.h> - -#include <nuttx/spinlock.h> - -#include "arch_atomic.h" - -/**************************************************************************** - * Private Data - ****************************************************************************/ - -static spinlock_t g_atomic_lock = SP_UNLOCKED; - -/**************************************************************************** - * Private Functions - ****************************************************************************/ - -#if !defined(CONFIG_BUILD_FLAT) && !defined(__KERNEL__) -/* User code cannot disable the interrupts. Lock the scheduler instead: - * as with up_irq_save(), no other thread runs on this CPU meanwhile. The - * spinlock keeps out the other CPUs of an SMP build. spin_lock() can call - * kernel functions, so try the lock in a loop. - */ - -static inline irqstate_t atomic_lock(void) -{ - sched_lock(); - while (!spin_trylock_notrace(&g_atomic_lock)) - { - } - - return 0; -} - -static inline void atomic_unlock(irqstate_t flags) -{ - spin_unlock_notrace(&g_atomic_lock); - sched_unlock(); -} -#else -static inline irqstate_t atomic_lock(void) -{ - return spin_lock_irqsave(&g_atomic_lock); -} - -static inline void atomic_unlock(irqstate_t flags) -{ - spin_unlock_irqrestore(&g_atomic_lock, flags); -} -#endif - -/**************************************************************************** - * Public Functions - ****************************************************************************/ - -/**************************************************************************** - * Name: __atomic_store_8 - ****************************************************************************/ - -STORE(__atomic_store_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -STORE(atomic_store_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_load_8 - ****************************************************************************/ - -LOAD(__atomic_load_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -LOAD(atomic_load_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_exchange_8 - ****************************************************************************/ - -EXCHANGE(__atomic_exchange_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -EXCHANGE(atomic_exchange_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_compare_exchange_8 - ****************************************************************************/ - -CMP_EXCHANGE(__atomic_compare_exchange_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -CMP_EXCHANGE(atomic_compare_exchange_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_flag_test_and_set_8 - ****************************************************************************/ - -FLAG_TEST_AND_SET(__atomic_flags_test_and_set_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FLAG_TEST_AND_SET(atomic_flags_test_and_set_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_fetch_add_8 - ****************************************************************************/ - -FETCH_ADD(__atomic_fetch_add_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FETCH_ADD(atomic_fetch_add_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_fetch_sub_8 - ****************************************************************************/ - -FETCH_SUB(__atomic_fetch_sub_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FETCH_SUB(atomic_fetch_sub_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_fetch_and_8 - ****************************************************************************/ - -FETCH_AND(__atomic_fetch_and_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FETCH_AND(atomic_fetch_and_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_fetch_or_8 - ****************************************************************************/ - -FETCH_OR(__atomic_fetch_or_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FETCH_OR(atomic_fetch_or_, 8, int64_t) -#endif - -/**************************************************************************** - * Name: __atomic_fetch_xor_8 - ****************************************************************************/ - -FETCH_XOR(__atomic_fetch_xor_, 8, uint64_t) -#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN -FETCH_XOR(atomic_fetch_xor_, 8, int64_t) -#endif - -/* Clang define the __sync builtins, add #ifndef to avoid - * redefined/redeclared problem. - */ - -#ifndef __clang__ - -/**************************************************************************** - * Name: __sync_add_and_fetch_8 - ****************************************************************************/ - -SYNC_ADD_FETCH(__sync_add_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_sub_and_fetch_8 - ****************************************************************************/ - -SYNC_SUB_FETCH(__sync_sub_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_or_and_fetch_8 - ****************************************************************************/ - -SYNC_OR_FETCH(__sync_or_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_and_and_fetch_8 - ****************************************************************************/ - -SYNC_AND_FETCH(__sync_and_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_xor_and_fetch_8 - ****************************************************************************/ - -SYNC_XOR_FETCH(__sync_xor_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_nand_and_fetch_8 - ****************************************************************************/ - -SYNC_NAND_FETCH(__sync_nand_and_fetch_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_bool_compare_and_swap_8 - ****************************************************************************/ - -SYNC_BOOL_CMP_SWAP(__sync_bool_compare_and_swap_, 8, uint64_t) - -/**************************************************************************** - * Name: __sync_val_compare_and_swap_8 - ****************************************************************************/ - -SYNC_VAL_CMP_SWAP(__sync_val_compare_and_swap_, 8, uint64_t) - -#endif /* __clang__ */
