This is an automated email from the ASF dual-hosted git repository. xiaoxiang781216 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit fccaaa5cfc6f46b173674038c695450dc2a45a21 Author: ouyangxiangzhen <[email protected]> AuthorDate: Mon Nov 9 21:09:06 2026 +0800 spinlock: add lock wait policy The WFE/SEV lock waiting made every unlock pay for a DSB before SEV, a full barrier on the unlock hot path. Measured with the ostest spinlock loop on qemu-armv8a (4 CPUs), an unconditional DSB/SEV cost 155.9 -> 11.3 M ops/s single-threaded and 26.9 -> 3.5 M ops/s with 4 threads versus busy wait, and the trade-off differs per system. Add a lock wait policy choice and route the spinlock paths through SPIN_WAIT() in the spin loop and SPIN_KICK() after the release store (9 call sites). SPINWAIT_RELAX (default) waits with the UP_RELAX() relax hint and never kicks: the unlock stays a plain release store. Where an architecture defines no hint, the tier degenerates to a plain busy wait, which makes a separate NOP tier redundant. SPINWAIT_PM (default when CONFIG_PM is enabled) waits with WFE() and wakes with SEV(): the DSB before SEV is carried by the Arm UP_SEV() (see the previous commit, ARM DDI 0406C), so no extra barrier is emitted here -- an explicit UP_DSB() would double the barrier on Arm and pay an mfence per unlock on x86_64 (UP_DSB=mfence, UP_SEV empty). Evidence: the qemu-armv8a numbers above; under SPINWAIT_RELAX the arm64 unlock path contains no SEV at all, while armv7-a with SPINWAIT_PM keeps the WFE/SEV waiting. Assisted-By: mimo-v2.6-pro Signed-off-by: ouyangxiangzhen <[email protected]> --- include/nuttx/spinlock.h | 31 ++++++++++++++++++++++--------- sched/Kconfig | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/include/nuttx/spinlock.h b/include/nuttx/spinlock.h index c4bac30e2cf..0ab876633a6 100644 --- a/include/nuttx/spinlock.h +++ b/include/nuttx/spinlock.h @@ -54,6 +54,19 @@ extern "C" # define SPINLOCK_INLINE static inline_function #endif +/* Lock wait policy, see "Spinlock wait policy" in sched/Kconfig. + * SPIN_WAIT() runs in the spin loop, SPIN_KICK() after the lock + * release store to wake the waiters. + */ + +#ifdef CONFIG_SPINWAIT_PM +# define SPIN_WAIT() UP_WFE() +# define SPIN_KICK() UP_SEV() +#else /* SPINWAIT_RELAX */ +# define SPIN_WAIT() UP_RELAX() +# define SPIN_KICK() +#endif + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ @@ -149,12 +162,12 @@ void spin_lock_notrace(FAR volatile spinlock_t *lock) int32_t ticket = atomic_add_relaxed(&lock->next, 1); while (atomic_read_acquire(&lock->owner) != ticket) { - UP_WFE(); + SPIN_WAIT(); } #else /* CONFIG_TICKET_SPINLOCK */ while (atomic_xchg_acquire((FAR atomic_t *)lock, SP_LOCKED) == SP_LOCKED) { - UP_WFE(); + SPIN_WAIT(); } #endif } @@ -338,7 +351,7 @@ spin_unlock_notrace(FAR volatile spinlock_t *lock) #else atomic_set_release((FAR atomic_t *)lock, SP_UNLOCKED); #endif - UP_SEV(); + SPIN_KICK(); } #else # define spin_unlock_notrace(lock) @@ -590,7 +603,7 @@ void rspin_lock(FAR rspinlock_t *lock) break; } - UP_WFE(); + SPIN_WAIT(); } /* The local value shares the same memory location with the @@ -923,7 +936,7 @@ bool rspin_unlock(FAR rspinlock_t *lock) if (RSPINLOCK_COUNT(val) == 1u) { atomic_set_release(&lock->val, 0u); - UP_SEV(); + SPIN_KICK(); ret = true; } else @@ -1054,7 +1067,7 @@ SPINLOCK_INLINE void read_lock(FAR volatile rwlock_t *lock) if (old == RW_SP_WRITE_LOCKED) { - UP_WFE(); + SPIN_WAIT(); } } } @@ -1137,7 +1150,7 @@ static inline_function void read_unlock(FAR volatile rwlock_t *lock) DEBUGASSERT(atomic_read(lock) >= RW_SP_READ_LOCKED); atomic_sub_release(lock, 1); - UP_SEV(); + SPIN_KICK(); } /**************************************************************************** @@ -1180,7 +1193,7 @@ SPINLOCK_INLINE void write_lock(FAR volatile rwlock_t *lock) break; } - UP_WFE(); + SPIN_WAIT(); } nxsched_critmon_busywait(false, return_address(0)); @@ -1246,7 +1259,7 @@ static inline_function void write_unlock(FAR volatile rwlock_t *lock) DEBUGASSERT(atomic_read(lock) == RW_SP_WRITE_LOCKED); atomic_set_release(lock, RW_SP_UNLOCKED); - UP_SEV(); + SPIN_KICK(); } /**************************************************************************** diff --git a/sched/Kconfig b/sched/Kconfig index 57c00029c12..61e88756f24 100644 --- a/sched/Kconfig +++ b/sched/Kconfig @@ -306,6 +306,39 @@ config SPINLOCK_INLINE this option to emit callable non-inline versions and reduce header-inline code growth. +choice + prompt "Spinlock wait policy" + default SPINWAIT_PM if PM + default SPINWAIT_RELAX + +config SPINWAIT_RELAX + bool "Relax hint wait" + ---help--- + Spinlock waiters poll the lock with the architecture relax + hint (PAUSE on x86_64, YIELD on aarch64) and unlocking sends + no wake event. The hint reduces bus and pipeline + contention without sleeping, and the unlock stays a plain + release store. + +config SPINWAIT_PM + bool "WFE/SEV wait (power saving)" + ---help--- + Spinlock waiters may sleep in the spin loop (WFE on Arm) + and unlocking sends a wake event (DSB+SEV). Waiting CPUs + can enter a low-power state instead of hammering the + interconnect while a lock is held, at the cost of a more + expensive unlock. + + On Arm, SEV is not ordered with memory accesses, so the + unlock needs a DSB before SEV to make the release store + visible before the event (ARM DDI 0406C, "Use of WFE and + SEV with Locks"); expect lower lock/unlock throughput. + + On architectures without WFE/SEV semantics this policy + degrades to busy wait. + +endchoice + endif # SPINLOCK config IRQCHAIN
