To save text thus and thus avoid cache misses in the worst-case path, this patch moves the slow path of xnlock_get out-of-line.
--- include/asm-generic/bits/pod.h | 18 ++++++++++++++++++ include/asm-generic/system.h | 29 ++++++++++++++--------------- 2 files changed, 32 insertions(+), 15 deletions(-) Index: b/include/asm-generic/bits/pod.h =================================================================== --- a/include/asm-generic/bits/pod.h +++ b/include/asm-generic/bits/pod.h @@ -295,4 +295,22 @@ unsigned long long xnarch_get_cpu_time(v EXPORT_SYMBOL(xnarch_get_cpu_time); +#ifdef CONFIG_SMP +void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS) +{ + unsigned int spin_limit; + int cpu = xnarch_current_cpu(); + + xnlock_dbg_prepare_spin(&spin_limit); + + while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0) + do { + cpu_relax(); + xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */ + XNLOCK_DBG_PASS_CONTEXT); + } while(atomic_read(&lock->owner) != ~0); +} +EXPORT_SYMBOL(__xnlock_spin); +#endif /* CONFIG_SMP */ + #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */ Index: b/include/asm-generic/system.h =================================================================== --- a/include/asm-generic/system.h +++ b/include/asm-generic/system.h @@ -130,10 +130,13 @@ typedef struct { #define XNLOCK_DBG_PASS_CONTEXT , file, line, function #define XNLOCK_DBG_MAX_SPINS 10000000 -static inline void -xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit) +static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { *start = rthal_rdtsc(); +} + +static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit) +{ *spin_limit = XNLOCK_DBG_MAX_SPINS; } @@ -204,15 +207,14 @@ typedef struct { atomic_t owner; } xnloc #define XNLOCK_DBG_CONTEXT_ARGS #define XNLOCK_DBG_PASS_CONTEXT -static inline void -xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit) -{ } +static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { } +static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit) { } static inline void -xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { } +xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit) { } static inline void -xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { } +xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start) { } static inline int xnlock_dbg_release(xnlock_t *lock) { @@ -321,23 +323,20 @@ static inline void xnlock_init (xnlock_t #define DEFINE_XNLOCK(lock) xnlock_t lock = XNARCH_LOCK_UNLOCKED #define DEFINE_PRIVATE_XNLOCK(lock) static DEFINE_XNLOCK(lock) +void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS); + static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS) { unsigned long long start; - unsigned int spin_limit; int cpu = xnarch_current_cpu(); if (atomic_read(&lock->owner) == cpu) return 1; - xnlock_dbg_prepare_acquire(&start, &spin_limit); + xnlock_dbg_prepare_acquire(&start); - while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0) - do { - cpu_relax(); - xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */ - XNLOCK_DBG_PASS_CONTEXT); - } while(atomic_read(&lock->owner) != ~0); + if (unlikely(atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)) + __xnlock_spin(lock /*, */ XNLOCK_DBG_PASS_CONTEXT); xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
signature.asc
Description: OpenPGP digital signature
_______________________________________________ Xenomai-core mailing list Xenomai-core@gna.org https://mail.gna.org/listinfo/xenomai-core