Extend the contended_release tracepoint to queued rwlocks, using the same out-of-line traced unlock approach as queued spinlocks.
Signed-off-by: Dmitry Ilvokhin <[email protected]> --- include/asm-generic/qrwlock.h | 22 ++++++++++++++++++++++ kernel/locking/qrwlock.c | 16 ++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h index 4b627bafba8b..274c19006125 100644 --- a/include/asm-generic/qrwlock.h +++ b/include/asm-generic/qrwlock.h @@ -14,6 +14,7 @@ #define __ASM_GENERIC_QRWLOCK_H #include <linux/atomic.h> +#include <linux/tracepoint-defs.h> #include <asm/barrier.h> #include <asm/processor.h> @@ -35,6 +36,10 @@ */ extern void queued_read_lock_slowpath(struct qrwlock *lock); extern void queued_write_lock_slowpath(struct qrwlock *lock); +extern void queued_read_unlock_traced(struct qrwlock *lock); +extern void queued_write_unlock_traced(struct qrwlock *lock); + +DECLARE_TRACEPOINT(contended_release); /** * queued_read_trylock - try to acquire read lock of a queued rwlock @@ -115,6 +120,17 @@ static __always_inline void __queued_read_unlock(struct qrwlock *lock) */ static inline void queued_read_unlock(struct qrwlock *lock) { + /* + * Trace and unlock are combined in the traced unlock variant so + * the compiler does not need to preserve the lock pointer across + * the function call, avoiding callee-saved register save/restore + * on the hot path. + */ + if (tracepoint_enabled(contended_release)) { + queued_read_unlock_traced(lock); + return; + } + __queued_read_unlock(lock); } @@ -129,6 +145,12 @@ static __always_inline void __queued_write_unlock(struct qrwlock *lock) */ static inline void queued_write_unlock(struct qrwlock *lock) { + /* See comment in queued_read_unlock(). */ + if (tracepoint_enabled(contended_release)) { + queued_write_unlock_traced(lock); + return; + } + __queued_write_unlock(lock); } diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c index d2ef312a8611..5ae4b0372719 100644 --- a/kernel/locking/qrwlock.c +++ b/kernel/locking/qrwlock.c @@ -90,3 +90,19 @@ void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) trace_contention_end(lock, 0); } EXPORT_SYMBOL(queued_write_lock_slowpath); + +void __lockfunc queued_read_unlock_traced(struct qrwlock *lock) +{ + if (queued_rwlock_is_contended(lock)) + trace_call__contended_release(lock); + __queued_read_unlock(lock); +} +EXPORT_SYMBOL(queued_read_unlock_traced); + +void __lockfunc queued_write_unlock_traced(struct qrwlock *lock) +{ + if (queued_rwlock_is_contended(lock)) + trace_call__contended_release(lock); + __queued_write_unlock(lock); +} +EXPORT_SYMBOL(queued_write_unlock_traced); -- 2.52.0
