Author: mjg
Date: Sun Feb 5 09:54:16 2017
New Revision: 313280
URL: https://svnweb.freebsd.org/changeset/base/313280
Log:
sx: move lockstat handling out of inline primitives
See r313275 for details.
Modified:
head/sys/kern/kern_sx.c
head/sys/sys/sx.h
Modified: head/sys/kern/kern_sx.c
==============================================================================
--- head/sys/kern/kern_sx.c Sun Feb 5 09:53:13 2017 (r313279)
+++ head/sys/kern/kern_sx.c Sun Feb 5 09:54:16 2017 (r313280)
@@ -310,6 +310,7 @@ sx_try_slock_(struct sx *sx, const char
int
_sx_xlock(struct sx *sx, int opts, const char *file, int line)
{
+ uintptr_t tid, x;
int error = 0;
if (SCHEDULER_STOPPED())
@@ -321,7 +322,13 @@ _sx_xlock(struct sx *sx, int opts, const
("sx_xlock() of destroyed sx @ %s:%d", file, line));
WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
line, NULL);
- error = __sx_xlock(sx, curthread, opts, file, line);
+ tid = (uintptr_t)curthread;
+ x = SX_LOCK_UNLOCKED;
+ if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
+ error = _sx_xlock_hard(sx, x, tid, opts, file, line);
+ else
+ LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+ 0, 0, file, line, LOCKSTAT_WRITER);
if (!error) {
LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
file, line);
@@ -379,7 +386,7 @@ _sx_xunlock(struct sx *sx, const char *f
WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
line);
- __sx_xunlock(sx, curthread, file, line);
+ _sx_xunlock_hard(sx, (uintptr_t)curthread, file, line);
TD_LOCKS_DEC(curthread);
}
@@ -757,8 +764,13 @@ _sx_xunlock_hard(struct sx *sx, uintptr_
MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
- /* If the lock is recursed, then unrecurse one level. */
- if (sx_xlocked(sx) && sx_recursed(sx)) {
+ if (!sx_recursed(sx)) {
+ LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx,
+ LOCKSTAT_WRITER);
+ if (atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
+ return;
+ } else {
+ /* The lock is recursed, unrecurse one level. */
if ((--sx->sx_recurse) == 0)
atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
if (LOCK_LOG_TEST(&sx->lock_object, 0))
Modified: head/sys/sys/sx.h
==============================================================================
--- head/sys/sys/sx.h Sun Feb 5 09:53:13 2017 (r313279)
+++ head/sys/sys/sx.h Sun Feb 5 09:54:16 2017 (r313280)
@@ -145,21 +145,19 @@ struct sx_args {
* deferred to 'tougher' functions.
*/
+#if (LOCK_DEBUG == 0) && !defined(SX_NOINLINE)
/* Acquire an exclusive lock. */
static __inline int
__sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
int line)
{
uintptr_t tid = (uintptr_t)td;
- uintptr_t v;
+ uintptr_t v = SX_LOCK_UNLOCKED;
int error = 0;
- v = SX_LOCK_UNLOCKED;
- if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid))
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
+ !atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid)))
error = _sx_xlock_hard(sx, v, tid, opts, file, line);
- else
- LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
- 0, 0, file, line, LOCKSTAT_WRITER);
return (error);
}
@@ -170,12 +168,11 @@ __sx_xunlock(struct sx *sx, struct threa
{
uintptr_t tid = (uintptr_t)td;
- if (sx->sx_recurse == 0)
- LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx,
- LOCKSTAT_WRITER);
- if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
+ !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)))
_sx_xunlock_hard(sx, tid, file, line);
}
+#endif
/*
* Public interface for lock operations.
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
