Author: mjg
Date: Wed Nov 22 23:52:05 2017
New Revision: 326110
URL: https://svnweb.freebsd.org/changeset/base/326110
Log:
rwlock: don't check for curthread's read lock count in the fast path
Modified:
head/sys/kern/kern_rwlock.c
Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Wed Nov 22 23:10:20 2017 (r326109)
+++ head/sys/kern/kern_rwlock.c Wed Nov 22 23:52:05 2017 (r326110)
@@ -364,12 +364,20 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *
* is unlocked and has no writer waiters or spinners. Failing otherwise
* prioritizes writers before readers.
*/
-#define RW_CAN_READ(td, _rw)
\
- (((_rw) & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER))
==\
- RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ))
+static bool __always_inline
+__rw_can_read(struct thread *td, uintptr_t v, bool fp)
+{
+ if ((v & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER))
+ == RW_LOCK_READ)
+ return (true);
+ if (!fp && td->td_rw_rlocks && (v & RW_LOCK_READ))
+ return (true);
+ return (false);
+}
+
static bool __always_inline
-__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp
+__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp
LOCK_FILE_LINE_ARG_DEF)
{
@@ -383,7 +391,7 @@ __rw_rlock_try(struct rwlock *rw, struct thread *td, u
* completely unlocked rwlock since such a lock is encoded
* as a read lock with no waiters.
*/
- while (RW_CAN_READ(td, *vp)) {
+ while (__rw_can_read(td, *vp, fp)) {
if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
*vp + RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
@@ -452,7 +460,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td,
#endif
for (;;) {
- if (__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG))
+ if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
@@ -492,7 +500,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td,
n = RW_READERS(v);
lock_delay_spin(n);
v = RW_READ_VALUE(rw);
- if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td,
v))
+ if ((v & RW_LOCK_READ) == 0 ||
__rw_can_read(td, v, false))
break;
}
#ifdef KDTRACE_HOOKS
@@ -518,7 +526,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td,
* recheck its state and restart the loop if needed.
*/
v = RW_READ_VALUE(rw);
- if (RW_CAN_READ(td, v)) {
+ if (__rw_can_read(td, v, false)) {
turnstile_cancel(ts);
continue;
}
@@ -630,7 +638,7 @@ __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DE
v = RW_READ_VALUE(rw);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
- !__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG)))
+ !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG)))
__rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
