Module Name: src
Committed By: ad
Date: Mon Jan 6 11:12:56 UTC 2020
Modified Files:
src/sys/kern: kern_mutex.c
Log Message:
mutex_vector_enter(): avoid some unneeded reads of mtx_owner.
To generate a diff of this commit:
cvs rdiff -u -r1.86 -r1.87 src/sys/kern/kern_mutex.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_mutex.c
diff -u src/sys/kern/kern_mutex.c:1.86 src/sys/kern/kern_mutex.c:1.87
--- src/sys/kern/kern_mutex.c:1.86 Wed Dec 11 20:46:06 2019
+++ src/sys/kern/kern_mutex.c Mon Jan 6 11:12:55 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_mutex.c,v 1.86 2019/12/11 20:46:06 ad Exp $ */
+/* $NetBSD: kern_mutex.c,v 1.87 2020/01/06 11:12:55 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
#define __MUTEX_PRIVATE
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.86 2019/12/11 20:46:06 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.87 2020/01/06 11:12:55 ad Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -186,8 +186,8 @@ do { \
#define MUTEX_OWNER(owner) \
(owner & MUTEX_THREAD)
-#define MUTEX_HAS_WAITERS(mtx) \
- (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0)
+#define MUTEX_HAS_WAITERS(owner) \
+ ((owner & MUTEX_BIT_WAITERS) != 0)
#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \
do { \
@@ -209,10 +209,10 @@ do { \
(mtx)->mtx_owner = MUTEX_THREAD; \
} while (/* CONSTCOND */ 0)
-#define MUTEX_SPIN_P(mtx) \
- (((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0)
-#define MUTEX_ADAPTIVE_P(mtx) \
- (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0)
+#define MUTEX_SPIN_P(owner) \
+ (((owner) & MUTEX_BIT_SPIN) != 0)
+#define MUTEX_ADAPTIVE_P(owner) \
+ (((owner) & MUTEX_BIT_SPIN) == 0)
#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0)
#if defined(LOCKDEBUG)
@@ -310,10 +310,11 @@ static void
mutex_dump(const volatile void *cookie, lockop_printer_t pr)
{
const volatile kmutex_t *mtx = cookie;
+ uintptr_t owner = mtx->mtx_owner;
pr("owner field : %#018lx wait/spin: %16d/%d\n",
- (long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx),
- MUTEX_SPIN_P(mtx));
+ (long)MUTEX_OWNER(owner), MUTEX_HAS_WAITERS(owner),
+ MUTEX_SPIN_P(owner));
}
/*
@@ -327,7 +328,7 @@ static void __noinline
mutex_abort(const char *func, size_t line, const kmutex_t *mtx, const char *msg)
{
- LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx) ?
+ LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx->mtx_owner) ?
&mutex_spin_lockops : &mutex_adaptive_lockops), msg);
}
@@ -380,10 +381,11 @@ mutex_init(kmutex_t *mtx, kmutex_type_t
void
mutex_destroy(kmutex_t *mtx)
{
+ uintptr_t owner = mtx->mtx_owner;
- if (MUTEX_ADAPTIVE_P(mtx)) {
- MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) &&
- !MUTEX_HAS_WAITERS(mtx));
+ if (MUTEX_ADAPTIVE_P(owner)) {
+ MUTEX_ASSERT(mtx, !MUTEX_OWNED(owner) &&
+ !MUTEX_HAS_WAITERS(owner));
} else {
MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx));
}
@@ -454,7 +456,8 @@ mutex_vector_enter(kmutex_t *mtx)
/*
* Handle spin mutexes.
*/
- if (MUTEX_SPIN_P(mtx)) {
+ owner = mtx->mtx_owner;
+ if (MUTEX_SPIN_P(owner)) {
#if defined(LOCKDEBUG) && defined(MULTIPROCESSOR)
u_int spins = 0;
#endif
@@ -501,7 +504,7 @@ mutex_vector_enter(kmutex_t *mtx)
curthread = (uintptr_t)curlwp;
- MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+ MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(owner));
MUTEX_ASSERT(mtx, curthread != 0);
MUTEX_ASSERT(mtx, !cpu_intr_p());
MUTEX_WANTLOCK(mtx);
@@ -519,7 +522,7 @@ mutex_vector_enter(kmutex_t *mtx)
* then we stop spinning, and sleep instead.
*/
KPREEMPT_DISABLE(curlwp);
- for (owner = mtx->mtx_owner;;) {
+ for (;;) {
if (!MUTEX_OWNED(owner)) {
/*
* Mutex owner clear could mean two things:
@@ -667,12 +670,18 @@ mutex_vector_enter(kmutex_t *mtx)
* If the waiters bit is not set it's unsafe to go asleep,
* as we might never be awoken.
*/
- if ((membar_consumer(), mutex_oncpu(owner)) ||
- (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) {
+ membar_consumer();
+ if (mutex_oncpu(owner)) {
turnstile_exit(mtx);
owner = mtx->mtx_owner;
continue;
}
+ membar_consumer();
+ owner = mtx->mtx_owner;
+ if (!MUTEX_HAS_WAITERS(owner)) {
+ turnstile_exit(mtx);
+ continue;
+ }
#endif /* MULTIPROCESSOR */
LOCKSTAT_START_TIMER(lsflag, slptime);
@@ -707,7 +716,7 @@ mutex_vector_exit(kmutex_t *mtx)
turnstile_t *ts;
uintptr_t curthread;
- if (MUTEX_SPIN_P(mtx)) {
+ if (MUTEX_SPIN_P(mtx->mtx_owner)) {
#ifdef FULL
if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {
MUTEX_ABORT(mtx, "exiting unheld spin mutex");
@@ -748,7 +757,7 @@ mutex_vector_exit(kmutex_t *mtx)
*/
{
int s = splhigh();
- if (!MUTEX_HAS_WAITERS(mtx)) {
+ if (!MUTEX_HAS_WAITERS(mtx->mtx_owner)) {
MUTEX_RELEASE(mtx);
splx(s);
return;
@@ -810,7 +819,7 @@ mutex_owned(const kmutex_t *mtx)
if (mtx == NULL)
return 0;
- if (MUTEX_ADAPTIVE_P(mtx))
+ if (MUTEX_ADAPTIVE_P(mtx->mtx_owner))
return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp;
#ifdef FULL
return MUTEX_SPINBIT_LOCKED_P(mtx);
@@ -829,7 +838,7 @@ lwp_t *
mutex_owner(const kmutex_t *mtx)
{
- MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+ MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx->mtx_owner));
return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner);
}
@@ -846,7 +855,7 @@ mutex_owner_running(const kmutex_t *mtx)
uintptr_t owner;
bool rv;
- MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
+ MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx->mtx_owner));
kpreempt_disable();
owner = mtx->mtx_owner;
rv = !MUTEX_OWNED(owner) || mutex_oncpu(MUTEX_OWNER(owner));
@@ -887,7 +896,7 @@ mutex_tryenter(kmutex_t *mtx)
/*
* Handle spin mutexes.
*/
- if (MUTEX_SPIN_P(mtx)) {
+ if (MUTEX_SPIN_P(mtx->mtx_owner)) {
MUTEX_SPIN_SPLRAISE(mtx);
#ifdef FULL
if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
