Module Name: src
Committed By: ad
Date: Fri Jan 17 20:26:22 UTC 2020
Modified Files:
src/sys/kern: kern_lock.c
Log Message:
kernel_lock:
- Defer setting ci_biglock_wanted for a bit, because if curlwp holds a mutex
or rwlock, and otherlwp is spinning waiting for the mutex/rwlock, setting
ci_biglock_wanted causes otherlwp to block to avoid deadlock. If the spin
on kernel_lock is short there's no point causing trouble.
- Do exponential backoff.
- Put the spinout check under LOCKDEBUG to match the others.
To generate a diff of this commit:
cvs rdiff -u -r1.164 -r1.165 src/sys/kern/kern_lock.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_lock.c
diff -u src/sys/kern/kern_lock.c:1.164 src/sys/kern/kern_lock.c:1.165
--- src/sys/kern/kern_lock.c:1.164 Tue Dec 3 15:20:59 2019
+++ src/sys/kern/kern_lock.c Fri Jan 17 20:26:22 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $ */
+/* $NetBSD: kern_lock.c,v 1.165 2020/01/17 20:26:22 ad Exp $ */
/*-
- * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
+ * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.165 2020/01/17 20:26:22 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -164,7 +164,10 @@ _kernel_lock(int nlocks)
LOCKSTAT_TIMER(spintime);
LOCKSTAT_FLAG(lsflag);
struct lwp *owant;
- u_int spins;
+ u_int count;
+#ifdef LOCKDEBUG
+ u_int spins = 0;
+#endif
int s;
struct lwp *l = curlwp;
@@ -184,7 +187,7 @@ _kernel_lock(int nlocks)
LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS,
0);
- if (__cpu_simple_lock_try(kernel_lock)) {
+ if (__predict_true(__cpu_simple_lock_try(kernel_lock))) {
ci->ci_biglock_count = nlocks;
l->l_blcnt = nlocks;
LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
@@ -200,10 +203,13 @@ _kernel_lock(int nlocks)
* is required to ensure that the result of any mutex_exit()
* by the current LWP becomes visible on the bus before the set
* of ci->ci_biglock_wanted becomes visible.
+ *
+ * However, we won't set ci_biglock_wanted until we've spun for
+ * a bit, as we don't want to make any lock waiters in rw_oncpu()
+ * or mutex_oncpu() block prematurely.
*/
membar_producer();
owant = ci->ci_biglock_wanted;
- ci->ci_biglock_wanted = l;
/*
* Spin until we acquire the lock. Once we have it, record the
@@ -212,23 +218,30 @@ _kernel_lock(int nlocks)
LOCKSTAT_ENTER(lsflag);
LOCKSTAT_START_TIMER(lsflag, spintime);
- spins = 0;
+ count = SPINLOCK_BACKOFF_MIN;
do {
splx(s);
while (__SIMPLELOCK_LOCKED_P(kernel_lock)) {
+#ifdef LOCKDEBUG
if (SPINLOCK_SPINOUT(spins)) {
extern int start_init_exec;
if (!start_init_exec)
_KERNEL_LOCK_ABORT("spinout");
}
- SPINLOCK_BACKOFF_HOOK;
- SPINLOCK_SPIN_HOOK;
+#endif
+ SPINLOCK_BACKOFF(count);
+ if (count == SPINLOCK_BACKOFF_MAX) {
+ /* Ok, waiting for real. */
+ ci->ci_biglock_wanted = l;
+ }
}
s = splvm();
} while (!__cpu_simple_lock_try(kernel_lock));
ci->ci_biglock_count = nlocks;
l->l_blcnt = nlocks;
+ splx(s);
+
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
RETURN_ADDRESS, 0);
@@ -237,7 +250,6 @@ _kernel_lock(int nlocks)
LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS);
}
LOCKSTAT_EXIT(lsflag);
- splx(s);
/*
* Now that we have kernel_lock, reset ci_biglock_wanted. This
@@ -257,7 +269,9 @@ _kernel_lock(int nlocks)
* prevents stores from a following mutex_exit() being reordered
* to occur before our store to ci_biglock_wanted above.
*/
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_enter();
+#endif
}
/*
