Module Name: src
Committed By: ad
Date: Sat Nov 23 16:36:39 UTC 2019
Modified Files:
src/sys/arch/amd64/amd64: lock_stubs.S
src/sys/arch/i386/i386: lock_stubs.S
Log Message:
For this case during build.sh:
rw_enter(lock, RW_READ);
Having instrumented it, it turns out that >99.5% of the time the lock is
completely unknowned. Make this assumption in the assembly stub for
rw_enter(), and avoid the initial read of the lock word. Where there are
existing read holds, we'll do an additional CMPXCHG but should already have
the cache line in the EXCLUSIVE state.
To generate a diff of this commit:
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/amd64/amd64/lock_stubs.S
cvs rdiff -u -r1.30 -r1.31 src/sys/arch/i386/i386/lock_stubs.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/lock_stubs.S
diff -u src/sys/arch/amd64/amd64/lock_stubs.S:1.33 src/sys/arch/amd64/amd64/lock_stubs.S:1.34
--- src/sys/arch/amd64/amd64/lock_stubs.S:1.33 Thu Nov 14 16:23:52 2019
+++ src/sys/arch/amd64/amd64/lock_stubs.S Sat Nov 23 16:36:38 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.S,v 1.33 2019/11/14 16:23:52 maxv Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.34 2019/11/23 16:36:38 ad Exp $ */
/*
* Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -191,13 +191,15 @@ END(mutex_spin_exit)
* Acquire one hold on a RW lock.
*/
ENTRY(rw_enter)
- cmpl $RW_READER, %esi
+ xorl %eax, %eax
+ testl %esi, %esi /* RW_READER = 0 */
jne 2f
/*
- * Reader: this is the most common case.
+ * Reader, and no existing readers on the lock: this is a most
+ * common case. Instead of reading from the lock word, use cmpxchg
+ * and get the cache line into the EXCLUSIVE state to begin with.
*/
- movq (%rdi), %rax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
jnz 3f
@@ -213,7 +215,6 @@ ENTRY(rw_enter)
* Writer: if the compare-and-set fails, don't bother retrying.
*/
2: movq CPUVAR(CURLWP), %rcx
- xorq %rax, %rax
orq $RW_WRITE_LOCKED, %rcx
LOCK
cmpxchgq %rcx, (%rdi)
@@ -268,13 +269,15 @@ END(rw_exit)
* Try to acquire one hold on a RW lock.
*/
ENTRY(rw_tryenter)
- cmpl $RW_READER, %esi
+ xorl %eax, %eax
+ testl %esi, %esi /* RW_READER = 0 */
jne 2f
/*
- * Reader: this is the most common case.
+ * Reader, and no existing readers on the lock: this is a most
+ * common case. Instead of reading from the lock word, use cmpxchg
+ * and get the cache line into the EXCLUSIVE state to begin with.
*/
- movq (%rdi), %rax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
jnz 4f
@@ -291,7 +294,6 @@ ENTRY(rw_tryenter)
* Writer: if the compare-and-set fails, don't bother retrying.
*/
2: movq CPUVAR(CURLWP), %rcx
- xorq %rax, %rax
orq $RW_WRITE_LOCKED, %rcx
LOCK
cmpxchgq %rcx, (%rdi)
Index: src/sys/arch/i386/i386/lock_stubs.S
diff -u src/sys/arch/i386/i386/lock_stubs.S:1.30 src/sys/arch/i386/i386/lock_stubs.S:1.31
--- src/sys/arch/i386/i386/lock_stubs.S:1.30 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/i386/i386/lock_stubs.S Sat Nov 23 16:36:38 2019
@@ -1,7 +1,7 @@
-/* $NetBSD: lock_stubs.S,v 1.30 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.31 2019/11/23 16:36:38 ad Exp $ */
/*-
- * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
+ * Copyright (c) 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -35,7 +35,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.30 2019/02/11 14:59:32 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.31 2019/11/23 16:36:38 ad Exp $");
#include "opt_lockdebug.h"
@@ -104,13 +104,15 @@ END(mutex_exit)
*/
ENTRY(rw_enter)
movl 4(%esp), %edx
+ xorl %eax, %eax
cmpl $RW_READER, 8(%esp)
jne 2f
/*
- * Reader
+ * Reader, and no existing readers on the lock: this is a most
+ * common case. Instead of reading from the lock word, use cmpxchg
+ * and get the cache line into the EXCLUSIVE state to begin with.
*/
- movl (%edx), %eax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
jnz 3f
@@ -125,7 +127,7 @@ ENTRY(rw_enter)
/*
* Writer
*/
-2: xorl %eax, %eax
+2:
movl %fs:CPU_INFO_CURLWP(%eax), %ecx
orl $RW_WRITE_LOCKED, %ecx
LOCK(3)
@@ -186,13 +188,15 @@ END(rw_exit)
*/
ENTRY(rw_tryenter)
movl 4(%esp), %edx
+ xorl %eax, %eax
cmpl $RW_READER, 8(%esp)
jne 2f
/*
- * Reader
+ * Reader, and no existing readers on the lock: this is a most
+ * common case. Instead of reading from the lock word, use cmpxchg
+ * and get the cache line into the EXCLUSIVE state to begin with.
*/
- movl (%edx), %eax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
jnz 4f
@@ -209,7 +213,6 @@ ENTRY(rw_tryenter)
* Writer
*/
2:
- xorl %eax, %eax
movl %fs:CPU_INFO_CURLWP(%eax), %ecx
orl $RW_WRITE_LOCKED, %ecx
LOCK(13)
