Module Name: src
Committed By: riastradh
Date: Sat Apr 9 12:07:01 UTC 2022
Modified Files:
src/common/lib/libc/arch/i386/atomic: atomic.S
src/common/lib/libc/arch/x86_64/atomic: atomic.S
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/i386/include: frameasm.h
src/sys/arch/x86/x86: patch.c
Log Message:
x86: Every load is a load-acquire, so membar_consumer is a noop.
lfence is only needed for MD logic, such as operations on I/O memory
rather than normal cacheable memory, or special instructions like
RDTSC -- never for MI synchronization between threads/CPUs. No need
for hot-patching to do lfence here.
(The x86_lfence function might reasonably be patched on i386 to do
lfence for MD logic, but it isn't now and this doesn't change that.)
To generate a diff of this commit:
cvs rdiff -u -r1.30 -r1.31 src/common/lib/libc/arch/i386/atomic/atomic.S
cvs rdiff -u -r1.23 -r1.24 src/common/lib/libc/arch/x86_64/atomic/atomic.S
cvs rdiff -u -r1.53 -r1.54 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/i386/include/frameasm.h
cvs rdiff -u -r1.49 -r1.50 src/sys/arch/x86/x86/patch.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/arch/i386/atomic/atomic.S
diff -u src/common/lib/libc/arch/i386/atomic/atomic.S:1.30 src/common/lib/libc/arch/i386/atomic/atomic.S:1.31
--- src/common/lib/libc/arch/i386/atomic/atomic.S:1.30 Wed Apr 6 22:47:56 2022
+++ src/common/lib/libc/arch/i386/atomic/atomic.S Sat Apr 9 12:07:00 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.30 2022/04/06 22:47:56 riastradh Exp $ */
+/* $NetBSD: atomic.S,v 1.31 2022/04/09 12:07:00 riastradh Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -46,12 +46,10 @@
#include "opt_xen.h"
#include <machine/frameasm.h>
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7);
#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7);
#define HOTPATCH_CAS_64 HOTPATCH(HP_NAME_CAS_64, 49);
#else
#define LOCK lock
-#define HOTPATCH_SSE2_LFENCE /* nothing */
#define HOTPATCH_SSE2_MFENCE /* nothing */
#define HOTPATCH_CAS_64 /* nothing */
#endif
@@ -181,10 +179,11 @@ ENTRY(_atomic_cas_32_ni)
END(_atomic_cas_32_ni)
ENTRY(_membar_consumer)
- HOTPATCH_SSE2_LFENCE
- /* 7 bytes of instructions */
- LOCK
- addl $0, -4(%esp)
+ /*
+ * Every load from normal memory is a load-acquire on x86, so
+ * there is never any need for explicit barriers to order
+ * load-before-anything.
+ */
ret
END(_membar_consumer)
@@ -396,12 +395,6 @@ STRONG_ALIAS(_membar_exit,_membar_produc
#ifdef _HARDKERNEL
.section .rodata
-LABEL(sse2_lfence)
- lfence
- ret
- nop; nop; nop;
-LABEL(sse2_lfence_end)
-
LABEL(sse2_mfence)
mfence
ret
Index: src/common/lib/libc/arch/x86_64/atomic/atomic.S
diff -u src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.23 src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.24
--- src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.23 Wed Apr 6 22:47:57 2022
+++ src/common/lib/libc/arch/x86_64/atomic/atomic.S Sat Apr 9 12:07:00 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.23 2022/04/06 22:47:57 riastradh Exp $ */
+/* $NetBSD: atomic.S,v 1.24 2022/04/09 12:07:00 riastradh Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -41,11 +41,9 @@
#ifdef _HARDKERNEL
#include <machine/frameasm.h>
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 8);
#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8);
#else
#define LOCK lock
-#define HOTPATCH_SSE2_LFENCE /* nothing */
#define HOTPATCH_SSE2_MFENCE /* nothing */
#endif
@@ -256,10 +254,11 @@ END(_atomic_cas_64_ni)
/* memory barriers */
ENTRY(_membar_consumer)
- HOTPATCH_SSE2_LFENCE
- /* 8 bytes of instructions */
- LOCK
- addq $0, -8(%rsp)
+ /*
+ * Every load from normal memory is a load-acquire on x86, so
+ * there is never any need for explicit barriers to order
+ * load-before-anything.
+ */
ret
END(_membar_consumer)
@@ -419,12 +418,6 @@ STRONG_ALIAS(_membar_exit,_membar_produc
#ifdef _HARDKERNEL
.section .rodata
-LABEL(sse2_lfence)
- lfence
- ret
- nop; nop; nop; nop;
-LABEL(sse2_lfence_end)
-
LABEL(sse2_mfence)
mfence
ret
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.53 src/sys/arch/amd64/include/frameasm.h:1.54
--- src/sys/arch/amd64/include/frameasm.h:1.53 Sat Apr 17 20:12:55 2021
+++ src/sys/arch/amd64/include/frameasm.h Sat Apr 9 12:07:00 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.53 2021/04/17 20:12:55 rillig Exp $ */
+/* $NetBSD: frameasm.h,v 1.54 2022/04/09 12:07:00 riastradh Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -63,8 +63,7 @@
#define HP_NAME_SVS_ENTER_NMI 11
#define HP_NAME_SVS_LEAVE_NMI 12
#define HP_NAME_MDS_LEAVE 13
-#define HP_NAME_SSE2_LFENCE 14
-#define HP_NAME_SSE2_MFENCE 15
+#define HP_NAME_SSE2_MFENCE 14
#define HOTPATCH(name, size) \
123: ; \
Index: src/sys/arch/i386/include/frameasm.h
diff -u src/sys/arch/i386/include/frameasm.h:1.33 src/sys/arch/i386/include/frameasm.h:1.34
--- src/sys/arch/i386/include/frameasm.h:1.33 Fri May 1 09:40:47 2020
+++ src/sys/arch/i386/include/frameasm.h Sat Apr 9 12:07:00 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.33 2020/05/01 09:40:47 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.34 2022/04/09 12:07:00 riastradh Exp $ */
#ifndef _I386_FRAMEASM_H_
#define _I386_FRAMEASM_H_
@@ -48,11 +48,10 @@
#define HP_NAME_STAC 2
#define HP_NAME_NOLOCK 3
#define HP_NAME_RETFENCE 4
-#define HP_NAME_SSE2_LFENCE 5
-#define HP_NAME_SSE2_MFENCE 6
-#define HP_NAME_CAS_64 7
-#define HP_NAME_SPLLOWER 8
-#define HP_NAME_MUTEX_EXIT 9
+#define HP_NAME_SSE2_MFENCE 5
+#define HP_NAME_CAS_64 6
+#define HP_NAME_SPLLOWER 7
+#define HP_NAME_MUTEX_EXIT 8
#define HOTPATCH(name, size) \
123: ; \
Index: src/sys/arch/x86/x86/patch.c
diff -u src/sys/arch/x86/x86/patch.c:1.49 src/sys/arch/x86/x86/patch.c:1.50
--- src/sys/arch/x86/x86/patch.c:1.49 Thu May 7 18:13:05 2020
+++ src/sys/arch/x86/x86/patch.c Sat Apr 9 12:07:00 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.49 2020/05/07 18:13:05 maxv Exp $ */
+/* $NetBSD: patch.c,v 1.50 2022/04/09 12:07:00 riastradh Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.49 2020/05/07 18:13:05 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.50 2022/04/09 12:07:00 riastradh Exp $");
#include "opt_lockdebug.h"
#ifdef i386
@@ -117,19 +117,6 @@ static const struct x86_hotpatch_descrip
};
__link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc);
-/* Use LFENCE if available, part of SSE2. */
-extern uint8_t sse2_lfence, sse2_lfence_end;
-static const struct x86_hotpatch_source hp_sse2_lfence_source = {
- .saddr = &sse2_lfence,
- .eaddr = &sse2_lfence_end
-};
-static const struct x86_hotpatch_descriptor hp_sse2_lfence_desc = {
- .name = HP_NAME_SSE2_LFENCE,
- .nsrc = 1,
- .srcs = { &hp_sse2_lfence_source }
-};
-__link_set_add_rodata(x86_hotpatch_descriptors, hp_sse2_lfence_desc);
-
/* Use MFENCE if available, part of SSE2. */
extern uint8_t sse2_mfence, sse2_mfence_end;
static const struct x86_hotpatch_source hp_sse2_mfence_source = {
@@ -342,12 +329,15 @@ x86_patch(bool early)
if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) {
/*
- * Faster memory barriers. We do not need to patch
- * membar_producer to use SFENCE because on x86
- * ordinary non-temporal stores are always issued in
- * program order to main memory and to other CPUs.
+ * Faster memory barriers. The only barrier x86 ever
+ * requires for MI synchronization between CPUs is
+ * MFENCE for store-before-load ordering; all other
+ * ordering is guaranteed already -- every load is a
+ * load-acquire and every store is a store-release.
+ *
+ * LFENCE and SFENCE are relevant only for MD logic
+ * involving I/O devices or non-temporal stores.
*/
- x86_hotpatch(HP_NAME_SSE2_LFENCE, 0);
x86_hotpatch(HP_NAME_SSE2_MFENCE, 0);
}
