Module Name: src
Committed By: ad
Date: Fri Apr 24 17:45:40 UTC 2009
Modified Files:
src/sys/arch/amd64/amd64: lock_stubs.S
src/sys/arch/i386/i386: lock_stubs.S
src/sys/arch/x86/x86: patch.c
Log Message:
A workaround for a bug with some Opteron revisions where locked operations
sometimes do not serve as memory barriers, allowing memory references to
bleed outside of critical sections. It's possible that this is the
reason for pkgbuild's longstanding crashiness.
This is not complete (atomic ops need some work too).
To generate a diff of this commit:
cvs rdiff -u -r1.21 -r1.22 src/sys/arch/amd64/amd64/lock_stubs.S
cvs rdiff -u -r1.22 -r1.23 src/sys/arch/i386/i386/lock_stubs.S
cvs rdiff -u -r1.17 -r1.18 src/sys/arch/x86/x86/patch.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/lock_stubs.S
diff -u src/sys/arch/amd64/amd64/lock_stubs.S:1.21 src/sys/arch/amd64/amd64/lock_stubs.S:1.22
--- src/sys/arch/amd64/amd64/lock_stubs.S:1.21 Fri Dec 5 13:08:11 2008
+++ src/sys/arch/amd64/amd64/lock_stubs.S Fri Apr 24 17:45:40 2009
@@ -1,7 +1,7 @@
-/* $NetBSD: lock_stubs.S,v 1.21 2008/12/05 13:08:11 ad Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.22 2009/04/24 17:45:40 ad Exp $ */
/*-
- * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -48,6 +48,7 @@
#define ENDLABEL(name,a) .align a; LABEL(name)
#define LOCK(num) .Lpatch/**/num: lock
+#define RET(num) .Lret/**/num: ret; nop; nop; ret
#ifndef LOCKDEBUG
@@ -64,7 +65,7 @@
LOCK(1)
cmpxchgq %rcx, (%rdi)
jnz 1f
- ret
+ RET(1)
1:
jmp _C_LABEL(mutex_vector_enter)
@@ -106,7 +107,7 @@
testb %al, %al
jnz 1f
#endif
- ret
+ RET(2)
1:
jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */
@@ -186,7 +187,7 @@
LOCK(2)
cmpxchgq %rdx, (%rdi)
jnz 1f
- ret
+ RET(3)
1:
jmp 0b
@@ -199,7 +200,7 @@
LOCK(3)
cmpxchgq %rcx, (%rdi)
jnz 3f
- ret
+ RET(4)
3:
jmp _C_LABEL(rw_vector_enter)
@@ -256,13 +257,13 @@
movq (%rdi), %rax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
- jnz 3f
+ jnz 4f
leaq RW_READ_INCR(%rax), %rdx
LOCK(8)
cmpxchgq %rdx, (%rdi)
jnz 1f
movl %edx, %eax /* nonzero */
- ret
+ RET(5)
1:
jmp 0b
@@ -276,10 +277,12 @@
cmpxchgq %rcx, (%rdi)
movl $0, %eax
setz %al
+3:
+ RET(6)
ret
-
-3: xorl %eax, %eax
- ret
+4:
+ xorl %eax, %eax
+ jmp 3b
#endif /* LOCKDEBUG */
@@ -296,7 +299,7 @@
LOCK(6)
cmpxchgb %ah, (%rdi)
jnz 2f
- ret
+ RET(7)
2:
movl $0x0100, %eax
pause
@@ -315,8 +318,8 @@
LOCK(7)
cmpxchgb %ah, (%rdi)
movl $0, %eax
- setz %al
- ret
+ setz %al
+ RET(8)
/*
* Patchpoints to replace with NOP when ncpu == 1.
@@ -328,3 +331,10 @@
.quad .Lpatch9
.quad 0
#endif
+
+LABEL(x86_retpatch)
+#ifndef LOCKDEBUG
+ .long .Lret1, .Lret2, .Lret3, .Lret4, .Lret5, .Lret6
+#endif
+ .long .Lret7, .Lret8
+ .long 0
Index: src/sys/arch/i386/i386/lock_stubs.S
diff -u src/sys/arch/i386/i386/lock_stubs.S:1.22 src/sys/arch/i386/i386/lock_stubs.S:1.23
--- src/sys/arch/i386/i386/lock_stubs.S:1.22 Fri Dec 5 13:08:11 2008
+++ src/sys/arch/i386/i386/lock_stubs.S Fri Apr 24 17:45:40 2009
@@ -1,7 +1,7 @@
-/* $NetBSD: lock_stubs.S,v 1.22 2008/12/05 13:08:11 ad Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.23 2009/04/24 17:45:40 ad Exp $ */
/*-
- * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -35,7 +35,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.22 2008/12/05 13:08:11 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.23 2009/04/24 17:45:40 ad Exp $");
#include "opt_lockdebug.h"
@@ -46,6 +46,7 @@
#define ALIGN64 .align 64
#define ALIGN32 .align 32
#define LOCK(num) .Lpatch/**/num: lock
+#define RET(num) .Lret/**/num: ret; nop; nop; ret
#define ENDLABEL(name,a) .align a; LABEL(name)
#if !defined(LOCKDEBUG)
@@ -59,12 +60,12 @@
ENTRY(mutex_enter)
movl 4(%esp), %edx
- movl CPUVAR(CURLWP), %ecx
xorl %eax, %eax
+ movl %fs:CPU_INFO_CURLWP(%eax), %ecx
LOCK(1)
cmpxchgl %ecx, (%edx)
jnz 1f
- ret
+ RET(1)
1:
jmp _C_LABEL(mutex_vector_enter)
END(mutex_enter)
@@ -80,8 +81,8 @@
*/
ENTRY(mutex_exit)
movl 4(%esp), %edx
- movl CPUVAR(CURLWP), %eax
xorl %ecx, %ecx
+ movl %fs:CPU_INFO_CURLWP(%ecx), %eax
cmpxchgl %ecx, (%edx)
jnz 1f
ret
@@ -110,20 +111,20 @@
LOCK(2)
cmpxchgl %ecx, (%edx)
jnz 1f
- ret
+ RET(2)
1:
jmp 0b
/*
* Writer
*/
-2: movl CPUVAR(CURLWP), %ecx
- xorl %eax, %eax
+2: xorl %eax, %eax
+ movl %fs:CPU_INFO_CURLWP(%eax), %ecx
orl $RW_WRITE_LOCKED, %ecx
LOCK(3)
cmpxchgl %ecx, (%edx)
jnz 3f
- ret
+ RET(3)
3:
jmp _C_LABEL(rw_vector_enter)
END(rw_enter)
@@ -187,13 +188,13 @@
movl (%edx), %eax
0:
testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
- jnz 3f
+ jnz 4f
leal RW_READ_INCR(%eax), %ecx
LOCK(12)
cmpxchgl %ecx, (%edx)
jnz 1f
movl %edx, %eax /* nonzero */
- ret
+ RET(4)
1:
jmp 0b
@@ -201,17 +202,18 @@
* Writer
*/
2:
- movl CPUVAR(CURLWP), %ecx
- orl $RW_WRITE_LOCKED, %ecx
xorl %eax, %eax
+ movl %fs:CPU_INFO_CURLWP(%eax), %ecx
+ orl $RW_WRITE_LOCKED, %ecx
LOCK(13)
cmpxchgl %ecx, (%edx)
movl $0, %eax
setz %al
- ret
3:
+ RET(5)
+4:
xorl %eax, %eax
- ret
+ jmp 3b
END(rw_tryenter)
#ifndef __XEN__
@@ -237,7 +239,7 @@
xchgb %al, MTX_LOCK(%edx) /* lock it */
testb %al, %al
jnz 3f
- ret
+ RET(6)
3:
jmp _C_LABEL(mutex_spin_retry)
@@ -326,7 +328,7 @@
LOCK(6)
cmpxchgb %ah, (%edx)
jnz 2f
- ret
+ RET(7)
2:
movl $0x0100, %eax
pause
@@ -350,7 +352,7 @@
cmpxchgb %ah, (%edx)
movl $0, %eax
setz %al
- ret
+ RET(8)
END(__cpu_simple_lock_try)
/*
@@ -365,3 +367,13 @@
.long .Lpatch13
.long 0
#endif
+
+LABEL(x86_retpatch)
+#ifndef LOCKDEBUG
+ .long .Lret1, .Lret2, .Lret3, .Lret4, .Lret5
+#ifndef __XEN__
+ .long .Lret6
+#endif
+#endif
+ .long .Lret7, .Lret8
+ .long 0
Index: src/sys/arch/x86/x86/patch.c
diff -u src/sys/arch/x86/x86/patch.c:1.17 src/sys/arch/x86/x86/patch.c:1.18
--- src/sys/arch/x86/x86/patch.c:1.17 Thu Apr 2 00:19:03 2009
+++ src/sys/arch/x86/x86/patch.c Fri Apr 24 17:45:40 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.17 2009/04/02 00:19:03 enami Exp $ */
+/* $NetBSD: patch.c,v 1.18 2009/04/24 17:45:40 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.17 2009/04/02 00:19:03 enami Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.18 2009/04/24 17:45:40 ad Exp $");
#include "opt_lockdebug.h"
@@ -74,6 +74,7 @@
void _atomic_cas_cx8_end(void);
extern void *x86_lockpatch[];
+extern void *x86_retpatch[];
extern void *atomic_lockpatch[];
#define X86_NOP 0x90
@@ -124,12 +125,14 @@
}
static inline void __unused
-patchbytes(void *addr, const int byte1, const int byte2)
+patchbytes(void *addr, const int byte1, const int byte2, const int byte3)
{
((uint8_t *)addr)[0] = (uint8_t)byte1;
if (byte2 != -1)
((uint8_t *)addr)[1] = (uint8_t)byte2;
+ if (byte3 != -1)
+ ((uint8_t *)addr)[2] = (uint8_t)byte3;
}
void
@@ -138,6 +141,7 @@
static bool first, second;
u_long psl;
u_long cr0;
+ int i;
if (early) {
if (first)
@@ -160,13 +164,11 @@
#if !defined(GPROF)
if (!early && ncpu == 1) {
#ifndef LOCKDEBUG
- int i;
-
/* Uniprocessor: kill LOCK prefixes. */
for (i = 0; x86_lockpatch[i] != 0; i++)
- patchbytes(x86_lockpatch[i], X86_NOP, -1);
+ patchbytes(x86_lockpatch[i], X86_NOP, -1, -1);
for (i = 0; atomic_lockpatch[i] != 0; i++)
- patchbytes(atomic_lockpatch[i], X86_NOP, -1);
+ patchbytes(atomic_lockpatch[i], X86_NOP, -1, -1);
#endif /* !LOCKDEBUG */
}
if (!early && (cpu_feature & CPUID_SSE2) != 0) {
@@ -214,6 +216,21 @@
#endif /* !LOCKDEBUG */
}
+ /*
+ * On some Opteron revisions, locked operations erroneously
+ * allow memory references to be `bled' outside of critical
+ * sections. Apply workaround.
+ */
+ if (cpu_vendor == CPUVENDOR_AMD &&
+ (CPUID2FAMILY(cpu_info_primary.ci_signature) == 0xe ||
+ (CPUID2FAMILY(cpu_info_primary.ci_signature) == 0xf &&
+ CPUID2EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) {
+ for (i = 0; x86_retpatch[i] != 0; i++) {
+ /* ret,nop,nop,ret -> lfence,ret */
+ patchbytes(x86_retpatch[i], 0x0f, 0xae, 0xe8);
+ }
+ }
+
/* Write back and invalidate cache, flush pipelines. */
wbinvd();
x86_flush();
