Author: kib
Date: Thu Jul 30 00:13:20 2015
New Revision: 286051
URL: https://svnweb.freebsd.org/changeset/base/286051
Log:
Use private cache line for the locked nop in *mb() on i386.
Suggested by: alc
Reviewed by: alc, bde
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Modified:
head/sys/i386/i386/vm_machdep.c
head/sys/i386/include/atomic.h
Modified: head/sys/i386/i386/vm_machdep.c
==============================================================================
--- head/sys/i386/i386/vm_machdep.c Wed Jul 29 23:59:17 2015
(r286050)
+++ head/sys/i386/i386/vm_machdep.c Thu Jul 30 00:13:20 2015
(r286051)
@@ -111,8 +111,8 @@ _Static_assert(OFFSETOF_CURTHREAD == off
"OFFSETOF_CURTHREAD does not correspond with offset of pc_curthread.");
_Static_assert(OFFSETOF_CURPCB == offsetof(struct pcpu, pc_curpcb),
"OFFSETOF_CURPCB does not correspond with offset of pc_curpcb.");
-_Static_assert(OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf),
- "OFFSETOF_MONINORBUF does not correspond with offset of pc_monitorbuf.");
+_Static_assert(__OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf),
+ "__OFFSETOF_MONINORBUF does not correspond with offset of pc_monitorbuf.");
static void cpu_reset_real(void);
#ifdef SMP
Modified: head/sys/i386/include/atomic.h
==============================================================================
--- head/sys/i386/include/atomic.h Wed Jul 29 23:59:17 2015
(r286050)
+++ head/sys/i386/include/atomic.h Thu Jul 30 00:13:20 2015
(r286051)
@@ -37,9 +37,31 @@
#include <machine/specialreg.h>
#endif
-#define mb() __asm __volatile("lock; addl $0,(%%esp)" : : :
"memory", "cc")
-#define wmb() __asm __volatile("lock; addl $0,(%%esp)" : : :
"memory", "cc")
-#define rmb() __asm __volatile("lock; addl $0,(%%esp)" : : :
"memory", "cc")
+#ifndef __OFFSETOF_MONITORBUF
+/*
+ * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
+ *
+ * The open-coded number is used instead of the symbolic expression to
+ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
+ * An assertion in i386/vm_machdep.c ensures that the value is correct.
+ */
+#define __OFFSETOF_MONITORBUF 0x180
+
+static __inline void
+__mbk(void)
+{
+
+ __asm __volatile("lock; addl $0,%%fs:%0"
+ : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc");
+}
+
+static __inline void
+__mbu(void)
+{
+
+ __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc");
+}
+#endif
/*
* Various simple operations on memory, each of which is atomic in the
@@ -246,40 +268,15 @@ atomic_testandset_int(volatile u_int *p,
* reordering accesses in a way that violates the semantics of acquire
* and release.
*/
-#if defined(_KERNEL)
-
-/*
- * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
- *
- * The open-coded number is used instead of the symbolic expression to
- * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
- * An assertion in i386/vm_machdep.c ensures that the value is correct.
- */
-#define OFFSETOF_MONITORBUF 0x180
+#if defined(_KERNEL)
#if defined(SMP)
-static __inline void
-__storeload_barrier(void)
-{
-
- __asm __volatile("lock; addl $0,%%fs:%0"
- : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc");
-}
+#define __storeload_barrier() __mbk()
#else /* _KERNEL && UP */
-static __inline void
-__storeload_barrier(void)
-{
-
- __compiler_membar();
-}
+#define __storeload_barrier() __compiler_membar()
#endif /* SMP */
#else /* !_KERNEL */
-static __inline void
-__storeload_barrier(void)
-{
-
- __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc");
-}
+#define __storeload_barrier() __mbu()
#endif /* _KERNEL*/
#define ATOMIC_LOAD(TYPE) \
@@ -776,4 +773,14 @@ u_long atomic_swap_long(volatile u_long
#endif /* !WANT_FUNCTIONS */
+#if defined(_KERNEL)
+#define mb() __mbk()
+#define wmb() __mbk()
+#define rmb() __mbk()
+#else
+#define mb() __mbu()
+#define wmb() __mbu()
+#define rmb() __mbu()
+#endif
+
#endif /* !_MACHINE_ATOMIC_H_ */
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
