Module Name: src
Committed By: maxv
Date: Sat Jun 16 17:11:13 UTC 2018
Modified Files:
src/sys/arch/x86/x86: fpu.c x86_machdep.c
Log Message:
Need IPIs when enabling eager fpu switch, to clear each fpu and get us
started. Otherwise it is possible that the first context switch on one of
the cpus will restore an invalid fpu state in the new lwp, if that lwp
had its fpu state stored on another cpu that didn't have time to do an
fpu save since eager-fpu was enabled.
Use barriers and all the related crap. The point is that we want to
ensure that no context switch occurs between [each fpu is cleared] and
[x86_fpu_eager is set to 'true'].
Also add KASSERTs.
To generate a diff of this commit:
cvs rdiff -u -r1.35 -r1.36 src/sys/arch/x86/x86/fpu.c
cvs rdiff -u -r1.116 -r1.117 src/sys/arch/x86/x86/x86_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/fpu.c
diff -u src/sys/arch/x86/x86/fpu.c:1.35 src/sys/arch/x86/x86/fpu.c:1.36
--- src/sys/arch/x86/x86/fpu.c:1.35 Sat Jun 16 05:52:17 2018
+++ src/sys/arch/x86/x86/fpu.c Sat Jun 16 17:11:13 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: fpu.c,v 1.35 2018/06/16 05:52:17 maxv Exp $ */
+/* $NetBSD: fpu.c,v 1.36 2018/06/16 17:11:13 maxv Exp $ */
/*
* Copyright (c) 2008 The NetBSD Foundation, Inc. All
@@ -96,7 +96,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.35 2018/06/16 05:52:17 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.36 2018/06/16 17:11:13 maxv Exp $");
#include "opt_multiprocessor.h"
@@ -107,6 +107,8 @@ __KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.35
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/xcall.h>
#include <machine/cpu.h>
#include <machine/cpuvar.h>
@@ -339,6 +341,8 @@ fpu_eagerrestore(struct lwp *l)
struct cpu_info *ci = curcpu();
clts();
+ KASSERT(ci->ci_fpcurlwp == NULL);
+ KASSERT(pcb->pcb_fpcpu == NULL);
ci->ci_fpcurlwp = l;
pcb->pcb_fpcpu = ci;
fpu_restore(l);
@@ -393,6 +397,10 @@ fputrap(struct trapframe *frame)
if (!USERMODE(frame->tf_cs))
panic("fpu trap from kernel, trapframe %p\n", frame);
+ if (__predict_false(x86_fpu_eager)) {
+ panic("%s: got DNA with EagerFPU enabled", __func__);
+ }
+
/*
* At this point, fpcurlwp should be curlwp. If it wasn't, the TS bit
* should be set, and we should have gotten a DNA exception.
@@ -890,3 +898,110 @@ process_read_fpregs_s87(struct lwp *l, s
memcpy(fpregs, &fpu_save->sv_87, sizeof(fpu_save->sv_87));
}
}
+
+/* -------------------------------------------------------------------------- */
+
+static volatile unsigned long eagerfpu_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long eagerfpu_cpu_barrier2 __cacheline_aligned;
+
+static void
+eager_change_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ bool enabled = (bool)arg1;
+ int s;
+
+ s = splhigh();
+
+ /* Rendez-vous 1. */
+ atomic_dec_ulong(&eagerfpu_cpu_barrier1);
+ while (atomic_cas_ulong(&eagerfpu_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ fpusave_cpu(true);
+ if (ci == &cpu_info_primary) {
+ x86_fpu_eager = enabled;
+ }
+
+ /* Rendez-vous 2. */
+ atomic_dec_ulong(&eagerfpu_cpu_barrier2);
+ while (atomic_cas_ulong(&eagerfpu_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ splx(s);
+}
+
+static int
+eager_change(bool enabled)
+{
+ struct cpu_info *ci = NULL;
+ CPU_INFO_ITERATOR cii;
+ uint64_t xc;
+
+ mutex_enter(&cpu_lock);
+
+ /*
+ * We expect all the CPUs to be online.
+ */
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ struct schedstate_percpu *spc = &ci->ci_schedstate;
+ if (spc->spc_flags & SPCF_OFFLINE) {
+ printf("[!] cpu%d offline, EagerFPU not changed\n",
+ cpu_index(ci));
+ mutex_exit(&cpu_lock);
+ return EOPNOTSUPP;
+ }
+ }
+
+ /* Initialize the barriers */
+ eagerfpu_cpu_barrier1 = ncpu;
+ eagerfpu_cpu_barrier2 = ncpu;
+
+ printf("[+] %s EagerFPU...",
+ enabled ? "Enabling" : "Disabling");
+ xc = xc_broadcast(0, eager_change_cpu,
+ (void *)enabled, NULL);
+ xc_wait(xc);
+ printf(" done!\n");
+
+ mutex_exit(&cpu_lock);
+
+ return 0;
+}
+
+static int
+sysctl_machdep_fpu_eager(SYSCTLFN_ARGS)
+{
+ struct sysctlnode node;
+ int error;
+ bool val;
+
+ val = *(bool *)rnode->sysctl_data;
+
+ node = *rnode;
+ node.sysctl_data = &val;
+
+ error = sysctl_lookup(SYSCTLFN_CALL(&node));
+ if (error != 0 || newp == NULL)
+ return error;
+
+ if (val == x86_fpu_eager)
+ return 0;
+ return eager_change(val);
+}
+
+void sysctl_eagerfpu_init(struct sysctllog **);
+
+void
+sysctl_eagerfpu_init(struct sysctllog **clog)
+{
+ sysctl_createv(clog, 0, NULL, NULL,
+ CTLFLAG_READWRITE,
+ CTLTYPE_BOOL, "fpu_eager",
+ SYSCTL_DESCR("Whether the kernel uses Eager FPU Switch"),
+ sysctl_machdep_fpu_eager, 0,
+ &x86_fpu_eager, 0,
+ CTL_MACHDEP, CTL_CREATE, CTL_EOL);
+}
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.116 src/sys/arch/x86/x86/x86_machdep.c:1.117
--- src/sys/arch/x86/x86/x86_machdep.c:1.116 Thu Jun 14 14:36:46 2018
+++ src/sys/arch/x86/x86/x86_machdep.c Sat Jun 16 17:11:13 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.116 2018/06/14 14:36:46 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.117 2018/06/16 17:11:13 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.116 2018/06/14 14:36:46 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.117 2018/06/16 17:11:13 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1277,12 +1277,8 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysc
#endif
#ifndef XEN
- sysctl_createv(clog, 0, NULL, NULL,
- CTLFLAG_READWRITE,
- CTLTYPE_BOOL, "fpu_eager",
- SYSCTL_DESCR("Whether the kernel uses Eager FPU Switch"),
- NULL, 0, &x86_fpu_eager, 0,
- CTL_MACHDEP, CTL_CREATE, CTL_EOL);
+ void sysctl_eagerfpu_init(struct sysctllog **);
+ sysctl_eagerfpu_init(clog);
#endif
/* None of these can ever change once the system has booted */
