Module Name: src
Committed By: maxv
Date: Thu Feb 22 10:42:11 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: locore.S
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/x86/x86: svs.c x86_machdep.c
Log Message:
Make the machdep.svs_enabled sysctl writable, and add the kernel code
needed to disable SVS at runtime.
We set 'svs_enabled' to false, and hotpatch the kernel entry/exit points
to eliminate the context switch code.
We need to make sure there is no remote CPU that is executing the code we
are hotpatching. So we use two barriers:
* After the first one each CPU is guaranteed to be executing in
svs_disable_cpu with interrupts disabled (this way it can't leave this
place).
* After the second one it is guaranteed that SVS is disabled, so we flush
the cache, enable interrupts and continue execution normally.
Between the two barriers, cpu0 will disable SVS (svs_enabled=false and
hotpatch), and each CPU will restore the generic syscall entry point.
Three notes:
* We should call svs_pgg_update(true) afterwards, to put back PG_G on
the kernel pages (for better performance). This will be done in another
commit.
* The fact that we disable interrupts does not prevent us from receiving
an NMI, and it would be problematic. So we need to add some code to
verify that PMCs are disabled before hotpatching. This will be done
in another commit.
* In svs_disable() we expect each CPU to be online. We need to add a
check to make sure they indeed are.
The sysctl allows only a 1->0 transition. There is no point in doing 0->1
transitions anyway, and it would be complicated to implement because we
need to re-synchronize the CPU user page tables with the current ones (we
lost track of them in the last 1->0 transition).
To generate a diff of this commit:
cvs rdiff -u -r1.152 -r1.153 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.35 -r1.36 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/x86/x86/svs.c
cvs rdiff -u -r1.105 -r1.106 src/sys/arch/x86/x86/x86_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.152 src/sys/arch/amd64/amd64/locore.S:1.153
--- src/sys/arch/amd64/amd64/locore.S:1.152 Thu Feb 22 08:56:51 2018
+++ src/sys/arch/amd64/amd64/locore.S Thu Feb 22 10:42:10 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.152 2018/02/22 08:56:51 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.153 2018/02/22 10:42:10 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1591,14 +1591,14 @@ END(intrfastexit)
TEXT_USER_END
#ifdef SVS
- .globl svs_enter
- .globl svs_enter_end
- .globl svs_enter_altstack
- .globl svs_enter_altstack_end
- .globl svs_leave
- .globl svs_leave_end
- .globl svs_leave_altstack
- .globl svs_leave_altstack_end
+ .globl svs_enter, svs_enter_end
+ .globl svs_enter_altstack, svs_enter_altstack_end
+ .globl svs_leave, svs_leave_end
+ .globl svs_leave_altstack, svs_leave_altstack_end
+ .globl nosvs_enter, nosvs_enter_end
+ .globl nosvs_enter_altstack, nosvs_enter_altstack_end
+ .globl nosvs_leave, nosvs_leave_end
+ .globl nosvs_leave_altstack, nosvs_leave_altstack_end
LABEL(svs_enter)
movq SVS_UTLS+UTLS_KPDIRPA,%rax
@@ -1630,4 +1630,20 @@ LABEL(svs_leave_altstack)
movq %rax,%cr3
1234:
LABEL(svs_leave_altstack_end)
+
+LABEL(nosvs_enter)
+ NOSVS_ENTER
+LABEL(nosvs_enter_end)
+
+LABEL(nosvs_enter_altstack)
+ NOSVS_ENTER_ALTSTACK
+LABEL(nosvs_enter_altstack_end)
+
+LABEL(nosvs_leave)
+ NOSVS_LEAVE
+LABEL(nosvs_leave_end)
+
+LABEL(nosvs_leave_altstack)
+ NOSVS_LEAVE_ALTSTACK
+LABEL(nosvs_leave_altstack_end)
#endif
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.35 src/sys/arch/amd64/include/frameasm.h:1.36
--- src/sys/arch/amd64/include/frameasm.h:1.35 Thu Feb 22 08:56:51 2018
+++ src/sys/arch/amd64/include/frameasm.h Thu Feb 22 10:42:11 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.35 2018/02/22 08:56:51 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.36 2018/02/22 10:42:11 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -112,28 +112,36 @@
#define UTLS_RSP0 16
#define SVS_ENTER_BYTES 22
-#define SVS_ENTER \
- HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \
+#define NOSVS_ENTER \
.byte 0xEB, (SVS_ENTER_BYTES-2) /* jmp */ ; \
.fill (SVS_ENTER_BYTES-2),1,0xCC
+#define SVS_ENTER \
+ HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \
+ NOSVS_ENTER
#define SVS_LEAVE_BYTES 31
-#define SVS_LEAVE \
- HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \
+#define NOSVS_LEAVE \
.byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \
.fill (SVS_LEAVE_BYTES-2),1,0xCC
+#define SVS_LEAVE \
+ HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \
+ NOSVS_LEAVE
#define SVS_ENTER_ALT_BYTES 23
-#define SVS_ENTER_ALTSTACK \
- HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \
+#define NOSVS_ENTER_ALTSTACK \
.byte 0xEB, (SVS_ENTER_ALT_BYTES-2) /* jmp */ ; \
.fill (SVS_ENTER_ALT_BYTES-2),1,0xCC
+#define SVS_ENTER_ALTSTACK \
+ HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \
+ NOSVS_ENTER_ALTSTACK
#define SVS_LEAVE_ALT_BYTES 22
-#define SVS_LEAVE_ALTSTACK \
- HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \
+#define NOSVS_LEAVE_ALTSTACK \
.byte 0xEB, (SVS_LEAVE_ALT_BYTES-2) /* jmp */ ; \
.fill (SVS_LEAVE_ALT_BYTES-2),1,0xCC
+#define SVS_LEAVE_ALTSTACK \
+ HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \
+ NOSVS_LEAVE_ALTSTACK
#else
#define SVS_ENTER /* nothing */
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.5 src/sys/arch/x86/x86/svs.c:1.6
--- src/sys/arch/x86/x86/svs.c:1.5 Thu Feb 22 09:41:06 2018
+++ src/sys/arch/x86/x86/svs.c Thu Feb 22 10:42:11 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.6 2018/02/22 10:42:11 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.6 2018/02/22 10:42:11 maxv Exp $");
#include "opt_svs.h"
@@ -38,6 +38,8 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/cpu.h>
+#include <sys/sysctl.h>
+#include <sys/xcall.h>
#include <x86/cputypes.h>
#include <machine/cpuvar.h>
@@ -541,6 +543,131 @@ svs_enable(void)
x86_patch_window_close(psl, cr0);
}
+static void
+svs_disable_hotpatch(void)
+{
+ extern uint8_t nosvs_enter, nosvs_enter_end;
+ extern uint8_t nosvs_enter_altstack, nosvs_enter_altstack_end;
+ extern uint8_t nosvs_leave, nosvs_leave_end;
+ extern uint8_t nosvs_leave_altstack, nosvs_leave_altstack_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &nosvs_enter;
+ size = (size_t)&nosvs_enter_end - (size_t)&nosvs_enter;
+ x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
+
+ bytes = &nosvs_enter_altstack;
+ size = (size_t)&nosvs_enter_altstack_end -
+ (size_t)&nosvs_enter_altstack;
+ x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
+
+ bytes = &nosvs_leave;
+ size = (size_t)&nosvs_leave_end - (size_t)&nosvs_leave;
+ x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
+
+ bytes = &nosvs_leave_altstack;
+ size = (size_t)&nosvs_leave_altstack_end -
+ (size_t)&nosvs_leave_altstack;
+ x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+
+static volatile unsigned long svs_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long svs_cpu_barrier2 __cacheline_aligned;
+typedef void (vector)(void);
+
+static void
+svs_disable_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ extern vector Xsyscall;
+ u_long psl;
+
+ psl = x86_read_psl();
+
+ atomic_dec_ulong(&svs_cpu_barrier1);
+ while (atomic_cas_ulong(&svs_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* cpu0 is the one that does the hotpatch job */
+ if (ci == &cpu_info_primary) {
+ svs_enabled = false;
+ svs_disable_hotpatch();
+ }
+
+ /* put back the non-SVS syscall entry point */
+ wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
+
+ atomic_dec_ulong(&svs_cpu_barrier2);
+ while (atomic_cas_ulong(&svs_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* Write back and invalidate cache, flush pipelines. */
+ wbinvd();
+ x86_flush();
+
+ x86_write_psl(psl);
+}
+
+static void
+svs_disable(void)
+{
+ uint64_t xc;
+
+ /*
+ * We expect all the CPUs to be online. XXX ensure they are.
+ */
+ svs_cpu_barrier1 = ncpu;
+ svs_cpu_barrier2 = ncpu;
+
+ printf("[+] Disabling SVS\n");
+ xc = xc_broadcast(0, svs_disable_cpu, NULL, NULL);
+ xc_wait(xc);
+
+ /*
+ * XXX printf("[+] Installing PG_G\n");
+ * XXX svs_pgg_update(true);
+ */
+
+ printf("[+] Done\n");
+}
+
+int sysctl_machdep_svs_enabled(SYSCTLFN_ARGS);
+
+int
+sysctl_machdep_svs_enabled(SYSCTLFN_ARGS)
+{
+ struct sysctlnode node;
+ int error, val;
+
+ val = *(int *)rnode->sysctl_data;
+
+ node = *rnode;
+ node.sysctl_data = &val;
+
+ error = sysctl_lookup(SYSCTLFN_CALL(&node));
+ if (error != 0 || newp == NULL)
+ return error;
+
+ if (val == 1) {
+ error = EINVAL;
+ } else {
+ if (svs_enabled) {
+ svs_disable();
+ }
+ error = 0;
+ }
+
+ return error;
+}
+
void
svs_init(bool early)
{
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.105 src/sys/arch/x86/x86/x86_machdep.c:1.106
--- src/sys/arch/x86/x86/x86_machdep.c:1.105 Thu Feb 22 09:41:06 2018
+++ src/sys/arch/x86/x86/x86_machdep.c Thu Feb 22 10:42:11 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1259,11 +1259,12 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysc
CTL_MACHDEP, CTL_CREATE, CTL_EOL);
#endif
#ifdef SVS
+ int sysctl_machdep_svs_enabled(SYSCTLFN_ARGS);
sysctl_createv(clog, 0, NULL, NULL,
- CTLFLAG_PERMANENT,
+ CTLFLAG_READWRITE,
CTLTYPE_BOOL, "svs_enabled",
SYSCTL_DESCR("Whether the kernel uses SVS"),
- NULL, 0, &svs_enabled, 0,
+ sysctl_machdep_svs_enabled, 0, &svs_enabled, 0,
CTL_MACHDEP, CTL_CREATE, CTL_EOL);
#endif
