Module Name: src
Committed By: martin
Date: Tue May 14 17:12:20 UTC 2019
Modified Files:
src/sys/arch/amd64/amd64 [netbsd-8]: amd64_trap.S locore.S
src/sys/arch/amd64/include [netbsd-8]: frameasm.h
src/sys/arch/x86/include [netbsd-8]: specialreg.h
src/sys/arch/x86/x86 [netbsd-8]: spectre.c
Log Message:
Pull up following revision(s) (requested by maxv in ticket #1269):
sys/arch/amd64/amd64/locore.S: revision 1.181 (adapted)
sys/arch/amd64/amd64/amd64_trap.S: revision 1.47 (adapted)
sys/arch/x86/include/specialreg.h: revision 1.144 (adapted)
sys/arch/amd64/include/frameasm.h: revision 1.43 (adapted)
sys/arch/x86/x86/spectre.c: revision 1.27 (adapted)
Mitigation for INTEL-SA-00233: Microarchitectural Data Sampling (MDS).
It requires a microcode update, now available on the Intel website. The
microcode modifies the behavior of the VERW instruction, and makes it flush
internal CPU buffers. We hotpatch the return-to-userland path to add VERW.
Two sysctls are added:
machdep.mds.mitigated = {0/1} user-settable
machdep.mds.method = {string} constructed by the kernel
The kernel will automatically enable the mitigation if the updated
microcode is present. If the new microcode is not present, the user can
load it via cpuctl, and set machdep.mds.mitigated=1.
To generate a diff of this commit:
cvs rdiff -u -r1.5.6.3 -r1.5.6.4 src/sys/arch/amd64/amd64/amd64_trap.S
cvs rdiff -u -r1.123.6.8 -r1.123.6.9 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.20.32.3 -r1.20.32.4 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.98.2.11 -r1.98.2.12 src/sys/arch/x86/include/specialreg.h
cvs rdiff -u -r1.19.2.2 -r1.19.2.3 src/sys/arch/x86/x86/spectre.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/amd64_trap.S
diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.5.6.3 src/sys/arch/amd64/amd64/amd64_trap.S:1.5.6.4
--- src/sys/arch/amd64/amd64/amd64_trap.S:1.5.6.3 Sat Apr 14 10:11:49 2018
+++ src/sys/arch/amd64/amd64/amd64_trap.S Tue May 14 17:12:19 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.5.6.3 2018/04/14 10:11:49 martin Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.5.6.4 2019/05/14 17:12:19 martin Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -151,6 +151,7 @@ IDTVEC(trap02)
movq %rsp,%rdi
incq CPUVAR(NTRAP)
call _C_LABEL(nmitrap)
+ MDS_LEAVE
SVS_LEAVE_ALTSTACK
swapgs
jmp .Lnmileave
@@ -159,6 +160,7 @@ IDTVEC(trap02)
movq %rsp,%rdi
incq CPUVAR(NTRAP)
call _C_LABEL(nmitrap)
+ MDS_LEAVE
SVS_LEAVE_ALTSTACK
.Lnmileave:
@@ -247,6 +249,7 @@ IDTVEC(trap08)
incq CPUVAR(NTRAP)
call _C_LABEL(doubletrap)
+ MDS_LEAVE
SVS_LEAVE_ALTSTACK
INTR_RESTORE_GPRS
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.123.6.8 src/sys/arch/amd64/amd64/locore.S:1.123.6.9
--- src/sys/arch/amd64/amd64/locore.S:1.123.6.8 Tue Jul 10 15:35:26 2018
+++ src/sys/arch/amd64/amd64/locore.S Tue May 14 17:12:19 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.123.6.8 2018/07/10 15:35:26 martin Exp $ */
+/* $NetBSD: locore.S,v 1.123.6.9 2019/05/14 17:12:19 martin Exp $ */
/*
* Copyright-o-rama!
@@ -1465,6 +1465,7 @@ IDTVEC_END(osyscall)
TEXT_USER_BEGIN
_ALIGN_TEXT
LABEL(syscall_sysret)
+ MDS_LEAVE
SVS_LEAVE
/* Set default the 64bit values in %ds and %es. */
@@ -1561,6 +1562,7 @@ END(pagezero)
_ALIGN_TEXT
LABEL(intrfastexit)
NOT_XEN(cli;)
+ MDS_LEAVE
SVS_LEAVE
INTR_RESTORE_GPRS
addq $(TF_REGSIZE+16),%rsp /* iret frame */
@@ -1637,3 +1639,18 @@ LABEL(nosvs_leave_altstack)
NOSVS_LEAVE_ALTSTACK
LABEL(nosvs_leave_altstack_end)
#endif
+
+ .globl mds_leave, mds_leave_end
+
+LABEL(mds_leave)
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1234f
+ pushq $GSEL(GDATA_SEL, SEL_KPL)
+ verw (%rsp)
+ addq $8,%rsp
+1234:
+LABEL(mds_leave_end)
+
+LABEL(nomds_leave)
+ NOMDS_LEAVE
+LABEL(nomds_leave_end)
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.20.32.3 src/sys/arch/amd64/include/frameasm.h:1.20.32.4
--- src/sys/arch/amd64/include/frameasm.h:1.20.32.3 Sat Apr 14 10:11:49 2018
+++ src/sys/arch/amd64/include/frameasm.h Tue May 14 17:12:19 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.20.32.3 2018/04/14 10:11:49 martin Exp $ */
+/* $NetBSD: frameasm.h,v 1.20.32.4 2019/05/14 17:12:19 martin Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -42,6 +42,7 @@
#define HP_NAME_SVS_LEAVE 6
#define HP_NAME_SVS_ENTER_ALT 7
#define HP_NAME_SVS_LEAVE_ALT 8
+#define HP_NAME_MDS_LEAVE 13
#define HOTPATCH(name, size) \
123: ; \
@@ -59,6 +60,18 @@
HOTPATCH(HP_NAME_STAC, 3) ; \
.byte 0x0F, 0x1F, 0x00 ; \
+/*
+ * MDS
+ */
+
+#define MDS_LEAVE_BYTES 20
+#define MDS_LEAVE \
+ HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \
+ NOMDS_LEAVE
+#define NOMDS_LEAVE \
+ .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \
+ .fill (MDS_LEAVE_BYTES-2),1,0xCC
+
#define SWAPGS NOT_XEN(swapgs)
/*
Index: src/sys/arch/x86/include/specialreg.h
diff -u src/sys/arch/x86/include/specialreg.h:1.98.2.11 src/sys/arch/x86/include/specialreg.h:1.98.2.12
--- src/sys/arch/x86/include/specialreg.h:1.98.2.11 Tue Feb 12 09:27:17 2019
+++ src/sys/arch/x86/include/specialreg.h Tue May 14 17:12:19 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: specialreg.h,v 1.98.2.11 2019/02/12 09:27:17 martin Exp $ */
+/* $NetBSD: specialreg.h,v 1.98.2.12 2019/05/14 17:12:19 martin Exp $ */
/*-
* Copyright (c) 1991 The Regents of the University of California.
@@ -426,6 +426,7 @@
/* %edx */
#define CPUID_SEF_AVX512_4VNNIW __BIT(2)
#define CPUID_SEF_AVX512_4FMAPS __BIT(3)
+#define CPUID_SEF_MD_CLEAR __BIT(10)
#define CPUID_SEF_IBRS __BIT(26) /* IBRS / IBPB Speculation Control */
#define CPUID_SEF_STIBP __BIT(27) /* STIBP Speculation Control */
#define CPUID_SEF_L1D_FLUSH __BIT(28) /* IA32_FLUSH_CMD MSR */
@@ -745,6 +746,7 @@
#define IA32_ARCH_RSBA 0x04
#define IA32_ARCH_SKIP_L1DFL_VMENTRY 0x08
#define IA32_ARCH_SSB_NO 0x10
+#define IA32_ARCH_MDS_NO 0x20
#define MSR_IA32_FLUSH_CMD 0x10b
#define IA32_FLUSH_CMD_L1D_FLUSH 0x01
#define MSR_BBL_CR_ADDR 0x116 /* PII+ only */
Index: src/sys/arch/x86/x86/spectre.c
diff -u src/sys/arch/x86/x86/spectre.c:1.19.2.2 src/sys/arch/x86/x86/spectre.c:1.19.2.3
--- src/sys/arch/x86/x86/spectre.c:1.19.2.2 Sat Jun 9 15:12:21 2018
+++ src/sys/arch/x86/x86/spectre.c Tue May 14 17:12:19 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: spectre.c,v 1.19.2.2 2018/06/09 15:12:21 martin Exp $ */
+/* $NetBSD: spectre.c,v 1.19.2.3 2019/05/14 17:12:19 martin Exp $ */
/*
* Copyright (c) 2018 NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.19.2.2 2018/06/09 15:12:21 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.19.2.3 2019/05/14 17:12:19 martin Exp $");
#include "opt_spectre.h"
@@ -428,6 +428,226 @@ sysctl_machdep_spectreV4_mitigated(SYSCT
/* -------------------------------------------------------------------------- */
+enum mds_mitigation {
+ MDS_MITIGATION_NONE,
+ MDS_MITIGATION_VERW,
+ MDS_MITIGATION_MDS_NO
+};
+
+static char mds_mitigation_name[64] = "(none)";
+
+static enum mds_mitigation mds_mitigation_method = MDS_MITIGATION_NONE;
+static bool mds_mitigation_enabled __read_mostly = false;
+
+static volatile unsigned long mds_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long mds_cpu_barrier2 __cacheline_aligned;
+
+#ifdef __x86_64__
+static void
+mds_disable_hotpatch(void)
+{
+ extern uint8_t nomds_leave, nomds_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &nomds_leave;
+ size = (size_t)&nomds_leave_end - (size_t)&nomds_leave;
+ x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+
+static void
+mds_enable_hotpatch(void)
+{
+ extern uint8_t mds_leave, mds_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &mds_leave;
+ size = (size_t)&mds_leave_end - (size_t)&mds_leave;
+ x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+#else
+/* MDS not supported on i386 */
+static void
+mds_disable_hotpatch(void)
+{
+ panic("%s: impossible", __func__);
+}
+static void
+mds_enable_hotpatch(void)
+{
+ panic("%s: impossible", __func__);
+}
+#endif
+
+static void
+mitigation_mds_apply_cpu(struct cpu_info *ci, bool enabled)
+{
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ case MDS_MITIGATION_MDS_NO:
+ panic("impossible");
+ case MDS_MITIGATION_VERW:
+ /* cpu0 is the one that does the hotpatch job */
+ if (ci == &cpu_info_primary) {
+ if (enabled) {
+ mds_enable_hotpatch();
+ } else {
+ mds_disable_hotpatch();
+ }
+ }
+ break;
+ }
+}
+
+static void
+mitigation_mds_change_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ bool enabled = (bool)arg1;
+ u_long psl = 0;
+
+ /* Rendez-vous 1. */
+ psl = x86_read_psl();
+ x86_disable_intr();
+
+ atomic_dec_ulong(&mds_cpu_barrier1);
+ while (atomic_cas_ulong(&mds_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ mitigation_mds_apply_cpu(ci, enabled);
+
+ /* Rendez-vous 2. */
+ atomic_dec_ulong(&mds_cpu_barrier2);
+ while (atomic_cas_ulong(&mds_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* Write back and invalidate cache, flush pipelines. */
+ wbinvd();
+ x86_flush();
+
+ x86_write_psl(psl);
+}
+
+static void
+mds_detect_method(void)
+{
+ u_int descs[4];
+ uint64_t msr;
+
+ if (cpu_vendor != CPUVENDOR_INTEL) {
+ mds_mitigation_method = MDS_MITIGATION_MDS_NO;
+ return;
+ }
+
+ x86_cpuid(0x7, descs);
+ if (descs[3] & CPUID_SEF_ARCH_CAP) {
+ msr = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
+ if (msr & IA32_ARCH_MDS_NO) {
+ mds_mitigation_method = MDS_MITIGATION_MDS_NO;
+ return;
+ }
+ }
+
+#ifdef __x86_64__
+ if (descs[3] & CPUID_SEF_MD_CLEAR) {
+ mds_mitigation_method = MDS_MITIGATION_VERW;
+ }
+#endif
+}
+
+static void
+mds_set_name(void)
+{
+ char name[64] = "";
+
+ if (!mds_mitigation_enabled) {
+ strlcat(name, "(none)", sizeof(name));
+ } else {
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ panic("%s: impossible", __func__);
+ case MDS_MITIGATION_MDS_NO:
+ strlcat(name, "[MDS_NO]", sizeof(name));
+ break;
+ case MDS_MITIGATION_VERW:
+ strlcat(name, "[VERW]", sizeof(name));
+ break;
+ }
+ }
+
+ strlcpy(mds_mitigation_name, name,
+ sizeof(mds_mitigation_name));
+}
+
+static int
+mitigation_mds_change(bool enabled)
+{
+ uint64_t xc;
+
+ mds_detect_method();
+
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ printf("[!] No mitigation available\n");
+ return EOPNOTSUPP;
+ case MDS_MITIGATION_VERW:
+ /* Initialize the barriers */
+ mds_cpu_barrier1 = ncpu;
+ mds_cpu_barrier2 = ncpu;
+
+ printf("[+] %s MDS Mitigation...",
+ enabled ? "Enabling" : "Disabling");
+ xc = xc_broadcast(XC_HIGHPRI, mitigation_mds_change_cpu,
+ (void *)enabled, NULL);
+ xc_wait(xc);
+ printf(" done!\n");
+ mds_mitigation_enabled = enabled;
+ mds_set_name();
+ return 0;
+ case MDS_MITIGATION_MDS_NO:
+ printf("[+] The CPU is not affected by MDS\n");
+ return 0;
+ default:
+ panic("impossible");
+ }
+}
+
+static int
+sysctl_machdep_mds_mitigated(SYSCTLFN_ARGS)
+{
+ struct sysctlnode node;
+ int error;
+ bool val;
+
+ val = *(bool *)rnode->sysctl_data;
+
+ node = *rnode;
+ node.sysctl_data = &val;
+
+ error = sysctl_lookup(SYSCTLFN_CALL(&node));
+ if (error != 0 || newp == NULL)
+ return error;
+
+ if (val == mds_mitigation_enabled)
+ return 0;
+ return mitigation_mds_change(val);
+}
+
+/* -------------------------------------------------------------------------- */
+
void
cpu_speculation_init(struct cpu_info *ci)
{
@@ -466,6 +686,23 @@ cpu_speculation_init(struct cpu_info *ci
mitigation_v4_apply_cpu(ci, true);
}
#endif
+
+ /*
+ * Microarchectural Data Sampling.
+ *
+ * cpu0 is the one that detects the method and sets the global
+ * variable.
+ */
+ if (ci == &cpu_info_primary) {
+ mds_detect_method();
+ mds_mitigation_enabled =
+ (mds_mitigation_method != MDS_MITIGATION_NONE);
+ mds_set_name();
+ }
+ if (mds_mitigation_method != MDS_MITIGATION_NONE &&
+ mds_mitigation_method != MDS_MITIGATION_MDS_NO) {
+ mitigation_mds_apply_cpu(ci, true);
+ }
}
void sysctl_speculation_init(struct sysctllog **);
@@ -543,4 +780,26 @@ sysctl_speculation_init(struct sysctllog
NULL, 0,
v4_mitigation_name, 0,
CTL_CREATE, CTL_EOL);
+
+ /* Microarchitectural Data Sampling */
+ spec_rnode = NULL;
+ sysctl_createv(clog, 0, NULL, &spec_rnode,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_NODE, "mds", NULL,
+ NULL, 0, NULL, 0,
+ CTL_MACHDEP, CTL_CREATE);
+ sysctl_createv(clog, 0, &spec_rnode, NULL,
+ CTLFLAG_READWRITE,
+ CTLTYPE_BOOL, "mitigated",
+ SYSCTL_DESCR("Whether MDS is mitigated"),
+ sysctl_machdep_mds_mitigated, 0,
+ &mds_mitigation_enabled, 0,
+ CTL_CREATE, CTL_EOL);
+ sysctl_createv(clog, 0, &spec_rnode, NULL,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_STRING, "method",
+ SYSCTL_DESCR("Mitigation method in use"),
+ NULL, 0,
+ mds_mitigation_name, 0,
+ CTL_CREATE, CTL_EOL);
}
