Module Name: src
Committed By: maxv
Date: Thu Jul 12 19:48:16 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: amd64_trap.S locore.S machdep.c
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/x86/x86: svs.c
Log Message:
Handle NMIs correctly when SVS is enabled. We store the kernel's CR3 at the
top of the NMI stack, and we unconditionally switch to it, because we don't
know with which page tables we received the NMI. Hotpatch the whole thing as
usual.
This restores the ability to use PMCs on Intel CPUs.
To generate a diff of this commit:
cvs rdiff -u -r1.42 -r1.43 src/sys/arch/amd64/amd64/amd64_trap.S
cvs rdiff -u -r1.169 -r1.170 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.305 -r1.306 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.38 -r1.39 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.18 -r1.19 src/sys/arch/x86/x86/svs.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/amd64_trap.S
diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.42 src/sys/arch/amd64/amd64/amd64_trap.S:1.43
--- src/sys/arch/amd64/amd64/amd64_trap.S:1.42 Fri May 25 15:33:56 2018
+++ src/sys/arch/amd64/amd64/amd64_trap.S Thu Jul 12 19:48:16 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.42 2018/05/25 15:33:56 maxv Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.43 2018/07/12 19:48:16 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -254,7 +254,6 @@ IDTVEC(trap02)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
IBRS_ENTER
- SVS_ENTER_ALTSTACK
cld
SMAP_ENABLE
movw %gs,TF_GS(%rsp)
@@ -262,6 +261,8 @@ IDTVEC(trap02)
movw %es,TF_ES(%rsp)
movw %ds,TF_DS(%rsp)
+ SVS_ENTER_NMI
+
movl $MSR_GSBASE,%ecx
rdmsr
cmpl $VM_MIN_KERNEL_ADDRESS_HIGH32,%edx
@@ -271,7 +272,6 @@ IDTVEC(trap02)
movq %rsp,%rdi
incq CPUVAR(NTRAP)
call _C_LABEL(nmitrap)
- SVS_LEAVE_ALTSTACK
swapgs
jmp .Lnmileave
@@ -279,9 +279,9 @@ IDTVEC(trap02)
movq %rsp,%rdi
incq CPUVAR(NTRAP)
call _C_LABEL(nmitrap)
- SVS_LEAVE_ALTSTACK
.Lnmileave:
+ SVS_LEAVE_NMI
IBRS_LEAVE
INTR_RESTORE_GPRS
addq $TF_REGSIZE+16,%rsp
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.169 src/sys/arch/amd64/amd64/locore.S:1.170
--- src/sys/arch/amd64/amd64/locore.S:1.169 Mon Jul 9 18:52:04 2018
+++ src/sys/arch/amd64/amd64/locore.S Thu Jul 12 19:48:16 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.169 2018/07/09 18:52:04 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.170 2018/07/12 19:48:16 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1633,6 +1633,13 @@ LABEL(svs_enter_altstack)
1234:
LABEL(svs_enter_altstack_end)
+LABEL(svs_enter_nmi)
+ movq %cr3,%rax
+ movq %rax,(FRAMESIZE+1*8)(%rsp) /* nmistore->scratch */
+ movq (FRAMESIZE+0*8)(%rsp),%rax /* nmistore->cr3 */
+ movq %rax,%cr3
+LABEL(svs_enter_nmi_end)
+
LABEL(svs_leave)
testb $SEL_UPL,TF_CS(%rsp)
jz 1234f
@@ -1650,6 +1657,11 @@ LABEL(svs_leave_altstack)
1234:
LABEL(svs_leave_altstack_end)
+LABEL(svs_leave_nmi)
+ movq (FRAMESIZE+1*8)(%rsp),%rax /* nmistore->scratch */
+ movq %rax,%cr3
+LABEL(svs_leave_nmi_end)
+
LABEL(nosvs_enter)
NOSVS_ENTER
LABEL(nosvs_enter_end)
@@ -1658,6 +1670,10 @@ LABEL(nosvs_enter_altstack)
NOSVS_ENTER_ALTSTACK
LABEL(nosvs_enter_altstack_end)
+LABEL(nosvs_enter_nmi)
+ NOSVS_ENTER_NMI
+LABEL(nosvs_enter_nmi_end)
+
LABEL(nosvs_leave)
NOSVS_LEAVE
LABEL(nosvs_leave_end)
@@ -1665,6 +1681,10 @@ LABEL(nosvs_leave_end)
LABEL(nosvs_leave_altstack)
NOSVS_LEAVE_ALTSTACK
LABEL(nosvs_leave_altstack_end)
+
+LABEL(nosvs_leave_nmi)
+ NOSVS_LEAVE_NMI
+LABEL(nosvs_leave_nmi_end)
#endif
.globl ibrs_enter, ibrs_enter_end
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.305 src/sys/arch/amd64/amd64/machdep.c:1.306
--- src/sys/arch/amd64/amd64/machdep.c:1.305 Wed Jun 20 11:49:37 2018
+++ src/sys/arch/amd64/amd64/machdep.c Thu Jul 12 19:48:16 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -281,6 +281,11 @@ void (*initclock_func)(void) = xen_initc
struct pool x86_dbregspl;
+struct nmistore {
+ uint64_t cr3;
+ uint64_t scratch;
+} __packed;
+
/*
* Size of memory segments, before any memory is stolen.
*/
@@ -506,6 +511,7 @@ cpu_init_tss(struct cpu_info *ci)
const cpuid_t cid = cpu_index(ci);
#endif
struct cpu_tss *cputss;
+ struct nmistore *store;
uintptr_t p;
#ifdef __HAVE_PCPU_AREA
@@ -533,13 +539,15 @@ cpu_init_tss(struct cpu_info *ci)
#endif
cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16;
- /* NMI */
+ /* NMI - store a structure at the top of the stack */
#ifdef __HAVE_PCPU_AREA
p = (vaddr_t)&pcpuarea->ent[cid].ist2;
#else
p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
#endif
- cputss->tss.tss_ist[2] = p + PAGE_SIZE - 16;
+ cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore);
+ store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore));
+ store->cr3 = pmap_pdirpa(pmap_kernel(), 0);
/* DB */
#ifdef __HAVE_PCPU_AREA
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.38 src/sys/arch/amd64/include/frameasm.h:1.39
--- src/sys/arch/amd64/include/frameasm.h:1.38 Wed Mar 28 16:02:49 2018
+++ src/sys/arch/amd64/include/frameasm.h Thu Jul 12 19:48:16 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.39 2018/07/12 19:48:16 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -46,6 +46,8 @@
#define HP_NAME_SVS_LEAVE_ALT 8
#define HP_NAME_IBRS_ENTER 9
#define HP_NAME_IBRS_LEAVE 10
+#define HP_NAME_SVS_ENTER_NMI 11
+#define HP_NAME_SVS_LEAVE_NMI 12
#define HOTPATCH(name, size) \
123: ; \
@@ -165,6 +167,22 @@
HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \
NOSVS_LEAVE_ALTSTACK
+#define SVS_ENTER_NMI_BYTES 22
+#define NOSVS_ENTER_NMI \
+ .byte 0xEB, (SVS_ENTER_NMI_BYTES-2) /* jmp */ ; \
+ .fill (SVS_ENTER_NMI_BYTES-2),1,0xCC
+#define SVS_ENTER_NMI \
+ HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES) ; \
+ NOSVS_ENTER_NMI
+
+#define SVS_LEAVE_NMI_BYTES 11
+#define NOSVS_LEAVE_NMI \
+ .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2) /* jmp */ ; \
+ .fill (SVS_LEAVE_NMI_BYTES-2),1,0xCC
+#define SVS_LEAVE_NMI \
+ HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES) ; \
+ NOSVS_LEAVE_NMI
+
#else
#define SVS_ENTER /* nothing */
#define SVS_LEAVE /* nothing */
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.18 src/sys/arch/x86/x86/svs.c:1.19
--- src/sys/arch/x86/x86/svs.c:1.18 Thu Apr 26 18:54:09 2018
+++ src/sys/arch/x86/x86/svs.c Thu Jul 12 19:48:16 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $ */
+/* $NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $");
#include "opt_svs.h"
@@ -571,8 +571,10 @@ svs_enable(void)
{
extern uint8_t svs_enter, svs_enter_end;
extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
+ extern uint8_t svs_enter_nmi, svs_enter_nmi_end;
extern uint8_t svs_leave, svs_leave_end;
extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
+ extern uint8_t svs_leave_nmi, svs_leave_nmi_end;
u_long psl, cr0;
uint8_t *bytes;
size_t size;
@@ -590,6 +592,11 @@ svs_enable(void)
(size_t)&svs_enter_altstack;
x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
+ bytes = &svs_enter_nmi;
+ size = (size_t)&svs_enter_nmi_end -
+ (size_t)&svs_enter_nmi;
+ x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size);
+
bytes = &svs_leave;
size = (size_t)&svs_leave_end - (size_t)&svs_leave;
x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
@@ -599,6 +606,11 @@ svs_enable(void)
(size_t)&svs_leave_altstack;
x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
+ bytes = &svs_leave_nmi;
+ size = (size_t)&svs_leave_nmi_end -
+ (size_t)&svs_leave_nmi;
+ x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size);
+
x86_patch_window_close(psl, cr0);
}
@@ -607,8 +619,10 @@ svs_disable_hotpatch(void)
{
extern uint8_t nosvs_enter, nosvs_enter_end;
extern uint8_t nosvs_enter_altstack, nosvs_enter_altstack_end;
+ extern uint8_t nosvs_enter_nmi, nosvs_enter_nmi_end;
extern uint8_t nosvs_leave, nosvs_leave_end;
extern uint8_t nosvs_leave_altstack, nosvs_leave_altstack_end;
+ extern uint8_t nosvs_leave_nmi, nosvs_leave_nmi_end;
u_long psl, cr0;
uint8_t *bytes;
size_t size;
@@ -624,6 +638,11 @@ svs_disable_hotpatch(void)
(size_t)&nosvs_enter_altstack;
x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
+ bytes = &nosvs_enter_nmi;
+ size = (size_t)&nosvs_enter_nmi_end -
+ (size_t)&nosvs_enter_nmi;
+ x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size);
+
bytes = &nosvs_leave;
size = (size_t)&nosvs_leave_end - (size_t)&nosvs_leave;
x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
@@ -633,6 +652,11 @@ svs_disable_hotpatch(void)
(size_t)&nosvs_leave_altstack;
x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
+ bytes = &nosvs_leave_nmi;
+ size = (size_t)&nosvs_leave_nmi_end -
+ (size_t)&nosvs_leave_nmi;
+ x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size);
+
x86_patch_window_close(psl, cr0);
}
