From: Joerg Roedel
Keep NMI state in SEV-ES code so the kernel can re-enable NMIs for the
vCPU when it reaches IRET.
Signed-off-by: Joerg Roedel
---
arch/x86/entry/entry_64.S | 48 +
arch/x86/include/asm/sev-es.h | 27 +++
arch/x86/include/uapi/asm/svm.h | 1 +
arch/x86/kernel/nmi.c | 8 ++
arch/x86/kernel/sev-es.c| 28 ++-
5 files changed, 111 insertions(+), 1 deletion(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 729876d368c5..355470b36896 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
#include
#include "calling.h"
@@ -629,6 +630,13 @@
SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
ud2
1:
#endif
+
+ /*
+* This code path is used by the NMI handler, so check if NMIs
+* need to be re-enabled when running as an SEV-ES guest.
+*/
+ SEV_ES_IRET_CHECK
+
POP_REGS pop_rdi=0
/*
@@ -1474,6 +1482,8 @@ SYM_CODE_START(nmi)
movq$-1, %rsi
calldo_nmi
+ SEV_ES_NMI_COMPLETE
+
/*
* Return back to user mode. We must *not* do the normal exit
* work, because we don't want to enable interrupts.
@@ -1599,6 +1609,7 @@ nested_nmi_out:
popq%rdx
/* We are returning to kernel mode, so this cannot result in a fault. */
+ SEV_ES_NMI_COMPLETE
iretq
first_nmi:
@@ -1687,6 +1698,12 @@ end_repeat_nmi:
movq$-1, %rsi
calldo_nmi
+ /*
+* When running as an SEV-ES guest, jump to the SEV-ES NMI IRET
+* path.
+*/
+ SEV_ES_NMI_COMPLETE
+
/* Always restore stashed CR3 value (see paranoid_entry) */
RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
@@ -1715,6 +1732,9 @@ nmi_restore:
std
movq$0, 5*8(%rsp) /* clear "NMI executing" */
+nmi_return:
+ UNWIND_HINT_IRET_REGS
+
/*
* iretq reads the "iret" frame and exits the NMI stack in a
* single instruction. We are returning to kernel mode, so this
@@ -1724,6 +1744,34 @@ nmi_restore:
iretq
SYM_CODE_END(nmi)
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+SYM_CODE_START(sev_es_iret_user)
+ UNWIND_HINT_IRET_REGS offset=8
+ /*
+* The kernel jumps here directly from
+* swapgs_restore_regs_and_return_to_usermode. %rsp points already to
+* trampoline stack, but %cr3 is still from kernel. User-regs are live
+* except %rdi. Switch to user CR3, restore user %rdi and user gs_base
+* and single-step over IRET
+*/
+ SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
+ popq%rdi
+ SWAPGS
+ /*
+* Enable single-stepping and execute IRET. When IRET is
+* finished the resulting #DB exception will cause a #VC
+* exception to be raised. The #VC exception handler will send a
+* NMI-complete message to the hypervisor to re-open the NMI
+* window.
+*/
+sev_es_iret_kernel:
+ pushf
+ btsq $X86_EFLAGS_TF_BIT, (%rsp)
+ popf
+ iretq
+SYM_CODE_END(sev_es_iret_user)
+#endif
+
#ifndef CONFIG_IA32_EMULATION
/*
* This handles SYSCALL from 32-bit code. There is no way to program
diff --git a/arch/x86/include/asm/sev-es.h b/arch/x86/include/asm/sev-es.h
index a4d7574c5c6a..22f45782149e 100644
--- a/arch/x86/include/asm/sev-es.h
+++ b/arch/x86/include/asm/sev-es.h
@@ -8,6 +8,8 @@
#ifndef __ASM_ENCRYPTED_STATE_H
#define __ASM_ENCRYPTED_STATE_H
+#ifndef __ASSEMBLY__
+
#include
#include
@@ -82,11 +84,36 @@ struct real_mode_header;
#ifdef CONFIG_AMD_MEM_ENCRYPT
int sev_es_setup_ap_jump_table(struct real_mode_header *rmh);
+void sev_es_nmi_enter(void);
#else /* CONFIG_AMD_MEM_ENCRYPT */
static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
{
return 0;
}
+static inline void sev_es_nmi_enter(void) { }
+#endif /* CONFIG_AMD_MEM_ENCRYPT*/
+
+#else /* !__ASSEMBLY__ */
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+#define SEV_ES_NMI_COMPLETE\
+ ALTERNATIVE "", "callq sev_es_nmi_complete",
X86_FEATURE_SEV_ES_GUEST
+
+.macro SEV_ES_IRET_CHECK
+ ALTERNATIVE "jmp.Lend_\@", "", X86_FEATURE_SEV_ES_GUEST
+ movqPER_CPU_VAR(sev_es_in_nmi), %rdi
+ testq %rdi, %rdi
+ jz .Lend_\@
+ callq sev_es_nmi_complete
+.Lend_\@:
+.endm
+
+#else /* CONFIG_AMD_MEM_ENCRYPT */
+#defineSEV_ES_NMI_RETURN
+.macro SEV_ES_IRET_CHECK
+.endm
#endif /* CONFIG_AMD_MEM_ENCRYPT*/
+#endif /* __ASSEMBLY__ */
+
#endif
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
index 20a05839dd9a..0f837339db66 100644
--- a/arch/x86/include/uapi/asm/svm.h
+++ b/arch/x86/include/uapi/asm/svm.h
@@ -84,6 +84,7 @@
/* SEV-ES software-defined VMGEXIT events */
#define
