Implement AMD SVM Caretaker VMCB preservation, exit decoding, and detach-time serialization in arch/x86/kvm/svm/caretaker.c.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/kvm/Makefile | 23 +- arch/x86/kvm/svm/caretaker.c | 467 +++++++++++++++++++++++++++ arch/x86/kvm/svm/caretaker_vmenter.S | 4 +- arch/x86/kvm/svm/svm.c | 11 +- 4 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 arch/x86/kvm/svm/caretaker.c diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index b550a3fd0708..9213d274817e 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -26,6 +26,7 @@ kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o kvm-intel-$(CONFIG_KVM_INTEL_TDX) += vmx/tdx.o kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o +kvm-amd-$(CONFIG_KVM_CARETAKER) += svm/caretaker.o svm/caretaker_vmenter.o kvm-amd-$(CONFIG_KVM_AMD_SEV) += svm/sev.o kvm-amd-$(CONFIG_KVM_HYPERV) += svm/hyperv.o @@ -43,6 +44,26 @@ obj-$(CONFIG_KVM_AMD) += kvm-amd.o AFLAGS_svm/vmenter.o := -iquote $(obj) $(obj)/svm/vmenter.o: $(obj)/kvm-asm-offsets.h +AFLAGS_svm/caretaker_vmenter.o := -iquote $(obj) +$(obj)/svm/caretaker_vmenter.o: $(obj)/kvm-asm-offsets.h + +KASAN_SANITIZE_caretaker.o := n +KCSAN_SANITIZE_caretaker.o := n +UBSAN_SANITIZE_caretaker.o := n +KCOV_INSTRUMENT_caretaker.o := n +KASAN_SANITIZE_svm/caretaker.o := n +KCSAN_SANITIZE_svm/caretaker.o := n +UBSAN_SANITIZE_svm/caretaker.o := n +KCOV_INSTRUMENT_svm/caretaker.o := n +KASAN_SANITIZE_vmx/caretaker.o := n +KCSAN_SANITIZE_vmx/caretaker.o := n +UBSAN_SANITIZE_vmx/caretaker.o := n +KCOV_INSTRUMENT_vmx/caretaker.o := n +CFLAGS_REMOVE_caretaker.o = $(CC_FLAGS_FTRACE) +CFLAGS_caretaker.o += -fno-stack-protector $(call cc-option,-ftrivial-auto-var-init=uninitialized) $(call cc-option,-fno-jump-tables) +CFLAGS_REMOVE_svm/caretaker.o = $(CC_FLAGS_FTRACE) +CFLAGS_svm/caretaker.o := -fno-stack-protector $(call cc-option,-ftrivial-auto-var-init=uninitialized) $(call cc-option,-fno-jump-tables) + AFLAGS_vmx/vmenter.o := -iquote $(obj) $(obj)/vmx/vmenter.o: $(obj)/kvm-asm-offsets.h @@ -50,7 +71,7 @@ AFLAGS_vmx/caretaker_vmenter.o := -iquote $(obj) $(obj)/vmx/caretaker_vmenter.o: $(obj)/kvm-asm-offsets.h CFLAGS_REMOVE_vmx/caretaker.o = $(CC_FLAGS_FTRACE) -CFLAGS_vmx/caretaker.o := -fno-stack-protector +CFLAGS_vmx/caretaker.o := -fno-stack-protector $(call cc-option,-ftrivial-auto-var-init=uninitialized) $(call cc-option,-fno-jump-tables) $(obj)/kvm-asm-offsets.h: $(obj)/kvm-asm-offsets.s FORCE diff --git a/arch/x86/kvm/svm/caretaker.c b/arch/x86/kvm/svm/caretaker.c new file mode 100644 index 000000000000..cb30e336f014 --- /dev/null +++ b/arch/x86/kvm/svm/caretaker.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD SVM Caretaker Standalone Execution Engine + * + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + * + * Runs the AMD SVM guest VMRUN loop in an isolated, KHO-preserved memory + * page that remains alive and executing across kexec relocation and + * kernel handover. + */ + +#include <linux/cpu_preserve.h> +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/kexec_handover.h> +#include <linux/kvm_host.h> +#include <linux/objtool.h> +#include <linux/oncore.h> + +#include <asm/apic.h> +#include <asm/cpu_entry_area.h> +#include <asm/desc.h> +#include <linux/pgtable.h> +#include <linux/processor.h> +#include <asm/set_memory.h> +#include <asm/svm.h> +#include <asm/tlbflush.h> + +#include "../caretaker.h" +#include "caretaker.h" +#include "mmu.h" +#include "svm.h" +#include "svm_ops.h" + +static int svm_caretaker_init_page(struct caretaker_svm_page *csp, struct kvm_vcpu *vcpu); +STACK_FRAME_NON_STANDARD(svm_caretaker_init_page); + +static int svm_caretaker_init_page(struct caretaker_svm_page *csp, struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + int ret; + + if (is_guest_mode(vcpu) || is_sev_guest(vcpu) || + kvm_vcpu_apicv_activated(vcpu)) + return -EOPNOTSUPP; + + ret = kvm_x86_caretaker_init_common_page(&csp->common, vcpu, sizeof(*csp)); + if (ret) + return ret; + + if (svm->vmcb01.ptr) + csp->vmcb = *svm->vmcb01.ptr; + csp->vmcb_pa = virt_to_phys(&csp->vmcb); + csp->hsave_pa = virt_to_phys(csp->hsave_area); + + if (svm->msrpm) + memcpy(csp->msrpm, svm->msrpm, MSRPM_SIZE); + else + memset(csp->msrpm, 0xff, MSRPM_SIZE); + memset(csp->iopm, 0xff, IOPM_SIZE); + csp->vmcb.control.msrpm_base_pa = virt_to_phys(csp->msrpm); + csp->vmcb.control.iopm_base_pa = virt_to_phys(csp->iopm); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_MSR_PROT); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_IOIO_PROT); + + /* Enable HLT/CPUID/VMMCALL intercepts handled by standalone loop */ + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_HLT); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_PAUSE); + csp->vmcb.control.pause_filter_count = 4096; + csp->vmcb.control.pause_filter_thresh = 128; + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_RDTSC); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_VMMCALL); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_CPUID); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_MONITOR); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_MWAIT); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_NMI); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_INTR); + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_INIT); + + /* + * Clear interrupt-window intercepts and virtual IRQ injection state + * left behind in vmcb01 if vcpu_enter_guest() was aborted while + * KVM had an active interrupt window open (matching VMX clearing + * CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING). + */ + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_VINTR); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_IRET); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_STGI); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_CR8_WRITE); + + /* Clear VMCB clean bits and flush TLB for execution on Caretaker CPU */ + csp->vmcb.control.clean = 0; + csp->vmcb.control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; + csp->vmcb.control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; + csp->vmcb.control.int_ctl |= V_INTR_MASKING_MASK; + if (csp->vmcb.control.int_ctl & V_GIF_ENABLE_MASK) + csp->vmcb.control.int_ctl |= V_GIF_MASK; + csp->vmcb.control.int_vector = 0; + csp->vmcb.control.int_state = 0; + csp->vmcb.control.event_inj = 0; + csp->vmcb.control.exit_int_info = 0; + if (csp->vmcb.control.asid == 0) + csp->vmcb.control.asid = 1; + csp->vmcb.save.rflags = kvm_get_rflags(vcpu); + csp->vmcb.save.rax = csp->common.rax; + csp->vmcb.save.rsp = kvm_rsp_read(vcpu); + csp->vmcb.save.rip = kvm_rip_read(vcpu); + csp->common.cr3 = csp->vmcb.save.cr3; + csp->common.cr0 = csp->vmcb.save.cr0; + csp->common.cr4 = csp->vmcb.save.cr4; + csp->common.efer = csp->vmcb.save.efer; + csp->common.last_exit_rsp = csp->vmcb.save.rsp; + csp->common.last_exit_rip = csp->vmcb.save.rip; + csp->common.last_exit_rflags = csp->vmcb.save.rflags; + + return 0; +} + +void svm_caretaker_init(struct kvm_vcpu *vcpu) +{ + struct caretaker_svm_page *csp; + + csp = kho_alloc_preserve(sizeof(*csp)); + if (IS_ERR(csp)) { + pr_err("caretaker svm: failed to allocate preserved page\n"); + return; + } + + if (svm_caretaker_init_page(csp, vcpu)) { + kvm_x86_caretaker_unpreserve_pages(&csp->common.abi); + vcpu->caretaker.cb = NULL; + kho_unpreserve_free(csp); + } +} + +static __cpu_preserved_text void +svm_caretaker_seg_to_kvm(struct kvm_segment *var, const struct vmcb_seg *s, int seg) +{ + var->base = s->base; + var->limit = s->limit; + var->selector = s->selector; + var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; + var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; + var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; + var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; + var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; + var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; + var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; + var->g = s->limit > 0xfffff; + var->unusable = !var->present; + if (seg == VCPU_SREG_TR) + var->type |= 0x2; +} + +static __cpu_preserved_text void +svm_caretaker_detach_serialize(void *page, struct kvm_vcpu_arch_ser *state) +{ + struct caretaker_svm_page *csp = page; + struct vmcb_save_area *save = &csp->vmcb.save; + + csp->common.cr0 = save->cr0; + csp->common.cr3 = save->cr3; + csp->common.cr4 = save->cr4; + csp->common.efer = save->efer; + csp->common.last_exit_rip = save->rip; + csp->common.last_exit_rsp = save->rsp; + csp->common.last_exit_rflags = save->rflags; + + kvm_x86_caretaker_detach_serialize_common(&csp->common, state); + + state->sregs.cr2 = save->cr2; + svm_caretaker_seg_to_kvm(&state->sregs.cs, &save->cs, VCPU_SREG_CS); + svm_caretaker_seg_to_kvm(&state->sregs.ds, &save->ds, VCPU_SREG_DS); + svm_caretaker_seg_to_kvm(&state->sregs.es, &save->es, VCPU_SREG_ES); + svm_caretaker_seg_to_kvm(&state->sregs.fs, &save->fs, VCPU_SREG_FS); + svm_caretaker_seg_to_kvm(&state->sregs.gs, &save->gs, VCPU_SREG_GS); + svm_caretaker_seg_to_kvm(&state->sregs.ss, &save->ss, VCPU_SREG_SS); + svm_caretaker_seg_to_kvm(&state->sregs.tr, &save->tr, VCPU_SREG_TR); + svm_caretaker_seg_to_kvm(&state->sregs.ldt, &save->ldtr, VCPU_SREG_LDTR); + state->sregs.gdt.base = save->gdtr.base; + state->sregs.gdt.limit = save->gdtr.limit; + state->sregs.idt.base = save->idtr.base; + state->sregs.idt.limit = save->idtr.limit; + + kvm_x86_caretaker_update_msr(state, MSR_STAR, save->star); + kvm_x86_caretaker_update_msr(state, MSR_LSTAR, save->lstar); + kvm_x86_caretaker_update_msr(state, MSR_CSTAR, save->cstar); + kvm_x86_caretaker_update_msr(state, MSR_SYSCALL_MASK, save->sfmask); + kvm_x86_caretaker_update_msr(state, MSR_KERNEL_GS_BASE, save->kernel_gs_base); + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_CS, save->sysenter_cs); + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_ESP, save->sysenter_esp); + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_EIP, save->sysenter_eip); +} + +static void svm_caretaker_sync_vcpu(struct kvm_vcpu *vcpu, void *vcpu_data) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, raw_smp_processor_id()); + + if (sd && sd->save_area_pa) + wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa); + + kvm_x86_caretaker_sync_vcpu_common(vcpu); + + if (svm->vmcb) { + if (!svm->vmcb->control.next_rip && svm->vmcb->control.insn_len) + svm->vmcb->control.next_rip = kvm_rip_read(vcpu) + + svm->vmcb->control.insn_len; + if (!svm->vmcb->control.next_rip) { + switch (svm->vmcb->control.exit_code) { + case SVM_EXIT_CPUID: + case SVM_EXIT_MSR: + case SVM_EXIT_PAUSE: + svm->vmcb->control.next_rip = kvm_rip_read(vcpu) + 2; + break; + case SVM_EXIT_HLT: + svm->vmcb->control.next_rip = kvm_rip_read(vcpu) + 1; + break; + case SVM_EXIT_VMMCALL: + case SVM_EXIT_XSETBV: + case SVM_EXIT_INVLPGA: + svm->vmcb->control.next_rip = kvm_rip_read(vcpu) + 3; + break; + default: + break; + } + } + + svm->vmcb->control.clean = 0; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; + svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; + svm->vmcb->control.int_ctl |= V_INTR_MASKING_MASK; + if (vgif) + svm_set_gif(svm, true); + svm_clr_intercept(svm, INTERCEPT_VINTR); + svm->vmcb->control.int_state = 0; + svm->nmi_masked = false; + svm->awaiting_iret_completion = false; + svm_set_intercept(svm, INTERCEPT_INTR); + svm_set_intercept(svm, INTERCEPT_NMI); + svm_set_intercept(svm, INTERCEPT_INIT); + svm_clr_intercept(svm, INTERCEPT_RDTSC); + svm_clr_intercept(svm, INTERCEPT_PAUSE); + svm_recalc_intercepts(vcpu); + svm->vmcb->control.event_inj = 0; + svm->vmcb->control.exit_int_info = 0; + } +} + +void __cpu_preserved_text +svm_caretaker_decode_exit(void *page, + struct kvm_caretaker_exit *exit) +{ + struct caretaker_svm_page *csp = page; + u64 exit_code = csp->vmcb.control.exit_code; + u64 info1 = csp->vmcb.control.exit_info_1; + u64 info2 = csp->vmcb.control.exit_info_2; + struct vmcb *vmcb = &csp->vmcb; + u64 rip = vmcb->save.rip; + u32 insn_len; + + if (vmcb->control.next_rip && vmcb->control.next_rip > rip) + insn_len = vmcb->control.next_rip - rip; + else + insn_len = vmcb->control.insn_len; + + csp->common.last_exit_rip = rip; + csp->common.last_exit_rsp = vmcb->save.rsp; + csp->common.last_exit_rflags = vmcb->save.rflags; + csp->common.cr3 = vmcb->save.cr3; + csp->common.cr0 = vmcb->save.cr0; + csp->common.cr4 = vmcb->save.cr4; + csp->common.efer = vmcb->save.efer; + vmcb->control.exit_int_info = 0; + + exit->rip = rip; + exit->insn_len = insn_len; + exit->raw_reason = exit_code; + exit->type = KVM_CARETAKER_EXIT_ARCH; + + switch (exit_code) { + case SVM_EXIT_IOIO: { + u16 port = (u16)(info1 >> 16); + + if (!exit->insn_len && info2 > rip) + exit->insn_len = (u32)(info2 - rip); + + if (port >= COM1_PORT_BASE && port <= COM1_PORT_END) { + exit->type = KVM_CARETAKER_EXIT_CONSOLE; + exit->mmio_io.addr = port; + exit->mmio_io.is_write = !(info1 & SVM_IOIO_TYPE_MASK); + exit->mmio_io.size = (u8)((info1 & SVM_IOIO_SIZE_MASK) >> + SVM_IOIO_SIZE_SHIFT); + exit->mmio_io.is_mmio = false; + exit->mmio_io.val_ptr = &csp->common.rax; + } + break; + } + case SVM_EXIT_HLT: + exit->type = KVM_CARETAKER_EXIT_IDLE; + if (!exit->insn_len) + exit->insn_len = 1; + break; + case SVM_EXIT_PAUSE: + exit->type = KVM_CARETAKER_EXIT_IDLE; + if (!exit->insn_len) + exit->insn_len = 2; + break; + case SVM_EXIT_CPUID: + exit->type = KVM_CARETAKER_EXIT_CPUID; + if (!exit->insn_len) + exit->insn_len = 2; + break; + case SVM_EXIT_MSR: + exit->type = KVM_CARETAKER_EXIT_MSR; + exit->msr.msr = (u32)csp->common.rcx; + exit->msr.is_write = (info1 != 0); + if (!exit->insn_len) + exit->insn_len = 2; + break; + case SVM_EXIT_INVD: + case SVM_EXIT_WBINVD: + exit->type = KVM_CARETAKER_EXIT_INSN_STEP; + if (!exit->insn_len) + exit->insn_len = 2; + break; + case SVM_EXIT_NPF: + exit->type = KVM_CARETAKER_EXIT_UNHANDLED; + break; + case SVM_EXIT_INTR: + kvm_x86_caretaker_disarm_timer(); + asm volatile("sti\n\tnop\n\tpause\n\tcli" : : : "memory"); + exit->type = KVM_CARETAKER_EXIT_PREEMPT_TIMER; + exit->insn_len = 0; + break; + case SVM_EXIT_NMI: + case SVM_EXIT_INIT: + exit->type = KVM_CARETAKER_EXIT_PREEMPT_TIMER; + exit->insn_len = 0; + break; + case SVM_EXIT_VINTR: + case SVM_EXIT_WRITE_CR8: + exit->type = KVM_CARETAKER_EXIT_CROSS_VCPU; + exit->insn_len = 0; + break; + case SVM_EXIT_VMMCALL: + exit->type = KVM_CARETAKER_EXIT_CROSS_VCPU; + if (!exit->insn_len) + exit->insn_len = 3; + break; + case SVM_EXIT_RDTSC: + exit->type = KVM_CARETAKER_EXIT_RDTSC; + if (!exit->insn_len) + exit->insn_len = 2; + break; + default: + /* + * Everything not decoded above keeps KVM_CARETAKER_EXIT_ARCH + * and stalls in kvm_x86_caretaker_handle_exit(). + * + * XSETBV and RDTSCP used to be decoded here and are + * deliberately not any more. INSN_STEP means "the instruction + * had no architectural effect worth emulating", which holds + * for INVD and WBINVD on a coherent, guest_memfd-backed VM but + * not for these: stepping XSETBV leaves XCR0 unchanged while + * the guest believes it changed. RDTSCP is the same story via + * KVM_CARETAKER_EXIT_RDTSC, which writes only RAX and RDX and + * would leave a stale RCX instead of IA32_TSC_AUX. + */ + break; + } +} + +static __cpu_preserved_text void +svm_caretaker_advance_rip(void *page, u64 rip) +{ + struct caretaker_svm_page *csp = page; + + csp->vmcb.save.rax = csp->common.rax; + csp->vmcb.save.rip = rip; + csp->common.last_exit_rip = rip; + csp->common.last_exit_rsp = csp->vmcb.save.rsp; +} + +static __cpu_preserved_text void +svm_caretaker_arm_timer(void *page, u64 deadline_ticks) +{ + struct caretaker_svm_page *csp = page; + + if (deadline_ticks) { + vmcb_set_intercept(&csp->vmcb.control, INTERCEPT_INTR); + kvm_x86_caretaker_arm_timer(deadline_ticks); + } else { + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_INTR); + } +} + +static __cpu_preserved_text void +svm_caretaker_disarm_timer(void *page) +{ + struct caretaker_svm_page *csp = page; + + kvm_x86_caretaker_disarm_timer(); + vmcb_clr_intercept(&csp->vmcb.control, INTERCEPT_INTR); + asm volatile("sti\n\tnop\n\tpause\n\tcli" : : : "memory"); +} + +static __cpu_preserved_text void +svm_caretaker_pre_enter(void *page) +{ + struct caretaker_svm_page *csp = page; + u64 efer; + + csp->orig_hsave_pa = native_rdmsrq(MSR_VM_HSAVE_PA); + + /* Ensure EFER_SVME is enabled and HSAVE is configured */ + efer = native_rdmsrq(MSR_EFER); + csp->orig_efer = efer; + native_wrmsrq(MSR_EFER, efer | EFER_SVME); + native_wrmsrq(MSR_VM_HSAVE_PA, csp->hsave_pa); +} + +static __cpu_preserved_text void +svm_caretaker_post_exit(void *page) +{ + struct caretaker_svm_page *csp = page; + + native_wrmsrq(MSR_VM_HSAVE_PA, csp->orig_hsave_pa); + asm volatile("1: stgi\n\t" + "2:\n\t" + _ASM_EXTABLE(1b, 2b) + : : : "memory"); + if (!(csp->orig_efer & EFER_SVME)) + native_wrmsrq(MSR_EFER, native_rdmsrq(MSR_EFER) & ~EFER_SVME); +} + +static const struct kvm_x86_caretaker_runtime_ops svm_caretaker_runtime_ops __cpu_preserved_data = { + .detach_serialize = svm_caretaker_detach_serialize, + .common = { + .enter_guest = svm_caretaker_enter, + .decode_exit = svm_caretaker_decode_exit, + .handle_arch_exit = kvm_x86_caretaker_handle_exit, + .advance_rip = svm_caretaker_advance_rip, + .arm_timer = svm_caretaker_arm_timer, + .disarm_timer = svm_caretaker_disarm_timer, + .pre_run = svm_caretaker_pre_enter, + .post_run = svm_caretaker_post_exit, + }, +}; + +static const struct kvm_x86_caretaker_ops svm_caretaker_ops = { + .name = "svm", + .init = svm_caretaker_init, + .sync_vcpu = svm_caretaker_sync_vcpu, + .runtime = &svm_caretaker_runtime_ops, +}; + +void svm_caretaker_register(void) +{ + kvm_x86_caretaker_register_ops(&svm_caretaker_ops); +} + +void svm_caretaker_unregister(void) +{ + kvm_x86_caretaker_unregister_ops(&svm_caretaker_ops); +} diff --git a/arch/x86/kvm/svm/caretaker_vmenter.S b/arch/x86/kvm/svm/caretaker_vmenter.S index e0276a34403e..10303268844e 100644 --- a/arch/x86/kvm/svm/caretaker_vmenter.S +++ b/arch/x86/kvm/svm/caretaker_vmenter.S @@ -27,7 +27,7 @@ SYM_TYPED_FUNC_START(svm_caretaker_enter) sti /* 0. Save host VMCB state (FS, GS, TR, LDTR) to host save area */ - movq CXP_HSAVE_PA(%rdi), %rax + movq CSP_HSAVE_PA_OFFSET(%rdi), %rax testq %rax, %rax jz 10f vmsave %rax @@ -69,7 +69,7 @@ SYM_TYPED_FUNC_START(svm_caretaker_enter) movq %rbx, CXP_REG_RAX(%rdi) /* 5a. Restore host VMCB state (FS, GS, TR, LDTR) from HSAVE */ - movq CXP_HSAVE_PA(%rdi), %rax + movq CSP_HSAVE_PA_OFFSET(%rdi), %rax testq %rax, %rax jz 40f vmload %rax diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..ca6b0d1e80fe 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -54,6 +54,7 @@ #include "svm.h" #include "svm_ops.h" +#include "caretaker.h" #include "hyperv.h" #include "kvm_onhyperv.h" #include "svm_onhyperv.h" @@ -973,6 +974,8 @@ static void svm_hardware_unsetup(void) { int cpu; + svm_caretaker_unregister(); + avic_hardware_unsetup(); sev_hardware_unsetup(); @@ -1117,7 +1120,7 @@ static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu) svm_clr_intercept(svm, INTERCEPT_RDPMC); } -static void svm_recalc_intercepts(struct kvm_vcpu *vcpu) +void svm_recalc_intercepts(struct kvm_vcpu *vcpu) { svm_recalc_instruction_intercepts(vcpu); svm_recalc_msr_intercepts(vcpu); @@ -4344,6 +4347,7 @@ static void svm_complete_interrupts(struct kvm_vcpu *vcpu) bool nmi_l1_to_l2 = svm->nmi_l1_to_l2; bool soft_int_injected = svm->soft_int_injected; + svm->vmcb->control.exit_int_info = 0; svm->nmi_l1_to_l2 = false; svm->soft_int_injected = false; @@ -4396,7 +4400,8 @@ static void svm_complete_interrupts(struct kvm_vcpu *vcpu) break; } case SVM_EXITINTINFO_TYPE_INTR: - kvm_queue_interrupt(vcpu, vector, false); + if (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) + kvm_queue_interrupt(vcpu, vector, false); break; case SVM_EXITINTINFO_TYPE_SOFT: kvm_queue_interrupt(vcpu, vector, true); @@ -5762,6 +5767,8 @@ static __init int svm_hardware_setup(void) goto err; } + svm_caretaker_register(); + return 0; err: -- 2.55.0.1082.g2b9226bbc0-goog
