Complete Intel VMX Caretaker detach-time serialization, vCPU sync, and ops registration in arch/x86/kvm/vmx/caretaker.c.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/kvm/Makefile | 9 + arch/x86/kvm/vmx/caretaker.c | 322 ++++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx/vmx.c | 4 + 3 files changed, 333 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 11e67c072258..b550a3fd0708 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -18,6 +18,8 @@ kvm-$(CONFIG_KVM_SMM) += smm.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/nested.o vmx/posted_intr.o vmx/main.o +kvm-intel-$(CONFIG_KVM_CARETAKER) += vmx/caretaker.o vmx/caretaker_vmenter.o + kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o @@ -44,6 +46,13 @@ $(obj)/svm/vmenter.o: $(obj)/kvm-asm-offsets.h AFLAGS_vmx/vmenter.o := -iquote $(obj) $(obj)/vmx/vmenter.o: $(obj)/kvm-asm-offsets.h +AFLAGS_vmx/caretaker_vmenter.o := -iquote $(obj) +$(obj)/vmx/caretaker_vmenter.o: $(obj)/kvm-asm-offsets.h + +CFLAGS_REMOVE_vmx/caretaker.o = $(CC_FLAGS_FTRACE) +CFLAGS_vmx/caretaker.o := -fno-stack-protector + + $(obj)/kvm-asm-offsets.h: $(obj)/kvm-asm-offsets.s FORCE $(call filechk,offsets,__KVM_ASM_OFFSETS_H__) diff --git a/arch/x86/kvm/vmx/caretaker.c b/arch/x86/kvm/vmx/caretaker.c index 47b4743650e5..efef1c30c0d1 100644 --- a/arch/x86/kvm/vmx/caretaker.c +++ b/arch/x86/kvm/vmx/caretaker.c @@ -49,8 +49,8 @@ static int vmx_caretaker_init_page(struct caretaker_vmx_page *cvp, u64 basic_msr, misc_msr; int ret; - if (!vmx->vmcs01.vmcs) - return -EINVAL; + if (!vmx->vmcs01.vmcs || is_guest_mode(vcpu)) + return -EOPNOTSUPP; ret = kvm_x86_caretaker_init_common_page(&cvp->common, vcpu, sizeof(*cvp)); if (ret) @@ -336,5 +336,323 @@ vmx_caretaker_init_host_vmcs(struct caretaker_vmx_page *cvp) vmx_vmwrite(PLE_WINDOW, 4096); } vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, cpu_ctl); + + /* + * Disable hardware MSR autoload lists while running in Caretaker so + * VM-entry/VM-exit do not dereference unpreserved host vmx->msr_autoload + * memory after kexec. + */ + vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0); + vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0); + vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0); +} + +/* + * Guest-visible VMCS fields carried from the VMCS the caretaker ran the vCPU + * on to the VMCS the new kernel allocated for it. + * + * The new kernel does not adopt the old VMCS: it belongs to the previous + * kernel's struct loaded_vmcs, whose layout is not part of any handover ABI, + * and the VMCS region itself is opaque and implementation defined. So the + * architecturally defined guest state is copied field by field instead. + * + * GUEST_IA32_EFER is handled separately because it is only written back when + * the caretaker actually recorded a value for it. + */ +static const u16 vmx_caretaker_guest_fields[] = { + GUEST_CS_SELECTOR, GUEST_CS_LIMIT, + GUEST_CS_AR_BYTES, GUEST_CS_BASE, + GUEST_SS_SELECTOR, GUEST_SS_LIMIT, + GUEST_SS_AR_BYTES, GUEST_SS_BASE, + GUEST_DS_SELECTOR, GUEST_DS_LIMIT, + GUEST_DS_AR_BYTES, GUEST_DS_BASE, + GUEST_ES_SELECTOR, GUEST_ES_LIMIT, + GUEST_ES_AR_BYTES, GUEST_ES_BASE, + GUEST_FS_SELECTOR, GUEST_FS_LIMIT, + GUEST_FS_AR_BYTES, GUEST_FS_BASE, + GUEST_GS_SELECTOR, GUEST_GS_LIMIT, + GUEST_GS_AR_BYTES, GUEST_GS_BASE, + GUEST_TR_SELECTOR, GUEST_TR_LIMIT, + GUEST_TR_AR_BYTES, GUEST_TR_BASE, + GUEST_LDTR_SELECTOR, GUEST_LDTR_LIMIT, + GUEST_LDTR_AR_BYTES, GUEST_LDTR_BASE, + GUEST_GDTR_LIMIT, GUEST_GDTR_BASE, + GUEST_IDTR_LIMIT, GUEST_IDTR_BASE, + GUEST_INTERRUPTIBILITY_INFO, + GUEST_ACTIVITY_STATE, + GUEST_IA32_DEBUGCTL, + GUEST_SYSENTER_CS, + GUEST_SYSENTER_ESP, + GUEST_SYSENTER_EIP, +}; + +static void +vmx_caretaker_sync_vcpu(struct kvm_vcpu *vcpu, void *vcpu_data) +{ + struct kvm_caretaker_arch_ser *abi = vcpu_data; + struct vcpu_vmx *vmx = to_vmx(vcpu); + phys_addr_t cur_vmcs_pa = vmx->loaded_vmcs ? virt_to_phys(vmx->loaded_vmcs->vmcs) : 0; + struct vmcs *prev_vmcs; + + guard(preempt)(); + prev_vmcs = this_cpu_read(current_vmcs); + + if (abi->vmcs_pa && cur_vmcs_pa && abi->vmcs_pa != cur_vmcs_pa) { + /* 42 * 8 bytes; this runs on the host stack, not a preserved one. */ + unsigned long val[ARRAY_SIZE(vmx_caretaker_guest_fields)]; + unsigned long guest_efer; + int i; + + asm volatile("vmptrld %0" : : "m" (abi->vmcs_pa) : "memory", "cc"); + + for (i = 0; i < ARRAY_SIZE(vmx_caretaker_guest_fields); i++) + val[i] = vmx_vmread(vmx_caretaker_guest_fields[i]); + guest_efer = vmx_caretaker_read_efer(); + + asm volatile("vmclear %0" : : "m" (abi->vmcs_pa) : "memory", "cc"); + asm volatile("vmptrld %0" : : "m" (cur_vmcs_pa) : "memory", "cc"); + + for (i = 0; i < ARRAY_SIZE(vmx_caretaker_guest_fields); i++) + vmx_vmwrite(vmx_caretaker_guest_fields[i], val[i]); + if (guest_efer) + vmx_vmwrite(GUEST_IA32_EFER, guest_efer); + + abi->vmcs_pa = cur_vmcs_pa; + } else if (cur_vmcs_pa) { + asm volatile("vmptrld %0" : : "m" (cur_vmcs_pa) : "memory", "cc"); + } + + kvm_x86_caretaker_sync_vcpu_common(vcpu); + + if (vmx->loaded_vmcs) { + pin_controls_clearbit(vmx, PIN_BASED_VMX_PREEMPTION_TIMER); + vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, pin_controls_get(vmx)); + vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_controls_get(vmx)); + vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); + memset(&vmx->loaded_vmcs->host_state, 0, + sizeof(struct vmcs_host_state)); + list_del_init(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link); + vmx->loaded_vmcs->cpu = -1; + vmx->loaded_vmcs->launched = 0; + } + vmx_segment_cache_clear(vmx); + vmx->vt.guest_state_loaded = false; + vmx->guest_uret_msrs_loaded = false; + + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); + vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); + vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); + vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); + + vmcs_writel(GUEST_RIP, kvm_rip_read(vcpu)); + vmcs_writel(GUEST_RSP, kvm_rsp_read(vcpu)); + vmcs_writel(GUEST_RFLAGS, kvm_get_rflags(vcpu)); + vmx_set_cr0(vcpu, vcpu->arch.cr0); + vmcs_writel(GUEST_CR3, vcpu->arch.cr3); + vmx_set_cr4(vcpu, vcpu->arch.cr4); + vmx_set_efer(vcpu, vcpu->arch.efer); + + if (vmx->loaded_vmcs) + vmx_set_constant_host_state(vmx); + + if (cur_vmcs_pa) + asm volatile("vmclear %0" : : "m" (cur_vmcs_pa) : "memory", "cc"); + + if (prev_vmcs && (!vmx->loaded_vmcs || prev_vmcs != vmx->loaded_vmcs->vmcs)) { + vmcs_load(prev_vmcs); + this_cpu_write(current_vmcs, prev_vmcs); + } else { + this_cpu_write(current_vmcs, NULL); + } +} + +static __cpu_preserved_text void +vmx_caretaker_detach_serialize(void *page, struct kvm_vcpu_arch_ser *state) +{ + struct caretaker_vmx_page *cvp = page; + + kvm_x86_caretaker_detach_serialize_common(&cvp->common, state); + kvm_x86_caretaker_update_msr(state, MSR_STAR, cvp->star); + kvm_x86_caretaker_update_msr(state, MSR_LSTAR, cvp->lstar); + kvm_x86_caretaker_update_msr(state, MSR_SYSCALL_MASK, cvp->fmask); + kvm_x86_caretaker_update_msr(state, MSR_KERNEL_GS_BASE, + cvp->common.kernel_gs_base); +} + +static __cpu_preserved_text void vmx_caretaker_arm_timer(void *page, u64 deadline_ticks) +{ + struct caretaker_vmx_page *cvp = page; + u32 shift = (cvp && cvp->timer_shift) ? cvp->timer_shift : VMX_PREEMPTION_TIMER_SHIFT; + u32 timer_value = 0; + u32 pin; + + if (deadline_ticks) { + u64 now = arch_oncore_read_counter(); + + if (deadline_ticks > now) { + u64 remaining = deadline_ticks - now; + + timer_value = (u32)(remaining >> shift); + if (timer_value == 0) + timer_value = 1; + } else { + timer_value = 1; + } + } + + if (timer_value > 0) { + vmx_vmwrite(VMX_PREEMPTION_TIMER_VALUE, timer_value); + pin = (u32)vmx_vmread(PIN_BASED_VM_EXEC_CONTROL); + pin |= PIN_BASED_VMX_PREEMPTION_TIMER; + vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, pin); + } else { + vmx_caretaker_disarm_timer(page); + } } +static __cpu_preserved_text void +vmx_caretaker_advance_rip(void *page, u64 rip) +{ + struct caretaker_vmx_page *cvp = page; + + cvp->common.last_exit_rip = rip; + vmx_vmwrite(GUEST_RIP, rip); +} + +static __cpu_preserved_text void vmx_caretaker_pre_enter(void *page) +{ + struct caretaker_vmx_page *cvp = page; + + /* Ensure VMX is active on this core */ + if (!(__read_cr4() & X86_CR4_VMXE)) { + asm volatile("mov %0, %%cr4" : : "r" (__read_cr4() | X86_CR4_VMXE) : "memory"); + if (cvp->vmxon_pa) { + asm volatile("1: vmxon %[vmxon_pa]\n\t" + "2:\n\t" + _ASM_EXTABLE(1b, 2b) + : : [vmxon_pa] "m" (cvp->vmxon_pa) + : "memory", "cc"); + } + } + + /* Activate VMCS on this pCPU */ + asm volatile("vmptrld %0" : : "m" (cvp->common.abi.vmcs_pa) : "memory", "cc"); + + /* Configure Caretaker host VMCS */ + vmx_caretaker_init_host_vmcs(cvp); + + if (cvp->common.arch_state) + native_write_cr2(cvp->common.arch_state->sregs.cr2); + + native_wrmsrq(MSR_STAR, cvp->star); + native_wrmsrq(MSR_LSTAR, cvp->lstar); + native_wrmsrq(MSR_SYSCALL_MASK, cvp->fmask); +} + +static __cpu_preserved_text void +vmx_caretaker_read_seg(struct kvm_segment *var, u16 sel_field, + u16 base_field, u16 limit_field, u16 ar_field) +{ + u32 ar = (u32)vmx_vmread(ar_field); + + var->base = vmx_vmread(base_field); + var->limit = (u32)vmx_vmread(limit_field); + var->selector = (u16)vmx_vmread(sel_field); + var->unusable = (ar >> 16) & 1; + var->type = ar & 15; + var->s = (ar >> 4) & 1; + var->dpl = (ar >> 5) & 3; + var->present = !var->unusable; + var->avl = (ar >> 12) & 1; + var->l = (ar >> 13) & 1; + var->db = (ar >> 14) & 1; + var->g = (ar >> 15) & 1; +} + +static __cpu_preserved_text void vmx_caretaker_post_exit(void *page) +{ + struct caretaker_vmx_page *cvp = page; + struct kvm_vcpu_arch_ser *state = cvp->common.arch_state; + u64 efer; + + cvp->star = native_rdmsrq(MSR_STAR); + cvp->lstar = native_rdmsrq(MSR_LSTAR); + cvp->fmask = native_rdmsrq(MSR_SYSCALL_MASK); + + cvp->common.cr0 = vmx_caretaker_read_cr0(); + cvp->common.cr3 = vmx_vmread(GUEST_CR3); + cvp->common.cr4 = vmx_caretaker_read_cr4(); + efer = vmx_caretaker_read_efer(); + if (efer) + cvp->common.efer = efer; + cvp->common.last_exit_rip = vmx_vmread(GUEST_RIP); + cvp->common.last_exit_rsp = vmx_vmread(GUEST_RSP); + cvp->common.last_exit_rflags = vmx_vmread(GUEST_RFLAGS); + + if (state) { + state->sregs.cr2 = native_read_cr2(); + vmx_caretaker_read_seg(&state->sregs.cs, GUEST_CS_SELECTOR, + GUEST_CS_BASE, GUEST_CS_LIMIT, GUEST_CS_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.ds, GUEST_DS_SELECTOR, + GUEST_DS_BASE, GUEST_DS_LIMIT, GUEST_DS_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.es, GUEST_ES_SELECTOR, + GUEST_ES_BASE, GUEST_ES_LIMIT, GUEST_ES_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.fs, GUEST_FS_SELECTOR, + GUEST_FS_BASE, GUEST_FS_LIMIT, GUEST_FS_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.gs, GUEST_GS_SELECTOR, + GUEST_GS_BASE, GUEST_GS_LIMIT, GUEST_GS_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.ss, GUEST_SS_SELECTOR, + GUEST_SS_BASE, GUEST_SS_LIMIT, GUEST_SS_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.tr, GUEST_TR_SELECTOR, + GUEST_TR_BASE, GUEST_TR_LIMIT, GUEST_TR_AR_BYTES); + vmx_caretaker_read_seg(&state->sregs.ldt, GUEST_LDTR_SELECTOR, + GUEST_LDTR_BASE, GUEST_LDTR_LIMIT, GUEST_LDTR_AR_BYTES); + state->sregs.gdt.base = vmx_vmread(GUEST_GDTR_BASE); + state->sregs.gdt.limit = (u16)vmx_vmread(GUEST_GDTR_LIMIT); + state->sregs.idt.base = vmx_vmread(GUEST_IDTR_BASE); + state->sregs.idt.limit = (u16)vmx_vmread(GUEST_IDTR_LIMIT); + + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_CS, + vmx_vmread(GUEST_SYSENTER_CS)); + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_ESP, + vmx_vmread(GUEST_SYSENTER_ESP)); + kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_EIP, + vmx_vmread(GUEST_SYSENTER_EIP)); + } + + /* Flush VMCS cache so host and incoming kernel see latest guest state */ + asm volatile("vmclear %0" : : "m" (cvp->common.abi.vmcs_pa) : "memory", "cc"); +} + +static const struct kvm_x86_caretaker_runtime_ops vmx_caretaker_runtime_ops __cpu_preserved_data = { + .detach_serialize = vmx_caretaker_detach_serialize, + .common = { + .enter_guest = vmx_caretaker_enter, + .decode_exit = vmx_caretaker_decode_exit, + .handle_arch_exit = kvm_x86_caretaker_handle_exit, + .advance_rip = vmx_caretaker_advance_rip, + .arm_timer = vmx_caretaker_arm_timer, + .disarm_timer = vmx_caretaker_disarm_timer, + .pre_run = vmx_caretaker_pre_enter, + .post_run = vmx_caretaker_post_exit, + }, +}; + +static const struct kvm_x86_caretaker_ops vmx_caretaker_ops = { + .name = "vmx", + .init = vmx_caretaker_init, + .sync_vcpu = vmx_caretaker_sync_vcpu, + .runtime = &vmx_caretaker_runtime_ops, +}; + +void vmx_caretaker_register(void) +{ + kvm_x86_caretaker_register_ops(&vmx_caretaker_ops); +} + +void vmx_caretaker_unregister(void) +{ + kvm_x86_caretaker_unregister_ops(&vmx_caretaker_ops); +} diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 48b6e69f99bf..817623f7429a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -54,6 +54,7 @@ #include <trace/events/ipi.h> #include "capabilities.h" +#include "caretaker.h" #include "common.h" #include "cpuid.h" #include "hyperv.h" @@ -8548,6 +8549,7 @@ void vmx_migrate_timers(struct kvm_vcpu *vcpu) void vmx_hardware_unsetup(void) { + vmx_caretaker_unregister(); kvm_set_posted_intr_wakeup_handler(NULL); if (nested) @@ -8855,6 +8857,8 @@ __init int vmx_hardware_setup(void) kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; + vmx_caretaker_register(); + return 0; } -- 2.55.0.1082.g2b9226bbc0-goog
