Implement Intel VMX Caretaker VMCS setup, exit decoding, and guest entry/exit loop in arch/x86/kvm/vmx/caretaker.c.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/kvm/vmx/caretaker.c | 340 +++++++++++++++++++++++++++ arch/x86/kvm/vmx/caretaker_vmenter.S | 12 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/vmx.h | 1 + 4 files changed, 347 insertions(+), 8 deletions(-) create mode 100644 arch/x86/kvm/vmx/caretaker.c diff --git a/arch/x86/kvm/vmx/caretaker.c b/arch/x86/kvm/vmx/caretaker.c new file mode 100644 index 000000000000..47b4743650e5 --- /dev/null +++ b/arch/x86/kvm/vmx/caretaker.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel VMX Caretaker Standalone Execution Engine + * + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + * + * Runs the Intel VMX guest execution loop in an isolated, KHO-preserved memory + * page that remains alive and executing across kexec relocation and + * kernel handover. + */ + +#include <linux/cleanup.h> +#include <linux/cpu_preserve.h> +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/kexec.h> +#include <linux/kexec_handover.h> +#include <linux/kvm_host.h> +#include <linux/objtool.h> +#include <linux/oncore.h> + +#include <asm/apic.h> +#include <asm/cpu_entry_area.h> +#include <asm/desc.h> +#include <asm/fixmap.h> +#include <linux/pgtable.h> +#include <linux/processor.h> +#include <asm/segment.h> +#include <asm/set_memory.h> +#include <asm/virt.h> +#include <asm/vmx.h> + +#include "../caretaker.h" +#include "caretaker.h" +#include "vmx.h" +#include "vmx_ops.h" +#include "x86.h" +#include "x86_ops.h" + +static int vmx_caretaker_init_page(struct caretaker_vmx_page *cvp, + struct kvm_vcpu *vcpu); +STACK_FRAME_NON_STANDARD(vmx_caretaker_init_page); + +static int vmx_caretaker_init_page(struct caretaker_vmx_page *cvp, + struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 basic_msr, misc_msr; + int ret; + + if (!vmx->vmcs01.vmcs) + return -EINVAL; + + ret = kvm_x86_caretaker_init_common_page(&cvp->common, vcpu, sizeof(*cvp)); + if (ret) + return ret; + cvp->common.abi.vmcs_pa = virt_to_phys(vmx->vmcs01.vmcs); + cvp->vmxon_pa = virt_to_phys(cvp->vmxon_area); + + memset(cvp->vmxon_area, 0, PAGE_SIZE); + basic_msr = native_rdmsrq(MSR_IA32_VMX_BASIC); + *(u32 *)cvp->vmxon_area = vmx_basic_vmcs_revision_id(basic_msr); + + if (!rdmsrq_safe(MSR_IA32_VMX_MISC, &misc_msr)) + cvp->timer_shift = vmx_misc_preemption_timer_rate(misc_msr); + else + cvp->timer_shift = VMX_PREEMPTION_TIMER_SHIFT; + + cvp->ple_supported = cpu_has_vmx_ple(); + + vcpu_load(vcpu); + + cvp->common.kernel_gs_base = vmx->msr_guest_kernel_gs_base; + kvm_msr_read(vcpu, MSR_STAR, &cvp->star); + kvm_msr_read(vcpu, MSR_LSTAR, &cvp->lstar); + kvm_msr_read(vcpu, MSR_SYSCALL_MASK, &cvp->fmask); + + /* Save current guest control registers from KVM */ + cvp->common.cr0 = kvm_read_cr0(vcpu); + cvp->common.cr3 = kvm_read_cr3(vcpu); + cvp->common.cr4 = kvm_read_cr4(vcpu); + cvp->common.efer = vcpu->arch.efer; + + cvp->common.last_exit_rip = kvm_rip_read(vcpu); + cvp->common.last_exit_rsp = kvm_rsp_read(vcpu); + cvp->common.last_exit_rflags = kvm_get_rflags(vcpu); + + if (kvm_host.efer & EFER_NX) + cvp->common.efer |= EFER_NX; + + vcpu_put(vcpu); + + if (vmx->loaded_vmcs) + loaded_vmcs_clear(vmx->loaded_vmcs); + + return 0; +} + +void vmx_caretaker_init(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct caretaker_vmx_page *cvp; + struct kvm_caretaker_arch_ser *abi; + + cvp = kho_alloc_preserve(sizeof(*cvp)); + if (IS_ERR(cvp)) { + pr_err("caretaker vmx: failed to allocate preserved page\n"); + return; + } + + if (vmx_caretaker_init_page(cvp, vcpu)) + goto err_free; + + abi = &cvp->common.abi; + if (vmx->vmcs01.vmcs && + kvm_x86_caretaker_preserve_page(abi, virt_to_page(vmx->vmcs01.vmcs))) + goto err_free; + if (vmx->vmcs01.msr_bitmap && + kvm_x86_caretaker_preserve_page(abi, virt_to_page(vmx->vmcs01.msr_bitmap))) + goto err_free; + if (vmx->pml_pg && + kvm_x86_caretaker_preserve_page(abi, vmx->pml_pg)) + goto err_free; + if (vmx->ve_info && + kvm_x86_caretaker_preserve_page(abi, virt_to_page(vmx->ve_info))) + goto err_free; + + return; + +err_free: + kvm_x86_caretaker_unpreserve_pages(&cvp->common.abi); + vcpu->caretaker.cb = NULL; + kho_unpreserve_free(cvp); +} + +static __cpu_preserved_text void vmx_caretaker_disarm_timer(void *page) +{ + u32 pin = (u32)vmx_vmread(PIN_BASED_VM_EXEC_CONTROL); + + if (pin & PIN_BASED_VMX_PREEMPTION_TIMER) { + vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, + pin & ~PIN_BASED_VMX_PREEMPTION_TIMER); + } +} + +static inline unsigned long vmx_caretaker_read_cr0(void) +{ + unsigned long mask = vmx_vmread(CR0_GUEST_HOST_MASK); + + return (vmx_vmread(CR0_READ_SHADOW) & mask) | + (vmx_vmread(GUEST_CR0) & ~mask); +} + +static inline unsigned long vmx_caretaker_read_cr4(void) +{ + unsigned long mask = vmx_vmread(CR4_GUEST_HOST_MASK); + + return (vmx_vmread(CR4_READ_SHADOW) & mask) | + (vmx_vmread(GUEST_CR4) & ~mask); +} + +static inline u64 vmx_caretaker_read_efer(void) +{ + u64 efer = vmx_vmread(GUEST_IA32_EFER); + + if (!efer) + efer = native_rdmsrq(MSR_EFER); + if (vmx_vmread(VM_ENTRY_CONTROLS) & VM_ENTRY_IA32E_MODE) + efer |= EFER_LMA | EFER_LME; + return efer; +} + +void __cpu_preserved_text +vmx_caretaker_decode_exit(void *page, + struct kvm_caretaker_exit *exit) +{ + struct caretaker_vmx_page *cvp = page; + u32 insn_len = (u32)vmx_vmread(VM_EXIT_INSTRUCTION_LEN); + u16 exit_reason = (u16)vmx_vmread(VM_EXIT_REASON); + u64 qual = vmx_vmread(EXIT_QUALIFICATION); + u64 rip = vmx_vmread(GUEST_RIP); + + cvp->common.last_exit_rip = rip; + cvp->common.last_exit_rsp = vmx_vmread(GUEST_RSP); + cvp->common.last_exit_rflags = vmx_vmread(GUEST_RFLAGS); + cvp->common.cr3 = vmx_vmread(GUEST_CR3); + cvp->common.cr0 = vmx_caretaker_read_cr0(); + cvp->common.cr4 = vmx_caretaker_read_cr4(); + + exit->rip = rip; + exit->insn_len = insn_len; + exit->raw_reason = exit_reason; + exit->type = KVM_CARETAKER_EXIT_ARCH; + + switch (exit_reason) { + case EXIT_REASON_IO_INSTRUCTION: { + u16 port = (u16)(qual >> VMX_IO_PORT_SHIFT); + + if (port >= COM1_PORT_BASE && port <= COM1_PORT_END) { + exit->type = KVM_CARETAKER_EXIT_CONSOLE; + exit->mmio_io.addr = port; + exit->mmio_io.is_write = !(qual & VMX_IO_DIRECTION_BIT); + exit->mmio_io.size = (u8)((qual & VMX_IO_SIZE_MASK) + 1); + exit->mmio_io.is_mmio = false; + exit->mmio_io.val_ptr = &cvp->common.rax; + } + break; + } + case EXIT_REASON_HLT: + exit->type = KVM_CARETAKER_EXIT_IDLE; + break; + case EXIT_REASON_PAUSE_INSTRUCTION: + exit->type = KVM_CARETAKER_EXIT_IDLE; + exit->insn_len = insn_len ? insn_len : PAUSE_INSN_LEN; + break; + case EXIT_REASON_CPUID: + exit->type = KVM_CARETAKER_EXIT_CPUID; + break; + case EXIT_REASON_VMCALL: + /* + * Do not write a return value. The Caretaker cannot run the + * hypercall, and reporting success for something like + * KVM_HC_SEND_IPI or a PV TLB flush is worse than not + * answering at all. CROSS_VCPU now stalls, so the guest + * stays parked on the VMCALL with RAX untouched until the + * incoming kernel services it. + */ + exit->type = KVM_CARETAKER_EXIT_CROSS_VCPU; + exit->insn_len = insn_len ? insn_len : VMCALL_INSN_LEN; + break; + case EXIT_REASON_MSR_READ: + exit->type = KVM_CARETAKER_EXIT_MSR; + exit->msr.msr = (u32)cvp->common.rcx; + exit->msr.is_write = false; + break; + case EXIT_REASON_MSR_WRITE: + exit->type = KVM_CARETAKER_EXIT_MSR; + exit->msr.msr = (u32)cvp->common.rcx; + exit->msr.is_write = true; + break; + case EXIT_REASON_RDTSC: + exit->type = KVM_CARETAKER_EXIT_RDTSC; + break; + case EXIT_REASON_EPT_VIOLATION: + exit->type = KVM_CARETAKER_EXIT_UNHANDLED; + break; + case EXIT_REASON_PREEMPTION_TIMER: + exit->type = KVM_CARETAKER_EXIT_PREEMPT_TIMER; + exit->insn_len = 0; + vmx_caretaker_disarm_timer(cvp); + break; + case EXIT_REASON_EOI_INDUCED: + case EXIT_REASON_APIC_WRITE: + case EXIT_REASON_APIC_ACCESS: + case EXIT_REASON_INTERRUPT_WINDOW: + exit->type = KVM_CARETAKER_EXIT_CROSS_VCPU; + exit->insn_len = 0; + break; + case EXIT_REASON_EXTERNAL_INTERRUPT: + case EXIT_REASON_EXCEPTION_NMI: + case EXIT_REASON_INIT_SIGNAL: + case EXIT_REASON_SIPI_SIGNAL: + exit->type = KVM_CARETAKER_EXIT_PREEMPT_TIMER; + exit->insn_len = 0; + break; + default: + break; + } +} + +void __cpu_preserved_text +vmx_caretaker_init_host_vmcs(struct caretaker_vmx_page *cvp) +{ + u64 fs_base = 0, gs_base = 0; + unsigned long pin, cpu_ctl; + + phys_addr_t host_cr3 = cvp->common.host_cr3; + + /* Configure Host Controls */ + vmx_vmwrite(HOST_CR0, read_cr0()); + vmx_vmwrite(HOST_CR4, __read_cr4()); + if (!host_cr3) { + struct cpu_preserved_stack_context *sctx = cpu_preserved_get_stack_context(); + + if (sctx && sctx->session_pgd_pa) + host_cr3 = sctx->session_pgd_pa; + else + host_cr3 = x86_caretaker_pgd_pa; + } + if (host_cr3) + vmx_vmwrite(HOST_CR3, host_cr3); + vmx_vmwrite(HOST_RSP, (unsigned long)&cvp->common.stack[CXP_STACK_SIZE]); + vmx_vmwrite(HOST_RIP, (unsigned long)&vmx_caretaker_exit_handler); + + /* Configure Host Selectors */ + vmx_vmwrite(HOST_CS_SELECTOR, __KERNEL_CS); + vmx_vmwrite(HOST_SS_SELECTOR, __KERNEL_DS); + vmx_vmwrite(HOST_DS_SELECTOR, __KERNEL_DS); + vmx_vmwrite(HOST_ES_SELECTOR, __KERNEL_DS); + vmx_vmwrite(HOST_FS_SELECTOR, 0); + vmx_vmwrite(HOST_GS_SELECTOR, 0); + vmx_vmwrite(HOST_TR_SELECTOR, GDT_ENTRY_TSS * 8); + + /* Configure Host Bases */ + fs_base = native_rdmsrq(MSR_FS_BASE); + gs_base = native_rdmsrq(MSR_GS_BASE); + vmx_vmwrite(HOST_FS_BASE, fs_base); + vmx_vmwrite(HOST_GS_BASE, gs_base); + vmx_vmwrite(HOST_TR_BASE, (unsigned long)&cvp->common.tss); + vmx_vmwrite(HOST_GDTR_BASE, (unsigned long)&cvp->common.gdt[0]); + vmx_vmwrite(HOST_IDTR_BASE, (unsigned long)&caretaker_x86_idt[0]); + + /* Configure PIN and CPU execution controls */ + pin = vmx_vmread(PIN_BASED_VM_EXEC_CONTROL); + pin |= (PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING); + pin &= ~(PIN_BASED_VMX_PREEMPTION_TIMER | PIN_BASED_POSTED_INTR); + vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, pin); + + cpu_ctl = vmx_vmread(CPU_BASED_VM_EXEC_CONTROL); + cpu_ctl &= ~(CPU_BASED_INTR_WINDOW_EXITING | + CPU_BASED_NMI_WINDOW_EXITING); + cpu_ctl |= (CPU_BASED_HLT_EXITING | + CPU_BASED_PAUSE_EXITING | + CPU_BASED_MWAIT_EXITING | + CPU_BASED_MONITOR_EXITING | + CPU_BASED_UNCOND_IO_EXITING); + + if (cvp && cvp->ple_supported && + (cpu_ctl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { + unsigned long sec_ctl = vmx_vmread(SECONDARY_VM_EXEC_CONTROL); + + sec_ctl |= SECONDARY_EXEC_PAUSE_LOOP_EXITING; + vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_ctl); + vmx_vmwrite(PLE_GAP, 4096); + vmx_vmwrite(PLE_WINDOW, 4096); + } + vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, cpu_ctl); +} + diff --git a/arch/x86/kvm/vmx/caretaker_vmenter.S b/arch/x86/kvm/vmx/caretaker_vmenter.S index ff135cba854d..51781e3e61e2 100644 --- a/arch/x86/kvm/vmx/caretaker_vmenter.S +++ b/arch/x86/kvm/vmx/caretaker_vmenter.S @@ -26,8 +26,8 @@ SYM_TYPED_FUNC_START(vmx_caretaker_enter) CARETAKER_PUSH_HOST_REGS movq %rsp, CXP_STACK_ORIG(%rdi) - /* Switch to private preserved stack */ - movq CXP_STACK_TOP(%rdi), %rsp + /* Switch to private preserved stack at top of caretaker_x86_page */ + leaq PAGE_SIZE(%rdi), %rsp /* Activate VMCS on this pCPU */ vmptrld CXP_VMCS_PA(%rdi) @@ -60,11 +60,10 @@ SYM_TYPED_FUNC_START(vmx_caretaker_enter) vmresume vmlaunch - /* If both fail, record error and return */ + /* If both fail, return error in %rax */ popq %rdi mov $VMCS_VM_INSTRUCTION_ERROR, %eax vmread %rax, %rax - movq %rax, CXP_LAST_EXIT_CODE(%rdi) btsq $31, %rax jmp .Lvmx_ret SYM_FUNC_END(vmx_caretaker_enter) @@ -73,15 +72,14 @@ SYM_FUNC_START(vmx_caretaker_exit_handler) ENDBR /* * Hardware jumps here on VM-Exit with: - * RSP = HOST_RSP (CXP_STACK_TOP) + * RSP = HOST_RSP (top of 4 KB struct caretaker_x86_page) * CR3 = HOST_CR3 */ pushq %rdi - /* Recompute CVP pointer from RSP: CVP = (RSP & PAGE_MASK) - CXP_STACK_OFFSET */ + /* Recover CVP base from RSP: stack lives in the upper half of page 0 */ movq %rsp, %rdi andq $PAGE_MASK, %rdi - subq $CXP_STACK_OFFSET, %rdi /* Save guest GPRs into CVP */ CARETAKER_SAVE_GPRS %rdi diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..48b6e69f99bf 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -839,7 +839,7 @@ static void __loaded_vmcs_clear(void *arg) loaded_vmcs->launched = 0; } -static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) +void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) { int cpu = loaded_vmcs->cpu; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..5cb64ae4c8ee 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -342,6 +342,7 @@ static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) } void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); +void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs); int allocate_vpid(void); void free_vpid(int vpid); void vmx_set_constant_host_state(struct vcpu_vmx *vmx); -- 2.55.0.1082.g2b9226bbc0-goog
