Add arch/x86/kvm/vmx/caretaker_vmenter.S and caretaker.h for standalone Intel VMX VMLAUNCH/VMRESUME world switch on preserved physical cores.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/kvm/msrs.c | 2 + arch/x86/kvm/vmx/caretaker.h | 78 +++++++++++++++++++ arch/x86/kvm/vmx/caretaker_vmenter.S | 112 +++++++++++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 arch/x86/kvm/vmx/caretaker.h create mode 100644 arch/x86/kvm/vmx/caretaker_vmenter.S diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index ae849e0b7d14..536c8d1959dc 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -882,11 +882,13 @@ int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) { return __kvm_set_msr(vcpu, index, data, true); } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_write); int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) { return __kvm_get_msr(vcpu, index, data, true); } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_read); int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) { diff --git a/arch/x86/kvm/vmx/caretaker.h b/arch/x86/kvm/vmx/caretaker.h new file mode 100644 index 000000000000..d4272e202e73 --- /dev/null +++ b/arch/x86/kvm/vmx/caretaker.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#ifndef __VMX_CARETAKER_H +#define __VMX_CARETAKER_H + +#define VMCS_VM_INSTRUCTION_ERROR 0x4400 + +#ifndef __ASSEMBLY__ +#include <linux/types.h> +#include <linux/kvm_host.h> +#include <linux/oncore.h> +#include <asm/vmx.h> +#endif + +#include "../caretaker.h" + +#ifndef __ASSEMBLY__ +#include <asm/desc.h> +#include <linux/processor.h> + +/* Length of instructions */ +#define PAUSE_INSN_LEN 2 +#define VMCALL_INSN_LEN 3 + +/* Exit qualification bitfields for I/O instructions (Intel SDM Vol 3C) */ +#define VMX_IO_SIZE_MASK 0x7 +#define VMX_IO_DIRECTION_BIT BIT(3) +#define VMX_IO_PORT_SHIFT 16 + +/* Default VMX preemption timer shift (counts down every 2^5 TSC ticks) */ +#define VMX_PREEMPTION_TIMER_SHIFT 5 + +struct caretaker_vmx_page { + struct caretaker_x86_page common; + /* Guest syscall state not automatically switched by VMCS */ + u64 star; + u64 lstar; + u64 fmask; + u64 vmxon_pa; + u32 timer_shift; + u32 ple_supported; + u8 vmxon_area[PAGE_SIZE] __aligned(PAGE_SIZE); +} __aligned(PAGE_SIZE); + +static inline unsigned long vmx_vmread(unsigned long field) +{ + unsigned long val; + + asm volatile("vmread %1, %0" : "=r" (val) : "r" (field) : "cc"); + return val; +} + +static inline void vmx_vmwrite(unsigned long field, unsigned long val) +{ + asm volatile("vmwrite %1, %0" : : "r" (field), "r" (val) : "cc"); +} + +#ifdef CONFIG_KVM_CARETAKER +int vmx_caretaker_enter(void *page); +void vmx_caretaker_exit_handler(void); +void vmx_caretaker_register(void); +void vmx_caretaker_unregister(void); +void vmx_caretaker_decode_exit(void *page, + struct kvm_caretaker_exit *exit); +void vmx_caretaker_init_host_vmcs(struct caretaker_vmx_page *cvp); +void vmx_caretaker_init(struct kvm_vcpu *vcpu); +#else +static inline void vmx_caretaker_register(void) {} +static inline void vmx_caretaker_unregister(void) {} +static inline void vmx_caretaker_init(struct kvm_vcpu *vcpu) {} +#endif + +#endif /* !__ASSEMBLY__ */ + +#endif /* __VMX_CARETAKER_H */ diff --git a/arch/x86/kvm/vmx/caretaker_vmenter.S b/arch/x86/kvm/vmx/caretaker_vmenter.S new file mode 100644 index 000000000000..ff135cba854d --- /dev/null +++ b/arch/x86/kvm/vmx/caretaker_vmenter.S @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#include <linux/linkage.h> +#include <linux/cfi_types.h> +#include <linux/objtool.h> +#include <asm/asm.h> +#include <asm/bitsperlong.h> +#include <asm/kvm_vcpu_regs.h> +#include <asm/nospec-branch.h> +#include <asm/msr-index.h> +#include <asm/page_types.h> +#include "caretaker.h" +#include "../caretaker_asm.h" + +.section .text.cpu_preserved, "ax" + +/** + * vmx_caretaker_enter - Low-level transition to VMX guest mode and back + * @cvp: Pointer to struct caretaker_vmx_page (%rdi) + */ +SYM_TYPED_FUNC_START(vmx_caretaker_enter) + ENDBR + CARETAKER_PUSH_HOST_REGS + movq %rsp, CXP_STACK_ORIG(%rdi) + + /* Switch to private preserved stack */ + movq CXP_STACK_TOP(%rdi), %rsp + + /* Activate VMCS on this pCPU */ + vmptrld CXP_VMCS_PA(%rdi) + + /* Restore guest MSR_KERNEL_GS_BASE if valid/canonical */ + movq CXP_KERNEL_GS_BASE(%rdi), %rax + movq %rax, %rdx + sarq $56, %rdx + incq %rdx + cmpq $1, %rdx + ja 2f + movl $MSR_KERNEL_GS_BASE, %ecx + movq CXP_KERNEL_GS_BASE(%rdi), %rax + movq %rax, %rdx + shrq $32, %rdx +1: wrmsr +2: + _ASM_EXTABLE(1b, 2b) + + /* Restore guest GPRs */ + CARETAKER_RESTORE_GPRS %rdi + + pushq %rdi + movq CXP_REG_RDI(%rdi), %rdi + + /* Clear CPU microarchitectural buffers before guest entry */ + CARETAKER_CLEAR_CPU_BUFFERS + + /* Launch or resume */ + vmresume + vmlaunch + + /* If both fail, record error and return */ + popq %rdi + mov $VMCS_VM_INSTRUCTION_ERROR, %eax + vmread %rax, %rax + movq %rax, CXP_LAST_EXIT_CODE(%rdi) + btsq $31, %rax + jmp .Lvmx_ret +SYM_FUNC_END(vmx_caretaker_enter) + +SYM_FUNC_START(vmx_caretaker_exit_handler) + ENDBR + /* + * Hardware jumps here on VM-Exit with: + * RSP = HOST_RSP (CXP_STACK_TOP) + * CR3 = HOST_CR3 + */ + pushq %rdi + + /* Recompute CVP pointer from RSP: CVP = (RSP & PAGE_MASK) - CXP_STACK_OFFSET */ + movq %rsp, %rdi + andq $PAGE_MASK, %rdi + subq $CXP_STACK_OFFSET, %rdi + + /* Save guest GPRs into CVP */ + CARETAKER_SAVE_GPRS %rdi + popq CXP_REG_RDI(%rdi) + + /* Stuff Return Stack Buffer to prevent guest RSB poisoning */ + CARETAKER_FILL_RETURN_BUFFER %rax + + /* Read guest MSR_KERNEL_GS_BASE */ + movl $MSR_KERNEL_GS_BASE, %ecx + rdmsr + shlq $32, %rdx + orq %rax, %rdx + movq %rdx, CXP_KERNEL_GS_BASE(%rdi) + + xorq %rax, %rax + jmp .Lvmx_ret + +.Lvmx_ret: + /* Restore caller stack */ + movq CXP_STACK_ORIG(%rdi), %rsp + + /* Restore host callee-saved registers */ + CARETAKER_POP_HOST_REGS + ret +SYM_FUNC_END(vmx_caretaker_exit_handler) +STACK_FRAME_NON_STANDARD(vmx_caretaker_enter) +STACK_FRAME_NON_STANDARD(vmx_caretaker_exit_handler) -- 2.55.0.1082.g2b9226bbc0-goog
