Similar to the non-VHE case, save and restore the SPE register state at each world switch for VHE enabled systems if the VCPU has the SPE feature.
Signed-off-by: Alexandru Elisei <[email protected]> --- arch/arm64/include/asm/kvm_hyp.h | 24 +++++- arch/arm64/include/asm/sysreg.h | 2 + arch/arm64/kvm/hyp/vhe/Makefile | 1 + arch/arm64/kvm/hyp/vhe/spe-sr.c | 128 +++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp/vhe/switch.c | 8 ++ 5 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/kvm/hyp/vhe/spe-sr.c diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 06e77a739458..03bc51049996 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -106,8 +106,28 @@ static inline void __spe_restore_host_state_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt) {} static inline void __spe_restore_guest_state_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *guest_ctxt) {} -#endif -#endif +#endif /* CONFIG_KVM_ARM_SPE */ +#else +#ifdef CONFIG_KVM_ARM_SPE +void __spe_save_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt); +void __spe_save_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt); +void __spe_restore_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt); +void __spe_restore_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt); +#else +static inline void __spe_save_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt) {} +static inline void __spe_save_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt) {} +static inline void __spe_restore_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt) {} +static inline void __spe_restore_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt) {} +#endif /* CONFIG_KVM_ARM_SPE */ +#endif /* __KVM_NVHE_HYPERVISOR__ */ void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7b9c3acba684..b2d691bc1049 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -267,6 +267,8 @@ #define SYS_PMSCR_EL1_TS_SHIFT 5 #define SYS_PMSCR_EL1_PCT_SHIFT 6 +#define SYS_PMSCR_EL12 sys_reg(3, 5, 9, 9, 0) + #define SYS_PMSCR_EL2 sys_reg(3, 4, 9, 9, 0) #define SYS_PMSCR_EL2_E0HSPE_SHIFT 0 #define SYS_PMSCR_EL2_E2SPE_SHIFT 1 diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile index 96bec0ecf9dd..7cb4a9e5ceb0 100644 --- a/arch/arm64/kvm/hyp/vhe/Makefile +++ b/arch/arm64/kvm/hyp/vhe/Makefile @@ -7,5 +7,6 @@ asflags-y := -D__KVM_VHE_HYPERVISOR__ ccflags-y := -D__KVM_VHE_HYPERVISOR__ obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o +obj-$(CONFIG_KVM_ARM_SPE) += spe-sr.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o diff --git a/arch/arm64/kvm/hyp/vhe/spe-sr.c b/arch/arm64/kvm/hyp/vhe/spe-sr.c new file mode 100644 index 000000000000..00eab9e2ec60 --- /dev/null +++ b/arch/arm64/kvm/hyp/vhe/spe-sr.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 - ARM Ltd + */ + +#include <linux/kvm_host.h> + +#include <asm/kvm_hyp.h> +#include <asm/kprobes.h> + +#include <hyp/spe-sr.h> + +/* + * Disable host profiling, drain the buffer and save the host SPE context. + * Extra care must be taken because profiling might be in progress. + */ +void __spe_save_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt) +{ + u64 pmblimitr, pmscr_el2; + + /* Disable profiling while the SPE context is being switched. */ + pmscr_el2 = read_sysreg_el2(SYS_PMSCR); + write_sysreg_el2(0, SYS_PMSCR); + isb(); + + pmblimitr = read_sysreg_s(SYS_PMBLIMITR_EL1); + if (pmblimitr & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)) { + psb_csync(); + dsb(nsh); + /* Ensure hardware updates to PMBPTR_EL1 are visible. */ + isb(); + } + + ctxt_sys_reg(host_ctxt, PMBPTR_EL1) = read_sysreg_s(SYS_PMBPTR_EL1); + ctxt_sys_reg(host_ctxt, PMBSR_EL1) = read_sysreg_s(SYS_PMBSR_EL1); + ctxt_sys_reg(host_ctxt, PMBLIMITR_EL1) = pmblimitr; + ctxt_sys_reg(host_ctxt, PMSCR_EL2) = pmscr_el2; + + __spe_save_common_state(host_ctxt); +} +NOKPROBE_SYMBOL(__spe_save_host_state_vhe); + +/* + * Drain the guest's buffer and save the SPE state. Profiling is disabled + * because we're at EL2 and the buffer owning exceptions level is EL1. + */ +void __spe_save_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt) +{ + u64 pmblimitr; + + /* + * We're at EL2 and the buffer owning regime is EL1, which means that + * profiling is disabled. After we disable traps and restore the host's + * MDCR_EL2, profiling will remain disabled because we've disabled it + * via PMSCR_EL2 when we saved the host's SPE state. All it's needed + * here is to drain the buffer. + */ + pmblimitr = read_sysreg_s(SYS_PMBLIMITR_EL1); + if (pmblimitr & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)) { + psb_csync(); + dsb(nsh); + /* Ensure hardware updates to PMBPTR_EL1 are visible. */ + isb(); + } + + ctxt_sys_reg(guest_ctxt, PMBPTR_EL1) = read_sysreg_s(SYS_PMBPTR_EL1); + ctxt_sys_reg(guest_ctxt, PMBSR_EL1) = read_sysreg_s(SYS_PMBSR_EL1); + /* PMBLIMITR_EL1 is updated only on a trapped write. */ + ctxt_sys_reg(guest_ctxt, PMSCR_EL1) = read_sysreg_el1(SYS_PMSCR); + + __spe_save_common_state(guest_ctxt); +} +NOKPROBE_SYMBOL(__spe_save_guest_state_vhe); + +/* + * Restore the host SPE context. Special care must be taken because we're + * potentially resuming a profiling session which was stopped when we saved the + * host SPE register state. + */ +void __spe_restore_host_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *host_ctxt) +{ + __spe_restore_common_state(host_ctxt); + + write_sysreg_s(ctxt_sys_reg(host_ctxt, PMBPTR_EL1), SYS_PMBPTR_EL1); + write_sysreg_s(ctxt_sys_reg(host_ctxt, PMBLIMITR_EL1), SYS_PMBLIMITR_EL1); + write_sysreg_s(ctxt_sys_reg(host_ctxt, PMBSR_EL1), SYS_PMBSR_EL1); + + /* + * Make sure buffer pointer and limit is updated first, so we don't end + * up in a situation where profiling is enabled and the buffer uses the + * values programmed by the guest. + * + * This also serves to make sure the write to MDCR_EL2 which changes the + * buffer owning Exception level is visible. + * + * After the synchronization, profiling is still disabled at EL2, + * because we cleared PMSCR_EL2 when we saved the host context. + */ + isb(); + + write_sysreg_el2(ctxt_sys_reg(host_ctxt, PMSCR_EL2), SYS_PMSCR); +} +NOKPROBE_SYMBOL(__spe_restore_host_state_vhe); + +/* + * Restore the guest SPE context while profiling is disabled at EL2. + */ +void __spe_restore_guest_state_vhe(struct kvm_vcpu *vcpu, + struct kvm_cpu_context *guest_ctxt) +{ + __spe_restore_common_state(guest_ctxt); + + /* + * No synchronization needed here. Profiling is disabled at EL2 because + * PMSCR_EL2 has been cleared when saving the host's context, and the + * buffer has already been drained. + */ + + write_sysreg_s(ctxt_sys_reg(guest_ctxt, PMBPTR_EL1), SYS_PMBPTR_EL1); + write_sysreg_s(ctxt_sys_reg(guest_ctxt, PMBSR_EL1), SYS_PMBSR_EL1); + write_sysreg_s(ctxt_sys_reg(guest_ctxt, PMBLIMITR_EL1), SYS_PMBLIMITR_EL1); + write_sysreg_el1(ctxt_sys_reg(guest_ctxt, PMSCR_EL1), SYS_PMSCR); + /* PMSCR_EL2 has been cleared when saving the host state. */ +} +NOKPROBE_SYMBOL(__spe_restore_guest_state_vhe); diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index ec4e179d56ae..46da018f4a5a 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -135,6 +135,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) guest_ctxt = &vcpu->arch.ctxt; sysreg_save_host_state_vhe(host_ctxt); + if (kvm_vcpu_has_spe(vcpu)) + __spe_save_host_state_vhe(vcpu, host_ctxt); /* * ARM erratum 1165522 requires us to configure both stage 1 and @@ -153,6 +155,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) __kvm_adjust_pc(vcpu); sysreg_restore_guest_state_vhe(guest_ctxt); + if (kvm_vcpu_has_spe(vcpu)) + __spe_restore_guest_state_vhe(vcpu, guest_ctxt); __debug_switch_to_guest(vcpu); do { @@ -163,10 +167,14 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) } while (fixup_guest_exit(vcpu, &exit_code)); sysreg_save_guest_state_vhe(guest_ctxt); + if (kvm_vcpu_has_spe(vcpu)) + __spe_save_guest_state_vhe(vcpu, guest_ctxt); __deactivate_traps(vcpu); sysreg_restore_host_state_vhe(host_ctxt); + if (kvm_vcpu_has_spe(vcpu)) + __spe_restore_host_state_vhe(vcpu, host_ctxt); if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) __fpsimd_save_fpexc32(vcpu); -- 2.33.0 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
