When userspace requests that a VCPU is not allowed to profile anymore via the KVM_ARM_VCPU_SPE_STOP attribute, keep all the register state in memory and trap all registers, not just the buffer registers, and don't copy any of this shadow state on the hardware.
Signed-off-by: Alexandru Elisei <[email protected]> --- arch/arm64/include/asm/kvm_hyp.h | 2 + arch/arm64/include/asm/kvm_spe.h | 14 +++++++ arch/arm64/include/uapi/asm/kvm.h | 3 ++ arch/arm64/kvm/arm.c | 9 ++++ arch/arm64/kvm/debug.c | 13 ++++-- arch/arm64/kvm/hyp/nvhe/debug-sr.c | 4 +- arch/arm64/kvm/hyp/nvhe/spe-sr.c | 24 +++++++++++ arch/arm64/kvm/hyp/vhe/spe-sr.c | 56 +++++++++++++++++++++++++ arch/arm64/kvm/spe.c | 67 ++++++++++++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 2 +- 10 files changed, 188 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 03bc51049996..ce365427b483 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -86,8 +86,10 @@ void __debug_switch_to_host(struct kvm_vcpu *vcpu); #ifdef __KVM_NVHE_HYPERVISOR__ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt); +void __debug_save_spe(u64 *pmscr_el1); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt); +void __debug_restore_spe(u64 pmscr_el1); #ifdef CONFIG_KVM_ARM_SPE void __spe_save_host_state_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt); diff --git a/arch/arm64/include/asm/kvm_spe.h b/arch/arm64/include/asm/kvm_spe.h index d7d7b9e243de..f51561e3b43f 100644 --- a/arch/arm64/include/asm/kvm_spe.h +++ b/arch/arm64/include/asm/kvm_spe.h @@ -16,13 +16,23 @@ static __always_inline bool kvm_supports_spe(void) return static_branch_likely(&kvm_spe_available); } +/* Guest profiling disabled by the user. */ +#define KVM_VCPU_SPE_STOP_USER (1 << 0) +/* Stop profiling and exit to userspace when guest starts profiling. */ +#define KVM_VCPU_SPE_STOP_USER_EXIT (1 << 1) + struct kvm_vcpu_spe { bool initialized; /* SPE initialized for the VCPU */ int irq_num; /* Buffer management interrut number */ bool irq_level; /* 'true' if the interrupt is asserted at the VGIC */ bool hwirq_level; /* 'true' if the SPE hardware is asserting the interrupt */ + u64 flags; }; +#define kvm_spe_profiling_stopped(vcpu) \ + (((vcpu)->arch.spe.flags & KVM_VCPU_SPE_STOP_USER) || \ + ((vcpu)->arch.spe.flags & KVM_VCPU_SPE_STOP_USER_EXIT)) \ + struct kvm_spe { bool perfmon_capable; /* Is the VM perfmon_capable()? */ }; @@ -31,6 +41,7 @@ void kvm_spe_init_supported_cpus(void); void kvm_spe_vm_init(struct kvm *kvm); int kvm_spe_vcpu_first_run_init(struct kvm_vcpu *vcpu); void kvm_spe_sync_hwstate(struct kvm_vcpu *vcpu); +bool kvm_spe_exit_to_user(struct kvm_vcpu *vcpu); void kvm_spe_write_sysreg(struct kvm_vcpu *vcpu, int reg, u64 val); u64 kvm_spe_read_sysreg(struct kvm_vcpu *vcpu, int reg); @@ -48,6 +59,8 @@ int kvm_spe_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); struct kvm_vcpu_spe { }; +#define kvm_spe_profiling_stopped(vcpu) (false) + struct kvm_spe { }; @@ -55,6 +68,7 @@ static inline void kvm_spe_init_supported_cpus(void) {} static inline void kvm_spe_vm_init(struct kvm *kvm) {} static inline int kvm_spe_vcpu_first_run_init(struct kvm_vcpu *vcpu) { return -ENOEXEC; } static inline void kvm_spe_sync_hwstate(struct kvm_vcpu *vcpu) {} +static inline bool kvm_spe_exit_to_user(struct kvm_vcpu *vcpu) { return false; } static inline void kvm_spe_write_sysreg(struct kvm_vcpu *vcpu, int reg, u64 val) {} static inline u64 kvm_spe_read_sysreg(struct kvm_vcpu *vcpu, int reg) { return 0; } diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 75a5113f610e..63599ee39a7b 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -376,6 +376,9 @@ struct kvm_arm_copy_mte_tags { #define KVM_ARM_VCPU_SPE_STOP_EXIT (1 << 1) #define KVM_ARM_VCPU_SPE_RESUME (1 << 2) +/* run->fail_entry.hardware_entry_failure_reason codes. */ +#define KVM_EXIT_FAIL_ENTRY_SPE (1 << 0) + /* KVM_IRQ_LINE irq field index values */ #define KVM_ARM_IRQ_VCPU2_SHIFT 28 #define KVM_ARM_IRQ_VCPU2_MASK 0xf diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index ec449bc5f811..b7aae25bb9da 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -873,6 +873,15 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) continue; } + if (unlikely(kvm_spe_exit_to_user(vcpu))) { + run->exit_reason = KVM_EXIT_FAIL_ENTRY; + run->fail_entry.hardware_entry_failure_reason + = KVM_EXIT_FAIL_ENTRY_SPE; + ret = -EAGAIN; + preempt_enable(); + continue; + } + kvm_pmu_flush_hwstate(vcpu); local_irq_disable(); diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c index 6e5fc1887215..6a4277a23bbb 100644 --- a/arch/arm64/kvm/debug.c +++ b/arch/arm64/kvm/debug.c @@ -96,11 +96,18 @@ static void kvm_arm_setup_mdcr_el2(struct kvm_vcpu *vcpu) if (kvm_supports_spe() && kvm_vcpu_has_spe(vcpu)) { /* * Use EL1&0 for the profiling buffer translation regime and - * trap accesses to the buffer control registers; leave - * MDCR_EL2.TPMS unset and do not trap accesses to the profiling - * control registers. + * trap accesses to the buffer control registers; if profiling + * is stopped, also set MSCR_EL2.TMPS to trap accesses to the + * rest of the registers, otherwise leave it clear. + * + * Leaving MDCR_EL2.E2P unset, like we do when the VCPU does not + * have SPE, means that the PMBIDR_EL1.P (which KVM does not + * trap) will be set and the guest will detect SPE as being + * unavailable. */ vcpu->arch.mdcr_el2 |= MDCR_EL2_E2PB_EL1_TRAP << MDCR_EL2_E2PB_SHIFT; + if (kvm_spe_profiling_stopped(vcpu)) + vcpu->arch.mdcr_el2 |= MDCR_EL2_TPMS; } else { /* * Trap accesses to the profiling control registers; leave diff --git a/arch/arm64/kvm/hyp/nvhe/debug-sr.c b/arch/arm64/kvm/hyp/nvhe/debug-sr.c index 1622615954b2..944972de0944 100644 --- a/arch/arm64/kvm/hyp/nvhe/debug-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/debug-sr.c @@ -14,7 +14,7 @@ #include <asm/kvm_hyp.h> #include <asm/kvm_mmu.h> -static void __debug_save_spe(u64 *pmscr_el1) +void __debug_save_spe(u64 *pmscr_el1) { u64 reg; @@ -40,7 +40,7 @@ static void __debug_save_spe(u64 *pmscr_el1) dsb(nsh); } -static void __debug_restore_spe(u64 pmscr_el1) +void __debug_restore_spe(u64 pmscr_el1) { if (!pmscr_el1) return; diff --git a/arch/arm64/kvm/hyp/nvhe/spe-sr.c b/arch/arm64/kvm/hyp/nvhe/spe-sr.c index b74131486a75..8ed03aa4f965 100644 --- a/arch/arm64/kvm/hyp/nvhe/spe-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/spe-sr.c @@ -23,6 +23,11 @@ void __spe_save_host_state_nvhe(struct kvm_vcpu *vcpu, { u64 pmblimitr; + if (kvm_spe_profiling_stopped(vcpu)) { + __debug_save_spe(__ctxt_sys_reg(host_ctxt, PMSCR_EL1)); + return; + } + pmblimitr = read_sysreg_s(SYS_PMBLIMITR_EL1); if (pmblimitr & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)) { psb_csync(); @@ -49,6 +54,13 @@ void __spe_save_guest_state_nvhe(struct kvm_vcpu *vcpu, { u64 pmbsr; + /* + * Profiling is stopped and all register accesses are trapped, nothing + * to save here. + */ + if (kvm_spe_profiling_stopped(vcpu)) + return; + if (read_sysreg_s(SYS_PMBLIMITR_EL1) & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)) { psb_csync(); dsb(nsh); @@ -82,6 +94,11 @@ void __spe_save_guest_state_nvhe(struct kvm_vcpu *vcpu, void __spe_restore_host_state_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt) { + if (kvm_spe_profiling_stopped(vcpu)) { + __debug_restore_spe(ctxt_sys_reg(host_ctxt, PMSCR_EL1)); + return; + } + __spe_restore_common_state(host_ctxt); write_sysreg_s(ctxt_sys_reg(host_ctxt, PMBPTR_EL1), SYS_PMBPTR_EL1); @@ -94,6 +111,13 @@ void __spe_restore_host_state_nvhe(struct kvm_vcpu *vcpu, void __spe_restore_guest_state_nvhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *guest_ctxt) { + /* + * Profiling is stopped and all register accesses are trapped, nothing + * to restore here. + */ + if (kvm_spe_profiling_stopped(vcpu)) + return; + __spe_restore_common_state(guest_ctxt); write_sysreg_s(ctxt_sys_reg(guest_ctxt, PMBPTR_EL1), SYS_PMBPTR_EL1); diff --git a/arch/arm64/kvm/hyp/vhe/spe-sr.c b/arch/arm64/kvm/hyp/vhe/spe-sr.c index ea4b3b69bb32..024a4c0618cc 100644 --- a/arch/arm64/kvm/hyp/vhe/spe-sr.c +++ b/arch/arm64/kvm/hyp/vhe/spe-sr.c @@ -10,6 +10,34 @@ #include <hyp/spe-sr.h> +static void __spe_save_host_buffer(u64 *pmscr_el2) +{ + u64 pmblimitr; + + /* Disable guest profiling. */ + write_sysreg_el1(0, SYS_PMSCR); + + pmblimitr = read_sysreg_s(SYS_PMBLIMITR_EL1); + if (!(pmblimitr & BIT(SYS_PMBLIMITR_EL1_E_SHIFT))) { + *pmscr_el2 = 0; + return; + } + + *pmscr_el2 = read_sysreg_el2(SYS_PMSCR); + + /* Disable profiling at EL2 so we can drain the buffer. */ + write_sysreg_el2(0, SYS_PMSCR); + isb(); + + /* + * We're going to change the buffer owning exception level when we + * activate traps, drain the buffer now. + */ + psb_csync(); + dsb(nsh); +} +NOKPROBE_SYMBOL(__spe_save_host_buffer); + /* * Disable host profiling, drain the buffer and save the host SPE context. * Extra care must be taken because profiling might be in progress. @@ -19,6 +47,11 @@ void __spe_save_host_state_vhe(struct kvm_vcpu *vcpu, { u64 pmblimitr, pmscr_el2; + if (kvm_spe_profiling_stopped(vcpu)) { + __spe_save_host_buffer(__ctxt_sys_reg(host_ctxt, PMSCR_EL2)); + return; + } + /* Disable profiling while the SPE context is being switched. */ pmscr_el2 = read_sysreg_el2(SYS_PMSCR); write_sysreg_el2(__vcpu_sys_reg(vcpu, PMSCR_EL2), SYS_PMSCR); @@ -50,6 +83,9 @@ void __spe_save_guest_state_vhe(struct kvm_vcpu *vcpu, { u64 pmblimitr, pmbsr; + if (kvm_spe_profiling_stopped(vcpu)) + return; + /* * We're at EL2 and the buffer owning regime is EL1, which means that * profiling is disabled. After we disable traps and restore the host's @@ -78,6 +114,18 @@ void __spe_save_guest_state_vhe(struct kvm_vcpu *vcpu, } NOKPROBE_SYMBOL(__spe_save_guest_state_vhe); +static void __spe_restore_host_buffer(u64 pmscr_el2) +{ + if (!pmscr_el2) + return; + + /* Synchronize MDCR_EL2 write. */ + isb(); + + write_sysreg_el2(pmscr_el2, SYS_PMSCR); +} +NOKPROBE_SYMBOL(__spe_restore_host_buffer); + /* * Restore the host SPE context. Special care must be taken because we're * potentially resuming a profiling session which was stopped when we saved the @@ -86,6 +134,11 @@ NOKPROBE_SYMBOL(__spe_save_guest_state_vhe); void __spe_restore_host_state_vhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt) { + if (kvm_spe_profiling_stopped(vcpu)) { + __spe_restore_host_buffer(ctxt_sys_reg(host_ctxt, PMSCR_EL2)); + return; + } + __spe_restore_common_state(host_ctxt); write_sysreg_s(ctxt_sys_reg(host_ctxt, PMBPTR_EL1), SYS_PMBPTR_EL1); @@ -115,6 +168,9 @@ NOKPROBE_SYMBOL(__spe_restore_host_state_vhe); void __spe_restore_guest_state_vhe(struct kvm_vcpu *vcpu, struct kvm_cpu_context *guest_ctxt) { + if (kvm_spe_profiling_stopped(vcpu)) + return; + __spe_restore_common_state(guest_ctxt); /* diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c index 2630e777fe1d..69ca731ba9d3 100644 --- a/arch/arm64/kvm/spe.c +++ b/arch/arm64/kvm/spe.c @@ -140,6 +140,28 @@ void kvm_spe_sync_hwstate(struct kvm_vcpu *vcpu) kvm_spe_update_irq(vcpu, true); } +static bool kvm_spe_buffer_enabled(struct kvm_vcpu *vcpu) +{ + return !vcpu->arch.spe.irq_level && + (__vcpu_sys_reg(vcpu, PMBLIMITR_EL1) & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)); +} + +bool kvm_spe_exit_to_user(struct kvm_vcpu *vcpu) +{ + u64 pmscr_enabled_mask = BIT(SYS_PMSCR_EL1_E0SPE_SHIFT) | + BIT(SYS_PMSCR_EL1_E1SPE_SHIFT); + + if (!(vcpu->arch.spe.flags & KVM_VCPU_SPE_STOP_USER_EXIT)) + return false; + + /* + * We don't trap the guest dropping to EL0, so exit even if profiling is + * disabled at EL1, but enabled at EL0. + */ + return kvm_spe_buffer_enabled(vcpu) && + (__vcpu_sys_reg(vcpu, PMSCR_EL1) & pmscr_enabled_mask); +} + void kvm_spe_write_sysreg(struct kvm_vcpu *vcpu, int reg, u64 val) { __vcpu_sys_reg(vcpu, reg) = val; @@ -215,6 +237,31 @@ static bool kvm_spe_irq_is_valid(struct kvm *kvm, int irq) return true; } +static int kvm_spe_stop_user(struct kvm_vcpu *vcpu, int flags) +{ + struct kvm_vcpu_spe *spe = &vcpu->arch.spe; + + if (flags & KVM_ARM_VCPU_SPE_STOP_TRAP) { + if (flags & ~KVM_ARM_VCPU_SPE_STOP_TRAP) + return -EINVAL; + spe->flags = KVM_VCPU_SPE_STOP_USER; + } + + if (flags & KVM_ARM_VCPU_SPE_STOP_EXIT) { + if (flags & ~KVM_ARM_VCPU_SPE_STOP_EXIT) + return -EINVAL; + spe->flags = KVM_VCPU_SPE_STOP_USER_EXIT; + } + + if (flags & KVM_ARM_VCPU_SPE_RESUME) { + if (flags & ~KVM_ARM_VCPU_SPE_RESUME) + return -EINVAL; + spe->flags = 0; + } + + return 0; +} + int kvm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) { if (!kvm_vcpu_supports_spe(vcpu)) @@ -268,6 +315,8 @@ int kvm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) if (!flags) return -EINVAL; + + return kvm_spe_stop_user(vcpu, flags); } } @@ -293,6 +342,24 @@ int kvm_spe_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) return 0; } + case KVM_ARM_VCPU_SPE_STOP: { + int __user *uaddr = (int __user *)(long)attr->addr; + struct kvm_vcpu_spe *spe = &vcpu->arch.spe; + int flag = 0; + + if (!vcpu->arch.spe.initialized) + return -EAGAIN; + + if (spe->flags & KVM_VCPU_SPE_STOP_USER) + flag = KVM_ARM_VCPU_SPE_STOP_TRAP; + else if (spe->flags & KVM_VCPU_SPE_STOP_USER_EXIT) + flag = KVM_ARM_VCPU_SPE_STOP_EXIT; + + if (put_user(flag, uaddr)) + return -EFAULT; + + return 0; + } } return -ENXIO; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 064742cee425..cc711b081f31 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -608,7 +608,7 @@ static bool access_spe_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *p, { int reg = r->reg; u64 val = p->regval; - if (reg < PMBLIMITR_EL1) { + if (reg < PMBLIMITR_EL1 && !kvm_spe_profiling_stopped(vcpu)) { print_sys_reg_msg(p, "Unsupported guest SPE register access at: %lx [%08lx]\n", *vcpu_pc(vcpu), *vcpu_cpsr(vcpu)); } -- 2.33.0 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
