From: Peter Zijlstra <[email protected]> queued_spin_lock_slowpath() and queued_spin_unlock() are dispatched through pv_ops_lock via the paravirt-ops ALTERNATIVE machinery, which picks the target (native inline store / hypervisor call) once at boot and cannot change at runtime.
Convert both to static_call(). The site becomes a direct call patched in place (one byte smaller), and on native the unlock still collapses to the inline "movb $0, (%rdi)" store, so the fast path is unchanged. Unlike the ALTERNATIVE mechanism, a static_call() target can also be updated at runtime via static_call_update(). This is a prerequisite for the contended_release tracepoint, which has to swap in a traced unlock while the system is running. [ ilvokhin: commit message; fix PARAVIRT_SPINLOCKS=n build; teach __static_call_validate() about the inline unlock insn; make the slowpath site module-safe: static_call_mod() + EXPORT_STATIC_CALL_TRAMP(); pass @lock to the callee-save unlock, fixing a boot hang under CALL_DEPTH_TRACKING. Boot tested native + KVM PV guest. ] Link: https://lore.kernel.org/all/[email protected]/ Co-developed-by: Dmitry Ilvokhin <[email protected]> Signed-off-by: Dmitry Ilvokhin <[email protected]> --- arch/x86/hyperv/hv_spinlock.c | 4 ++-- arch/x86/include/asm/cpufeatures.h | 1 - arch/x86/include/asm/paravirt-spinlock.h | 19 +++++++++++------ arch/x86/kernel/kvm.c | 5 ++--- arch/x86/kernel/paravirt-spinlocks.c | 12 +++++------ arch/x86/kernel/static_call.c | 27 ++++++++++++++++++++++++ arch/x86/xen/spinlock.c | 5 ++--- tools/arch/x86/include/asm/cpufeatures.h | 1 - 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 210b494e4de0..6b4bdea18218 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -78,8 +78,8 @@ void __init hv_init_spinlocks(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = hv_qlock_wait; pv_ops_lock.kick = hv_qlock_kick; pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1b4a48bff18f..e41fe5c24841 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -225,7 +225,6 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* "ept_ad" Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* VMware prefers VMMCALL hypercall instruction */ -#define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h index 7beffcb08ed6..ff735830de4a 100644 --- a/arch/x86/include/asm/paravirt-spinlock.h +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -3,6 +3,7 @@ #define _ASM_X86_PARAVIRT_SPINLOCK_H #include <asm/paravirt_types.h> +#include <linux/static_call_types.h> #ifdef CONFIG_SMP #include <asm/spinlock_types.h> @@ -11,9 +12,6 @@ struct qspinlock; struct pv_lock_ops { - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); - struct paravirt_callee_save queued_spin_unlock; - void (*wait)(u8 *ptr, u8 val); void (*kick)(int cpu); @@ -26,20 +24,27 @@ extern struct pv_lock_ops pv_ops_lock; extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); extern void __pv_init_lock_hash(void); extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); extern bool nopvspin; +DECLARE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +DECLARE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); + static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) { - PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); + static_call_mod(queued_spin_lock_slowpath)(lock, val); } static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) { - PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, - "movb $0, (%%" _ASM_ARG1 ")", - ALT_NOT(X86_FEATURE_PVUNLOCK)); + PVOP_CALL_ARGS; + __STATIC_CALL_MOD_ADDRESSABLE(queued_spin_unlock); + asm volatile ("call " STATIC_CALL_TRAMP_STR(queued_spin_unlock) + : PVOP_VCALLEE_CLOBBERS, ASM_CALL_CONSTRAINT + : PVOP_CALL_ARG1(lock) + : "memory", "cc"); } static __always_inline bool pv_vcpu_is_preempted(long cpu) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index dcef84da304b..253c159c4abe 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1136,9 +1136,8 @@ void __init kvm_spinlock_init(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = - PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = kvm_wait; pv_ops_lock.kick = kvm_kick_cpu; diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index 95452444868f..ddc19dc28ba1 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -25,9 +25,14 @@ __visible void __native_queued_spin_unlock(struct qspinlock *lock) } PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); +DEFINE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +EXPORT_STATIC_CALL_TRAMP(queued_spin_lock_slowpath); +DEFINE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); +EXPORT_STATIC_CALL_TRAMP(queued_spin_unlock); + bool pv_is_native_spin_unlock(void) { - return pv_ops_lock.queued_spin_unlock.func == + return static_call_query(queued_spin_unlock) == __raw_callee_save___native_queued_spin_unlock; } @@ -45,16 +50,11 @@ bool pv_is_native_vcpu_is_preempted(void) void __init paravirt_set_cap(void) { - if (!pv_is_native_spin_unlock()) - setup_force_cpu_cap(X86_FEATURE_PVUNLOCK); - if (!pv_is_native_vcpu_is_preempted()) setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); } struct pv_lock_ops pv_ops_lock = { - .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, - .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), .wait = paravirt_nop, .kick = paravirt_nop, .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 61592e41a6b1..bab9406e6d6a 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -4,6 +4,12 @@ #include <linux/bug.h> #include <asm/text-patching.h> +/* Declared locally to avoid pulling asm/paravirt-spinlock.h header. */ +#ifdef CONFIG_PARAVIRT_SPINLOCKS +struct qspinlock; +void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); +#endif + enum insn_type { CALL = 0, /* site call */ NOP = 1, /* site cond-call */ @@ -31,6 +37,17 @@ static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; */ static const u8 warninsn[] = { 0x67, 0x48, 0x0f, 0xb9, 0x3a }; +#ifdef CONFIG_PARAVIRT_SPINLOCKS +/* + * ds ds movb $0, (_ASM_ARG1) + */ +#ifdef CONFIG_64BIT +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x07, 0x00 }; +#else +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x00, 0x00 }; +#endif +#endif + static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ { u8 ret = 0; @@ -78,6 +95,12 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, emulate = code; code = &warninsn; } +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (func == &__raw_callee_save___native_queued_spin_unlock) { + emulate = code; + code = &unlockinsn; + } +#endif break; case NOP: @@ -139,6 +162,10 @@ static void __static_call_validate(u8 *insn, bool tail, bool tramp) !memcmp(insn, xor5rax, 5) || !memcmp(insn, warninsn, 5)) return; +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (!memcmp(insn, unlockinsn, 5)) + return; +#endif } /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 83ac24ead289..f718e535ea7c 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -134,9 +134,8 @@ void __init xen_init_spinlocks(void) printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = - PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = xen_qlock_wait; pv_ops_lock.kick = xen_qlock_kick; pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 86d17b195e79..61541f042f74 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -225,7 +225,6 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* "ept_ad" Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* VMware prefers VMMCALL hypercall instruction */ -#define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ -- 2.53.0-Meta
