For HAVE_RCU_TRAMPOLINE_READERS the ftrace trampolines must be Tasks Trace RCU readers while they call out, since that -- and not the absence of a voluntary context switch -- is what synchronize_rcu_tasks() will wait for before ftrace_shutdown() frees a dynamic trampoline or its ops.
Open-code rcu_read_lock_trace() and rcu_read_unlock_trace() in ftrace_caller and ftrace_regs_caller: bump current->trc_reader_nesting and, for the outermost reader, do the SRCU-fast per-CPU increment on rcu_tasks_trace_srcu_struct and stash the counter pointer in current->trc_reader_scp, exactly as the C inlines do (including the smp_mb() when CONFIG_TASKS_TRACE_RCU_NO_MB is not set). The lock sits before the function_trace_op load, because between that load and the call the ops pointer is protected only by Tasks RCU, and the unlock after the call returns. Note at rcu_read_lock_trace() that it now has open-coded copies that have to be kept in step. The sequences are inside the region that create_trampoline() copies for per-ops trampolines; their %rip-relative references are fixed up by text_poke_apply_relocation() like CALL_DEPTH_ACCOUNT's. %rax and %rcx are dead at both points. Two pieces of core text still run outside that reader while holding the address of a Tasks-RCU-protected trampoline they are about to enter: the static stubs themselves, whose direct-call tails keep a BPF trampoline address on the stack until the final RET, and, under CONFIG_MITIGATION_RETHUNK, the return thunk that RET expands to. Add an ftrace_static_tramp_end marker after ftrace_stub_direct_tramp and linker symbols around .text..__x86.return_thunk and .text..__x86.rethunk_safe, and provide arch_rcu_tasks_trampoline_text() covering [ftrace_caller, ftrace_static_tramp_end) and both thunk ranges so the irq-exit check treats a task interrupted there as a holdout. All of this is built only under CONFIG_TASKS_RCU_TRAMPOLINE_READERS, which x86 does not enable until a later patch. Assisted-by: LLM Signed-off-by: Josef Bacik <[email protected]> --- arch/x86/kernel/asm-offsets.c | 8 +++++ arch/x86/kernel/ftrace.c | 43 ++++++++++++++++++++++++++ arch/x86/kernel/ftrace_64.S | 69 ++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 4 +++ include/linux/rcupdate_trace.h | 6 ++++ 5 files changed, 130 insertions(+) diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 081816888f7a..876c3986419a 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include <linux/crypto.h> #include <crypto/aria.h> #include <linux/sched.h> +#include <linux/srcu.h> #include <linux/stddef.h> #include <linux/hardirq.h> #include <linux/suspend.h> @@ -46,6 +47,13 @@ static void __used common(void) #ifdef CONFIG_STACKPROTECTOR OFFSET(TASK_stack_canary, task_struct, stack_canary); #endif +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + OFFSET(TASK_trc_reader_nesting, task_struct, trc_reader_nesting); + OFFSET(TASK_trc_reader_scp, task_struct, trc_reader_scp); + OFFSET(SRCU_srcu_ctrp, srcu_struct, srcu_ctrp); + OFFSET(SRCU_CTR_srcu_locks, srcu_ctr, srcu_locks); + OFFSET(SRCU_CTR_srcu_unlocks, srcu_ctr, srcu_unlocks); +#endif BLANK(); OFFSET(pbe_address, pbe, address); diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 17d6edfcb7e0..9babaed483eb 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -275,6 +275,49 @@ static inline void tramp_free(void *tramp) execmem_free(tramp); } +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS +extern void ftrace_static_tramp_end(void); +extern char __return_thunk_start[], __return_thunk_end[]; +extern char __rethunk_safe_start[], __rethunk_safe_end[]; + +/* + * The SRCU-fast increments in TRACE_RCU_READ_LOCK/UNLOCK (ftrace_64.S) are the + * this_cpu_inc() form. + */ +static_assert(!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)); + +/* + * See rcu_tasks_trampoline_text(). Some core kernel text behaves like a + * trampoline for Tasks RCU purposes because a task executing there outside + * any Tasks Trace reader may still be about to enter a Tasks-RCU-protected + * trampoline whose address it already holds: + * + * - the static ftrace_caller / ftrace_regs_caller / ftrace_stub_direct_tramp + * stubs, which carry a direct-call target on the stack until their final + * RET, and + * - the return thunks that RET expands to under CONFIG_MITIGATION_RETHUNK, + * which run after leaving the stubs above and before landing in that + * target. + */ +bool arch_rcu_tasks_trampoline_text(unsigned long ip) +{ + if (ip >= (unsigned long)ftrace_caller && + ip < (unsigned long)ftrace_static_tramp_end) + return true; +#ifdef CONFIG_MITIGATION_RETPOLINE + if (ip >= (unsigned long)__return_thunk_start && + ip < (unsigned long)__return_thunk_end) + return true; +#endif +#ifdef CONFIG_MITIGATION_SRSO + if (ip >= (unsigned long)__rethunk_safe_start && + ip < (unsigned long)__rethunk_safe_end) + return true; +#endif + return false; +} +#endif /* CONFIG_TASKS_RCU_TRAMPOLINE_READERS */ + /* Defined as markers to the end of the ftrace default trampolines */ extern void ftrace_regs_caller_end(void); extern void ftrace_caller_end(void); diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 62c1c93aa1c6..5d8cb3861978 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -7,6 +7,7 @@ #include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm-offsets.h> +#include <asm/percpu.h> #include <asm/ptrace.h> #include <asm/ftrace.h> #include <asm/nospec-branch.h> @@ -145,6 +146,53 @@ SYM_FUNC_END(ftrace_stub_graph) #ifdef CONFIG_DYNAMIC_FTRACE +/* + * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace(), see + * include/linux/rcupdate_trace.h and CONFIG_HAVE_RCU_TRAMPOLINE_READERS: the + * trampoline and the ftrace_ops it is about to load are kept alive by Tasks + * RCU only while we are inside this reader, so the lock must precede the + * function_trace_op load and the unlock must follow the call. These live + * inside the region copied into dynamic trampolines; the %rip-relative + * references are fixed up by text_poke_apply_relocation() in + * create_trampoline(). Clobbers %rax, %rcx and flags. + */ +.macro TRACE_RCU_READ_LOCK +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + movq PER_CPU_VAR(current_task), %rcx + movl TASK_trc_reader_nesting(%rcx), %eax + incl TASK_trc_reader_nesting(%rcx) + testl %eax, %eax + jnz .Ltrl_nested_\@ + movq rcu_tasks_trace_srcu_struct+SRCU_srcu_ctrp(%rip), %rax + incq %gs:SRCU_CTR_srcu_locks(%rax) + movq %rax, TASK_trc_reader_scp(%rcx) +#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB + lock addl $0, -4(%rsp) /* smp_mb() */ +#endif +.Ltrl_nested_\@: +#endif +.endm + +.macro TRACE_RCU_READ_UNLOCK +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + movq PER_CPU_VAR(current_task), %rcx + movl TASK_trc_reader_nesting(%rcx), %eax + subl $1, %eax + jnz .Ltru_nested_\@ + /* Outermost: pick up scp before an interrupt can see nesting == 0. */ + movq TASK_trc_reader_scp(%rcx), %rax + movl $0, TASK_trc_reader_nesting(%rcx) +#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB + lock addl $0, -4(%rsp) /* smp_mb() */ +#endif + incq %gs:SRCU_CTR_srcu_unlocks(%rax) + jmp .Ltru_done_\@ +.Ltru_nested_\@: + movl %eax, TASK_trc_reader_nesting(%rcx) +.Ltru_done_\@: +#endif +.endm + SYM_FUNC_START(__fentry__) ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT @@ -163,6 +211,8 @@ SYM_FUNC_START(ftrace_caller) leaq MCOUNT_REG_SIZE+8(%rsp), %rcx movq %rcx, RSP(%rsp) + TRACE_RCU_READ_LOCK + SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ @@ -181,6 +231,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) ANNOTATE_NOENDBR call ftrace_stub + TRACE_RCU_READ_UNLOCK + /* Handlers can change the RIP */ movq RIP(%rsp), %rax movq %rax, MCOUNT_REG_SIZE(%rsp) @@ -209,6 +261,8 @@ SYM_FUNC_START(ftrace_regs_caller) CALL_DEPTH_ACCOUNT + TRACE_RCU_READ_LOCK + SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ @@ -246,6 +300,8 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) ANNOTATE_NOENDBR call ftrace_stub + TRACE_RCU_READ_UNLOCK + /* Copy flags back to SS, to restore them */ movq EFLAGS(%rsp), %rax movq %rax, MCOUNT_REG_SIZE(%rsp) @@ -328,6 +384,19 @@ SYM_FUNC_START(ftrace_stub_direct_tramp) RET SYM_FUNC_END(ftrace_stub_direct_tramp) +/* + * [ftrace_caller, ftrace_static_tramp_end) is treated as trampoline text by + * rcu_tasks_trampoline_text(): outside TRACE_RCU_READ_LOCK/UNLOCK the stubs + * may still hold a direct-call trampoline address (ORIG_RAX / the return + * address they RET to) that only Tasks RCU keeps alive. With return thunks + * the RET itself runs elsewhere; arch_rcu_tasks_trampoline_text() covers + * those too. + */ +SYM_CODE_START_NOALIGN(ftrace_static_tramp_end) + UNWIND_HINT_UNDEFINED + ANNOTATE_NOENDBR +SYM_CODE_END(ftrace_static_tramp_end) + #else /* ! CONFIG_DYNAMIC_FTRACE */ SYM_FUNC_START(__fentry__) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 2438b89a4620..e546283dc267 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -151,7 +151,9 @@ SECTIONS * definition. */ . = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); + __rethunk_safe_start = .; *(.text..__x86.rethunk_safe) + __rethunk_safe_end = .; #endif ALIGN_ENTRY_TEXT_END @@ -162,7 +164,9 @@ SECTIONS SOFTIRQENTRY_TEXT #ifdef CONFIG_MITIGATION_RETPOLINE *(.text..__x86.indirect_thunk) + __return_thunk_start = .; *(.text..__x86.return_thunk) + __return_thunk_end = .; #endif STATIC_CALL_TEXT *(.gnu.warning) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 273c59a03251..dcdb11643496 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -92,6 +92,12 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) * the all the other tasks exit their critical sections. * * For more details, please see the documentation for rcu_read_lock(). + * + * CONFIG_HAVE_RCU_TRAMPOLINE_READERS architectures open-code this pair in + * their ftrace and BPF trampolines (arch/x86/kernel/ftrace_64.S, + * arch/x86/kernel/kprobes/opt.c, arch/x86/net/bpf_jit_comp.c, + * arch/arm64/kernel/entry-ftrace.S, arch/arm64/net/bpf_jit_comp.c, + * samples/ftrace/ftrace-direct.h); changes here need to be mirrored there. */ static inline void rcu_read_lock_trace(void) { -- 2.55.0
