For HAVE_RCU_TRAMPOLINE_READERS the ftrace trampoline must be a Tasks Trace RCU reader while it calls out, since that is what synchronize_rcu_tasks() will wait for before ftrace_shutdown() frees an ftrace_ops (or, with CALL_OPS, lets its owner free it) under a task preempted in the callback.
Open-code rcu_read_lock_trace() and rcu_read_unlock_trace() around the call to ops->func in ftrace_caller: bump current->trc_reader_nesting via sp_el0 and, for the outermost reader, do the SRCU-fast per-CPU increment on rcu_tasks_trace_srcu_struct and stash the counter pointer in current->trc_reader_scp, as the C inlines do (including the dmb when CONFIG_TASKS_TRACE_RCU_NO_MB is not set). The per-CPU increment is an LL/SC add on this CPU's counter; being migrated between reading the per-CPU offset and the store-exclusive only means another CPU's counter is incremented atomically instead, which SRCU sums over anyway. x12-x16 are free at both points. arm64 has no return thunks and, with CALL_OPS, no dynamic ftrace trampolines, but ftrace_caller itself carries the ops pointer in x11 from before the reader is entered and a direct-call BPF trampoline address in x17 until the final br/ret after it is left, so mark the end of the static trampoline text and provide arch_rcu_tasks_trampoline_text() covering [ftrace_caller, ftrace_static_tramp_end). Built only under CONFIG_TASKS_RCU_TRAMPOLINE_READERS, which arm64 does not enable until a later patch. Assisted-by: LLM Signed-off-by: Josef Bacik <[email protected]> --- arch/arm64/kernel/asm-offsets.c | 8 +++++ arch/arm64/kernel/entry-ftrace.S | 74 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 20 +++++++++++ 3 files changed, 102 insertions(+) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 9c853ed3ceab..f6a8fb1f9b43 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -10,6 +10,7 @@ #include <linux/arm_sdei.h> #include <linux/sched.h> +#include <linux/srcu.h> #include <linux/ftrace.h> #include <linux/kexec.h> #include <linux/mm.h> @@ -39,6 +40,13 @@ int main(void) DEFINE(TSK_STACK, offsetof(struct task_struct, stack)); #ifdef CONFIG_STACKPROTECTOR DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary)); +#endif +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + DEFINE(TSK_TRC_READER_NESTING, offsetof(struct task_struct, trc_reader_nesting)); + DEFINE(TSK_TRC_READER_SCP, offsetof(struct task_struct, trc_reader_scp)); + DEFINE(SRCU_SRCU_CTRP, offsetof(struct srcu_struct, srcu_ctrp)); + DEFINE(SRCU_CTR_SRCU_LOCKS, offsetof(struct srcu_ctr, srcu_locks)); + DEFINE(SRCU_CTR_SRCU_UNLOCKS, offsetof(struct srcu_ctr, srcu_unlocks)); #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 025140caafe7..fc2805eb9e15 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -14,6 +14,72 @@ #include <asm/insn.h> #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS +/* + * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace(), see + * include/linux/rcupdate_trace.h and CONFIG_HAVE_RCU_TRAMPOLINE_READERS. The + * whole of ftrace_caller is treated as trampoline text by the irq-exit check + * (see arch_rcu_tasks_trampoline_text()), so these only need to bracket the + * call out to ops->func; everything before the lock and after the unlock, + * including the direct-call tails that carry a BPF trampoline address in x17, + * is covered by that. + * + * The SRCU-fast per-CPU increment is done LL/SC on this CPU's counter; being + * migrated between reading the per-CPU offset and the store-exclusive only + * means another CPU's counter is (atomically) incremented, which SRCU sums + * over anyway. Ordering between the nesting count and the scp stash only + * matters against interrupts on this CPU, which observe program order. + * Clobbers x12-x16 and the flags. + */ + .macro trace_rcu_srcu_inc, addr:req, tmp:req, wtmp2:req +8888: ldxr \tmp, [\addr] + add \tmp, \tmp, #1 + stxr \wtmp2, \tmp, [\addr] + cbnz \wtmp2, 8888b + .endm + + .macro trace_rcu_read_lock +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + mrs x12, sp_el0 // current + ldr w13, [x12, #TSK_TRC_READER_NESTING] + add w14, w13, #1 + str w14, [x12, #TSK_TRC_READER_NESTING] + cbnz w13, .Ltrl_nested\@ // interrupted a reader: done + ldr_l x13, rcu_tasks_trace_srcu_struct + SRCU_SRCU_CTRP + str x13, [x12, #TSK_TRC_READER_SCP] + get_this_cpu_offset x14 + add x14, x14, x13 + add x14, x14, #SRCU_CTR_SRCU_LOCKS + trace_rcu_srcu_inc x14, x15, w16 +#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB + dmb ish +#endif +.Ltrl_nested\@: +#endif + .endm + + .macro trace_rcu_read_unlock +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + mrs x12, sp_el0 // current + ldr w13, [x12, #TSK_TRC_READER_NESTING] + subs w13, w13, #1 + b.ne .Ltru_nested\@ + /* Outermost: pick up scp before an interrupt can see nesting == 0. */ + ldr x14, [x12, #TSK_TRC_READER_SCP] + str wzr, [x12, #TSK_TRC_READER_NESTING] +#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB + dmb ish +#endif + get_this_cpu_offset x15 + add x14, x14, x15 + add x14, x14, #SRCU_CTR_SRCU_UNLOCKS + trace_rcu_srcu_inc x14, x15, w16 + b .Ltru_done\@ +.Ltru_nested\@: + str w13, [x12, #TSK_TRC_READER_NESTING] +.Ltru_done\@: +#endif + .endm + /* * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before * the regular function prologue. For an enabled callsite, ftrace_init_nop() and @@ -94,6 +160,8 @@ SYM_CODE_START(ftrace_caller) stp x29, x30, [sp, #FREGS_SIZE] add x29, sp, #FREGS_SIZE + trace_rcu_read_lock + /* Prepare arguments for the tracer func */ sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) mov x1, x9 // parent_ip (callsite's LR) @@ -111,6 +179,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) bl ftrace_stub // func(ip, parent_ip, op, regs) #endif + trace_rcu_read_unlock + /* * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved * x19-x29 per the AAPCS, and we created frame records upon entry, so we need @@ -178,6 +248,10 @@ SYM_CODE_START(ftrace_stub_direct_tramp) SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +/* End of [ftrace_caller, ...) for arch_rcu_tasks_trampoline_text(). */ +SYM_CODE_START(ftrace_static_tramp_end) +SYM_CODE_END(ftrace_static_tramp_end) + #else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ /* diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index e1a3c0b3a051..5f4193f15cd9 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -17,6 +17,26 @@ #include <asm/insn.h> #include <asm/text-patching.h> +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS +extern void ftrace_static_tramp_end(void); + +/* The SRCU-fast increments in entry-ftrace.S are the this_cpu_inc() form. */ +static_assert(!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)); + +/* + * See rcu_tasks_trampoline_text(). ftrace_caller and ftrace_stub_direct_tramp + * are core kernel text but must be treated as trampolines: a task interrupted + * in them outside the Tasks Trace reader may be carrying an ops pointer (x11) + * or a direct-call BPF trampoline address (x17) whose lifetime is guarded only + * by Tasks RCU. + */ +bool arch_rcu_tasks_trampoline_text(unsigned long ip) +{ + return ip >= (unsigned long)ftrace_caller && + ip < (unsigned long)ftrace_static_tramp_end; +} +#endif + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS struct fregs_offset { const char *name; -- 2.55.0
