The jump-optimized kprobe template calls optimized_callback() from a dynamically allocated slot with preemption enabled, and only Tasks RCU keeps that slot alive under a task preempted in the callback. For HAVE_RCU_TRAMPOLINE_READERS that means the template must be a Tasks Trace reader across the call, so open-code rcu_read_lock_trace() and rcu_read_unlock_trace() around it as ftrace_64.S does. The template lives in .rodata and is memcpy()d into each slot without relocation processing, so the references to current_task and rcu_tasks_trace_srcu_struct are absolute (R_X86_64_32S, relocated for KASLR like any other) rather than %rip-relative. %rax and %rcx have already been saved by SAVE_REGS_STRING and are dead after the call.
The slot itself is dynamically allocated text, so the instructions before the lock and after the unlock are covered by the irq-exit check. 64-bit only; 32-bit x86 does not take part. Assisted-by: LLM Signed-off-by: Josef Bacik <[email protected]> --- arch/x86/kernel/kprobes/opt.c | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 3f8fea52619f..68a5de6cdabe 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -31,6 +31,7 @@ #include <asm/set_memory.h> #include <asm/sections.h> #include <asm/nospec-branch.h> +#include <asm/asm-offsets.h> #include "common.h" @@ -101,6 +102,47 @@ static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) *(unsigned long *)addr = val; } +/* + * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() around the call + * to optimized_callback(), see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the + * equivalent macros in ftrace_64.S. The template is memcpy()d into the slot + * without relocation processing, so memory references must be absolute + * rather than %rip-relative. %rax and %rcx are free at both points. + */ +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS +#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB +#define OPTPROBE_TRACE_RCU_MB " lock addl $0, -4(%rsp)\n" +#else +#define OPTPROBE_TRACE_RCU_MB +#endif +#define OPTPROBE_TRACE_RCU_READ_LOCK \ + " movq %gs:current_task, %rcx\n" \ + " movl " __stringify(TASK_trc_reader_nesting) "(%rcx), %eax\n" \ + " incl " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \ + " testl %eax, %eax\n" \ + " jnz 1f\n" \ + " movq rcu_tasks_trace_srcu_struct+" __stringify(SRCU_srcu_ctrp) ", %rax\n" \ + " incq %gs:" __stringify(SRCU_CTR_srcu_locks) "(%rax)\n" \ + " movq %rax, " __stringify(TASK_trc_reader_scp) "(%rcx)\n" \ + OPTPROBE_TRACE_RCU_MB \ + "1:\n" +#define OPTPROBE_TRACE_RCU_READ_UNLOCK \ + " movq %gs:current_task, %rcx\n" \ + " movl " __stringify(TASK_trc_reader_nesting) "(%rcx), %eax\n" \ + " subl $1, %eax\n" \ + " jnz 2f\n" \ + " movq " __stringify(TASK_trc_reader_scp) "(%rcx), %rax\n" \ + " movl $0, " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \ + OPTPROBE_TRACE_RCU_MB \ + " incq %gs:" __stringify(SRCU_CTR_srcu_unlocks) "(%rax)\n" \ + " jmp 3f\n" \ + "2: movl %eax, " __stringify(TASK_trc_reader_nesting) "(%rcx)\n" \ + "3:\n" +#else +#define OPTPROBE_TRACE_RCU_READ_LOCK +#define OPTPROBE_TRACE_RCU_READ_UNLOCK +#endif + asm ( ".pushsection .rodata\n" ".global optprobe_template_entry\n" @@ -114,6 +156,7 @@ asm ( "optprobe_template_clac:\n" ASM_NOP3 SAVE_REGS_STRING + OPTPROBE_TRACE_RCU_READ_LOCK " movq %rsp, %rsi\n" ".global optprobe_template_val\n" "optprobe_template_val:\n" @@ -122,6 +165,7 @@ asm ( ".global optprobe_template_call\n" "optprobe_template_call:\n" ASM_NOP5 + OPTPROBE_TRACE_RCU_READ_UNLOCK /* Copy 'regs->flags' into 'regs->ss'. */ " movq 18*8(%rsp), %rdx\n" " movq %rdx, 20*8(%rsp)\n" -- 2.55.0
