uretprobes_srcu currently uses normal SRCU, which issues two smp_mb() per read lock/unlock pair. This overhead is paid on every uretprobe hit.
Switch to SRCU-fast-updown, which eliminates the per-reader memory barriers by moving the ordering cost to the grace-period side (synchronize_rcu() instead of smp_mb()). This is acceptable because grace periods (uprobe unregistration) are infrequent compared to reader-side uretprobe hits. The updown flavor is required because the SRCU read lock is taken in prepare_uretprobe() when a return instance is created and is held until that return instance is finalized. The traced thread returns to user space in between, so the lock is inherently released in a different context from where it was acquired: on the normal return path via uprobe_handle_trampoline() -> hprobe_finalize(), or from ri_timer() (expiry) or dup_utask() (fork) via hprobe_expire(). srcu_down_read_fast() / srcu_up_read_fast() are designed for this acquire-here / release-elsewhere pattern and, unlike the same-context srcu_read_lock_fast() variant, do not carry the lockdep read-side tracking that would warn on it. The short, same-context SRCU sections in ri_timer() and dup_utask() (which guard the uprobe against reuse across the hprobe_expire() cmpxchg) instead use guard(srcu_fast_updown) for proper lockdep coverage. Signed-off-by: Puranjay Mohan <[email protected]> --- include/linux/uprobes.h | 5 +++-- kernel/events/uprobes.c | 29 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8..f3b07753c2f3d 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -25,6 +25,7 @@ struct mm_struct; struct inode; struct notifier_block; struct page; +struct srcu_ctr; /* * Allowed return values from uprobe consumer's handler callback @@ -106,7 +107,7 @@ enum hprobe_state { * underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an * internal marker and is handled transparently by hprobe_fetch() helper. * - * When uprobe is SRCU-protected, we also record srcu_idx value, necessary for + * When uprobe is SRCU-protected, we also record srcu_scp value, necessary for * SRCU unlocking. * * See hprobe_expire() and hprobe_fetch() for details of race-free uprobe @@ -115,7 +116,7 @@ enum hprobe_state { */ struct hprobe { enum hprobe_state state; - int srcu_idx; + struct srcu_ctr __percpu *srcu_scp; struct uprobe *uprobe; }; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e2844..afa491b0bd3f9 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -54,7 +54,7 @@ static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); /* Covers return_instance's uprobe lifetime. */ -DEFINE_STATIC_SRCU(uretprobes_srcu); +DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu); /* Have a copy of original instruction */ #define UPROBE_COPY_INSN 0 @@ -707,12 +707,13 @@ static void put_uprobe(struct uprobe *uprobe) } /* Initialize hprobe as SRCU-protected "leased" uprobe */ -static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx) +static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, + struct srcu_ctr __percpu *srcu_scp) { WARN_ON(!uprobe); hprobe->state = HPROBE_LEASED; hprobe->uprobe = uprobe; - hprobe->srcu_idx = srcu_idx; + hprobe->srcu_scp = srcu_scp; } /* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */ @@ -720,7 +721,7 @@ static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe) { hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE; hprobe->uprobe = uprobe; - hprobe->srcu_idx = -1; + hprobe->srcu_scp = NULL; } /* @@ -757,7 +758,7 @@ static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate) { switch (hstate) { case HPROBE_LEASED: - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); break; case HPROBE_STABLE: put_uprobe(hprobe->uprobe); @@ -829,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get) */ if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) { /* We won the race, we are the ones to unlock SRCU */ - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); return get ? get_uprobe(uprobe) : uprobe; } @@ -2045,7 +2046,7 @@ static void ri_timer(struct timer_list *timer) struct return_instance *ri; /* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); /* RCU protects return_instance from freeing. */ guard(rcu)(); @@ -2142,7 +2143,7 @@ static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) t->utask = n_utask; /* protect uprobes from freeing, we'll need try_get_uprobe() them */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); p = &n_utask->return_instances; for (o = o_utask->return_instances; o; o = o->next) { @@ -2254,8 +2255,8 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, { struct uprobe_task *utask = current->utask; unsigned long orig_ret_vaddr, trampoline_vaddr; + struct srcu_ctr __percpu *srcu_scp; bool chained; - int srcu_idx; if (!get_xol_area()) goto free; @@ -2293,8 +2294,12 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; } - /* __srcu_read_lock() because SRCU lock survives switch to user space */ - srcu_idx = __srcu_read_lock(&uretprobes_srcu); + /* + * Use srcu_down_read_fast() because the SRCU lock survives a switch to + * user space and can be unlocked from a different context by ri_timer() + * or dup_utask(). + */ + srcu_scp = srcu_down_read_fast(&uretprobes_srcu); ri->func = instruction_pointer(regs); ri->stack = user_stack_pointer(regs); @@ -2303,7 +2308,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, utask->depth++; - hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx); + hprobe_init_leased(&ri->hprobe, uprobe, srcu_scp); ri->next = utask->return_instances; rcu_assign_pointer(utask->return_instances, ri); -- 2.53.0-Meta
