From: Feng Yang <[email protected]>

bpf_get_stack() has two implementations.  bpf_get_stack() passes
may_fault == false and resolves build IDs with build_id_parse_nofault(),
so an IP whose backing file page is not resident falls back to
BPF_STACK_BUILD_ID_IP.  bpf_get_stack_sleepable() passes may_fault == true
and uses build_id_parse(), which faults the page in and returns a real
build ID.

Only kprobe/uprobe programs are routed to the sleepable implementation
today, so the other program types that can be sleepable keep losing
build IDs for non-resident pages:

 - tracing_prog_func_proto() serves BPF_TRACE_ITER, fentry/fexit/fmod_ret
   and BPF_TRACE_RAW_TP programs, and falls back to
   raw_tp_prog_func_proto() for BPF_FUNC_get_stack, which is also the
   get_func_proto() of BPF_PROG_TYPE_RAW_TRACEPOINT and is reached by
   BPF_PROG_TYPE_LSM and BPF_PROG_TYPE_SYSCALL through the default cases
   of bpf_lsm_func_proto() and syscall_prog_func_proto().

 - tp_prog_func_proto() serves BPF_PROG_TYPE_TRACEPOINT programs.

Route them to bpf_get_stack_sleepable() as well and mark the new
prototypes might_sleep, following
commit 9d02927fdf4e ("bpf: Mark faultable stack helpers as sleepable"),
which did the same for bpf_get_task_stack() and uprobe programs.

The verifier then rejects bpf_get_stack() from these program types
whenever in_sleepable_context() is false: inside bpf_rcu_read_lock(),
bpf_preempt_disable(), bpf_local_irq_save() and bpf_spin_lock() regions,
and in callback subprograms running under rcu_read_lock().  This also
covers calls that do not pass BPF_F_USER_BUILD_ID and would not sleep;
such programs load today.  This is the same trade off that making the
faultable stack helpers sleepable made for bpf_get_task_stack().

Signed-off-by: Feng Yang <[email protected]>
---
v3: Add selftest. Update commit messages.

v2: Move regs onto the stack.
https://lore.kernel.org/all/[email protected]/

v1: https://lore.kernel.org/all/[email protected]/
---
 kernel/trace/bpf_trace.c | 54 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 2aec56d7f3d3..87f7758ffd04 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -84,6 +84,7 @@ static struct bpf_raw_event_map 
*bpf_get_raw_tracepoint_module(const char *name)
 
 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
+u64 bpf_get_stack_sleepable(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 
 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
                                  u64 flags, const struct btf **btf,
@@ -1477,6 +1478,26 @@ static const struct bpf_func_proto 
bpf_get_stack_proto_tp = {
        .arg4_type      = ARG_ANYTHING,
 };
 
+BPF_CALL_4(bpf_get_stack_tp_sleepable, void *, tp_buff, void *, buf, u32, size,
+          u64, flags)
+{
+       struct pt_regs *regs = *(struct pt_regs **)tp_buff;
+
+       return bpf_get_stack_sleepable((unsigned long) regs, (unsigned long) 
buf,
+                                      (unsigned long) size, flags, 0);
+}
+
+static const struct bpf_func_proto bpf_get_stack_proto_tp_sleepable = {
+       .func           = bpf_get_stack_tp_sleepable,
+       .gpl_only       = true,
+       .might_sleep    = true,
+       .ret_type       = RET_INTEGER,
+       .arg1_type      = ARG_PTR_TO_CTX,
+       .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
+       .arg3_type      = ARG_MEM_SIZE_OR_ZERO,
+       .arg4_type      = ARG_ANYTHING,
+};
+
 static const struct bpf_func_proto *
 tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -1486,7 +1507,8 @@ tp_prog_func_proto(enum bpf_func_id func_id, const struct 
bpf_prog *prog)
        case BPF_FUNC_get_stackid:
                return &bpf_get_stackid_proto_tp;
        case BPF_FUNC_get_stack:
-               return &bpf_get_stack_proto_tp;
+               return prog->sleepable ? &bpf_get_stack_proto_tp_sleepable
+                                      : &bpf_get_stack_proto_tp;
        case BPF_FUNC_get_attach_cookie:
                return &bpf_get_attach_cookie_proto_trace;
        default:
@@ -1716,6 +1738,33 @@ static const struct bpf_func_proto 
bpf_get_stack_proto_raw_tp = {
        .arg4_type      = ARG_ANYTHING,
 };
 
+BPF_CALL_4(bpf_get_stack_raw_tp_sleepable, struct bpf_raw_tracepoint_args *, 
args,
+          void *, buf, u32, size, u64, flags)
+{
+       /*
+        * Don't take a per-CPU bpf_raw_tp_regs slot here: those slots are a
+        * LIFO nest counter and cannot be held across the sleep that
+        * build_id_parse() may take.  Sleepable programs never run in
+        * IRQ/NMI context, so a stack copy is sufficient.
+        */
+       struct pt_regs regs = {};
+
+       perf_fetch_caller_regs(&regs);
+       return bpf_get_stack_sleepable((unsigned long) &regs, (unsigned long) 
buf,
+                                      (unsigned long) size, flags, 0);
+}
+
+static const struct bpf_func_proto bpf_get_stack_proto_raw_tp_sleepable = {
+       .func           = bpf_get_stack_raw_tp_sleepable,
+       .gpl_only       = true,
+       .might_sleep    = true,
+       .ret_type       = RET_INTEGER,
+       .arg1_type      = ARG_PTR_TO_CTX,
+       .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
+       .arg3_type      = ARG_MEM_SIZE_OR_ZERO,
+       .arg4_type      = ARG_ANYTHING,
+};
+
 static const struct bpf_func_proto *
 raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -1725,7 +1774,8 @@ raw_tp_prog_func_proto(enum bpf_func_id func_id, const 
struct bpf_prog *prog)
        case BPF_FUNC_get_stackid:
                return &bpf_get_stackid_proto_raw_tp;
        case BPF_FUNC_get_stack:
-               return &bpf_get_stack_proto_raw_tp;
+               return prog->sleepable ? &bpf_get_stack_proto_raw_tp_sleepable
+                                      : &bpf_get_stack_proto_raw_tp;
        case BPF_FUNC_get_attach_cookie:
                return &bpf_get_attach_cookie_proto_tracing;
        default:
-- 
2.25.1


Reply via email to