This moves some functions and structs around to make the following patch easier to read.
Signed-off-by: Jing Liu <[email protected]> --- kernel/trace/bpf_trace.c | 304 +++++++++++++++++++-------------------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fe28d86f7c35..1fd07c10378f 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2291,67 +2291,6 @@ struct bpf_kprobe_multi_run_ctx { unsigned long entry_ip; }; -struct user_syms { - const char **syms; - char *buf; -}; - -#ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS -static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); -#define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) -#else -#define bpf_kprobe_multi_pt_regs_ptr() (NULL) -#endif - -static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) -{ - unsigned long ip = ftrace_get_symaddr(fentry_ip); - - return ip ? : fentry_ip; -} - -static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) -{ - unsigned long __user usymbol; - const char **syms = NULL; - char *buf = NULL, *p; - int err = -ENOMEM; - unsigned int i; - - syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); - if (!syms) - goto error; - - buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); - if (!buf) - goto error; - - for (p = buf, i = 0; i < cnt; i++) { - if (__get_user(usymbol, usyms + i)) { - err = -EFAULT; - goto error; - } - err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); - if (err == KSYM_NAME_LEN) - err = -E2BIG; - if (err < 0) - goto error; - syms[i] = p; - p += err + 1; - } - - us->syms = syms; - us->buf = buf; - return 0; - -error: - if (err) { - kvfree(syms); - kvfree(buf); - } - return err; -} - static void kprobe_multi_put_modules(struct module **mods, u32 cnt) { u32 i; @@ -2360,12 +2299,6 @@ static void kprobe_multi_put_modules(struct module **mods, u32 cnt) module_put(mods[i]); } -static void free_user_syms(struct user_syms *us) -{ - kvfree(us->syms); - kvfree(us->buf); -} - static void bpf_kprobe_multi_link_release(struct bpf_link *link) { struct bpf_kprobe_multi_link *kmulti_link; @@ -2469,6 +2402,152 @@ static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { #endif }; +static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) +{ + struct bpf_kprobe_multi_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, + session_ctx.run_ctx); + return run_ctx->entry_ip; +} + +struct modules_array { + struct module **mods; + int mods_cnt; + int mods_cap; +}; + +static int add_module(struct modules_array *arr, struct module *mod) +{ + struct module **mods; + + if (arr->mods_cnt == arr->mods_cap) { + arr->mods_cap = max(16, arr->mods_cap * 3 / 2); + mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); + if (!mods) + return -ENOMEM; + arr->mods = mods; + } + + arr->mods[arr->mods_cnt] = mod; + arr->mods_cnt++; + return 0; +} + +static bool has_module(struct modules_array *arr, struct module *mod) +{ + int i; + + for (i = arr->mods_cnt - 1; i >= 0; i--) { + if (arr->mods[i] == mod) + return true; + } + return false; +} + +static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) +{ + struct modules_array arr = {}; + u32 i, err = 0; + + for (i = 0; i < addrs_cnt; i++) { + bool skip_add = false; + struct module *mod; + + scoped_guard(rcu) { + mod = __module_address(addrs[i]); + /* Either no module or it's already stored */ + if (!mod || has_module(&arr, mod)) { + skip_add = true; + break; /* scoped_guard */ + } + if (!try_module_get(mod)) + err = -EINVAL; + } + if (skip_add) + continue; + if (err) + break; + err = add_module(&arr, mod); + if (err) { + module_put(mod); + break; + } + } + + /* We return either err < 0 in case of error, ... */ + if (err) { + kprobe_multi_put_modules(arr.mods, arr.mods_cnt); + kfree(arr.mods); + return err; + } + + /* or number of modules found if everything is ok. */ + *mods = arr.mods; + return arr.mods_cnt; +} + +struct user_syms { + const char **syms; + char *buf; +}; + +#ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS +static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); +#define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) +#else +#define bpf_kprobe_multi_pt_regs_ptr() (NULL) +#endif + +static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) +{ + unsigned long ip = ftrace_get_symaddr(fentry_ip); + + return ip ? : fentry_ip; +} + +static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) +{ + unsigned long __user usymbol; + const char **syms = NULL; + char *buf = NULL, *p; + int err = -ENOMEM; + unsigned int i; + + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); + if (!syms) + goto error; + + buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); + if (!buf) + goto error; + + for (p = buf, i = 0; i < cnt; i++) { + if (__get_user(usymbol, usyms + i)) { + err = -EFAULT; + goto error; + } + err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); + if (err == KSYM_NAME_LEN) + err = -E2BIG; + if (err < 0) + goto error; + syms[i] = p; + p += err + 1; + } + + us->syms = syms; + us->buf = buf; + return 0; + +error: + if (err) { + kvfree(syms); + kvfree(buf); + } + return err; +} + static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv) { const struct bpf_kprobe_multi_link *link = priv; @@ -2520,15 +2599,6 @@ static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) return *cookie; } -static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) -{ - struct bpf_kprobe_multi_run_ctx *run_ctx; - - run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, - session_ctx.run_ctx); - return run_ctx->entry_ip; -} - static __always_inline int kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, unsigned long entry_ip, struct ftrace_regs *fregs, @@ -2597,6 +2667,12 @@ kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip, fregs, true, data); } +static void free_user_syms(struct user_syms *us) +{ + kvfree(us->syms); + kvfree(us->buf); +} + static int symbols_cmp_r(const void *a, const void *b, const void *priv) { const char **str_a = (const char **) a; @@ -2627,82 +2703,6 @@ static void symbols_swap_r(void *a, void *b, int size, const void *priv) } } -struct modules_array { - struct module **mods; - int mods_cnt; - int mods_cap; -}; - -static int add_module(struct modules_array *arr, struct module *mod) -{ - struct module **mods; - - if (arr->mods_cnt == arr->mods_cap) { - arr->mods_cap = max(16, arr->mods_cap * 3 / 2); - mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); - if (!mods) - return -ENOMEM; - arr->mods = mods; - } - - arr->mods[arr->mods_cnt] = mod; - arr->mods_cnt++; - return 0; -} - -static bool has_module(struct modules_array *arr, struct module *mod) -{ - int i; - - for (i = arr->mods_cnt - 1; i >= 0; i--) { - if (arr->mods[i] == mod) - return true; - } - return false; -} - -static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) -{ - struct modules_array arr = {}; - u32 i, err = 0; - - for (i = 0; i < addrs_cnt; i++) { - bool skip_add = false; - struct module *mod; - - scoped_guard(rcu) { - mod = __module_address(addrs[i]); - /* Either no module or it's already stored */ - if (!mod || has_module(&arr, mod)) { - skip_add = true; - break; /* scoped_guard */ - } - if (!try_module_get(mod)) - err = -EINVAL; - } - if (skip_add) - continue; - if (err) - break; - err = add_module(&arr, mod); - if (err) { - module_put(mod); - break; - } - } - - /* We return either err < 0 in case of error, ... */ - if (err) { - kprobe_multi_put_modules(arr.mods, arr.mods_cnt); - kfree(arr.mods); - return err; - } - - /* or number of modules found if everything is ok. */ - *mods = arr.mods; - return arr.mods_cnt; -} - static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt) { u32 i; -- 2.25.1
