On Wed Jul 15, 2026 at 11:32 AM EDT, Leon Hwang wrote:
> Introduce global percpu data, inspired by the commit
> 6316f78306c1 ("Merge branch 'support-global-data'"). It enables the
> definition of global percpu variables in BPF, similar to the
> include/linux/percpu-defs.h::DEFINE_PER_CPU() macro.
>
> For example, in BPF, it is able to define a global percpu variable like:
>
> int data SEC(".percpu");
>
> With this patch, tools like retsnoop [1] and bpfsnoop [2] can simplify
> their BPF code for handling LBRs. The code can be updated from
>
> static struct perf_branch_entry lbrs[1][MAX_LBR_ENTRIES] SEC(".data.lbrs");
>
> to
>
> static struct perf_branch_entry lbrs[MAX_LBR_ENTRIES] SEC(".percpu.lbrs");
>
> This eliminates the need to retrieve the CPU ID using the
> bpf_get_smp_processor_id() helper.
>
> Additionally, by reusing global percpu data map, sharing information
> between tail callers and callees or freplace callers and callees becomes
> simpler compared to reusing percpu_array maps.
>
> Links:
> [1] https://github.com/anakryiko/retsnoop
> [2] https://github.com/bpfsnoop/bpfsnoop
>
> Signed-off-by: Leon Hwang <[email protected]>
> ---
> kernel/bpf/arraymap.c | 38 ++++++++++++++++++++++++++++++++++++--
> kernel/bpf/const_fold.c | 1 -
> kernel/bpf/fixups.c | 38 ++++++++++++++++++++++++++++++++++++++
> kernel/bpf/verifier.c | 15 +++++++++++++++
> 4 files changed, 89 insertions(+), 3 deletions(-)
>
> diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
> index 248b4818178c..c4e9430941e5 100644
> --- a/kernel/bpf/arraymap.c
> +++ b/kernel/bpf/arraymap.c
> @@ -259,6 +259,37 @@ static void *percpu_array_map_lookup_elem(struct bpf_map
> *map, void *key)
> return this_cpu_ptr(array->pptrs[index & array->index_mask]);
> }
>
> +static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64
> *imm, u32 off)
> +{
> + struct bpf_array *array = container_of(map, struct bpf_array, map);
> +
> + if (map->max_entries != 1)
> + return -EOPNOTSUPP;
> + if (off >= map->value_size)
> + return -EINVAL;
> + if (!bpf_jit_supports_percpu_insn())
> + return -EOPNOTSUPP;
Nit: This should be first, if the JIT doesn't support the instruction
all else is moot. Same below.
> +
> + *imm = (u64)(__force unsigned long) array->pptrs[0];
> + return 0;
> +}
> +
> +static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64
> imm, u32 *off)
> +{
> + struct bpf_array *array = container_of(map, struct bpf_array, map);
> + u64 base = (u64)(__force unsigned long) array->pptrs[0];
> +
> + if (map->max_entries != 1)
> + return -EOPNOTSUPP;
> + if (imm < base || imm >= base + array->elem_size)
> + return -ENOENT;
> + if (!bpf_jit_supports_percpu_insn())
> + return -EOPNOTSUPP;
> +
> + *off = imm - base;
> + return 0;
> +}
> +
> /* emit BPF instructions equivalent to C code of
> percpu_array_map_lookup_elem() */
> static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn
> *insn_buf)
> {
> @@ -551,9 +582,10 @@ static int array_map_check_btf(struct bpf_map *map,
> const struct btf_type *key_type,
> const struct btf_type *value_type)
> {
> - /* One exception for keyless BTF: .bss/.data/.rodata map */
> + /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */
> if (btf_type_is_void(key_type)) {
> - if (map->map_type != BPF_MAP_TYPE_ARRAY ||
> + if ((map->map_type != BPF_MAP_TYPE_ARRAY &&
> + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) ||
> map->max_entries != 1)
> return -EINVAL;
>
> @@ -832,6 +864,8 @@ const struct bpf_map_ops percpu_array_map_ops = {
> .map_get_next_key = bpf_array_get_next_key,
> .map_lookup_elem = percpu_array_map_lookup_elem,
> .map_gen_lookup = percpu_array_map_gen_lookup,
> + .map_direct_value_addr = percpu_array_map_direct_value_addr,
> + .map_direct_value_meta = percpu_array_map_direct_value_meta,
> .map_update_elem = array_map_update_elem,
> .map_delete_elem = array_map_delete_elem,
> .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem,
> diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
> index b2a19acadb91..5787246bef30 100644
> --- a/kernel/bpf/const_fold.c
> +++ b/kernel/bpf/const_fold.c
> @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env,
> struct const_arg_info *
> u64 val = 0;
>
> if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr
> ||
> - map->map_type == BPF_MAP_TYPE_INSN_ARRAY ||
> off < 0 || off + size > map->value_size ||
> bpf_map_direct_read(map, off, size, &val, is_ldsx)) {
> *dst = unknown;
> diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
> index d3be972714b2..02642bba73e5 100644
> --- a/kernel/bpf/fixups.c
> +++ b/kernel/bpf/fixups.c
> @@ -1818,6 +1818,44 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
> goto next_insn;
> }
>
> + if (env->prog->jit_requested &&
> + bpf_jit_supports_percpu_insn() &&
> + insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
> + (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
> + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) {
> + struct bpf_map *map;
> +
> + aux = &env->insn_aux_data[i + delta];
> + map = env->used_maps[aux->map_index];
> + if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
> + goto next_insn;
> +
> + prog->jit_required = true;
> +
> + /*
> + * We are *skipping* first half of ld_imm64 insn
> + * with 'i++;', patching over second half of it
> + * with that same half + mov64_percpu_reg insn.
> + * All because bpf_patch_insn_data() can only
> + * replace one 8-byte insn, which does not work
> + * well for ld_imm64 insn.
> + */
> +
> + insn_buf[0] = insn[1];
> + insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg,
> insn->dst_reg);
> + cnt = 2;
> +
> + i++;
> + new_prog = bpf_patch_insn_data(env, i + delta,
> insn_buf, cnt);
> + if (!new_prog)
> + return -ENOMEM;
> +
> + delta += cnt - 1;
> + env->prog = prog = new_prog;
> + insn = new_prog->insnsi + i + delta;
> + goto next_insn;
> + }
> +
> if (insn->code != (BPF_JMP | BPF_CALL))
> goto next_insn;
> if (insn->src_reg == BPF_PSEUDO_CALL)
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index e1244a721194..ae442ea217c4 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -5615,6 +5615,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off,
> int size, u64 *val,
> u64 addr;
> int err;
>
> + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type ==
> BPF_MAP_TYPE_PERCPU_ARRAY)
> + return -EINVAL;
> err = map->ops->map_direct_value_addr(map, &addr, off);
> if (err)
> return err;
> @@ -6174,6 +6176,7 @@ static int check_mem_access(struct bpf_verifier_env
> *env, int insn_idx, struct b
> if (tnum_is_const(reg->var_off) &&
> bpf_map_is_rdonly(map) &&
> map->ops->map_direct_value_addr &&
> + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
> map->map_type != BPF_MAP_TYPE_INSN_ARRAY) {
An issue I have with the patch is that it takes the special-casing we
already have for INSN_ARRAY and makes it worse. The instruction arrays
implement direct_value_addr, but it's not actually valid to call
direct_value_addr from a bunch of places where it's called, so we have
to special case check the map type to prevent it. Case in point the
special casing in check_arg_const_str in the thunk below.
Imo we should consider making a different map method different from
direct_value_addr that INSN_ARRAY and PERCPU_ARRAY implement. That way
we remove most of the map type checks in those paths. I'm not exactly
sure how that would look like, but I could take a look at it tomorrow.
> int map_off = off + reg->var_off.value;
> u64 val = 0;
> @@ -8176,6 +8179,12 @@ static int check_arg_const_str(struct bpf_verifier_env
> *env,
> return -EACCES;
> }
>
> + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
> + verbose(env, "%s points to percpu_array map which cannot be
> used as const string\n",
> + reg_arg_name(env, argno));
> + return -EACCES;
> + }
> +
> if (!bpf_map_is_rdonly(map)) {
> verbose(env, "%s does not point to a readonly map'\n",
> reg_arg_name(env, argno));
> return -EACCES;
> @@ -18279,6 +18288,12 @@ static int check_and_resolve_insns(struct
> bpf_verifier_env *env)
> return -EINVAL;
> }
>
> + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY
> &&
> + !env->prog->jit_requested) {
> + verbose(env, "JIT is required to use
> global percpu data\n");
> + return -EOPNOTSUPP;
> + }
> +
Another place where we add map-specific checks where there are none.
> err = map->ops->map_direct_value_addr(map,
> &addr, off);
> if (err) {
> verbose(env, "invalid access to map
> value pointer, value_size=%u off=%u\n",
