From: Jim Cromie <[email protected]> Embed a struct folio_pool inside struct bpf_verifier_env to allocate transient bpf_verifier_stack_elem frames using direct-map large folios, releasing all frames in bulk at the end of bpf_check().
Use folio_pool_alloc_obj() to preserve strict type specificity at callsites. Signed-off-by: Jim Cromie <[email protected]> --- include/linux/bpf_verifier.h | 3 +++ kernel/bpf/verifier.c | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 39a851e690ec..43f5f0eaffac 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -898,6 +898,8 @@ struct bpf_scc_info { struct bpf_liveness; +#include <linux/folio_pool.h> + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -907,6 +909,7 @@ struct bpf_verifier_env { struct bpf_prog *prog; /* eBPF program being verified */ const struct bpf_verifier_ops *ops; struct module *attach_btf_mod; /* The owner module of prog->aux->attach_btf */ + struct folio_pool state_pool; /* pool for transient state nodes */ struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */ int stack_size; /* number of states to be processed */ bool strict_alignment; /* perform strict pointer alignment checks */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdc5fbb1f78c..8a9e66ce4dc8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1725,7 +1725,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, *prev_insn_idx = head->prev_insn_idx; elem = head->next; bpf_free_verifier_state(&head->st, false); - kfree(head); + folio_pool_free_elem(head); env->head = elem; env->stack_size--; return 0; @@ -1743,6 +1743,8 @@ static bool error_recoverable_with_nospec(int err) return err == -EPERM || err == -EACCES || err == -EINVAL; } +DEFINE_STATIC_KEY_TRUE(bpf_state_pool_key); + static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx, bool speculative) @@ -1751,7 +1753,9 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, struct bpf_verifier_stack_elem *elem; int err; - elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); + elem = folio_pool_alloc_obj(env, state_pool, + struct bpf_verifier_stack_elem, + GFP_KERNEL_ACCOUNT); if (!elem) return ERR_PTR(-ENOMEM); @@ -2275,7 +2279,9 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, struct bpf_verifier_stack_elem *elem; struct bpf_func_state *frame; - elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); + elem = folio_pool_alloc_obj(env, state_pool, + struct bpf_verifier_stack_elem, + GFP_KERNEL_ACCOUNT); if (!elem) return ERR_PTR(-ENOMEM); @@ -19789,6 +19795,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (!env) return -ENOMEM; + folio_pool_init_key(&env->state_pool, sizeof(struct bpf_verifier_stack_elem), + get_order(SZ_64K), &bpf_state_pool_key); + env->bt.env = env; len = (*prog)->len; @@ -20055,6 +20064,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, bpf_clear_insn_aux_data(env, 0, env->prog->len); err_free_env: bpf_stack_liveness_free(env); + folio_pool_free(&env->state_pool); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); -- 2.55.0
