Following steps are taken in the driver to offload an XDP program:
XDP_SETUP_PROG:
* prepare:
- allocate program state;
- run verifier (bpf_analyzer());
- run translation;
* load:
- stop old program if needed;
- load program;
- enable BPF if not enabled;
* clean up:
- free program image.
With new infrastructure the flow will look like this:
BPF_OFFLOAD_VERIFIER_PREP:
- allocate program state;
BPF_OFFLOAD_TRANSLATE:
- run translation;
XDP_SETUP_PROG:
- stop old program if needed;
- load program;
- enable BPF if not enabled;
BPF_OFFLOAD_DESTROY:
- free program image.
Take advantage of the new infrastructure. Allocation of driver
metadata has to be moved from jit.c to offload.c since it's now
done at a different stage. Since there is no separate driver
private data for verification step, move temporary nfp_meta
pointer into nfp_prog. We will now use user space context
offsets.
Signed-off-by: Jakub Kicinski <jakub.kicin...@netronome.com>
Reviewed-by: Quentin Monnet <quentin.mon...@netronome.com>
---
drivers/net/ethernet/netronome/nfp/bpf/jit.c | 35 ++++-----
drivers/net/ethernet/netronome/nfp/bpf/main.c | 4 +
drivers/net/ethernet/netronome/nfp/bpf/main.h | 15 +++-
drivers/net/ethernet/netronome/nfp/bpf/offload.c | 85 ++++++++++------------
drivers/net/ethernet/netronome/nfp/bpf/verifier.c | 43 ++---------
drivers/net/ethernet/netronome/nfp/nfp_app.h | 37 ++++++++++
.../net/ethernet/netronome/nfp/nfp_net_common.c | 8 ++
7 files changed, 121 insertions(+), 106 deletions(-)
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index eae7a137a7a8..995e95410b11 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -1427,19 +1427,18 @@ static int mem_ldx_skb(struct nfp_prog *nfp_prog,
struct nfp_insn_meta *meta,
swreg dst = reg_both(meta->insn.dst_reg * 2);
switch (meta->insn.off) {
- case offsetof(struct sk_buff, len):
- if (size != FIELD_SIZEOF(struct sk_buff, len))
+ case offsetof(struct __sk_buff, len):
+ if (size != FIELD_SIZEOF(struct __sk_buff, len))
return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
break;
- case offsetof(struct sk_buff, data):
- if (size != sizeof(void *))
+ case offsetof(struct __sk_buff, data):
+ if (size != FIELD_SIZEOF(struct __sk_buff, data))
return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
break;
- case offsetof(struct sk_buff, cb) +
- offsetof(struct bpf_skb_data_end, data_end):
- if (size != sizeof(void *))
+ case offsetof(struct __sk_buff, data_end):
+ if (size != FIELD_SIZEOF(struct __sk_buff, data_end))
return -EOPNOTSUPP;
emit_alu(nfp_prog, dst,
plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
@@ -1458,14 +1457,15 @@ static int mem_ldx_xdp(struct nfp_prog *nfp_prog,
struct nfp_insn_meta *meta,
{
swreg dst = reg_both(meta->insn.dst_reg * 2);
- if (size != sizeof(void *))
- return -EINVAL;
-
switch (meta->insn.off) {
- case offsetof(struct xdp_buff, data):
+ case offsetof(struct xdp_md, data):
+ if (size != FIELD_SIZEOF(struct xdp_md, data))
+ return -EOPNOTSUPP;
wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
break;
- case offsetof(struct xdp_buff, data_end):
+ case offsetof(struct xdp_md, data_end):
+ if (size != FIELD_SIZEOF(struct xdp_md, data_end))
+ return -EOPNOTSUPP;
emit_alu(nfp_prog, dst,
plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
break;
@@ -2243,19 +2243,10 @@ static int nfp_bpf_ustore_calc(struct nfp_prog
*nfp_prog, __le64 *ustore)
return 0;
}
-/**
- * nfp_bpf_jit() - translate BPF code into NFP assembly
- * @nfp_prog: nfp_prog prepared based on @filter
- * @filter: kernel BPF filter struct
- */
-int nfp_bpf_jit(struct nfp_prog *nfp_prog, struct bpf_prog *filter)
+int nfp_bpf_jit(struct nfp_prog *nfp_prog)
{
int ret;
- ret = nfp_prog_verify(nfp_prog, filter);
- if (ret)
- return ret;
-
ret = nfp_bpf_optimize(nfp_prog);
if (ret)
return ret;
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c
b/drivers/net/ethernet/netronome/nfp/bpf/main.c
index 7ae7528cd96b..e379b78e86ef 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c
@@ -173,4 +173,8 @@ const struct nfp_app_type app_bpf = {
.setup_tc = nfp_bpf_setup_tc,
.tc_busy = nfp_bpf_tc_busy,
.xdp_offload = nfp_bpf_xdp_offload,
+
+ .bpf_verifier_prep = nfp_bpf_verifier_prep,
+ .bpf_translate = nfp_bpf_translate,
+ .bpf_destroy = nfp_bpf_destroy,
};
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h
b/drivers/net/ethernet/netronome/nfp/bpf/main.h
index 36b4eda2d3f8..082a15f6dfb5 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h
@@ -139,6 +139,7 @@ static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
* @prog: machine code
* @prog_len: number of valid instructions in @prog array
* @__prog_alloc_len: alloc size of @prog array
+ * @verifier_meta: temporary storage for verifier's insn meta
* @type: BPF program type
* @start_off: address of the first instruction in the memory
* @tgt_out: jump target for normal exit
@@ -154,6 +155,8 @@ struct nfp_prog {
unsigned int prog_len;
unsigned int __prog_alloc_len;
+ struct nfp_insn_meta *verifier_meta;
+
enum bpf_prog_type type;
unsigned int start_off;
@@ -169,13 +172,21 @@ struct nfp_prog {
struct list_head insns;
};
-int nfp_bpf_jit(struct nfp_prog *nfp_prog, struct bpf_prog *filter);
+int nfp_bpf_jit(struct nfp_prog *prog);
-int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog);
+extern const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops;
+struct netdev_bpf;
+struct nfp_app;
struct nfp_net;
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
bool old_prog);
+int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
+ struct netdev_bpf *bpf);
+int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog);
+int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog);
#endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c
b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
index c5546c0e87d8..b6cee71f49d3 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
@@ -84,14 +84,17 @@ static void nfp_prog_free(struct nfp_prog *nfp_prog)
kfree(nfp_prog);
}
-static struct nfp_prog *nfp_bpf_verifier_prep(struct bpf_prog *prog)
+int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
+ struct netdev_bpf *bpf)
{
+ struct bpf_prog *prog = bpf->verifier.prog;
struct nfp_prog *nfp_prog;
int ret;
nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
if (!nfp_prog)
- return NULL;
+ return -ENOMEM;
+ prog->aux->offload->dev_priv = nfp_prog;
INIT_LIST_HEAD(&nfp_prog->insns);
nfp_prog->type = prog->type;
@@ -100,18 +103,21 @@ static struct nfp_prog *nfp_bpf_verifier_prep(struct
bpf_prog *prog)
if (ret)
goto err_free;
- return nfp_prog;
+ nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
+ bpf->verifier.ops = &nfp_bpf_analyzer_ops;
+
+ return 0;
err_free:
nfp_prog_free(nfp_prog);
- return NULL;
+ return ret;
}
-static int
-nfp_bpf_translate(struct nfp_net *nn, struct nfp_prog *nfp_prog,
- struct bpf_prog *prog)
+int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog)
{
+ struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
unsigned int stack_size;
unsigned int max_instr;
@@ -133,55 +139,38 @@ nfp_bpf_translate(struct nfp_net *nn, struct nfp_prog
*nfp_prog,
if (!nfp_prog->prog)
return -ENOMEM;
- return nfp_bpf_jit(nfp_prog, prog);
+ return nfp_bpf_jit(nfp_prog);
}
-static void nfp_bpf_destroy(struct nfp_prog *nfp_prog)
+int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog)
{
+ struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
+
kfree(nfp_prog->prog);
nfp_prog_free(nfp_prog);
+
+ return 0;
}
-static struct nfp_prog *
-nfp_net_bpf_offload_prepare(struct nfp_net *nn, struct bpf_prog *prog,
- dma_addr_t *dma_addr)
+static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
{
- struct nfp_prog *nfp_prog;
+ struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
unsigned int max_mtu;
+ dma_addr_t dma_addr;
int err;
max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
if (max_mtu < nn->dp.netdev->mtu) {
nn_info(nn, "BPF offload not supported with MTU larger than HW
packet split boundary\n");
- return NULL;
+ return -EOPNOTSUPP;
}
- nfp_prog = nfp_bpf_verifier_prep(prog);
- if (!nfp_prog)
- return NULL;
-
- err = nfp_bpf_translate(nn, nfp_prog, prog);
- if (err)
- goto err_destroy_prog;
-
- *dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
- nfp_prog->prog_len * sizeof(u64),
- DMA_TO_DEVICE);
- if (dma_mapping_error(nn->dp.dev, *dma_addr))
- goto err_destroy_prog;
-
- return 0;
-
-err_destroy_prog:
- nfp_bpf_destroy(nfp_prog);
- return NULL;
-}
-
-static void
-nfp_net_bpf_load(struct nfp_net *nn, struct nfp_prog *nfp_prog,
- dma_addr_t dma_addr)
-{
- int err;
+ dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
+ nfp_prog->prog_len * sizeof(u64),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(nn->dp.dev, dma_addr))
+ return -ENOMEM;
nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
@@ -193,7 +182,8 @@ nfp_net_bpf_load(struct nfp_net *nn, struct nfp_prog
*nfp_prog,
dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
DMA_TO_DEVICE);
- nfp_bpf_destroy(nfp_prog);
+
+ return err;
}
static void nfp_net_bpf_start(struct nfp_net *nn)
@@ -222,8 +212,10 @@ static int nfp_net_bpf_stop(struct nfp_net *nn)
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
bool old_prog)
{
- struct nfp_prog *nfp_prog;
- dma_addr_t dma_addr;
+ int err;
+
+ if (prog && !prog->aux->offload)
+ return -EINVAL;
if (prog && old_prog) {
u8 cap;
@@ -242,11 +234,10 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct
bpf_prog *prog,
if (old_prog && !prog)
return nfp_net_bpf_stop(nn);
- nfp_prog = nfp_net_bpf_offload_prepare(nn, prog, &dma_addr);
- if (!nfp_prog)
- return -EINVAL;
+ err = nfp_net_bpf_load(nn, prog);
+ if (err)
+ return err;
- nfp_net_bpf_load(nn, nfp_prog, dma_addr);
if (!old_prog)
nfp_net_bpf_start(nn);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 4f31bdefd331..8d43491ddd6b 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -40,12 +40,6 @@
#include "main.h"
-/* Analyzer/verifier definitions */
-struct nfp_bpf_analyzer_priv {
- struct nfp_prog *prog;
- struct nfp_insn_meta *meta;
-};
-
static struct nfp_insn_meta *
nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
unsigned int insn_idx, unsigned int n_insns)
@@ -171,11 +165,11 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct
nfp_insn_meta *meta,
static int
nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
{
- struct nfp_bpf_analyzer_priv *priv = env->analyzer_priv;
- struct nfp_insn_meta *meta = priv->meta;
+ struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
+ struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
- meta = nfp_bpf_goto_meta(priv->prog, meta, insn_idx, env->prog->len);
- priv->meta = meta;
+ meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len);
+ nfp_prog->verifier_meta = meta;
if (meta->insn.src_reg >= MAX_BPF_REG ||
meta->insn.dst_reg >= MAX_BPF_REG) {
@@ -184,39 +178,18 @@ nfp_verify_insn(struct bpf_verifier_env *env, int
insn_idx, int prev_insn_idx)
}
if (meta->insn.code == (BPF_JMP | BPF_EXIT))
- return nfp_bpf_check_exit(priv->prog, env);
+ return nfp_bpf_check_exit(nfp_prog, env);
if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM))
- return nfp_bpf_check_ptr(priv->prog, meta, env,
+ return nfp_bpf_check_ptr(nfp_prog, meta, env,
meta->insn.src_reg);
if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM))
- return nfp_bpf_check_ptr(priv->prog, meta, env,
+ return nfp_bpf_check_ptr(nfp_prog, meta, env,
meta->insn.dst_reg);
return 0;
}
-static const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
+const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
.insn_hook = nfp_verify_insn,
};
-
-int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog)
-{
- struct nfp_bpf_analyzer_priv *priv;
- int ret;
-
- nfp_prog->stack_depth = prog->aux->stack_depth;
-
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
-
- priv->prog = nfp_prog;
- priv->meta = nfp_prog_first_meta(nfp_prog);
-
- ret = bpf_analyzer(prog, &nfp_bpf_analyzer_ops, priv);
-
- kfree(priv);
-
- return ret;
-}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h
b/drivers/net/ethernet/netronome/nfp/nfp_app.h
index 857bb33020ba..54b67c9b8d5b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h
@@ -42,6 +42,7 @@
struct bpf_prog;
struct net_device;
+struct netdev_bpf;
struct pci_dev;
struct sk_buff;
struct sk_buff;
@@ -83,6 +84,9 @@ extern const struct nfp_app_type app_flower;
* @setup_tc: setup TC ndo
* @tc_busy: TC HW offload busy (rules loaded)
* @xdp_offload: offload an XDP program
+ * @bpf_verifier_prep: verifier prep for dev-specific BPF programs
+ * @bpf_translate: translate call for dev-specific BPF programs
+ * @bpf_destroy: destroy for dev-specific BPF programs
* @eswitch_mode_get: get SR-IOV eswitch mode
* @sriov_enable: app-specific sriov initialisation
* @sriov_disable: app-specific sriov clean-up
@@ -118,6 +122,12 @@ struct nfp_app_type {
bool (*tc_busy)(struct nfp_app *app, struct nfp_net *nn);
int (*xdp_offload)(struct nfp_app *app, struct nfp_net *nn,
struct bpf_prog *prog);
+ int (*bpf_verifier_prep)(struct nfp_app *app, struct nfp_net *nn,
+ struct netdev_bpf *bpf);
+ int (*bpf_translate)(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog);
+ int (*bpf_destroy)(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog);
int (*sriov_enable)(struct nfp_app *app, int num_vfs);
void (*sriov_disable)(struct nfp_app *app);
@@ -271,6 +281,33 @@ static inline int nfp_app_xdp_offload(struct nfp_app *app,
struct nfp_net *nn,
return app->type->xdp_offload(app, nn, prog);
}
+static inline int
+nfp_app_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
+ struct netdev_bpf *bpf)
+{
+ if (!app || !app->type->bpf_verifier_prep)
+ return -EOPNOTSUPP;
+ return app->type->bpf_verifier_prep(app, nn, bpf);
+}
+
+static inline int
+nfp_app_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog)
+{
+ if (!app || !app->type->bpf_translate)
+ return -EOPNOTSUPP;
+ return app->type->bpf_translate(app, nn, prog);
+}
+
+static inline int
+nfp_app_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
+ struct bpf_prog *prog)
+{
+ if (!app || !app->type->bpf_destroy)
+ return -EOPNOTSUPP;
+ return app->type->bpf_destroy(app, nn, prog);
+}
+
static inline bool nfp_app_ctrl_tx(struct nfp_app *app, struct sk_buff *skb)
{
trace_devlink_hwmsg(priv_to_devlink(app->pf), false, 0,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index f6c6ad4e8a59..232044b1b7aa 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -3393,6 +3393,14 @@ static int nfp_net_xdp(struct net_device *netdev, struct
netdev_bpf *xdp)
xdp->prog_attached = XDP_ATTACHED_HW;
xdp->prog_id = nn->xdp_prog ? nn->xdp_prog->aux->id : 0;
return 0;
+ case BPF_OFFLOAD_VERIFIER_PREP:
+ return nfp_app_bpf_verifier_prep(nn->app, nn, xdp);
+ case BPF_OFFLOAD_TRANSLATE:
+ return nfp_app_bpf_translate(nn->app, nn,
+ xdp->offload.prog);
+ case BPF_OFFLOAD_DESTROY:
+ return nfp_app_bpf_destroy(nn->app, nn,
+ xdp->offload.prog);
default:
return -EINVAL;
}
--
2.14.1
