The arm64 JIT rejected BPF_LD | BPF_ABS and BPF_LD | BPF_IND with "invalid opcode", so cBPF programs converted by rte_bpf_convert() could not be JITed. Add these opcodes, mirroring the x86 JIT: a fast path for data held in the first mbuf segment and a __rte_pktmbuf_read() slow path for everything else. Programs using these opcodes now use the call register layout, since the slow path makes a function call.
Bugzilla ID: 1427 Signed-off-by: Stephen Hemminger <[email protected]> --- lib/bpf/bpf_jit_arm64.c | 147 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 1 deletion(-) diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c index 099822e9f1..6952c61806 100644 --- a/lib/bpf/bpf_jit_arm64.c +++ b/lib/bpf/bpf_jit_arm64.c @@ -1123,6 +1123,133 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off) emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off)); } +/* LD_ABS/LD_IND code block offsets (in arm64 instructions) */ +enum { + LDMB_FAST_OFS, /* fast path */ + LDMB_SLOW_OFS, /* slow path */ + LDMB_FIN_OFS, /* common tail */ + LDMB_OFS_NUM +}; + +/* + * Helper for emit_ld_mbuf(): fast path. + * Compute the packet offset; if it lies inside the first segment leave the + * data pointer in R0, otherwise branch to the slow path. + */ +static void +emit_ldmb_fast_path(struct a64_jit_ctx *ctx, uint8_t src, uint8_t mode, + uint32_t sz, int32_t imm, const uint32_t ofs[LDMB_OFS_NUM]) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); + uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); + uint8_t tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2); + uint8_t tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3); + + /* off = imm (+ src for BPF_IND) */ + emit_mov_imm(ctx, 1, tmp1, imm); + if (mode == BPF_IND) + emit_add(ctx, 1, tmp1, src); + + /* if ((int64_t)(mbuf->data_len - off) < sz) goto slow_path */ + emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_len)); + emit_ldr(ctx, BPF_H, tmp2, r6, tmp2); + emit_sub(ctx, 1, tmp2, tmp1); + emit_mov_imm(ctx, 1, tmp3, sz); + emit_cmp(ctx, 1, tmp2, tmp3); + emit_b_cond(ctx, A64_LT, (int32_t)(ofs[LDMB_SLOW_OFS] - ctx->idx)); + + /* R0 = mbuf->buf_addr + mbuf->data_off + off */ + emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_off)); + emit_ldr(ctx, BPF_H, tmp2, r6, tmp2); + emit_mov_imm(ctx, 1, r0, offsetof(struct rte_mbuf, buf_addr)); + emit_ldr(ctx, EBPF_DW, r0, r6, r0); + emit_add(ctx, 1, r0, tmp2); + emit_add(ctx, 1, r0, tmp1); + + emit_b(ctx, (int32_t)(ofs[LDMB_FIN_OFS] - ctx->idx)); +} + +/* + * Helper for emit_ld_mbuf(): slow path. + * R0 = __rte_pktmbuf_read(mbuf, off, sz, buf); return 0 if NULL. + * The scratch buffer is the space reserved by __rte_bpf_validate() at the + * bottom of the eBPF stack frame, i.e. (frame_pointer - stack_ofs). + */ +static void +emit_ldmb_slow_path(struct a64_jit_ctx *ctx, uint32_t sz, uint32_t stack_ofs) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); + uint8_t fp = ebpf_to_a64_reg(ctx, EBPF_FP); + uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); + + /* arguments of __rte_pktmbuf_read(mbuf, off, len, buf) */ + emit_mov_64(ctx, A64_R(1), tmp1); /* off (held in tmp1) */ + emit_mov_64(ctx, A64_R(0), r6); /* mbuf */ + emit_mov_imm(ctx, 0, A64_R(2), sz); /* len */ + emit_sub_imm_64(ctx, A64_R(3), fp, stack_ofs); /* buf */ + + emit_call(ctx, tmp1, (void *)(uintptr_t)__rte_pktmbuf_read); + emit_return_zero_if_src_zero(ctx, 1, r0); +} + +/* + * Helper for emit_ld_mbuf(): common tail. + * Load the value pointed to by R0 and convert from network byte order. + */ +static void +emit_ldmb_fin(struct a64_jit_ctx *ctx, uint8_t opsz, uint32_t sz) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + + emit_ldr(ctx, opsz, r0, r0, A64_ZR); + if (opsz != BPF_B) + emit_be(ctx, r0, sz * 8); +} + +/* + * Emit code for BPF_LD | BPF_ABS and BPF_LD | BPF_IND packet loads: + * + * off = imm (+ src for BPF_IND) + * if (mbuf->data_len - off >= sz) -- fast path + * ptr = mbuf->buf_addr + mbuf->data_off + off; + * else -- slow path + * ptr = __rte_pktmbuf_read(mbuf, off, sz, buf); + * if (ptr == NULL) + * return 0; + * R0 = ntoh(*(size *)ptr); -- common tail + * + * The three blocks are sized in a dry run so the forward branches can be + * resolved, then emitted for real (arm64 instructions are fixed width, so + * the dry run reproduces the real instruction count exactly). + */ +static void +emit_ld_mbuf(struct a64_jit_ctx *ctx, uint8_t op, uint8_t src, int32_t imm, + uint32_t stack_ofs) +{ + uint8_t mode = BPF_MODE(op); + uint8_t opsz = BPF_SIZE(op); + uint32_t sz = bpf_size(opsz); + uint32_t ofs[LDMB_OFS_NUM]; + + /* seed offsets so the dry-run branches stay in range */ + ofs[LDMB_FAST_OFS] = ofs[LDMB_SLOW_OFS] = ofs[LDMB_FIN_OFS] = ctx->idx; + + /* dry run to record block offsets */ + emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs); + ofs[LDMB_SLOW_OFS] = ctx->idx; + emit_ldmb_slow_path(ctx, sz, stack_ofs); + ofs[LDMB_FIN_OFS] = ctx->idx; + emit_ldmb_fin(ctx, opsz, sz); + + /* rewind and emit for real with resolved offsets */ + ctx->idx = ofs[LDMB_FAST_OFS]; + emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs); + emit_ldmb_slow_path(ctx, sz, stack_ofs); + emit_ldmb_fin(ctx, opsz, sz); +} + static void check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) { @@ -1135,8 +1262,17 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) op = ins->code; switch (op) { - /* Call imm */ + /* + * BPF_ABS/BPF_IND can fall through to __rte_pktmbuf_read(), + * so they need the call-clobbered register layout as well. + */ case (BPF_JMP | EBPF_CALL): + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): ctx->foundcall = 1; return; } @@ -1338,6 +1474,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) emit_mov_imm(ctx, 1, dst, u64); i++; break; + /* R0 = ntoh(*(size *)(mbuf data + (src) + imm)) */ + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + emit_ld_mbuf(ctx, op, src, imm, bpf->stack_sz); + break; /* *(size *)(dst + off) = src */ case (BPF_STX | BPF_MEM | BPF_B): case (BPF_STX | BPF_MEM | BPF_H): -- 2.53.0
