The existing code for emitting bpf atomic instruction sequences for
atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been
refactored into a reusable function, bpf_jit_emit_ppc_atomic_op().
It also computes the jump offset and tracks the instruction index for jited
LDARX/LWARX to be used in case it causes a fault.
Signed-off-by: Saket Kumar Bhaskar <sk...@linux.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 203 +++++++++++++++++-------------
1 file changed, 115 insertions(+), 88 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index d4fe4dacf2d6..6a85cd847075 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -423,6 +423,111 @@ asm (
" blr ;"
);
+static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,
+ const struct bpf_insn *insn, u32 *jmp_off,
+ u32 *tmp_idx, u32 *addrp)
+{
+ u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
+ u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
+ u32 size = BPF_SIZE(insn->code);
+ u32 src_reg = bpf_to_ppc(insn->src_reg);
+ u32 dst_reg = bpf_to_ppc(insn->dst_reg);
+ s32 imm = insn->imm;
+
+ u32 save_reg = tmp2_reg;
+ u32 ret_reg = src_reg;
+ u32 fixup_idx;
+
+ /* Get offset into TMP_REG_1 */
+ EMIT(PPC_RAW_LI(tmp1_reg, insn->off));
+ /*
+ * Enforce full ordering for operations with BPF_FETCH by emitting a
'sync'
+ * before and after the operation.
+ *
+ * This is a requirement in the Linux Kernel Memory Model.
+ * See __cmpxchg_u64() in asm/cmpxchg.h as an example.
+ */
+ if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
+ EMIT(PPC_RAW_SYNC());
+
+ *tmp_idx = ctx->idx;
+
+ /* load value from memory into TMP_REG_2 */
+ if (size == BPF_DW)
+ EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
+ else
+ EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
+ /* Save old value in _R0 */
+ if (imm & BPF_FETCH)
+ EMIT(PPC_RAW_MR(_R0, tmp2_reg));
+
+ switch (imm) {
+ case BPF_ADD:
+ case BPF_ADD | BPF_FETCH:
+ EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
+ break;
+ case BPF_AND:
+ case BPF_AND | BPF_FETCH:
+ EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
+ break;
+ case BPF_OR:
+ case BPF_OR | BPF_FETCH:
+ EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
+ break;
+ case BPF_XOR:
+ case BPF_XOR | BPF_FETCH:
+ EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
+ break;
+ case BPF_CMPXCHG:
+ /*
+ * Return old value in BPF_REG_0 for BPF_CMPXCHG &
+ * in src_reg for other cases.
+ */
+ ret_reg = bpf_to_ppc(BPF_REG_0);
+
+ /* Compare with old value in BPF_R0 */
+ if (size == BPF_DW)
+ EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
+ else
+ EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
+ /* Don't set if different from old value */
+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
+ fallthrough;
+ case BPF_XCHG:
+ save_reg = src_reg;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ /* store new value */
+ if (size == BPF_DW)
+ EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
+ else
+ EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
+ /* we're done if this succeeded */
+ PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);
+ fixup_idx = ctx->idx;
+
+ if (imm & BPF_FETCH) {
+ /* Emit 'sync' to enforce full ordering */
+ if (IS_ENABLED(CONFIG_SMP))
+ EMIT(PPC_RAW_SYNC());
+ EMIT(PPC_RAW_MR(ret_reg, _R0));
+ /*
+ * Skip unnecessary zero-extension for 32-bit cmpxchg.
+ * For context, see commit 39491867ace5.
+ */
+ if (size != BPF_DW && imm == BPF_CMPXCHG &&
+ insn_is_zext(insn + 1))
+ *addrp = ctx->idx * 4;
+ }
+
+ *jmp_off = (fixup_idx - *tmp_idx) * 4;
+
+ return 0;
+}
+
static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32
src_reg, s16 off,
u32 code, u32 *image)
{
@@ -538,7 +643,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32
*fimage, struct code
u32 size = BPF_SIZE(code);
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
- u32 save_reg, ret_reg;
s16 off = insn[i].off;
s32 imm = insn[i].imm;
bool func_addr_fixed;
@@ -546,6 +650,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32
*fimage, struct code
u64 imm64;
u32 true_cond;
u32 tmp_idx;
+ u32 jmp_off;
/*
* addrs[] maps a BPF bytecode address into a real offset from
@@ -1081,93 +1186,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
u32 *fimage, struct code
return -EOPNOTSUPP;
}
- save_reg = tmp2_reg;
- ret_reg = src_reg;
-
- /* Get offset into TMP_REG_1 */
- EMIT(PPC_RAW_LI(tmp1_reg, off));
- /*
- * Enforce full ordering for operations with BPF_FETCH
by emitting a 'sync'
- * before and after the operation.
- *
- * This is a requirement in the Linux Kernel Memory
Model.
- * See __cmpxchg_u64() in asm/cmpxchg.h as an example.
- */
- if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
- EMIT(PPC_RAW_SYNC());
- tmp_idx = ctx->idx * 4;
- /* load value from memory into TMP_REG_2 */
- if (size == BPF_DW)
- EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg,
0));
- else
- EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg,
0));
-
- /* Save old value in _R0 */
- if (imm & BPF_FETCH)
- EMIT(PPC_RAW_MR(_R0, tmp2_reg));
-
- switch (imm) {
- case BPF_ADD:
- case BPF_ADD | BPF_FETCH:
- EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
- break;
- case BPF_AND:
- case BPF_AND | BPF_FETCH:
- EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
- break;
- case BPF_OR:
- case BPF_OR | BPF_FETCH:
- EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
- break;
- case BPF_XOR:
- case BPF_XOR | BPF_FETCH:
- EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
- break;
- case BPF_CMPXCHG:
- /*
- * Return old value in BPF_REG_0 for
BPF_CMPXCHG &
- * in src_reg for other cases.
- */
- ret_reg = bpf_to_ppc(BPF_REG_0);
-
- /* Compare with old value in BPF_R0 */
- if (size == BPF_DW)
-
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
- else
-
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
- /* Don't set if different from old value */
- PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
- fallthrough;
- case BPF_XCHG:
- save_reg = src_reg;
- break;
- default:
- pr_err_ratelimited(
- "eBPF filter atomic op code %02x (@%d)
unsupported\n",
- code, i);
- return -EOPNOTSUPP;
- }
-
- /* store new value */
- if (size == BPF_DW)
- EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg,
dst_reg));
- else
- EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg,
dst_reg));
- /* we're done if this succeeded */
- PPC_BCC_SHORT(COND_NE, tmp_idx);
-
- if (imm & BPF_FETCH) {
- /* Emit 'sync' to enforce full ordering */
- if (IS_ENABLED(CONFIG_SMP))
- EMIT(PPC_RAW_SYNC());
- EMIT(PPC_RAW_MR(ret_reg, _R0));
- /*
- * Skip unnecessary zero-extension for 32-bit
cmpxchg.
- * For context, see commit 39491867ace5.
- */
- if (size != BPF_DW && imm == BPF_CMPXCHG &&
- insn_is_zext(&insn[i + 1]))
- addrs[++i] = ctx->idx * 4;
+ ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
+ &jmp_off, &tmp_idx,
&addrs[i + 1]);
+ if (ret) {
+ if (ret == -EOPNOTSUPP) {
+ pr_err_ratelimited(
+ "eBPF filter atomic op code
%02x (@%d) unsupported\n",
+ code, i);
+ }
+ return ret;
}
break;
--
2.43.5
