In order to prepare for KASAN checks insertion before every memory-related load or store, group all BPF_ST instructions that indeed access memory in a single helper to allow instrumenting those in one call, rather than having to instrument all cases individually.
Signed-off-by: Alexis Lothoré (eBPF Foundation) <[email protected]> --- Changes in v9: - update BPF_ST management to resync it with BPF_STX Changes in v7: - align dst_reg type with same type as other emit helpers - fix line wrapping - add missing doc on top of new helper - remove unneeded cast Changes in v3: - new patch --- arch/x86/net/bpf_jit_comp.c | 73 ++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 48429fae0641..f907be3988b2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1315,6 +1315,46 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int *pprog = prog; } +/* ST: *(u8*)(dst_reg + off) = imm */ +static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + s32 insn_off) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + if (is_imm8(insn_off)) + EMIT2(add_1reg(0x40, dst_reg), insn_off); + else + EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -2250,49 +2290,28 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + imm32); break; } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; + insn_off = insn->off; if (dst_reg == BPF_REG_PARAMS) { /* * Args 7+: reverse BPF negative offsets to * x86 positive rsp offsets. * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; dst_reg = BPF_REG_FP; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); - - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, insn_off); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ -- 2.55.0
