---
arch/powerpc/net/bpf_jit.h | 7 +++---
arch/powerpc/net/bpf_jit_comp.c | 38 ++++++++++++++++++++++++++-----
arch/powerpc/net/bpf_jit_comp32.c | 7 +++---
arch/powerpc/net/bpf_jit_comp64.c | 7 +++---
4 files changed, 44 insertions(+), 15 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 6d58df361648..4da8bde92e1e 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h
@@ -227,10 +227,11 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage,
struct codegen_context *
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct
codegen_context *ctx,
u32 *addrs, int pass, bool extra_pass);
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);
-void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
-void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx);
+void bpf_jit_build_epilogue(u32 *image, u32 *fimage, struct codegen_context
*ctx);
+void bpf_jit_build_fentry_stubs(u32 *image, u32 *fimage, struct
codegen_context *ctx);
void bpf_jit_realloc_regs(struct codegen_context *ctx);
-int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int
tmp_reg, long exit_addr);
+int bpf_jit_emit_exit_insn(u32 *image, u32 *fimage, struct codegen_context
*ctx, int tmp_reg,
+
long exit_addr);
void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int
cookie_cnt,
int cookie_off,
int retval_off);
void store_func_meta(u32 *image, struct codegen_context *ctx, u64 func_meta,
int func_meta_off);
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 11981d2270a9..8ca36a933c7a 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -49,11 +49,35 @@ asm (
" .popsection ;"
);
-void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
+void bpf_jit_build_fentry_stubs(u32 *image, u32 *fimage, struct
codegen_context *ctx)
{
int ool_stub_idx, long_branch_stub_idx;
+ int ool_stub_sz;
/*
+ * Align the mis-aligned dummy_tramp_addr field in the fimage.
+ * The alignment NOP must appear before OOL stub, to make
+ * ool_stub_idx & long_branch_stub_idx constant from end.
+ *
+ * The fimage can be non 8-byte aligned, so final alignment depends
+ * on start of fimage and the stub's instruction count offset. The
+ * OOL stub size is 4 instructions (with CONFIG_PPC_FTRACE_OUT_OF_LINE)
+ * or 3 instructions (without) before dummy_tramp_addr.
+ *
+ * Emit a NOP here if address is not SZL aligned.
+ *
+ * In pass=0 when image==NULL, conservatively account for space
+ * required to accommodate alignment NOP. In case final pass skips
+ * emitting alignment NOP, the image buffer have 4 spare bytes and
+ * jited_len signifies correct program size.
+ */
+
+ ool_stub_sz = IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) ? 16 : 12;
+ if (!image || !IS_ALIGNED((unsigned long)fimage + ctx->idx*4 +
ool_stub_sz, SZL))
+ EMIT(PPC_RAW_NOP());
+
+ /*
+ * nop // optional, for alignment of dummy_tramp_addr
* Out-of-line stub:
* mflr r0
* [b|bl] tramp
@@ -70,7 +94,7 @@ void bpf_jit_build_fentry_stubs(u32 *image, struct
codegen_context *ctx)
/*
* Long branch stub:
- * .long <dummy_tramp_addr>
+ * .long <dummy_tramp_addr> // 8-byte aligned
* mflr r11
* bcl 20,31,$+4
* mflr r12
@@ -81,6 +105,7 @@ void bpf_jit_build_fentry_stubs(u32 *image, struct
codegen_context *ctx)
*/
if (image)
*((unsigned long *)&image[ctx->idx]) = (unsigned
long)dummy_tramp;
+
ctx->idx += SZL / 4;
long_branch_stub_idx = ctx->idx;
EMIT(PPC_RAW_MFLR(_R11));
@@ -97,7 +122,8 @@ void bpf_jit_build_fentry_stubs(u32 *image, struct
codegen_context *ctx)
}
}
-int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
+int bpf_jit_emit_exit_insn(u32 *image, u32 *fimage, struct codegen_context
*ctx,
+ int tmp_reg, long
exit_addr)
{
if (exit_addr && is_offset_in_branch_range(exit_addr - (long)(ctx->idx
* 4))) {
PPC_JMP(exit_addr);
@@ -106,7 +132,7 @@ int bpf_jit_emit_exit_insn(u32 *image, struct
codegen_context *ctx, int tmp_reg,
PPC_JMP(ctx->alt_exit_addr);
} else {
ctx->alt_exit_addr = ctx->idx * 4;
- bpf_jit_build_epilogue(image, ctx);
+ bpf_jit_build_epilogue(image, fimage, ctx);
}
return 0;
@@ -286,7 +312,7 @@ struct bpf_prog *bpf_int_jit_compile(struct
bpf_verifier_env *env, struct bpf_pr
*/
bpf_jit_build_prologue(NULL, &cgctx);
addrs[fp->len] = cgctx.idx * 4;
- bpf_jit_build_epilogue(NULL, &cgctx);
+ bpf_jit_build_epilogue(NULL, NULL, &cgctx);
fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
extable_len = fp->aux->num_exentries * sizeof(struct
exception_table_entry);
@@ -322,7 +348,7 @@ struct bpf_prog *bpf_int_jit_compile(struct
bpf_verifier_env *env, struct bpf_pr
goto out_err;
}
addrs[fp->len] = cgctx.idx * 4;
- bpf_jit_build_epilogue(code_base, &cgctx);
+ bpf_jit_build_epilogue(code_base, fcode_base, &cgctx);
proglen = cgctx.idx * 4;
diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index bfdc50740da8..f5b9441cf46a 100644
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c
@@ -229,7 +229,7 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct
codegen_context *ctx
}
-void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
+void bpf_jit_build_epilogue(u32 *image, u32 *fimage, struct codegen_context
*ctx)
{
EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
@@ -237,7 +237,7 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
EMIT(PPC_RAW_BLR());
- bpf_jit_build_fentry_stubs(image, ctx);
+ bpf_jit_build_fentry_stubs(image, fimage, ctx);
}
/* Relative offset needs to be calculated based on final image location */
@@ -1149,7 +1149,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
u32 *fimage, struct code
* we'll just fall through to the epilogue.
*/
if (i != flen - 1) {
- ret = bpf_jit_emit_exit_insn(image, ctx, _R0,
exit_addr);
+ ret = bpf_jit_emit_exit_insn(image, fimage,
+ ctx, _R0,
exit_addr);
if (ret)
return ret;
}
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 59302cabc466..e80312171aa7 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -398,7 +398,7 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct
codegen_context *ctx
}
}
-void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
+void bpf_jit_build_epilogue(u32 *image, u32 *fimage, struct codegen_context
*ctx)
{
bpf_jit_emit_common_epilogue(image, ctx);
@@ -407,7 +407,7 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
EMIT(PPC_RAW_BLR());
- bpf_jit_build_fentry_stubs(image, ctx);
+ bpf_jit_build_fentry_stubs(image, fimage, ctx);
}
/*
@@ -1737,7 +1737,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
u32 *fimage, struct code
* we'll just fall through to the epilogue.
*/
if (i != flen - 1) {
- ret = bpf_jit_emit_exit_insn(image, ctx,
tmp1_reg, exit_addr);
+ ret = bpf_jit_emit_exit_insn(image, fimage, ctx,
+ tmp1_reg,
exit_addr);
if (ret)
return ret;
}
