[PATCH bpf] bpf: fix default unprivileged allocation limit
When using a large page size, the default value of the bpf_jit_limit
knob becomes invalid and users are not able to run unprivileged bpf
programs.
The bpf_jit_limit knob is represented internally as a 32-bit signed
integer because of which the default value, i.e. PAGE_SIZE * 4,
overflows in case of an architecture like powerpc64 which uses 64K
as the default page size (i.e. CONFIG_PPC_64K_PAGES is set).
So, instead of depending on the page size, use a constant value.
Fixes: ede95a63b5e8 ("bpf: add bpf_jit_limit knob to restrict unpriv
allocations")
Signed-off-by: Sandipan Das
---
kernel/bpf/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index b1a3545d0ec8..a81d097a17fb 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -365,7 +365,7 @@ void bpf_prog_kallsyms_del_all(struct bpf_prog *fp)
}
#ifdef CONFIG_BPF_JIT
-# define BPF_JIT_LIMIT_DEFAULT (PAGE_SIZE * 4)
+# define BPF_JIT_LIMIT_DEFAULT (4096 * 4)
/* All BPF JIT sysctl knobs here. */
int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON);
--
2.19.2
[PATCH bpf] bpf: powerpc: fix broken uapi for BPF_PROG_TYPE_PERF_EVENT
Now that there are different variants of pt_regs for userspace and
kernel, the uapi for the BPF_PROG_TYPE_PERF_EVENT program type must
be changed by exporting the user_pt_regs structure instead of the
pt_regs structure that is in-kernel only.
Fixes: 002af9391bfb ("powerpc: Split user/kernel definitions of struct pt_regs")
Signed-off-by: Sandipan Das
---
arch/powerpc/include/asm/perf_event.h | 2 ++
arch/powerpc/include/uapi/asm/Kbuild | 1 -
arch/powerpc/include/uapi/asm/bpf_perf_event.h | 9 +
3 files changed, 11 insertions(+), 1 deletion(-)
create mode 100644 arch/powerpc/include/uapi/asm/bpf_perf_event.h
diff --git a/arch/powerpc/include/asm/perf_event.h
b/arch/powerpc/include/asm/perf_event.h
index 8bf1b6351716..16a49819da9a 100644
--- a/arch/powerpc/include/asm/perf_event.h
+++ b/arch/powerpc/include/asm/perf_event.h
@@ -26,6 +26,8 @@
#include
#include
+#define perf_arch_bpf_user_pt_regs(regs) >user_regs
+
/*
* Overload regs->result to specify whether we should use the MSR (result
* is zero) or the SIAR (result is non zero).
diff --git a/arch/powerpc/include/uapi/asm/Kbuild
b/arch/powerpc/include/uapi/asm/Kbuild
index a658091a19f9..3712152206f3 100644
--- a/arch/powerpc/include/uapi/asm/Kbuild
+++ b/arch/powerpc/include/uapi/asm/Kbuild
@@ -1,7 +1,6 @@
# UAPI Header export list
include include/uapi/asm-generic/Kbuild.asm
-generic-y += bpf_perf_event.h
generic-y += param.h
generic-y += poll.h
generic-y += resource.h
diff --git a/arch/powerpc/include/uapi/asm/bpf_perf_event.h
b/arch/powerpc/include/uapi/asm/bpf_perf_event.h
new file mode 100644
index ..b551b741653d
--- /dev/null
+++ b/arch/powerpc/include/uapi/asm/bpf_perf_event.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__
+#define _UAPI__ASM_BPF_PERF_EVENT_H__
+
+#include
+
+typedef struct user_pt_regs bpf_user_pt_regs_t;
+
+#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */
--
2.19.2
Re: [PATCH v2 bpf-next 2/7] ppc: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_*
On 06/12/18 12:22 AM, Jiong Wang wrote: > This patch implements code-gen for BPF_ALU | BPF_ARSH | BPF_*. > > Cc: Naveen N. Rao > Cc: Sandipan Das > Signed-off-by: Jiong Wang > --- > arch/powerpc/include/asm/ppc-opcode.h | 2 ++ > arch/powerpc/net/bpf_jit.h| 4 > arch/powerpc/net/bpf_jit_comp64.c | 6 ++ > 3 files changed, 12 insertions(+) > > [...] Looks good to me. All tests are passing as well. Reviewed-by: Sandipan Das
Re: [PATCH bpf-next 2/7] ppc: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_*
Hi Jiong, On 05/12/18 2:25 AM, Jiong Wang wrote: > This patch implements code-gen for BPF_ALU | BPF_ARSH | BPF_*. > > Cc: Naveen N. Rao > Cc: Sandipan Das > Signed-off-by: Jiong Wang > --- [...] > diff --git a/arch/powerpc/net/bpf_jit_comp64.c > b/arch/powerpc/net/bpf_jit_comp64.c > index 17482f5..c685b4f 100644 > --- a/arch/powerpc/net/bpf_jit_comp64.c > +++ b/arch/powerpc/net/bpf_jit_comp64.c > @@ -529,9 +529,15 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 > *image, > if (imm != 0) > PPC_SRDI(dst_reg, dst_reg, imm); > break; > + case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ > + PPC_SRAW(dst_reg, dst_reg, src_reg); > + break; On ppc64, the sraw and srawi instructions also use sign extension. So, you will have to ensure that upper 32 bits are cleared. We already have a label in our JIT code called bpf_alu32_trunc that takes care of this. Replacing the break statement with a goto bpf_alu32_trunc will fix this. > case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ > PPC_SRAD(dst_reg, dst_reg, src_reg); > break; > + case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ > + PPC_SRAWI(dst_reg, dst_reg, imm); > + break; Same here. > case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ > if (imm != 0) > PPC_SRADI(dst_reg, dst_reg, imm); > With Regards, Sandipan
Re: [PATCH bpf] bpf: powerpc64: optimize JIT passes for bpf function calls
Hi Daniel,
On 03/12/18 6:18 PM, Daniel Borkmann wrote:
>
> Thanks for the patch, just to clarify, it's targeted at bpf-next and
> not bpf, correct?
>
This patch is targeted at the bpf tree.
This depends on commit e2c95a61656d ("bpf, ppc64: generalize fetching
subprog into bpf_jit_get_func_addr") which is already available in the
bpf tree.
Thanks,
Sandipan
Re: [PATCH bpf] bpf: fix bpf_prog_get_info_by_fd to return 0 func_lens for unpriv
On 02/11/18 4:05 PM, Daniel Borkmann wrote:
> While dbecd7388476 ("bpf: get kernel symbol addresses via syscall")
> zeroed info.nr_jited_ksyms in bpf_prog_get_info_by_fd() for queries
> from unprivileged users, commit 815581c11cc2 ("bpf: get JITed image
> lengths of functions via syscall") forgot about doing so and therefore
> returns the #elems of the user set up buffer which is incorrect. It
> also needs to indicate a info.nr_jited_func_lens of zero.
>
> Fixes: 815581c11cc2 ("bpf: get JITed image lengths of functions via syscall")
> Signed-off-by: Daniel Borkmann
> Cc: Sandipan Das
> Cc: Song Liu
> ---
> kernel/bpf/syscall.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
> index ccb9327..1ea5ce1 100644
> --- a/kernel/bpf/syscall.c
> +++ b/kernel/bpf/syscall.c
> @@ -2078,6 +2078,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
> *prog,
> info.jited_prog_len = 0;
> info.xlated_prog_len = 0;
> info.nr_jited_ksyms = 0;
> + info.nr_jited_func_lens = 0;
> goto done;
> }
>
Looks good. Thanks for fixing this.
- Sandipan
Re: [PATCH bpf-next v4 02/10] bpf: powerpc64: pad function address loads with NOPs
On 05/24/2018 01:04 PM, Daniel Borkmann wrote:
> On 05/24/2018 08:56 AM, Sandipan Das wrote:
>> For multi-function programs, loading the address of a callee
>> function to a register requires emitting instructions whose
>> count varies from one to five depending on the nature of the
>> address.
>>
>> Since we come to know of the callee's address only before the
>> extra pass, the number of instructions required to load this
>> address may vary from what was previously generated. This can
>> make the JITed image grow or shrink.
>>
>> To avoid this, we should generate a constant five-instruction
>> when loading function addresses by padding the optimized load
>> sequence with NOPs.
>>
>> Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
>> ---
>> arch/powerpc/net/bpf_jit_comp64.c | 34 +++---
>> 1 file changed, 23 insertions(+), 11 deletions(-)
>>
>> diff --git a/arch/powerpc/net/bpf_jit_comp64.c
>> b/arch/powerpc/net/bpf_jit_comp64.c
>> index 1bdb1aff0619..e4582744a31d 100644
>> --- a/arch/powerpc/net/bpf_jit_comp64.c
>> +++ b/arch/powerpc/net/bpf_jit_comp64.c
>> @@ -167,25 +167,37 @@ static void bpf_jit_build_epilogue(u32 *image, struct
>> codegen_context *ctx)
>>
>> static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx,
>> u64 func)
>> {
>> +unsigned int i, ctx_idx = ctx->idx;
>> +
>> +/* Load function address into r12 */
>> +PPC_LI64(12, func);
>> +
>> +/* For bpf-to-bpf function calls, the callee's address is unknown
>> + * until the last extra pass. As seen above, we use PPC_LI64() to
>> + * load the callee's address, but this may optimize the number of
>> + * instructions required based on the nature of the address.
>> + *
>> + * Since we don't want the number of instructions emitted to change,
>> + * we pad the optimized PPC_LI64() call with NOPs to guarantee that
>> + * we always have a five-instruction sequence, which is the maximum
>> + * that PPC_LI64() can emit.
>> + */
>> +for (i = ctx->idx - ctx_idx; i < 5; i++)
>> +PPC_NOP();
>
> By the way, I think you can still optimize this. The nops are not really
> needed in case of insn->src_reg != BPF_PSEUDO_CALL since the address of
> a normal BPF helper call will always be at a fixed location and known a
> priori.
>
Ah, true. Thanks for pointing this out. There are a few other things that
we are planning to do for the ppc64 JIT compiler. Will put out a patch for
this with that series.
- Sandipan
>> #ifdef PPC64_ELF_ABI_v1
>> -/* func points to the function descriptor */
>> -PPC_LI64(b2p[TMP_REG_2], func);
>> -/* Load actual entry point from function descriptor */
>> -PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
>> -/* ... and move it to LR */
>> -PPC_MTLR(b2p[TMP_REG_1]);
>> /*
>> * Load TOC from function descriptor at offset 8.
>> * We can clobber r2 since we get called through a
>> * function pointer (so caller will save/restore r2)
>> * and since we don't use a TOC ourself.
>> */
>> -PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
>> -#else
>> -/* We can clobber r12 */
>> -PPC_FUNC_ADDR(12, func);
>> -PPC_MTLR(12);
>> +PPC_BPF_LL(2, 12, 8);
>> +/* Load actual entry point from function descriptor */
>> +PPC_BPF_LL(12, 12, 0);
>> #endif
>> +
>> +PPC_MTLR(12);
>> PPC_BLRL();
>> }
>>
>>
>
[PATCH bpf-next v4 05/10] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
addresses of the kernel symbols corresponding to each
function in a program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Move new fields to the end of bpf_prog_info to avoid
breaking userspace.
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index c3e502d06bc3..0be90965867d 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2205,6 +2205,8 @@ struct bpf_prog_info {
__u32 gpl_compatible:1;
__u64 netns_dev;
__u64 netns_ino;
+ __u32 nr_jited_ksyms;
+ __aligned_u64 jited_ksyms;
} __attribute__((aligned(8)));
struct bpf_map_info {
--
2.14.3
[PATCH bpf-next v4 03/10] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls in the powerpc64
JIT compiler. The JIT compiler converts the bpf call instructions
to native branch instructions. After a round of the usual passes,
the start addresses of the JITed images for the callee functions
are known. Finally, to fixup the branch target addresses, we need
to perform an extra pass.
Because of the address range in which JITed images are allocated
on powerpc64, the offsets of the start addresses of these images
from __bpf_call_base are as large as 64 bits. So, for a function
call, we cannot use the imm field of the instruction to determine
the callee's address. Instead, we use the alternative method of
getting it from the list of function addresses in the auxiliary
data of the caller by using the off field as an index.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Fix memory leak for jit_data when we fail to allocated addrs.
- Remove unnecessary bpf_jit_binary_lock_ro() call.
---
arch/powerpc/net/bpf_jit_comp64.c | 76 +--
1 file changed, 66 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index e4582744a31d..f1c95779843b 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -268,7 +268,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -724,11 +724,25 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+
+ /* bpf function call */
+ if (insn[i].src_reg == BPF_PSEUDO_CALL)
+ if (!extra_pass)
+ func = NULL;
+ else if (fp->aux->func && off <
fp->aux->func_cnt)
+ /* use the subprog id from the off
+* field to lookup the callee address
+*/
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ /* kernel helper call */
+ else
+ func = (u8 *) __bpf_call_base + imm;
bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -876,6 +890,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -883,6 +905,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -890,6 +913,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -903,11 +927,32 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
[PATCH bpf-next v4 07/10] bpf: fix multi-function JITed dump obtained via syscall
Currently, for multi-function programs, we cannot get the JITed
instructions using the bpf system call's BPF_OBJ_GET_INFO_BY_FD
command. Because of this, userspace tools such as bpftool fail
to identify a multi-function program as being JITed or not.
With the JIT enabled and the test program running, this can be
verified as follows:
# cat /proc/sys/net/core/bpf_jit_enable
1
Before applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T11:43:38+0530 uid 0
xlated 216B not jited memlock 65536B
...
# bpftool prog dump jited id 1
no instructions returned
After applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T12:13:01+0530 uid 0
xlated 216B jited 308B memlock 65536B
...
# bpftool prog dump jited id 1
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
18: addir31,r1,48
1c: li r3,10
...
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v4:
- If the image allocated in userspace is not large enough,
fill it up till the end even if the last JITed image can
only be copied partially.
---
kernel/bpf/syscall.c | 37 ++---
1 file changed, 34 insertions(+), 3 deletions(-)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 068a4fc79ddb..c8e987a612b5 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1970,13 +1970,44 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
*prog,
* for offload.
*/
ulen = info.jited_prog_len;
- info.jited_prog_len = prog->jited_len;
+ if (prog->aux->func_cnt) {
+ u32 i;
+
+ info.jited_prog_len = 0;
+ for (i = 0; i < prog->aux->func_cnt; i++)
+ info.jited_prog_len += prog->aux->func[i]->jited_len;
+ } else {
+ info.jited_prog_len = prog->jited_len;
+ }
+
if (info.jited_prog_len && ulen) {
if (bpf_dump_raw_ok()) {
uinsns = u64_to_user_ptr(info.jited_prog_insns);
ulen = min_t(u32, info.jited_prog_len, ulen);
- if (copy_to_user(uinsns, prog->bpf_func, ulen))
- return -EFAULT;
+
+ /* for multi-function programs, copy the JITed
+* instructions for all the functions
+*/
+ if (prog->aux->func_cnt) {
+ u32 len, free, i;
+ u8 *img;
+
+ free = ulen;
+ for (i = 0; i < prog->aux->func_cnt; i++) {
+ len = prog->aux->func[i]->jited_len;
+ len = min_t(u32, len, free);
+ img = (u8 *)
prog->aux->func[i]->bpf_func;
+ if (copy_to_user(uinsns, img, len))
+ return -EFAULT;
+ uinsns += len;
+ free -= len;
+ if (!free)
+ break;
+ }
+ } else {
+ if (copy_to_user(uinsns, prog->bpf_func, ulen))
+ return -EFAULT;
+ }
} else {
info.jited_prog_insns = 0;
}
--
2.14.3
[PATCH bpf-next v4 08/10] bpf: get JITed image lengths of functions via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of the JITed image lengths of each function for a
given program to userspace using the bpf system call with
the BPF_OBJ_GET_INFO_BY_FD command.
This can be used by userspace applications like bpftool
to split up the contiguous JITed dump, also obtained via
the system call, into more relatable chunks corresponding
to each function.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 20
2 files changed, 22 insertions(+)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 0be90965867d..344d2ddcef49 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2206,7 +2206,9 @@ struct bpf_prog_info {
__u64 netns_dev;
__u64 netns_ino;
__u32 nr_jited_ksyms;
+ __u32 nr_jited_func_lens;
__aligned_u64 jited_ksyms;
+ __aligned_u64 jited_func_lens;
} __attribute__((aligned(8)));
struct bpf_map_info {
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index c8e987a612b5..788456c18617 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2037,6 +2037,26 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_func_lens;
+ info.nr_jited_func_lens = prog->aux->func_cnt;
+ if (info.nr_jited_func_lens && ulen) {
+ if (bpf_dump_raw_ok()) {
+ u32 __user *user_lens;
+ u32 func_len, i;
+
+ /* copy the JITed image lengths for each function */
+ ulen = min_t(u32, info.nr_jited_func_lens, ulen);
+ user_lens = u64_to_user_ptr(info.jited_func_lens);
+ for (i = 0; i < ulen; i++) {
+ func_len = prog->aux->func[i]->jited_len;
+ if (put_user(func_len, _lens[i]))
+ return -EFAULT;
+ }
+ } else {
+ info.jited_func_lens = 0;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
--
2.14.3
[PATCH bpf-next v4 02/10] bpf: powerpc64: pad function address loads with NOPs
For multi-function programs, loading the address of a callee
function to a register requires emitting instructions whose
count varies from one to five depending on the nature of the
address.
Since we come to know of the callee's address only before the
extra pass, the number of instructions required to load this
address may vary from what was previously generated. This can
make the JITed image grow or shrink.
To avoid this, we should generate a constant five-instruction
when loading function addresses by padding the optimized load
sequence with NOPs.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 34 +++---
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 1bdb1aff0619..e4582744a31d 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -167,25 +167,37 @@ static void bpf_jit_build_epilogue(u32 *image, struct
codegen_context *ctx)
static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx,
u64 func)
{
+ unsigned int i, ctx_idx = ctx->idx;
+
+ /* Load function address into r12 */
+ PPC_LI64(12, func);
+
+ /* For bpf-to-bpf function calls, the callee's address is unknown
+* until the last extra pass. As seen above, we use PPC_LI64() to
+* load the callee's address, but this may optimize the number of
+* instructions required based on the nature of the address.
+*
+* Since we don't want the number of instructions emitted to change,
+* we pad the optimized PPC_LI64() call with NOPs to guarantee that
+* we always have a five-instruction sequence, which is the maximum
+* that PPC_LI64() can emit.
+*/
+ for (i = ctx->idx - ctx_idx; i < 5; i++)
+ PPC_NOP();
+
#ifdef PPC64_ELF_ABI_v1
- /* func points to the function descriptor */
- PPC_LI64(b2p[TMP_REG_2], func);
- /* Load actual entry point from function descriptor */
- PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
- /* ... and move it to LR */
- PPC_MTLR(b2p[TMP_REG_1]);
/*
* Load TOC from function descriptor at offset 8.
* We can clobber r2 since we get called through a
* function pointer (so caller will save/restore r2)
* and since we don't use a TOC ourself.
*/
- PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
-#else
- /* We can clobber r12 */
- PPC_FUNC_ADDR(12, func);
- PPC_MTLR(12);
+ PPC_BPF_LL(2, 12, 8);
+ /* Load actual entry point from function descriptor */
+ PPC_BPF_LL(12, 12, 0);
#endif
+
+ PPC_MTLR(12);
PPC_BLRL();
}
--
2.14.3
[PATCH bpf-next v4 04/10] bpf: get kernel symbol addresses via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of kernel symbol addresses for all functions in a
given program to userspace using the bpf system call with
the BPF_OBJ_GET_INFO_BY_FD command.
When bpf_jit_kallsyms is enabled, we can get the address
of the corresponding kernel symbol for a callee function
and resolve the symbol's name. The address is determined
by adding the value of the call instruction's imm field
to __bpf_call_base. This offset gets assigned to the imm
field by the verifier.
For some architectures, such as powerpc64, the imm field
is not large enough to hold this offset.
We resolve this by:
[1] Assigning the subprog id to the imm field of a call
instruction in the verifier instead of the offset of
the callee's symbol's address from __bpf_call_base.
[2] Determining the address of a callee's corresponding
symbol by using the imm field as an index for the
list of kernel symbol addresses now available from
the program info.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Copy addresses to jited_ksyms only if bpf_dump_raw_ok()
is true.
- Move new fields to the end of bpf_prog_info to avoid
breaking userspace.
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 25 +
kernel/bpf/verifier.c| 7 +--
3 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index c3e502d06bc3..0be90965867d 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2205,6 +2205,8 @@ struct bpf_prog_info {
__u32 gpl_compatible:1;
__u64 netns_dev;
__u64 netns_ino;
+ __u32 nr_jited_ksyms;
+ __aligned_u64 jited_ksyms;
} __attribute__((aligned(8)));
struct bpf_map_info {
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 0b4c94551001..068a4fc79ddb 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1933,6 +1933,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
+ info.nr_jited_ksyms = 0;
goto done;
}
@@ -1981,6 +1982,30 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_ksyms;
+ info.nr_jited_ksyms = prog->aux->func_cnt;
+ if (info.nr_jited_ksyms && ulen) {
+ if (bpf_dump_raw_ok()) {
+ u64 __user *user_ksyms;
+ ulong ksym_addr;
+ u32 i;
+
+ /* copy the address of the kernel symbol
+* corresponding to each function
+*/
+ ulen = min_t(u32, info.nr_jited_ksyms, ulen);
+ user_ksyms = u64_to_user_ptr(info.jited_ksyms);
+ for (i = 0; i < ulen; i++) {
+ ksym_addr = (ulong)
prog->aux->func[i]->bpf_func;
+ ksym_addr &= PAGE_MASK;
+ if (put_user((u64) ksym_addr, _ksyms[i]))
+ return -EFAULT;
+ }
+ } else {
+ info.jited_ksyms = 0;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 559cb74ba29e..8c4d9d0fd3ab 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5426,17 +5426,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* later look the same as if they were interpreted only.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
- unsigned long addr;
-
if (insn->code != (BPF_JMP | BPF_CALL) ||
insn->src_reg != BPF_PSEUDO_CALL)
continue;
insn->off = env->insn_aux_data[i].call_imm;
subprog = find_subprog(env, i + insn->off + 1);
- addr = (unsigned long)func[subprog]->bpf_func;
- addr &= PAGE_MASK;
- insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
- addr - __bpf_call_base;
+ insn->imm = subprog;
}
prog->jited = 1;
--
2.14.3
[PATCH bpf-next v4 06/10] tools: bpftool: resolve calls without using imm field
Currently, we resolve the callee's address for a JITed function
call by using the imm field of the call instruction as an offset
from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
use this address to get the callee's kernel symbol's name.
For some architectures, such as powerpc64, the imm field is not
large enough to hold this offset. So, instead of assigning this
offset to the imm field, the verifier now assigns the subprog
id. Also, a list of kernel symbol addresses for all the JITed
functions is provided in the program info. We now use the imm
field as an index for this list to lookup a callee's symbol's
address and resolve its name.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
Reviewed-by: Jakub Kicinski <jakub.kicin...@netronome.com>
---
v3:
- Avoid using redundant pointers.
- Fix indentation.
v2:
- Order variables from longest to shortest.
- Make sure that ksyms_ptr and ksyms_len are always initialized.
- Simplify code.
---
tools/bpf/bpftool/prog.c | 24
tools/bpf/bpftool/xlated_dumper.c | 10 +-
tools/bpf/bpftool/xlated_dumper.h | 2 ++
3 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 9bdfdf2d3fbe..e05ab58d39e2 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -420,7 +420,9 @@ static int do_show(int argc, char **argv)
static int do_dump(int argc, char **argv)
{
+ unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {};
+ unsigned int nr_func_ksyms;
struct dump_data dd = {};
__u32 len = sizeof(info);
unsigned int buf_size;
@@ -496,10 +498,22 @@ static int do_dump(int argc, char **argv)
return -1;
}
+ nr_func_ksyms = info.nr_jited_ksyms;
+ if (nr_func_ksyms) {
+ func_ksyms = malloc(nr_func_ksyms * sizeof(__u64));
+ if (!func_ksyms) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
+ info.jited_ksyms = ptr_to_u64(func_ksyms);
+ info.nr_jited_ksyms = nr_func_ksyms;
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
@@ -513,6 +527,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (info.nr_jited_ksyms > nr_func_ksyms) {
+ p_err("too many addresses returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -558,6 +577,9 @@ static int do_dump(int argc, char **argv)
dump_xlated_cfg(buf, *member_len);
} else {
kernel_syms_load();
+ dd.nr_jited_ksyms = info.nr_jited_ksyms;
+ dd.jited_ksyms = (__u64 *) info.jited_ksyms;
+
if (json_output)
dump_xlated_json(, buf, *member_len, opcodes);
else
@@ -566,10 +588,12 @@ static int do_dump(int argc, char **argv)
}
free(buf);
+ free(func_ksyms);
return 0;
err_free:
free(buf);
+ free(func_ksyms);
return -1;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c
b/tools/bpf/bpftool/xlated_dumper.c
index 7a3173b76c16..efdc8fecf2bb 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -174,7 +174,11 @@ static const char *print_call_pcrel(struct dump_data *dd,
unsigned long address,
const struct bpf_insn *insn)
{
- if (sym)
+ if (!dd->nr_jited_ksyms)
+ /* Do not show address for interpreted programs */
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+ "%+d", insn->off);
+ else if (sym)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"%+d#%s", insn->off, sym->name);
else
@@ -203,6 +207,10 @@ static const char *print_call(void *private_data,
unsigned long address = dd->address_call_base + insn->imm;
struct kernel_sym *sym;
+ if (insn->src_reg == BPF_PSEUDO_CALL &&
+ (__u32) insn->imm < dd->nr_jited_ksyms)
+ address = dd->jited_ksyms[insn->imm];
+
sym = kernel_syms_search(dd, address);
if (insn->src_reg == BPF_PSEUDO_CALL)
return print_call_pcrel(dd, sym, address, insn);
diff --git a/tools/bpf/bpftool/xlated_dumper.h
b/tools/bpf/bpftool/xlated_dumper.h
index b34affa7ef2d
[PATCH bpf-next v4 09/10] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
JITed image lengths of each function in a multi-function
program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 0be90965867d..344d2ddcef49 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2206,7 +2206,9 @@ struct bpf_prog_info {
__u64 netns_dev;
__u64 netns_ino;
__u32 nr_jited_ksyms;
+ __u32 nr_jited_func_lens;
__aligned_u64 jited_ksyms;
+ __aligned_u64 jited_func_lens;
} __attribute__((aligned(8)));
struct bpf_map_info {
--
2.14.3
[PATCH bpf-next v4 10/10] tools: bpftool: add delimiters to multi-function JITed dumps
This splits up the contiguous JITed dump obtained via the bpf
system call into more relatable chunks for each function in
the program. If the kernel symbols corresponding to these are
known, they are printed in the header for each JIT image dump
otherwise the masked start address is printed.
Before applying this patch:
# bpftool prog dump jited id 1
0: push %rbp
1: mov%rsp,%rbp
...
70: leaveq
71: retq
72: push %rbp
73: mov%rsp,%rbp
...
dd: leaveq
de: retq
# bpftool -p prog dump jited id 1
[{
"pc": "0x0",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0x71",
"operation": "retq",
"operands": [null
]
},{
"pc": "0x72",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0xde",
"operation": "retq",
"operands": [null
]
}
]
After applying this patch:
# echo 0 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog dump jited id 1
0xc02c7000:
0: push %rbp
1: mov%rsp,%rbp
...
70: leaveq
71: retq
0xc02cf000:
0: push %rbp
1: mov%rsp,%rbp
...
6b: leaveq
6c: retq
# bpftool -p prog dump jited id 1
[{
"name": "0xc02c7000",
"insns": [{
"pc": "0x0",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0x71",
"operation": "retq",
"operands": [null
]
}
]
},{
"name": "0xc02cf000",
"insns": [{
"pc": "0x0",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0x6c",
"operation": "retq",
"operands": [null
]
}
]
}
]
# echo 1 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog dump jited id 1
bpf_prog_b811aab41a39ad3d_foo:
0: push %rbp
1: mov%rsp,%rbp
...
70: leaveq
71: retq
bpf_prog_cf418ac8b67bebd9_F:
0: push %rbp
1: mov%rsp,%rbp
...
6b: leaveq
6c: retq
# bpftool -p prog dump jited id 1
[{
"name": "bpf_prog_b811aab41a39ad3d_foo",
"insns": [{
"pc": "0x0",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0x71",
"operation": "retq",
"operands": [null
]
}
]
},{
"name": "bpf_prog_cf418ac8b67bebd9_F",
"insns": [{
"pc": "0x0",
"operation": "push",
"operands": ["%rbp"
]
},{
...
},{
"pc": "0x6c",
"operation": "retq",
"operands": [null
]
}
]
}
]
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v4:
- Fix JSON output.
---
tools/bpf/bpftool/prog.c | 73 ++-
tools/bpf/bpftool/xlated_dumper.c | 4 +--
tools/bpf/bpftool/xlated_dumper.h | 1 +
3 files changed, 75 insertions(+), 3 deletions(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index e05ab58d39e2..39b88e760367 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -422,7 +422,9 @@ static int do_dump(int argc, char **argv)
{
unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {};
+ unsigned int *func_lens = NULL;
unsigned int nr_func_ksyms;
+ unsigned int nr_func_lens;
struct dump_data dd = {};
__u32 len = sizeof(info);
unsigned int buf_size;
@@ -508,12 +510,24 @@ static int do_d
[PATCH bpf-next v4 00/10] bpf: enhancements for multi-function programs
[1] Support for bpf-to-bpf function calls in the powerpc64 JIT compiler. [2] Provide a way for resolving function calls because of the way JITed images are allocated in powerpc64. [3] Fix to get JITed instruction dumps for multi-function programs from the bpf system call. [4] Fix for bpftool to show delimited multi-function JITed image dumps. v4: - Incorporate review comments from Jakub. - Fix JSON output for bpftool. v3: - Change base tree tag to bpf-next. - Incorporate review comments from Alexei, Daniel and Jakub. - Make sure that the JITed image does not grow or shrink after the last pass due to the way the instruction sequence used to load a callee's address maybe optimized. - Make additional changes to the bpf system call and bpftool to make multi-function JITed dumps easier to correlate. v2: - Incorporate review comments from Jakub. Sandipan Das (10): bpf: support 64-bit offsets for bpf function calls bpf: powerpc64: pad function address loads with NOPs bpf: powerpc64: add JIT support for multi-function programs bpf: get kernel symbol addresses via syscall tools: bpf: sync bpf uapi header tools: bpftool: resolve calls without using imm field bpf: fix multi-function JITed dump obtained via syscall bpf: get JITed image lengths of functions via syscall tools: bpf: sync bpf uapi header tools: bpftool: add delimiters to multi-function JITed dumps arch/powerpc/net/bpf_jit_comp64.c | 110 ++ include/uapi/linux/bpf.h | 4 ++ kernel/bpf/syscall.c | 82 ++-- kernel/bpf/verifier.c | 22 +--- tools/bpf/bpftool/prog.c | 97 - tools/bpf/bpftool/xlated_dumper.c | 14 +++-- tools/bpf/bpftool/xlated_dumper.h | 3 ++ tools/include/uapi/linux/bpf.h| 4 ++ 8 files changed, 301 insertions(+), 35 deletions(-) -- 2.14.3
[PATCH bpf-next v4 01/10] bpf: support 64-bit offsets for bpf function calls
The imm field of a bpf instruction is a signed 32-bit integer.
For JITed bpf-to-bpf function calls, it holds the offset of the
start address of the callee's JITed image from __bpf_call_base.
For some architectures, such as powerpc64, this offset may be
as large as 64 bits and cannot be accomodated in the imm field
without truncation.
We resolve this by:
[1] Additionally using the auxiliary data of each function to
keep a list of start addresses of the JITed images for all
functions determined by the verifier.
[2] Retaining the subprog id inside the off field of the call
instructions and using it to index into the list mentioned
above and lookup the callee's address.
To make sure that the existing JIT compilers continue to work
without requiring changes, we keep the imm field as it is.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/verifier.c | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a9e4b1372da6..559cb74ba29e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5383,11 +5383,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* we use the aux data to keep a list of the start addresses
+* of the JITed images for each function in the program
+*
+* for some architectures, such as powerpc64, the imm field
+* might not be large enough to hold the offset of the start
+* address of the callee's JITed image from __bpf_call_base
+*
+* in such cases, we can lookup the start address of a callee
+* by using its subprog id, available from the off field of
+* the call instruction, as an index for this list
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt;
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
Re: [PATCH bpf-next v3 10/10] tools: bpftool: add delimiters to multi-function JITed dumps
On 05/23/2018 07:20 PM, Daniel Borkmann wrote: > On 05/23/2018 12:37 PM, Sandipan Das wrote: > [...] >> Other than that, for powerpc64, there is a problem with the way the >> binutils disassembler code (in "opcodes/ppc-dis.c") passes arguments >> to the callback fprintf_json(). >> >> In fprintf_json(), we always expect the va_list elements to resolve >> to strings (char *). But for powerpc64, the register or immediate >> operands are always passed as integers. So, when the code attempts >> to resolve these operands using va_arg(ap, char *), bpftool crashes. >> For now, I am using a workaround based on vsnprintf() but this does >> not get the semantics correct for memory operands. You can probably >> see that for the store instructions in the JSON dump above this. >> >> Daniel, >> >> Would it be okay if I send out a fix for this in a different series? > > I'm fine either way with regards to the fix. Feels like a portability bug > in the binutils disassembler? > > We could probably have a feature test like in test-disassembler-four-args > and select a workaround in bpftool based on that outcome. > > Thanks Sandipan! > > [1] tools/build/feature/test-disassembler-four-args.c > Cool. Thanks for the tip! - Sandipan
Re: [PATCH bpf-next v3 10/10] tools: bpftool: add delimiters to multi-function JITed dumps
On 05/23/2018 02:38 PM, Daniel Borkmann wrote:
> On 05/22/2018 09:55 PM, Jakub Kicinski wrote:
>> On Tue, 22 May 2018 22:46:13 +0530, Sandipan Das wrote:
>>> + if (info.nr_jited_func_lens && info.jited_func_lens) {
>>> + struct kernel_sym *sym = NULL;
>>> + unsigned char *img = buf;
>>> + __u64 *ksyms = NULL;
>>> + __u32 *lens;
>>> + __u32 i;
>>> +
>>> + if (info.nr_jited_ksyms) {
>>> + kernel_syms_load();
>>> + ksyms = (__u64 *) info.jited_ksyms;
>>> + }
>>> +
>>> + lens = (__u32 *) info.jited_func_lens;
>>> + for (i = 0; i < info.nr_jited_func_lens; i++) {
>>> + if (ksyms) {
>>> + sym = kernel_syms_search(, ksyms[i]);
>>> + if (sym)
>>> + printf("%s:\n", sym->name);
>>> + else
>>> + printf("%016llx:\n", ksyms[i]);
>>> + }
>>> +
>>> + disasm_print_insn(img, lens[i], opcodes, name);
>>> + img += lens[i];
>>> + printf("\n");
>>> + }
>>> + } else {
>>
>> The output doesn't seem to be JSON-compatible :( We try to make sure
>> all bpftool command can produce valid JSON when run with -j (or -p)
>> switch.
>>
>> Would it be possible to make each function a separate JSON object with
>> "name" and "insn" array? Would that work?
>
> Sandipan, could you take a look at this? Given there's json output today we
> should definitely try not to break it; presumably this would be one final
> respin of your series with this fixed.
>
>
Sure. With a few changes, I am able get JSON output like the following:
# echo 0 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog -p dump jited id 1
[{
"name": "0xdaa8",
"insns": [{
"pc": "0x0",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x4",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x8",
"operation": "mflr",
"operands": ["r0"
]
},{
"pc": "0xc",
"operation": "std",
"operands": ["r0","16","(","r1",")"
]
},{
"pc": "0x10",
"operation": "stdu",
"operands": ["r1","-112","(","r1",")"
]
},{
...
}
]
},{
"name": "0xdaae",
"insns": [{
"pc": "0x0",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x4",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x8",
"operation": "mflr",
"operands": ["r0"
]
},{
...
}
]
}
]
# echo 1 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog -p dump jited id 1
[{
"name": "bpf_prog_b811aab41a39ad3d_foo",
"insns": [{
"pc": "0x0",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x4",
"operation": "nop",
"operands": [null
]
},{
"pc": "0x8",
"operation":
[PATCH bpf-next v3 01/10] bpf: support 64-bit offsets for bpf function calls
The imm field of a bpf instruction is a signed 32-bit integer.
For JITed bpf-to-bpf function calls, it holds the offset of the
start address of the callee's JITed image from __bpf_call_base.
For some architectures, such as powerpc64, this offset may be
as large as 64 bits and cannot be accomodated in the imm field
without truncation.
We resolve this by:
[1] Additionally using the auxillary data of each function to
keep a list of start addresses of the JITed images for all
functions determined by the verifier.
[2] Retaining the subprog id inside the off field of the call
instructions and using it to index into the list mentioned
above and lookup the callee's address.
To make sure that the existing JIT compilers continue to work
without requiring changes, we keep the imm field as it is.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/verifier.c | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a9e4b1372da6..559cb74ba29e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5383,11 +5383,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* we use the aux data to keep a list of the start addresses
+* of the JITed images for each function in the program
+*
+* for some architectures, such as powerpc64, the imm field
+* might not be large enough to hold the offset of the start
+* address of the callee's JITed image from __bpf_call_base
+*
+* in such cases, we can lookup the start address of a callee
+* by using its subprog id, available from the off field of
+* the call instruction, as an index for this list
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt;
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
[PATCH bpf-next v3 03/10] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls in the powerpc64
JIT compiler. The JIT compiler converts the bpf call instructions
to native branch instructions. After a round of the usual passes,
the start addresses of the JITed images for the callee functions
are known. Finally, to fixup the branch target addresses, we need
to perform an extra pass.
Because of the address range in which JITed images are allocated
on powerpc64, the offsets of the start addresses of these images
from __bpf_call_base are as large as 64 bits. So, for a function
call, we cannot use the imm field of the instruction to determine
the callee's address. Instead, we use the alternative method of
getting it from the list of function addresses in the auxiliary
data of the caller by using the off field as an index.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Fix memory leak for jit_data when we fail to allocated addrs.
- Remove unnecessary bpf_jit_binary_lock_ro() call.
---
arch/powerpc/net/bpf_jit_comp64.c | 76 +--
1 file changed, 66 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index e4582744a31d..f1c95779843b 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -268,7 +268,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -724,11 +724,25 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+
+ /* bpf function call */
+ if (insn[i].src_reg == BPF_PSEUDO_CALL)
+ if (!extra_pass)
+ func = NULL;
+ else if (fp->aux->func && off <
fp->aux->func_cnt)
+ /* use the subprog id from the off
+* field to lookup the callee address
+*/
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ /* kernel helper call */
+ else
+ func = (u8 *) __bpf_call_base + imm;
bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -876,6 +890,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -883,6 +905,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -890,6 +913,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -903,11 +927,32 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
[PATCH bpf-next v3 02/10] bpf: powerpc64: pad function address loads with NOPs
For multi-function programs, loading the address of a callee
function to a register requires emitting instructions whose
count varies from one to five depending on the nature of the
address.
Since we come to know of the callee's address only before the
extra pass, the number of instructions required to load this
address may vary from what was previously generated. This can
make the JITed image grow or shrink.
To avoid this, we should generate a constant five-instruction
when loading function addresses by padding the optimized load
sequence with NOPs.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 34 +++---
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 1bdb1aff0619..e4582744a31d 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -167,25 +167,37 @@ static void bpf_jit_build_epilogue(u32 *image, struct
codegen_context *ctx)
static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx,
u64 func)
{
+ unsigned int i, ctx_idx = ctx->idx;
+
+ /* Load function address into r12 */
+ PPC_LI64(12, func);
+
+ /* For bpf-to-bpf function calls, the callee's address is unknown
+* until the last extra pass. As seen above, we use PPC_LI64() to
+* load the callee's address, but this may optimize the number of
+* instructions required based on the nature of the address.
+*
+* Since we don't want the number of instructions emitted to change,
+* we pad the optimized PPC_LI64() call with NOPs to guarantee that
+* we always have a five-instruction sequence, which is the maximum
+* that PPC_LI64() can emit.
+*/
+ for (i = ctx->idx - ctx_idx; i < 5; i++)
+ PPC_NOP();
+
#ifdef PPC64_ELF_ABI_v1
- /* func points to the function descriptor */
- PPC_LI64(b2p[TMP_REG_2], func);
- /* Load actual entry point from function descriptor */
- PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
- /* ... and move it to LR */
- PPC_MTLR(b2p[TMP_REG_1]);
/*
* Load TOC from function descriptor at offset 8.
* We can clobber r2 since we get called through a
* function pointer (so caller will save/restore r2)
* and since we don't use a TOC ourself.
*/
- PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
-#else
- /* We can clobber r12 */
- PPC_FUNC_ADDR(12, func);
- PPC_MTLR(12);
+ PPC_BPF_LL(2, 12, 8);
+ /* Load actual entry point from function descriptor */
+ PPC_BPF_LL(12, 12, 0);
#endif
+
+ PPC_MTLR(12);
PPC_BLRL();
}
--
2.14.3
[PATCH bpf-next v3 00/10] bpf: enhancements for multi-function programs
v3: - Change base tree tag to bpf-next. - Incorporate review comments from Alexei, Daniel and Jakub. - Make sure that the JITed image does not grow or shrink after the last pass due to the way the instruction sequence used to load a callee's address maybe optimized. - Make additional changes to the bpf system call and bpftool to make multi-function JITed dumps easier to correlate. v2: - Incorporate review comments from Jakub. Sandipan Das (10): bpf: support 64-bit offsets for bpf function calls bpf: powerpc64: pad function address loads with NOPs bpf: powerpc64: add JIT support for multi-function programs bpf: get kernel symbol addresses via syscall tools: bpf: sync bpf uapi header tools: bpftool: resolve calls without using imm field bpf: fix multi-function JITed dump obtained via syscall bpf: get JITed image lengths of functions via syscall tools: bpf: sync bpf uapi header tools: bpftool: add delimiters to multi-function JITed dumps arch/powerpc/net/bpf_jit_comp64.c | 110 ++ include/uapi/linux/bpf.h | 4 ++ kernel/bpf/syscall.c | 81 ++-- kernel/bpf/verifier.c | 22 +--- tools/bpf/bpftool/prog.c | 75 +- tools/bpf/bpftool/xlated_dumper.c | 14 +++-- tools/bpf/bpftool/xlated_dumper.h | 3 ++ tools/include/uapi/linux/bpf.h| 4 ++ 8 files changed, 278 insertions(+), 35 deletions(-) -- 2.14.3
[PATCH bpf-next v3 04/10] bpf: get kernel symbol addresses via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of kernel symbol addresses for all functions in a
given program to userspace using the bpf system call with
the BPF_OBJ_GET_INFO_BY_FD command.
When bpf_jit_kallsyms is enabled, we can get the address
of the corresponding kernel symbol for a callee function
and resolve the symbol's name. The address is determined
by adding the value of the call instruction's imm field
to __bpf_call_base. This offset gets assigned to the imm
field by the verifier.
For some architectures, such as powerpc64, the imm field
is not large enough to hold this offset.
We resolve this by:
[1] Assigning the subprog id to the imm field of a call
instruction in the verifier instead of the offset of
the callee's symbol's address from __bpf_call_base.
[2] Determining the address of a callee's corresponding
symbol by using the imm field as an index for the
list of kernel symbol addresses now available from
the program info.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Copy addresses to jited_ksyms only if bpf_dump_raw_ok()
is true.
- Move new fields to the end of bpf_prog_info to avoid
breaking userspace.
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 25 +
kernel/bpf/verifier.c| 7 +--
3 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 97446bbe2ca5..c44105f27da9 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2205,6 +2205,8 @@ struct bpf_prog_info {
__u32 gpl_compatible:1;
__u64 netns_dev;
__u64 netns_ino;
+ __u32 nr_jited_ksyms;
+ __aligned_u64 jited_ksyms;
} __attribute__((aligned(8)));
struct bpf_map_info {
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index bfcde949c7f8..f0ad4b5f0224 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1933,6 +1933,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
+ info.nr_jited_ksyms = 0;
goto done;
}
@@ -1981,6 +1982,30 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_ksyms;
+ info.nr_jited_ksyms = prog->aux->func_cnt;
+ if (info.nr_jited_ksyms && ulen) {
+ if (bpf_dump_raw_ok()) {
+ u64 __user *user_ksyms;
+ ulong ksym_addr;
+ u32 i;
+
+ /* copy the address of the kernel symbol
+* corresponding to each function
+*/
+ ulen = min_t(u32, info.nr_jited_ksyms, ulen);
+ user_ksyms = u64_to_user_ptr(info.jited_ksyms);
+ for (i = 0; i < ulen; i++) {
+ ksym_addr = (ulong)
prog->aux->func[i]->bpf_func;
+ ksym_addr &= PAGE_MASK;
+ if (put_user((u64) ksym_addr, _ksyms[i]))
+ return -EFAULT;
+ }
+ } else {
+ info.jited_ksyms = 0;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 559cb74ba29e..8c4d9d0fd3ab 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5426,17 +5426,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* later look the same as if they were interpreted only.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
- unsigned long addr;
-
if (insn->code != (BPF_JMP | BPF_CALL) ||
insn->src_reg != BPF_PSEUDO_CALL)
continue;
insn->off = env->insn_aux_data[i].call_imm;
subprog = find_subprog(env, i + insn->off + 1);
- addr = (unsigned long)func[subprog]->bpf_func;
- addr &= PAGE_MASK;
- insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
- addr - __bpf_call_base;
+ insn->imm = subprog;
}
prog->jited = 1;
--
2.14.3
[PATCH bpf-next v3 06/10] tools: bpftool: resolve calls without using imm field
Currently, we resolve the callee's address for a JITed function
call by using the imm field of the call instruction as an offset
from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
use this address to get the callee's kernel symbol's name.
For some architectures, such as powerpc64, the imm field is not
large enough to hold this offset. So, instead of assigning this
offset to the imm field, the verifier now assigns the subprog
id. Also, a list of kernel symbol addresses for all the JITed
functions is provided in the program info. We now use the imm
field as an index for this list to lookup a callee's symbol's
address and resolve its name.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Avoid using redundant pointers.
- Fix indentation.
v2:
- Order variables from longest to shortest.
- Make sure that ksyms_ptr and ksyms_len are always initialized.
- Simplify code.
---
tools/bpf/bpftool/prog.c | 24
tools/bpf/bpftool/xlated_dumper.c | 10 +-
tools/bpf/bpftool/xlated_dumper.h | 2 ++
3 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 9bdfdf2d3fbe..e05ab58d39e2 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -420,7 +420,9 @@ static int do_show(int argc, char **argv)
static int do_dump(int argc, char **argv)
{
+ unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {};
+ unsigned int nr_func_ksyms;
struct dump_data dd = {};
__u32 len = sizeof(info);
unsigned int buf_size;
@@ -496,10 +498,22 @@ static int do_dump(int argc, char **argv)
return -1;
}
+ nr_func_ksyms = info.nr_jited_ksyms;
+ if (nr_func_ksyms) {
+ func_ksyms = malloc(nr_func_ksyms * sizeof(__u64));
+ if (!func_ksyms) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
+ info.jited_ksyms = ptr_to_u64(func_ksyms);
+ info.nr_jited_ksyms = nr_func_ksyms;
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
@@ -513,6 +527,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (info.nr_jited_ksyms > nr_func_ksyms) {
+ p_err("too many addresses returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -558,6 +577,9 @@ static int do_dump(int argc, char **argv)
dump_xlated_cfg(buf, *member_len);
} else {
kernel_syms_load();
+ dd.nr_jited_ksyms = info.nr_jited_ksyms;
+ dd.jited_ksyms = (__u64 *) info.jited_ksyms;
+
if (json_output)
dump_xlated_json(, buf, *member_len, opcodes);
else
@@ -566,10 +588,12 @@ static int do_dump(int argc, char **argv)
}
free(buf);
+ free(func_ksyms);
return 0;
err_free:
free(buf);
+ free(func_ksyms);
return -1;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c
b/tools/bpf/bpftool/xlated_dumper.c
index 7a3173b76c16..efdc8fecf2bb 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -174,7 +174,11 @@ static const char *print_call_pcrel(struct dump_data *dd,
unsigned long address,
const struct bpf_insn *insn)
{
- if (sym)
+ if (!dd->nr_jited_ksyms)
+ /* Do not show address for interpreted programs */
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+ "%+d", insn->off);
+ else if (sym)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"%+d#%s", insn->off, sym->name);
else
@@ -203,6 +207,10 @@ static const char *print_call(void *private_data,
unsigned long address = dd->address_call_base + insn->imm;
struct kernel_sym *sym;
+ if (insn->src_reg == BPF_PSEUDO_CALL &&
+ (__u32) insn->imm < dd->nr_jited_ksyms)
+ address = dd->jited_ksyms[insn->imm];
+
sym = kernel_syms_search(dd, address);
if (insn->src_reg == BPF_PSEUDO_CALL)
return print_call_pcrel(dd, sym, address, insn);
diff --git a/tools/bpf/bpftool/xlated_dumper.h
b/tools/bpf/bpftool/xlated_dumper.h
index b34affa7ef2d..eafbb49c8d0b 100644
--- a/tools/bpf/bpftool/xlated_dumper.h
+++ b/tools/b
[PATCH bpf-next v3 07/10] bpf: fix multi-function JITed dump obtained via syscall
Currently, for multi-function programs, we cannot get the JITed
instructions using the bpf system call's BPF_OBJ_GET_INFO_BY_FD
command. Because of this, userspace tools such as bpftool fail
to identify a multi-function program as being JITed or not.
With the JIT enabled and the test program running, this can be
verified as follows:
# cat /proc/sys/net/core/bpf_jit_enable
1
Before applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T11:43:38+0530 uid 0
xlated 216B not jited memlock 65536B
...
# bpftool prog dump jited id 1
no instructions returned
After applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T12:13:01+0530 uid 0
xlated 216B jited 308B memlock 65536B
...
# bpftool prog dump jited id 1
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
18: addir31,r1,48
1c: li r3,10
...
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/syscall.c | 36 +---
1 file changed, 33 insertions(+), 3 deletions(-)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index f0ad4b5f0224..1c4cba91e523 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1970,13 +1970,43 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
*prog,
* for offload.
*/
ulen = info.jited_prog_len;
- info.jited_prog_len = prog->jited_len;
+ if (prog->aux->func_cnt) {
+ u32 i;
+
+ info.jited_prog_len = 0;
+ for (i = 0; i < prog->aux->func_cnt; i++)
+ info.jited_prog_len += prog->aux->func[i]->jited_len;
+ } else {
+ info.jited_prog_len = prog->jited_len;
+ }
+
if (info.jited_prog_len && ulen) {
if (bpf_dump_raw_ok()) {
uinsns = u64_to_user_ptr(info.jited_prog_insns);
ulen = min_t(u32, info.jited_prog_len, ulen);
- if (copy_to_user(uinsns, prog->bpf_func, ulen))
- return -EFAULT;
+
+ /* for multi-function programs, copy the JITed
+* instructions for all the functions
+*/
+ if (prog->aux->func_cnt) {
+ u32 len, free, i;
+ u8 *img;
+
+ free = ulen;
+ for (i = 0; i < prog->aux->func_cnt; i++) {
+ len = prog->aux->func[i]->jited_len;
+ img = (u8 *)
prog->aux->func[i]->bpf_func;
+ if (len > free)
+ break;
+ if (copy_to_user(uinsns, img, len))
+ return -EFAULT;
+ uinsns += len;
+ free -= len;
+ }
+ } else {
+ if (copy_to_user(uinsns, prog->bpf_func, ulen))
+ return -EFAULT;
+ }
} else {
info.jited_prog_insns = 0;
}
--
2.14.3
[PATCH bpf-next v3 10/10] tools: bpftool: add delimiters to multi-function JITed dumps
This splits up the contiguous JITed dump obtained via the bpf
system call into more relatable chunks for each function in
the program. If the kernel symbols corresponding to these are
known, they are printed in the header for each JIT image dump
otherwise the masked start address is printed.
Before applying this patch:
# bpftool prog dump jited id 1
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
...
a8: mr r3,r8
ac: blr
b0: nop
b4: nop
b8: mflrr0
bc: std r0,16(r1)
c0: stdur1,-112(r1)
c4: std r31,104(r1)
...
138: mr r3,r8
13c: blr
After applying this patch:
# echo 0 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog dump jited id 1
dacc:
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
...
a8: mr r3,r8
ac: blr
dad2:
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
...
88: mr r3,r8
8c: blr
# echo 1 > /proc/sys/net/core/bpf_jit_kallsyms
# bpftool prog dump jited id 1
bpf_prog_8852b2ccb8ec75a7_F:
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
...
a8: mr r3,r8
ac: blr
bpf_prog_196af774a3477707_F:
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
...
88: mr r3,r8
8c: blr
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/bpf/bpftool/prog.c | 51 ++-
tools/bpf/bpftool/xlated_dumper.c | 4 +--
tools/bpf/bpftool/xlated_dumper.h | 1 +
3 files changed, 53 insertions(+), 3 deletions(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index e05ab58d39e2..8ab7a683ac67 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -422,7 +422,9 @@ static int do_dump(int argc, char **argv)
{
unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {};
+ unsigned int *func_lens = NULL;
unsigned int nr_func_ksyms;
+ unsigned int nr_func_lens;
struct dump_data dd = {};
__u32 len = sizeof(info);
unsigned int buf_size;
@@ -508,12 +510,24 @@ static int do_dump(int argc, char **argv)
}
}
+ nr_func_lens = info.nr_jited_func_lens;
+ if (nr_func_lens) {
+ func_lens = malloc(nr_func_lens * sizeof(__u32));
+ if (!func_lens) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
info.jited_ksyms = ptr_to_u64(func_ksyms);
info.nr_jited_ksyms = nr_func_ksyms;
+ info.jited_func_lens = ptr_to_u64(func_lens);
+ info.nr_jited_func_lens = nr_func_lens;
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
@@ -532,6 +546,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (info.nr_jited_func_lens > nr_func_lens) {
+ p_err("too many values returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -569,7 +588,35 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
- disasm_print_insn(buf, *member_len, opcodes, name);
+ if (info.nr_jited_func_lens && info.jited_func_lens) {
+ struct kernel_sym *sym = NULL;
+ unsigned char *img = buf;
+ __u64 *ksyms = NULL;
+ __u32 *lens;
+ __u32 i;
+
+ if (info.nr_jited_ksyms) {
+ kernel_syms_load();
+ ksyms = (__u64 *) info.jited_ksyms;
+ }
+
+ lens = (__u32 *) info.jited_func_lens;
+ for (i = 0; i < info.nr_jited_func_lens; i++) {
+ if (ksyms) {
+ sym = kernel_syms_search(, ksyms[i]);
+ if (sym)
+ printf("%s:\n", sym->name);
+ else
+
[PATCH bpf-next v3 09/10] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
JITed image lengths of each function in a multi-function
program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index c44105f27da9..8c3109b5d6d3 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2206,7 +2206,9 @@ struct bpf_prog_info {
__u64 netns_dev;
__u64 netns_ino;
__u32 nr_jited_ksyms;
+ __u32 nr_jited_func_lens;
__aligned_u64 jited_ksyms;
+ __aligned_u64 jited_func_lens;
} __attribute__((aligned(8)));
struct bpf_map_info {
--
2.14.3
[PATCH bpf-next v3 08/10] bpf: get JITed image lengths of functions via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of the JITed image lengths of each function for a
given program to userspace using the bpf system call with
the BPF_OBJ_GET_INFO_BY_FD command.
This can be used by userspace applications like bpftool
to split up the contiguous JITed dump, also obtained via
the system call, into more relatable chunks corresponding
to each function.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 20
2 files changed, 22 insertions(+)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index c44105f27da9..8c3109b5d6d3 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2206,7 +2206,9 @@ struct bpf_prog_info {
__u64 netns_dev;
__u64 netns_ino;
__u32 nr_jited_ksyms;
+ __u32 nr_jited_func_lens;
__aligned_u64 jited_ksyms;
+ __aligned_u64 jited_func_lens;
} __attribute__((aligned(8)));
struct bpf_map_info {
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 1c4cba91e523..faadbcd90191 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2036,6 +2036,26 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_func_lens;
+ info.nr_jited_func_lens = prog->aux->func_cnt;
+ if (info.nr_jited_func_lens && ulen) {
+ if (bpf_dump_raw_ok()) {
+ u32 __user *user_lens;
+ u32 func_len, i;
+
+ /* copy the JITed image lengths for each function */
+ ulen = min_t(u32, info.nr_jited_func_lens, ulen);
+ user_lens = u64_to_user_ptr(info.jited_func_lens);
+ for (i = 0; i < ulen; i++) {
+ func_len = prog->aux->func[i]->jited_len;
+ if (put_user(func_len, _lens[i]))
+ return -EFAULT;
+ }
+ } else {
+ info.jited_func_lens = 0;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
--
2.14.3
[PATCH bpf-next v3 05/10] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
addresses of the kernel symbols corresponding to each
function in a program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3:
- Move new fields to the end of bpf_prog_info to avoid
breaking userspace.
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 97446bbe2ca5..c44105f27da9 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2205,6 +2205,8 @@ struct bpf_prog_info {
__u32 gpl_compatible:1;
__u64 netns_dev;
__u64 netns_ino;
+ __u32 nr_jited_ksyms;
+ __aligned_u64 jited_ksyms;
} __attribute__((aligned(8)));
struct bpf_map_info {
--
2.14.3
Re: [PATCH bpf v2 6/6] bpf: fix JITed dump for multi-function programs via syscall
Hi Daniel,
On 05/18/2018 09:21 PM, Daniel Borkmann wrote:
> On 05/18/2018 02:50 PM, Sandipan Das wrote:
>> Currently, for multi-function programs, we cannot get the JITed
>> instructions using the bpf system call's BPF_OBJ_GET_INFO_BY_FD
>> command. Because of this, userspace tools such as bpftool fail
>> to identify a multi-function program as being JITed or not.
>>
>> With the JIT enabled and the test program running, this can be
>> verified as follows:
>>
>> # cat /proc/sys/net/core/bpf_jit_enable
>> 1
>>
>> Before applying this patch:
>>
>> # bpftool prog list
>> 1: kprobe name foo tag b811aab41a39ad3d gpl
>> loaded_at 2018-05-16T11:43:38+0530 uid 0
>> xlated 216B not jited memlock 65536B
>> ...
>>
>> # bpftool prog dump jited id 1
>> no instructions returned
>>
>> After applying this patch:
>>
>> # bpftool prog list
>> 1: kprobe name foo tag b811aab41a39ad3d gpl
>> loaded_at 2018-05-16T12:13:01+0530 uid 0
>> xlated 216B jited 308B memlock 65536B
>> ...
>
> That's really nice! One comment inline below:
>
>> # bpftool prog dump jited id 1
>> 0: nop
>> 4: nop
>> 8: mflrr0
>> c: std r0,16(r1)
>> 10: stdur1,-112(r1)
>> 14: std r31,104(r1)
>> 18: addir31,r1,48
>> 1c: li r3,10
>> ...
>>
>> Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
>> ---
>> kernel/bpf/syscall.c | 38 --
>> 1 file changed, 32 insertions(+), 6 deletions(-)
>>
>> diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
>> index 54a72fafe57c..2430d159078c 100644
>> --- a/kernel/bpf/syscall.c
>> +++ b/kernel/bpf/syscall.c
>> @@ -1896,7 +1896,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
>> *prog,
>> struct bpf_prog_info info = {};
>> u32 info_len = attr->info.info_len;
>> char __user *uinsns;
>> -u32 ulen;
>> +u32 ulen, i;
>> int err;
>>
>> err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
>> @@ -1922,7 +1922,6 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
>> *prog,
>> ulen = min_t(u32, info.nr_map_ids, ulen);
>> if (ulen) {
>> u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
>> -u32 i;
>>
>> for (i = 0; i < ulen; i++)
>> if (put_user(prog->aux->used_maps[i]->id,
>> @@ -1970,13 +1969,41 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
>> *prog,
>> * for offload.
>> */
>> ulen = info.jited_prog_len;
>> -info.jited_prog_len = prog->jited_len;
>> +if (prog->aux->func_cnt) {
>> +info.jited_prog_len = 0;
>> +for (i = 0; i < prog->aux->func_cnt; i++)
>> +info.jited_prog_len += prog->aux->func[i]->jited_len;
>> +} else {
>> +info.jited_prog_len = prog->jited_len;
>> +}
>> +
>> if (info.jited_prog_len && ulen) {
>> if (bpf_dump_raw_ok()) {
>> uinsns = u64_to_user_ptr(info.jited_prog_insns);
>> ulen = min_t(u32, info.jited_prog_len, ulen);
>> -if (copy_to_user(uinsns, prog->bpf_func, ulen))
>> -return -EFAULT;
>> +
>> +/* for multi-function programs, copy the JITed
>> + * instructions for all the functions
>> + */
>> +if (prog->aux->func_cnt) {
>> +u32 len, free;
>> +u8 *img;
>> +
>> +free = ulen;
>> +for (i = 0; i < prog->aux->func_cnt; i++) {
>> +len = prog->aux->func[i]->jited_len;
>> +img = (u8 *)
>> prog->aux->func[i]->bpf_func;
>> +if (len > free)
>> +break;
>> +if (copy_to_user(uinsns, img, len))
>> +return -EFAULT;
>> +uinsns += len;
>> +free -= len;
>
> Is there any wa
Re: [PATCH bpf v2 1/6] bpf: support 64-bit offsets for bpf function calls
On 05/18/2018 08:45 PM, Daniel Borkmann wrote:
> On 05/18/2018 02:50 PM, Sandipan Das wrote:
>> The imm field of a bpf instruction is a signed 32-bit integer.
>> For JIT bpf-to-bpf function calls, it stores the offset of the
>> start address of the callee's JITed image from __bpf_call_base.
>>
>> For some architectures, such as powerpc64, this offset may be
>> as large as 64 bits and cannot be accomodated in the imm field
>> without truncation.
>>
>> We resolve this by:
>>
>> [1] Additionally using the auxillary data of each function to
>> keep a list of start addresses of the JITed images for all
>> functions determined by the verifier.
>>
>> [2] Retaining the subprog id inside the off field of the call
>> instructions and using it to index into the list mentioned
>> above and lookup the callee's address.
>>
>> To make sure that the existing JIT compilers continue to work
>> without requiring changes, we keep the imm field as it is.
>>
>> Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
>> ---
>> kernel/bpf/verifier.c | 15 ++-
>> 1 file changed, 14 insertions(+), 1 deletion(-)
>>
>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>> index a9e4b1372da6..6c56cce9c4e3 100644
>> --- a/kernel/bpf/verifier.c
>> +++ b/kernel/bpf/verifier.c
>> @@ -5383,11 +5383,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
>> insn->src_reg != BPF_PSEUDO_CALL)
>> continue;
>> subprog = insn->off;
>> -insn->off = 0;
>> insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
>> func[subprog]->bpf_func -
>> __bpf_call_base;
>> }
>> +
>> +/* we use the aux data to keep a list of the start addresses
>> + * of the JITed images for each function in the program
>> + *
>> + * for some architectures, such as powerpc64, the imm field
>> + * might not be large enough to hold the offset of the start
>> + * address of the callee's JITed image from __bpf_call_base
>> + *
>> + * in such cases, we can lookup the start address of a callee
>> + * by using its subprog id, available from the off field of
>> + * the call instruction, as an index for this list
>> + */
>> +func[i]->aux->func = func;
>> +func[i]->aux->func_cnt = env->subprog_cnt + 1;
>
> The target tree you have here is infact bpf, since in bpf-next there was a
> cleanup where the + 1 is removed. Just for the record that we need to keep
> this in mind for bpf into bpf-next merge since this would otherwise subtly
> break.
>
Sorry about the wrong tag. This series is indeed based off bpf-next.
- Sandipan
>> }
>> for (i = 0; i < env->subprog_cnt; i++) {
>> old_bpf_func = func[i]->bpf_func;
>>
>
>
[PATCH bpf v2 4/6] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
addresses of the kernel symbols corresponding to each
function in a JITed program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index d94d333a8225..040c9cac7303 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2188,6 +2188,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_ksyms;
+ __u32 nr_jited_ksyms;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
--
2.14.3
[PATCH bpf v2 2/6] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls in the powerpc64
JIT compiler. The JIT compiler converts the bpf call instructions
to native branch instructions. After a round of the usual passes,
the start addresses of the JITed images for the callee functions
are known. Finally, to fixup the branch target addresses, we need
to perform an extra pass.
Because of the address range in which JITed images are allocated
on powerpc64, the offsets of the start addresses of these images
from __bpf_call_base are as large as 64 bits. So, for a function
call, we cannot use the imm field of the instruction to determine
the callee's address. Instead, we use the alternative method of
getting it from the list of function addresses in the auxillary
data of the caller by using the off field as an index.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 79 ++-
1 file changed, 69 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 1bdb1aff0619..25939892d8f7 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -256,7 +256,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -712,11 +712,23 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+
+ /* bpf function call */
+ if (insn[i].src_reg == BPF_PSEUDO_CALL && extra_pass)
+ if (fp->aux->func && off < fp->aux->func_cnt)
+ /* use the subprog id from the off
+* field to lookup the callee address
+*/
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ /* kernel helper call */
+ else
+ func = (u8 *) __bpf_call_base + imm;
bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -864,6 +876,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -871,6 +891,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -878,6 +899,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -891,7 +913,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
fp = org_fp;
@@ -904,10 +947,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
/* Scouting faux-generat
[PATCH bpf v2 6/6] bpf: fix JITed dump for multi-function programs via syscall
Currently, for multi-function programs, we cannot get the JITed
instructions using the bpf system call's BPF_OBJ_GET_INFO_BY_FD
command. Because of this, userspace tools such as bpftool fail
to identify a multi-function program as being JITed or not.
With the JIT enabled and the test program running, this can be
verified as follows:
# cat /proc/sys/net/core/bpf_jit_enable
1
Before applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T11:43:38+0530 uid 0
xlated 216B not jited memlock 65536B
...
# bpftool prog dump jited id 1
no instructions returned
After applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T12:13:01+0530 uid 0
xlated 216B jited 308B memlock 65536B
...
# bpftool prog dump jited id 1
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
18: addir31,r1,48
1c: li r3,10
...
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/syscall.c | 38 --
1 file changed, 32 insertions(+), 6 deletions(-)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 54a72fafe57c..2430d159078c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1896,7 +1896,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
struct bpf_prog_info info = {};
u32 info_len = attr->info.info_len;
char __user *uinsns;
- u32 ulen;
+ u32 ulen, i;
int err;
err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
@@ -1922,7 +1922,6 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
ulen = min_t(u32, info.nr_map_ids, ulen);
if (ulen) {
u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
- u32 i;
for (i = 0; i < ulen; i++)
if (put_user(prog->aux->used_maps[i]->id,
@@ -1970,13 +1969,41 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
*prog,
* for offload.
*/
ulen = info.jited_prog_len;
- info.jited_prog_len = prog->jited_len;
+ if (prog->aux->func_cnt) {
+ info.jited_prog_len = 0;
+ for (i = 0; i < prog->aux->func_cnt; i++)
+ info.jited_prog_len += prog->aux->func[i]->jited_len;
+ } else {
+ info.jited_prog_len = prog->jited_len;
+ }
+
if (info.jited_prog_len && ulen) {
if (bpf_dump_raw_ok()) {
uinsns = u64_to_user_ptr(info.jited_prog_insns);
ulen = min_t(u32, info.jited_prog_len, ulen);
- if (copy_to_user(uinsns, prog->bpf_func, ulen))
- return -EFAULT;
+
+ /* for multi-function programs, copy the JITed
+* instructions for all the functions
+*/
+ if (prog->aux->func_cnt) {
+ u32 len, free;
+ u8 *img;
+
+ free = ulen;
+ for (i = 0; i < prog->aux->func_cnt; i++) {
+ len = prog->aux->func[i]->jited_len;
+ img = (u8 *)
prog->aux->func[i]->bpf_func;
+ if (len > free)
+ break;
+ if (copy_to_user(uinsns, img, len))
+ return -EFAULT;
+ uinsns += len;
+ free -= len;
+ }
+ } else {
+ if (copy_to_user(uinsns, prog->bpf_func, ulen))
+ return -EFAULT;
+ }
} else {
info.jited_prog_insns = 0;
}
@@ -1987,7 +2014,6 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (info.nr_jited_ksyms && ulen) {
u64 __user *user_jited_ksyms =
u64_to_user_ptr(info.jited_ksyms);
ulong ksym_addr;
- u32 i;
/* copy the address of the kernel symbol corresponding to
* each function
--
2.14.3
[PATCH bpf v2 3/6] bpf: get kernel symbol addresses via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of kernel symbol addresses for all functions in a
given program and to userspace using the bpf system call
with the BPF_OBJ_GET_INFO_BY_FD command.
When bpf_jit_kallsyms is enabled, we can get the address
of the corresponding kernel symbol for a callee function
and resolve the symbol's name. The address is determined
by adding the value of the call instruction's imm field
to __bpf_call_base. This offset gets assigned to the imm
field by the verifier.
For some architectures, such as powerpc64, the imm field
is not large enough to hold this offset.
We resolve this by:
[1] Assigning the subprog id to the imm field of a call
instruction in the verifier instead of the offset of
the callee's symbol's address from __bpf_call_base.
[2] Determining the address of a callee's corresponding
symbol by using the imm field as an index for the
list of kernel symbol addresses now available from
the program info.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 20
kernel/bpf/verifier.c| 7 +--
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index d94d333a8225..040c9cac7303 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2188,6 +2188,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_ksyms;
+ __u32 nr_jited_ksyms;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index bfcde949c7f8..54a72fafe57c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1933,6 +1933,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
+ info.nr_jited_ksyms = 0;
goto done;
}
@@ -1981,6 +1982,25 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_ksyms;
+ info.nr_jited_ksyms = prog->aux->func_cnt;
+ if (info.nr_jited_ksyms && ulen) {
+ u64 __user *user_jited_ksyms =
u64_to_user_ptr(info.jited_ksyms);
+ ulong ksym_addr;
+ u32 i;
+
+ /* copy the address of the kernel symbol corresponding to
+* each function
+*/
+ ulen = min_t(u32, info.nr_jited_ksyms, ulen);
+ for (i = 0; i < ulen; i++) {
+ ksym_addr = (ulong) prog->aux->func[i]->bpf_func;
+ ksym_addr &= PAGE_MASK;
+ if (put_user((u64) ksym_addr, _jited_ksyms[i]))
+ return -EFAULT;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6c56cce9c4e3..e826c396aba2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5426,17 +5426,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* later look the same as if they were interpreted only.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
- unsigned long addr;
-
if (insn->code != (BPF_JMP | BPF_CALL) ||
insn->src_reg != BPF_PSEUDO_CALL)
continue;
insn->off = env->insn_aux_data[i].call_imm;
subprog = find_subprog(env, i + insn->off + 1);
- addr = (unsigned long)func[subprog]->bpf_func;
- addr &= PAGE_MASK;
- insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
- addr - __bpf_call_base;
+ insn->imm = subprog;
}
prog->jited = 1;
--
2.14.3
[PATCH bpf v2 5/6] tools: bpftool: resolve calls without using imm field
Currently, we resolve the callee's address for a JITed function
call by using the imm field of the call instruction as an offset
from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
use this address to get the callee's kernel symbol's name.
For some architectures, such as powerpc64, the imm field is not
large enough to hold this offset. So, instead of assigning this
offset to the imm field, the verifier now assigns the subprog
id. Also, a list of kernel symbol addresses for all the JITed
functions is provided in the program info. We now use the imm
field as an index for this list to lookup a callee's symbol's
address and resolve its name.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v2:
- Order variables from longest to shortest
- Make sure that ksyms_ptr and ksyms_len are always initialized
- Simplify code
---
tools/bpf/bpftool/prog.c | 29 +
tools/bpf/bpftool/xlated_dumper.c | 10 +-
tools/bpf/bpftool/xlated_dumper.h | 2 ++
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 9bdfdf2d3fbe..e2f8f8f259fc 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -421,19 +421,26 @@ static int do_show(int argc, char **argv)
static int do_dump(int argc, char **argv)
{
struct bpf_prog_info info = {};
+ unsigned long *addrs = NULL;
struct dump_data dd = {};
__u32 len = sizeof(info);
unsigned int buf_size;
+ unsigned int nr_addrs;
char *filepath = NULL;
bool opcodes = false;
bool visual = false;
unsigned char *buf;
__u32 *member_len;
__u64 *member_ptr;
+ __u32 *ksyms_len;
+ __u64 *ksyms_ptr;
ssize_t n;
int err;
int fd;
+ ksyms_len = _jited_ksyms;
+ ksyms_ptr = _ksyms;
+
if (is_prefix(*argv, "jited")) {
member_len = _prog_len;
member_ptr = _prog_insns;
@@ -496,10 +503,22 @@ static int do_dump(int argc, char **argv)
return -1;
}
+ nr_addrs = *ksyms_len;
+ if (nr_addrs) {
+ addrs = malloc(nr_addrs * sizeof(__u64));
+ if (!addrs) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
+ *ksyms_ptr = ptr_to_u64(addrs);
+ *ksyms_len = nr_addrs;
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
@@ -513,6 +532,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (*ksyms_len > nr_addrs) {
+ p_err("too many addresses returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -558,6 +582,9 @@ static int do_dump(int argc, char **argv)
dump_xlated_cfg(buf, *member_len);
} else {
kernel_syms_load();
+ dd.nr_jited_ksyms = *ksyms_len;
+ dd.jited_ksyms = (__u64 *) *ksyms_ptr;
+
if (json_output)
dump_xlated_json(, buf, *member_len, opcodes);
else
@@ -566,10 +593,12 @@ static int do_dump(int argc, char **argv)
}
free(buf);
+ free(addrs);
return 0;
err_free:
free(buf);
+ free(addrs);
return -1;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c
b/tools/bpf/bpftool/xlated_dumper.c
index 7a3173b76c16..fb065b55db6d 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -174,7 +174,11 @@ static const char *print_call_pcrel(struct dump_data *dd,
unsigned long address,
const struct bpf_insn *insn)
{
- if (sym)
+ if (!dd->nr_jited_ksyms)
+ /* Do not show address for interpreted programs */
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+ "%+d", insn->off);
+ else if (sym)
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"%+d#%s", insn->off, sym->name);
else
@@ -203,6 +207,10 @@ static const char *print_call(void *private_data,
unsigned long address = dd->address_call_base + insn->imm;
struct kernel_sym *sym;
+ if (insn->src_reg == BPF_PSEUDO_CALL &&
+ (__u32) insn->imm < dd->nr_jited_ksyms)
+ address = dd->jited_ksyms[insn->imm];
+
sym = kernel_syms
[PATCH bpf v2 1/6] bpf: support 64-bit offsets for bpf function calls
The imm field of a bpf instruction is a signed 32-bit integer.
For JIT bpf-to-bpf function calls, it stores the offset of the
start address of the callee's JITed image from __bpf_call_base.
For some architectures, such as powerpc64, this offset may be
as large as 64 bits and cannot be accomodated in the imm field
without truncation.
We resolve this by:
[1] Additionally using the auxillary data of each function to
keep a list of start addresses of the JITed images for all
functions determined by the verifier.
[2] Retaining the subprog id inside the off field of the call
instructions and using it to index into the list mentioned
above and lookup the callee's address.
To make sure that the existing JIT compilers continue to work
without requiring changes, we keep the imm field as it is.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/verifier.c | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a9e4b1372da6..6c56cce9c4e3 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5383,11 +5383,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* we use the aux data to keep a list of the start addresses
+* of the JITed images for each function in the program
+*
+* for some architectures, such as powerpc64, the imm field
+* might not be large enough to hold the offset of the start
+* address of the callee's JITed image from __bpf_call_base
+*
+* in such cases, we can lookup the start address of a callee
+* by using its subprog id, available from the off field of
+* the call instruction, as an index for this list
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt + 1;
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
[PATCH bpf v2 0/6] bpf: enhancements for multi-function programs
This patch series introduces the following: [1] Support for bpf-to-bpf function calls in the powerpc64 JIT compiler. [2] Provide a way for resolving function calls because of the way JITed images are allocated in powerpc64. [3] Fix to get JITed instruction dumps for multi-function programs from the bpf system call. v2: - Incorporate review comments from Jakub Sandipan Das (6): bpf: support 64-bit offsets for bpf function calls bpf: powerpc64: add JIT support for multi-function programs bpf: get kernel symbol addresses via syscall tools: bpf: sync bpf uapi header tools: bpftool: resolve calls without using imm field bpf: fix JITed dump for multi-function programs via syscall arch/powerpc/net/bpf_jit_comp64.c | 79 ++- include/uapi/linux/bpf.h | 2 + kernel/bpf/syscall.c | 56 --- kernel/bpf/verifier.c | 22 +++ tools/bpf/bpftool/prog.c | 29 ++ tools/bpf/bpftool/xlated_dumper.c | 10 - tools/bpf/bpftool/xlated_dumper.h | 2 + tools/include/uapi/linux/bpf.h| 2 + 8 files changed, 179 insertions(+), 23 deletions(-) -- 2.14.3
Re: [PATCH bpf 5/6] tools: bpftool: resolve calls without using imm field
Hi Jakub,
On 05/18/2018 12:21 AM, Jakub Kicinski wrote:
> On Thu, 17 May 2018 12:05:47 +0530, Sandipan Das wrote:
>> Currently, we resolve the callee's address for a JITed function
>> call by using the imm field of the call instruction as an offset
>> from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
>> use this address to get the callee's kernel symbol's name.
>>
>> For some architectures, such as powerpc64, the imm field is not
>> large enough to hold this offset. So, instead of assigning this
>> offset to the imm field, the verifier now assigns the subprog
>> id. Also, a list of kernel symbol addresses for all the JITed
>> functions is provided in the program info. We now use the imm
>> field as an index for this list to lookup a callee's symbol's
>> address and resolve its name.
>>
>> Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
>> Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
>
> A few nit-picks below, thank you for the patch!
>
>> tools/bpf/bpftool/prog.c | 31 +++
>> tools/bpf/bpftool/xlated_dumper.c | 24 +---
>> tools/bpf/bpftool/xlated_dumper.h | 2 ++
>> 3 files changed, 50 insertions(+), 7 deletions(-)
>>
>> diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
>> index 9bdfdf2d3fbe..ac2f62a97e84 100644
>> --- a/tools/bpf/bpftool/prog.c
>> +++ b/tools/bpf/bpftool/prog.c
>> @@ -430,6 +430,10 @@ static int do_dump(int argc, char **argv)
>> unsigned char *buf;
>> __u32 *member_len;
>> __u64 *member_ptr;
>> +unsigned int nr_addrs;
>> +unsigned long *addrs = NULL;
>> +__u32 *ksyms_len;
>> +__u64 *ksyms_ptr;
>
> nit: please try to keep the variables ordered longest to shortest like
> we do in networking code (please do it in all functions).
>
>> ssize_t n;
>> int err;
>> int fd;
>> @@ -437,6 +441,8 @@ static int do_dump(int argc, char **argv)
>> if (is_prefix(*argv, "jited")) {
>> member_len = _prog_len;
>> member_ptr = _prog_insns;
>> +ksyms_len = _jited_ksyms;
>> +ksyms_ptr = _ksyms;
>> } else if (is_prefix(*argv, "xlated")) {
>> member_len = _prog_len;
>> member_ptr = _prog_insns;
>> @@ -496,10 +502,23 @@ static int do_dump(int argc, char **argv)
>> return -1;
>> }
>>
>> +nr_addrs = *ksyms_len;
>
> Here and ...
>
>> +if (nr_addrs) {
>> +addrs = malloc(nr_addrs * sizeof(__u64));
>> +if (!addrs) {
>> +p_err("mem alloc failed");
>> +free(buf);
>> +close(fd);
>> +return -1;
>
> You can just jump to err_free here.
>
>> +}
>> +}
>> +
>> memset(, 0, sizeof(info));
>>
>> *member_ptr = ptr_to_u64(buf);
>> *member_len = buf_size;
>> +*ksyms_ptr = ptr_to_u64(addrs);
>> +*ksyms_len = nr_addrs;
>
> ... here - this function is getting long, so maybe I'm not seeing
> something, but are ksyms_ptr and ksyms_len guaranteed to be initialized?
>
>> err = bpf_obj_get_info_by_fd(fd, , );
>> close(fd);
>> @@ -513,6 +532,11 @@ static int do_dump(int argc, char **argv)
>> goto err_free;
>> }
>>
>> +if (*ksyms_len > nr_addrs) {
>> +p_err("too many addresses returned");
>> +goto err_free;
>> +}
>> +
>> if ((member_len == _prog_len &&
>> info.jited_prog_insns == 0) ||
>> (member_len == _prog_len &&
>> @@ -558,6 +582,9 @@ static int do_dump(int argc, char **argv)
>> dump_xlated_cfg(buf, *member_len);
>> } else {
>> kernel_syms_load();
>> +dd.jited_ksyms = ksyms_ptr;
>> +dd.nr_jited_ksyms = *ksyms_len;
>> +
>> if (json_output)
>> dump_xlated_json(, buf, *member_len, opcodes);
>> else
>> @@ -566,10 +593,14 @@ static int do_dump(int argc, char **argv)
>> }
>>
>> free(buf);
>> +if (addrs)
>> +free(addrs);
>
> Free can deal with NULL pointers, no need for an if.
>
>> return 0;
>>
>> err_free:
>> free(buf);
>> +if (addrs)
>> +free(addrs)
[PATCH bpf 2/6] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls in the powerpc64
JIT compiler. The JIT compiler converts the bpf call instructions
to native branch instructions. After a round of the usual passes,
the start addresses of the JITed images for the callee functions
are known. Finally, to fixup the branch target addresses, we need
to perform an extra pass.
Because of the address range in which JITed images are allocated
on powerpc64, the offsets of the start addresses of these images
from __bpf_call_base are as large as 64 bits. So, for a function
call, we cannot use the imm field of the instruction to determine
the callee's address. Instead, we use the alternative method of
getting it from the list of function addresses in the auxillary
data of the caller by using the off field as an index.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 79 ++-
1 file changed, 69 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 1bdb1aff0619..25939892d8f7 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -256,7 +256,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -712,11 +712,23 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+
+ /* bpf function call */
+ if (insn[i].src_reg == BPF_PSEUDO_CALL && extra_pass)
+ if (fp->aux->func && off < fp->aux->func_cnt)
+ /* use the subprog id from the off
+* field to lookup the callee address
+*/
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ /* kernel helper call */
+ else
+ func = (u8 *) __bpf_call_base + imm;
bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -864,6 +876,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -871,6 +891,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -878,6 +899,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -891,7 +913,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
fp = org_fp;
@@ -904,10 +947,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
/* Scouting faux-generat
[PATCH bpf 1/6] bpf: support 64-bit offsets for bpf function calls
The imm field of a bpf instruction is a signed 32-bit integer.
For JIT bpf-to-bpf function calls, it stores the offset of the
start address of the callee's JITed image from __bpf_call_base.
For some architectures, such as powerpc64, this offset may be
as large as 64 bits and cannot be accomodated in the imm field
without truncation.
We resolve this by:
[1] Additionally using the auxillary data of each function to
keep a list of start addresses of the JITed images for all
functions determined by the verifier.
[2] Retaining the subprog id inside the off field of the call
instructions and using it to index into the list mentioned
above and lookup the callee's address.
To make sure that the existing JIT compilers continue to work
without requiring changes, we keep the imm field as it is.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/verifier.c | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index d5e1a6c4165d..aa76879f4fd1 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5373,11 +5373,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* we use the aux data to keep a list of the start addresses
+* of the JITed images for each function in the program
+*
+* for some architectures, such as powerpc64, the imm field
+* might not be large enough to hold the offset of the start
+* address of the callee's JITed image from __bpf_call_base
+*
+* in such cases, we can lookup the start address of a callee
+* by using its subprog id, available from the off field of
+* the call instruction, as an index for this list
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt + 1;
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
[PATCH bpf 4/6] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the two new fields for passing on the
addresses of the kernel symbols corresponding to each
function in a JITed program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 83a95ae388dd..c14a74eea910 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2107,6 +2107,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_ksyms;
+ __u32 nr_jited_ksyms;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
--
2.14.3
[PATCH bpf 6/6] bpf: fix JITed dump for multi-function programs via syscall
Currently, for multi-function programs, we cannot get the JITed
instructions using the bpf system call's BPF_OBJ_GET_INFO_BY_FD
command. Because of this, userspace tools such as bpftool fail
to identify a multi-function program as being JITed or not.
With the JIT enabled and the test program running, this can be
verified as follows:
# cat /proc/sys/net/core/bpf_jit_enable
1
Before applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T11:43:38+0530 uid 0
xlated 216B not jited memlock 65536B
...
# bpftool prog dump jited id 1
no instructions returned
After applying this patch:
# bpftool prog list
1: kprobe name foo tag b811aab41a39ad3d gpl
loaded_at 2018-05-16T12:13:01+0530 uid 0
xlated 216B jited 308B memlock 65536B
...
# bpftool prog dump jited id 1
0: nop
4: nop
8: mflrr0
c: std r0,16(r1)
10: stdur1,-112(r1)
14: std r31,104(r1)
18: addir31,r1,48
1c: li r3,10
...
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/syscall.c | 38 --
1 file changed, 32 insertions(+), 6 deletions(-)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 03c8437a2990..b2f70718aca7 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1896,7 +1896,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
struct bpf_prog_info info = {};
u32 info_len = attr->info.info_len;
char __user *uinsns;
- u32 ulen;
+ u32 ulen, i;
int err;
err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
@@ -1922,7 +1922,6 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
ulen = min_t(u32, info.nr_map_ids, ulen);
if (ulen) {
u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
- u32 i;
for (i = 0; i < ulen; i++)
if (put_user(prog->aux->used_maps[i]->id,
@@ -1970,13 +1969,41 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog
*prog,
* for offload.
*/
ulen = info.jited_prog_len;
- info.jited_prog_len = prog->jited_len;
+ if (prog->aux->func_cnt) {
+ info.jited_prog_len = 0;
+ for (i = 0; i < prog->aux->func_cnt; i++)
+ info.jited_prog_len += prog->aux->func[i]->jited_len;
+ } else {
+ info.jited_prog_len = prog->jited_len;
+ }
+
if (info.jited_prog_len && ulen) {
if (bpf_dump_raw_ok()) {
uinsns = u64_to_user_ptr(info.jited_prog_insns);
ulen = min_t(u32, info.jited_prog_len, ulen);
- if (copy_to_user(uinsns, prog->bpf_func, ulen))
- return -EFAULT;
+
+ /* for multi-function programs, copy the JITed
+* instructions for all the functions
+*/
+ if (prog->aux->func_cnt) {
+ u32 len, free;
+ u8 *img;
+
+ free = ulen;
+ for (i = 0; i < prog->aux->func_cnt; i++) {
+ len = prog->aux->func[i]->jited_len;
+ img = (u8 *)
prog->aux->func[i]->bpf_func;
+ if (len > free)
+ break;
+ if (copy_to_user(uinsns, img, len))
+ return -EFAULT;
+ uinsns += len;
+ free -= len;
+ }
+ } else {
+ if (copy_to_user(uinsns, prog->bpf_func, ulen))
+ return -EFAULT;
+ }
} else {
info.jited_prog_insns = 0;
}
@@ -1987,7 +2014,6 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (info.nr_jited_ksyms && ulen) {
u64 __user *user_jited_ksyms =
u64_to_user_ptr(info.jited_ksyms);
ulong ksym_addr;
- u32 i;
/* copy the address of the kernel symbol corresponding to
* each function
--
2.14.3
[PATCH bpf 3/6] bpf: get kernel symbol addresses via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of kernel symbol addresses for all functions in a
given program and to userspace using the bpf system call
with the BPF_OBJ_GET_INFO_BY_FD command.
When bpf_jit_kallsyms is enabled, we can get the address
of the corresponding kernel symbol for a callee function
and resolve the symbol's name. The address is determined
by adding the value of the call instruction's imm field
to __bpf_call_base. This offset gets assigned to the imm
field by the verifier.
For some architectures, such as powerpc64, the imm field
is not large enough to hold this offset.
We resolve this by:
[1] Assigning the subprog id to the imm field of a call
instruction in the verifier instead of the offset of
the callee's symbol's address from __bpf_call_base.
[2] Determining the address of a callee's corresponding
symbol by using the imm field as an index for the
list of kernel symbol addresses now available from
the program info.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 20
kernel/bpf/verifier.c| 7 +--
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 93d5a4eeec2a..061482d3be11 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2108,6 +2108,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_ksyms;
+ __u32 nr_jited_ksyms;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index c286e75ec087..03c8437a2990 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1933,6 +1933,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
+ info.nr_jited_ksyms = 0;
goto done;
}
@@ -1981,6 +1982,25 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_ksyms;
+ info.nr_jited_ksyms = prog->aux->func_cnt;
+ if (info.nr_jited_ksyms && ulen) {
+ u64 __user *user_jited_ksyms =
u64_to_user_ptr(info.jited_ksyms);
+ ulong ksym_addr;
+ u32 i;
+
+ /* copy the address of the kernel symbol corresponding to
+* each function
+*/
+ ulen = min_t(u32, info.nr_jited_ksyms, ulen);
+ for (i = 0; i < ulen; i++) {
+ ksym_addr = (ulong) prog->aux->func[i]->bpf_func;
+ ksym_addr &= PAGE_MASK;
+ if (put_user((u64) ksym_addr, _jited_ksyms[i]))
+ return -EFAULT;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index aa76879f4fd1..fc864eb3e29d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5416,17 +5416,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* later look the same as if they were interpreted only.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
- unsigned long addr;
-
if (insn->code != (BPF_JMP | BPF_CALL) ||
insn->src_reg != BPF_PSEUDO_CALL)
continue;
insn->off = env->insn_aux_data[i].call_imm;
subprog = find_subprog(env, i + insn->off + 1);
- addr = (unsigned long)func[subprog]->bpf_func;
- addr &= PAGE_MASK;
- insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
- addr - __bpf_call_base;
+ insn->imm = subprog;
}
prog->jited = 1;
--
2.14.3
[PATCH bpf 0/6] bpf: enhancements for multi-function programs
This patch series introduces the following: [1] Support for bpf-to-bpf function calls in the powerpc64 JIT compiler. [2] Provide a way for resolving function calls because of the way JITed images are allocated in powerpc64. [3] Fix to get JITed instruction dumps for multi-function programs from the bpf system call. Sandipan Das (6): bpf: support 64-bit offsets for bpf function calls bpf: powerpc64: add JIT support for multi-function programs bpf: get kernel symbol addresses via syscall tools: bpf: sync bpf uapi header tools: bpftool: resolve calls without using imm field bpf: fix JITed dump for multi-function programs via syscall arch/powerpc/net/bpf_jit_comp64.c | 79 ++- include/uapi/linux/bpf.h | 2 + kernel/bpf/syscall.c | 56 --- kernel/bpf/verifier.c | 22 +++ tools/bpf/bpftool/prog.c | 31 +++ tools/bpf/bpftool/xlated_dumper.c | 24 tools/bpf/bpftool/xlated_dumper.h | 2 + tools/include/uapi/linux/bpf.h| 2 + 8 files changed, 189 insertions(+), 29 deletions(-) -- 2.14.3
[PATCH bpf 5/6] tools: bpftool: resolve calls without using imm field
Currently, we resolve the callee's address for a JITed function
call by using the imm field of the call instruction as an offset
from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
use this address to get the callee's kernel symbol's name.
For some architectures, such as powerpc64, the imm field is not
large enough to hold this offset. So, instead of assigning this
offset to the imm field, the verifier now assigns the subprog
id. Also, a list of kernel symbol addresses for all the JITed
functions is provided in the program info. We now use the imm
field as an index for this list to lookup a callee's symbol's
address and resolve its name.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/bpf/bpftool/prog.c | 31 +++
tools/bpf/bpftool/xlated_dumper.c | 24 +---
tools/bpf/bpftool/xlated_dumper.h | 2 ++
3 files changed, 50 insertions(+), 7 deletions(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 9bdfdf2d3fbe..ac2f62a97e84 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -430,6 +430,10 @@ static int do_dump(int argc, char **argv)
unsigned char *buf;
__u32 *member_len;
__u64 *member_ptr;
+ unsigned int nr_addrs;
+ unsigned long *addrs = NULL;
+ __u32 *ksyms_len;
+ __u64 *ksyms_ptr;
ssize_t n;
int err;
int fd;
@@ -437,6 +441,8 @@ static int do_dump(int argc, char **argv)
if (is_prefix(*argv, "jited")) {
member_len = _prog_len;
member_ptr = _prog_insns;
+ ksyms_len = _jited_ksyms;
+ ksyms_ptr = _ksyms;
} else if (is_prefix(*argv, "xlated")) {
member_len = _prog_len;
member_ptr = _prog_insns;
@@ -496,10 +502,23 @@ static int do_dump(int argc, char **argv)
return -1;
}
+ nr_addrs = *ksyms_len;
+ if (nr_addrs) {
+ addrs = malloc(nr_addrs * sizeof(__u64));
+ if (!addrs) {
+ p_err("mem alloc failed");
+ free(buf);
+ close(fd);
+ return -1;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
+ *ksyms_ptr = ptr_to_u64(addrs);
+ *ksyms_len = nr_addrs;
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
@@ -513,6 +532,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (*ksyms_len > nr_addrs) {
+ p_err("too many addresses returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -558,6 +582,9 @@ static int do_dump(int argc, char **argv)
dump_xlated_cfg(buf, *member_len);
} else {
kernel_syms_load();
+ dd.jited_ksyms = ksyms_ptr;
+ dd.nr_jited_ksyms = *ksyms_len;
+
if (json_output)
dump_xlated_json(, buf, *member_len, opcodes);
else
@@ -566,10 +593,14 @@ static int do_dump(int argc, char **argv)
}
free(buf);
+ if (addrs)
+ free(addrs);
return 0;
err_free:
free(buf);
+ if (addrs)
+ free(addrs);
return -1;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c
b/tools/bpf/bpftool/xlated_dumper.c
index 7a3173b76c16..dc8e4eca0387 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -178,8 +178,12 @@ static const char *print_call_pcrel(struct dump_data *dd,
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"%+d#%s", insn->off, sym->name);
else
- snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
-"%+d#0x%lx", insn->off, address);
+ if (address)
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+"%+d#0x%lx", insn->off, address);
+ else
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+"%+d", insn->off);
return dd->scratch_buff;
}
@@ -200,14 +204,20 @@ static const char *print_call(void *private_data,
const struct bpf_insn *insn)
{
struct dump_data *dd = private_data;
- unsigned long address = dd->address_call_base + insn->imm;
- struct kernel_sym *sym;
+ unsigned long address = 0;
+ struct kern
[RFC][PATCH bpf v3 3/5] bpf: get JITed function addresses via syscall
This adds new two new fields to struct bpf_prog_info. For
multi-function programs, these fields can be used to pass
a list of kernel symbol addresses for all functions in a
given program and to userspace using the bpf system call
with the BPF_OBJ_GET_INFO_BY_FD command.
When bpf_jit_kallsyms is enabled, we can get the address
of the corresponding kernel symbol for a callee function
and resolve the symbol's name. The address is determined
by adding the value of the call instruction's imm field
to __bpf_call_base. This offset gets assigned to the imm
field by the verifier.
For some architectures, such as powerpc64, the imm field
is not large enough to hold this offset.
We resolve this by:
[1] Assigning the subprog id to the imm field of a call
instruction in the verifier instead of the offset of
the callee's symbol's address from __bpf_call_base.
[2] Determining the address of a callee's corresponding
symbol by using the imm field as an index for the
list of function addresses now available from the
program info.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/uapi/linux/bpf.h | 2 ++
kernel/bpf/syscall.c | 18 ++
kernel/bpf/verifier.c| 7 +--
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 93d5a4eeec2a..b9c9c7690337 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2108,6 +2108,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_funcs;
+ __u32 nr_jited_funcs;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 9b87198deea2..600da849233d 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1924,6 +1924,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
+ info.nr_jited_funcs = 0;
goto done;
}
@@ -1972,6 +1973,23 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
+ ulen = info.nr_jited_funcs;
+ info.nr_jited_funcs = prog->aux->func_cnt;
+ if (info.nr_jited_funcs && ulen) {
+ u64 __user *user_jited_funcs =
u64_to_user_ptr(info.jited_funcs);
+ u32 i;
+
+ /* copy the start addresses of the JITed images for all
+* functions
+*/
+ ulen = min_t(u32, info.nr_jited_funcs, ulen);
+ for (i = 0; i < ulen; i++) {
+ if (put_user((u64) prog->aux->func[i]->bpf_func &
PAGE_MASK,
+ _jited_funcs[i]))
+ return -EFAULT;
+ }
+ }
+
done:
if (copy_to_user(uinfo, , info_len) ||
put_user(info_len, >info.info_len))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index aa76879f4fd1..fc864eb3e29d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5416,17 +5416,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* later look the same as if they were interpreted only.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
- unsigned long addr;
-
if (insn->code != (BPF_JMP | BPF_CALL) ||
insn->src_reg != BPF_PSEUDO_CALL)
continue;
insn->off = env->insn_aux_data[i].call_imm;
subprog = find_subprog(env, i + insn->off + 1);
- addr = (unsigned long)func[subprog]->bpf_func;
- addr &= PAGE_MASK;
- insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
- addr - __bpf_call_base;
+ insn->imm = subprog;
}
prog->jited = 1;
--
2.14.3
[RFC][PATCH bpf v3 2/5] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls in the powerpc64
JIT compiler. The JIT compiler converts the bpf call instructions
to native branch instructions. After a round of the usual passes,
the start addresses of the JITed images for the callee functions
are known. Finally, to fixup the branch target addresses, we need
to perform an extra pass.
Because of the address range in which JITed images are allocated
on powerpc64, the offsets of the start addresses of these images
from __bpf_call_base are as large as 64 bits. So, for a function
call, we cannot use the imm field of the instruction to determine
the callee's address. Instead, we use the alternative method of
getting it from the list of function addresses in the auxillary
data of the caller by using the off field as an index.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3: Mention briefly about why the offsets are large.
v2: Use the off field of the instruction as an index for
aux->func to determine the start address of a callee
function.
---
arch/powerpc/net/bpf_jit_comp64.c | 79 ++-
1 file changed, 69 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 1bdb1aff0619..25939892d8f7 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -256,7 +256,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -712,11 +712,23 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+
+ /* bpf function call */
+ if (insn[i].src_reg == BPF_PSEUDO_CALL && extra_pass)
+ if (fp->aux->func && off < fp->aux->func_cnt)
+ /* use the subprog id from the off
+* field to lookup the callee address
+*/
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ /* kernel helper call */
+ else
+ func = (u8 *) __bpf_call_base + imm;
bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -864,6 +876,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -871,6 +891,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -878,6 +899,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -891,7 +913,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
[RFC][PATCH bpf v3 4/5] tools: bpf: sync bpf uapi header
Syncing the bpf.h uapi header with tools so that struct
bpf_prog_info has the new fields for storing the kernel
symbol addresses for the JITed functions in a program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/include/uapi/linux/bpf.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 83a95ae388dd..98857dc3469a 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -2107,6 +2107,8 @@ struct bpf_prog_info {
__u32 xlated_prog_len;
__aligned_u64 jited_prog_insns;
__aligned_u64 xlated_prog_insns;
+ __aligned_u64 jited_funcs;
+ __u32 nr_jited_funcs;
__u64 load_time;/* ns since boottime */
__u32 created_by_uid;
__u32 nr_map_ids;
--
2.14.3
[RFC][PATCH bpf v3 1/5] bpf: allow 64-bit offsets for bpf function calls
The imm field of a bpf instruction is a signed 32-bit integer.
For JIT bpf-to-bpf function calls, it stores the offset of the
start address of the callee's JITed image from __bpf_call_base.
For some architectures, such as powerpc64, this offset may be
as large as 64 bits and cannot be accomodated in the imm field
without truncation.
We resolve this by:
[1] Additionally using the auxillary data of each function to
keep a list of start addresses of the JITed images for all
functions determined by the verifier.
[2] Retaining the subprog id inside the off field of the call
instructions and using it to index into the list mentioned
above and lookup the callee's address.
To make sure that the existing JIT compilers continue to work
without requiring changes, we keep the imm field as it is.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v3: Rephrase commit message to explain solution in a point-
wise manner.
v2: Make aux->func point to the list of functions determined
by the verifier rather than allocating a separate callee
list for each function.
---
kernel/bpf/verifier.c | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index d5e1a6c4165d..aa76879f4fd1 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5373,11 +5373,24 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* we use the aux data to keep a list of the start addresses
+* of the JITed images for each function in the program
+*
+* for some architectures, such as powerpc64, the imm field
+* might not be large enough to hold the offset of the start
+* address of the callee's JITed image from __bpf_call_base
+*
+* in such cases, we can lookup the start address of a callee
+* by using its subprog id, available from the off field of
+* the call instruction, as an index for this list
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt + 1;
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
[RFC][PATCH bpf v3 5/5] tools: bpftool: resolve call addresses without using imm field
Currently, we resolve the callee's address for a JITed function
call by using the imm field of the call instruction as an offset
from __bpf_call_base. If bpf_jit_kallsyms is enabled, we further
use this address to get the callee's kernel symbol's name.
For some architectures, such as powerpc64, the imm field is not
large enough to hold this offset. So, instead of assigning this
offset to the imm field, the verifier now assigns the subprog
id. Also, a list of kernel symbol addresses for all the JITed
functions is provided in the program info. We now use the imm
field as an index for this list to lookup a callee's symbol's
address and resolve its name.
Suggested-by: Daniel Borkmann <dan...@iogearbox.net>
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/bpf/bpftool/prog.c | 33 +
tools/bpf/bpftool/xlated_dumper.c | 24 +---
tools/bpf/bpftool/xlated_dumper.h | 2 ++
3 files changed, 52 insertions(+), 7 deletions(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 9bdfdf2d3fbe..0ba947c7deec 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -428,6 +428,8 @@ static int do_dump(int argc, char **argv)
bool opcodes = false;
bool visual = false;
unsigned char *buf;
+ __u64 *addrs = NULL;
+ __u32 nr_addrs = 0;
__u32 *member_len;
__u64 *member_ptr;
ssize_t n;
@@ -496,11 +498,27 @@ static int do_dump(int argc, char **argv)
return -1;
}
+ if (info.nr_jited_funcs) {
+ nr_addrs = info.nr_jited_funcs;
+ addrs = (__u64 *) malloc(nr_addrs * sizeof(__u64));
+ if (!addrs) {
+ p_err("mem alloc failed");
+ free(buf);
+ close(fd);
+ return -1;
+ }
+ }
+
memset(, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
*member_len = buf_size;
+ if (nr_addrs) {
+ info.jited_funcs = ptr_to_u64(addrs);
+ info.nr_jited_funcs = nr_addrs;
+ }
+
err = bpf_obj_get_info_by_fd(fd, , );
close(fd);
if (err) {
@@ -513,6 +531,11 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (info.nr_jited_funcs > nr_addrs) {
+ p_err("too many addresses returned");
+ goto err_free;
+ }
+
if ((member_len == _prog_len &&
info.jited_prog_insns == 0) ||
(member_len == _prog_len &&
@@ -558,6 +581,12 @@ static int do_dump(int argc, char **argv)
dump_xlated_cfg(buf, *member_len);
} else {
kernel_syms_load();
+
+ if (info.nr_jited_funcs) {
+ dd.jited_funcs = (u64 *) info.jited_funcs;
+ dd.nr_jited_funcs = info.nr_jited_funcs;
+ }
+
if (json_output)
dump_xlated_json(, buf, *member_len, opcodes);
else
@@ -566,10 +595,14 @@ static int do_dump(int argc, char **argv)
}
free(buf);
+ if (nr_addrs)
+ free(addrs);
return 0;
err_free:
free(buf);
+ if (nr_addrs)
+ free(addrs);
return -1;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c
b/tools/bpf/bpftool/xlated_dumper.c
index 7a3173b76c16..70943612b667 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -178,8 +178,12 @@ static const char *print_call_pcrel(struct dump_data *dd,
snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
"%+d#%s", insn->off, sym->name);
else
- snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
-"%+d#0x%lx", insn->off, address);
+ if (address)
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+"%+d#0x%lx", insn->off, address);
+ else
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+"%+d", insn->off);
return dd->scratch_buff;
}
@@ -200,14 +204,20 @@ static const char *print_call(void *private_data,
const struct bpf_insn *insn)
{
struct dump_data *dd = private_data;
- unsigned long address = dd->address_call_base + insn->imm;
- struct kernel_sym *sym;
+ unsigned long address = 0;
+ struct kernel_sym *sym = NULL;
- sym = kernel_syms_search(dd, address);
- if (insn->src_reg == BPF_PSEUDO_CALL)
+ if (insn->src_reg == BPF_PSEUDO_CALL) {
+ if
[RFC][PATCH bpf] tools: bpftool: Fix tags for bpf-to-bpf calls
"Naveen N. Rao" wrote:
> I'm wondering if we can instead encode the bpf prog id in
> imm32. That way, we should be able to indicate the BPF
> function being called into. Daniel, is that something we
> can consider?
Since each subprog does not get a separate id, we cannot fetch
the fd and therefore the tag of a subprog. Instead we can use
the tag of the complete program as shown below.
"Daniel Borkmann" wrote:
> I think one limitation that would still need to be addressed later
> with such approach would be regarding the xlated prog dump in bpftool,
> see 'BPF calls via JIT' in 7105e828c087 ("bpf: allow for correlation
> of maps and helpers in dump"). Any ideas for this (potentially if we
> could use off + imm for calls, we'd get to 48 bits, but that seems
> still not be enough as you say)?
As an alternative, this is what I am thinking of:
Currently, for bpf-to-bpf calls, if bpf_jit_kallsyms is enabled,
bpftool looks up the name of the corresponding symbol for the
JIT-ed subprogram and shows it in the xlated dump alongside the
actual call instruction. However, the lookup is based on the
target address which is calculated using the imm field of the
instruction. So, once again, if imm is truncated, we will end
up with the wrong address. Also, the subprog aux data (which
has been proposed as a mitigation for this) is not accessible
from this tool.
We can still access the tag for the complete bpf program and use
this with the correct offset in an objdump-like notation as an
alterative for the name of the subprog that is the target of a
bpf-to-bpf call instruction.
Currently, an xlated dump looks like this:
0: (85) call pc+2#bpf_prog_5f76847930402518_F
1: (b7) r0 = 1
2: (95) exit
3: (b7) r0 = 2
4: (95) exit
With this patch, it will look like this:
0: (85) call pc+2#bpf_prog_8f85936f29a7790a+3
1: (b7) r0 = 1
2: (95) exit
3: (b7) r0 = 2
4: (95) exit
where 8f85936f29a7790a is the tag of the bpf program and 3 is
the offset to the start of the subprog from the start of the
program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
tools/bpf/bpftool/prog.c | 23 ++-
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index e8e2baaf93c2..93746d5d1e3c 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -415,9 +415,11 @@ struct kernel_sym {
};
struct dump_data {
+ unsigned char prog_tag[BPF_TAG_SIZE];
unsigned long address_call_base;
struct kernel_sym *sym_mapping;
__u32 sym_count;
+ unsigned int curr_line;
char scratch_buff[SYM_MAX_NAME];
};
@@ -499,16 +501,16 @@ static void print_insn(struct bpf_verifier_env *env,
const char *fmt, ...)
}
static const char *print_call_pcrel(struct dump_data *dd,
- struct kernel_sym *sym,
- unsigned long address,
const struct bpf_insn *insn)
{
- if (sym)
- snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
-"%+d#%s", insn->off, sym->name);
- else
- snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
-"%+d#0x%lx", insn->off, address);
+ snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
+"+%d#bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x+%d",
+insn->off,
+dd->prog_tag[0], dd->prog_tag[1],
+dd->prog_tag[2], dd->prog_tag[3],
+dd->prog_tag[4], dd->prog_tag[5],
+dd->prog_tag[6], dd->prog_tag[7],
+dd->curr_line + insn->off + 1);
return dd->scratch_buff;
}
@@ -534,7 +536,7 @@ static const char *print_call(void *private_data,
sym = kernel_syms_search(dd, address);
if (insn->src_reg == BPF_PSEUDO_CALL)
- return print_call_pcrel(dd, sym, address, insn);
+ return print_call_pcrel(dd, insn);
else
return print_call_helper(dd, sym, address);
}
@@ -576,6 +578,7 @@ static void dump_xlated_plain(struct dump_data *dd, void
*buf,
double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW);
printf("% 4d: ", i);
+ dd->curr_line = i;
print_bpf_insn(, NULL, insn + i, true);
if (opcodes) {
@@ -628,6 +631,7 @@ static void dump_xlated_json(struct dump_data *dd, void
*buf,
jsonw_start_object(json_wtr);
jsonw_name(json_wtr, "disasm");
+ dd->curr_line = i;
print_bpf_insn(, NULL, insn + i, true);
if (opcodes) {
@@ -788,6 +792,7 @@ static int do_dump(int arg
[RFC][PATCH bpf v2 1/2] bpf: allow 64-bit offsets for bpf function calls
The imm field of a bpf_insn is a signed 32-bit integer. For
JIT-ed bpf-to-bpf function calls, it stores the offset from
__bpf_call_base to the start of the callee function.
For some architectures, such as powerpc64, it was found that
this offset may be as large as 64 bits because of which this
cannot be accomodated in the imm field without truncation.
To resolve this, we additionally make aux->func within each
bpf_prog associated with the functions to point to the list
of all function addresses determined by the verifier.
We keep the value assigned to the off field of the bpf_insn
as a way to index into aux->func and also set aux->func_cnt
so that this can be used for performing basic upper bound
checks for the off field.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v2: Make aux->func point to the list of functions determined
by the verifier rather than allocating a separate callee
list for each function.
---
kernel/bpf/verifier.c | 16 +++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5fb69a85d967..1c4d9cd485ed 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5288,11 +5288,25 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->src_reg != BPF_PSEUDO_CALL)
continue;
subprog = insn->off;
- insn->off = 0;
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
}
+
+ /* the offset to a callee function from __bpf_call_base
+* may be larger than what the 32 bit integer imm can
+* accomodate which will truncate the higher order bits
+*
+* to avoid this, we additionally utilize the aux data
+* of each function to point to a list of all function
+* addresses determined by the verifier
+*
+* the off field of the instruction provides the index
+* in this list where the start address of a function
+* is available
+*/
+ func[i]->aux->func = func;
+ func[i]->aux->func_cnt = env->subprog_cnt + 1;
}
for (i = 0; i <= env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
--
2.14.3
[RFC][PATCH bpf v2 2/2] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls for the powerpc64
JIT compiler. After a round of the usual JIT passes, the offsets
to callee functions from __bpf_call_base are known. To update the
target addresses for the branch instructions associated with each
BPF_CALL, an extra pass is performed.
Since it is seen that the offsets may be as large as 64 bits for
powerpc64, we use the aux data associated with each caller to get
the correct branch target address rather than using the imm field
of the BPF_CALL instruction.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
v2: Use the off field of the instruction as an index for
aux->func to determine the start address of a callee
function.
---
arch/powerpc/net/bpf_jit_comp64.c | 73 +--
1 file changed, 63 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 0a34b0cec7b7..cf0d4e32aa52 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -290,7 +290,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -746,11 +746,17 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+ if (insn[i].src_reg == BPF_PSEUDO_CALL && extra_pass)
+ if (fp->aux->func && off < fp->aux->func_cnt)
+ func = (u8 *)
fp->aux->func[off]->bpf_func;
+ else
+ return -EINVAL;
+ else
+ func = (u8 *) __bpf_call_base + imm;
/* Save skb pointer if we need to re-cache skb data */
if ((ctx->seen & SEEN_SKB) &&
@@ -970,6 +976,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -977,6 +991,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -984,6 +999,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -997,7 +1013,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
fp = org_fp;
@@ -1010,10 +1047,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog
*fp)
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
/* Scouting faux-generate pass 0 */
- if (bpf_jit_build_body(fp, 0, , addrs)) {
+ if (bpf_jit_build_body(fp, 0, , addrs, false)) {
/* We hit something illegal or unsupported. */
fp = org_fp;
- goto out;
+ goto out_addrs;
}
/*
@@ -1031,9 +1068,10 @@ struct bpf_prog *bpf_int_jit_comp
Re: [RFC][PATCH bpf 1/2] bpf: allow 64-bit offsets for bpf function calls
On 02/10/2018 06:08 AM, Alexei Starovoitov wrote:
> On 2/9/18 8:54 AM, Naveen N. Rao wrote:
>> Naveen N. Rao wrote:
>>> Alexei Starovoitov wrote:
>>>> On 2/8/18 4:03 AM, Sandipan Das wrote:
>>>>> The imm field of a bpf_insn is a signed 32-bit integer. For
>>>>> JIT-ed bpf-to-bpf function calls, it stores the offset from
>>>>> __bpf_call_base to the start of the callee function.
>>>>>
>>>>> For some architectures, such as powerpc64, it was found that
>>>>> this offset may be as large as 64 bits because of which this
>>>>> cannot be accomodated in the imm field without truncation.
>>>>>
>>>>> To resolve this, we additionally use the aux data within each
>>>>> bpf_prog associated with the caller functions to store the
>>>>> addresses of their respective callees.
>>>>>
>>>>> Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
>>>>> ---
>>>>> kernel/bpf/verifier.c | 39 ++-
>>>>> 1 file changed, 38 insertions(+), 1 deletion(-)
>>>>>
>>>>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>>>>> index 5fb69a85d967..52088b4ca02f 100644
>>>>> --- a/kernel/bpf/verifier.c
>>>>> +++ b/kernel/bpf/verifier.c
>>>>> @@ -5282,6 +5282,19 @@ static int jit_subprogs(struct
>>>>> bpf_verifier_env *env)
>>>>> * run last pass of JIT
>>>>> */
>>>>> for (i = 0; i <= env->subprog_cnt; i++) {
>>>>> + u32 flen = func[i]->len, callee_cnt = 0;
>>>>> + struct bpf_prog **callee;
>>>>> +
>>>>> + /* for now assume that the maximum number of bpf function
>>>>> + * calls that can be made by a caller must be at most the
>>>>> + * number of bpf instructions in that function
>>>>> + */
>>>>> + callee = kzalloc(sizeof(func[i]) * flen, GFP_KERNEL);
>>>>> + if (!callee) {
>>>>> + err = -ENOMEM;
>>>>> + goto out_free;
>>>>> + }
>>>>> +
>>>>> insn = func[i]->insnsi;
>>>>> for (j = 0; j < func[i]->len; j++, insn++) {
>>>>> if (insn->code != (BPF_JMP | BPF_CALL) ||
>>>>> @@ -5292,6 +5305,26 @@ static int jit_subprogs(struct
>>>>> bpf_verifier_env *env)
>>>>> insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
>>>>> func[subprog]->bpf_func -
>>>>> __bpf_call_base;
>>>>> +
>>>>> + /* the offset to the callee from __bpf_call_base
>>>>> + * may be larger than what the 32 bit integer imm
>>>>> + * can accomodate which will truncate the higher
>>>>> + * order bits
>>>>> + *
>>>>> + * to avoid this, we additionally utilize the aux
>>>>> + * data of each caller function for storing the
>>>>> + * addresses of every callee associated with it
>>>>> + */
>>>>> + callee[callee_cnt++] = func[subprog];
>>>>
>>>> can you share typical /proc/kallsyms ?
>>>> Are you saying that kernel and kernel modules are allocated from
>>>> address spaces that are always more than 32-bit apart?
>>>
>>> Yes. On ppc64, kernel text is linearly mapped from 0xc000,
>>> while vmalloc'ed area starts from 0xd000 (for radix, this is
>>> different, but still beyond a 32-bit offset).
>>>
>>>> That would mean that all kernel calls into modules are far calls
>>>> and the other way around form .ko into kernel?
>>>> Performance is probably suffering because every call needs to be built
>>>> with full 64-bit offset. No ?
>>>
>>> Possibly, and I think Michael can give a better perspective, but I think
>>> this is due to our ABI. For inter-module calls, we need to setup the TOC
>>> pointer (or the address of the function being called with ABIv2),
>>> which would require us to load a full address regardless.
>>
>> Thinking more about this, as an optimization, for bpf-to-bpf calls, we
>> could detect a near call and just emit a relative branch since we don't
>> care about TOC with BPF. But, this will depend on whether the different
>> BPF functions are close enough (within 32MB) of one another.
>
> so that will be just an optimization. Understood.
> How about instead of doing callee = kzalloc(sizeof(func[i]) * flen..
> we keep insn->off pointing to subprog and move
> prog->aux->func = func;
> before the last JIT pass.
> Then you won't need to alloc this extra array.
>
Agreed. Will make the necessary changes in v2.
--
With Regards,
Sandipan
[RFC][PATCH bpf 2/2] bpf: powerpc64: add JIT support for multi-function programs
This adds support for bpf-to-bpf function calls for the powerpc64
JIT compiler. After a round of the usual JIT passes, the offsets
to callee functions from __bpf_call_base are known. To update the
target addresses for the branch instructions associated with each
BPF_CALL, an extra pass is performed.
Since it is seen that the offsets may be as large as 64 bits for
powerpc64, we use the aux data associated with each caller to get
the correct branch target address rather than using the imm field
of the BPF_CALL instruction.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit_comp64.c | 77 ++-
1 file changed, 67 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 0a34b0cec7b7..f31f22c8cb0b 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -290,7 +290,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct
codegen_context *ctx, u32
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
- u32 *addrs)
+ u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
@@ -299,6 +299,10 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
/* Start of epilogue code - will only be valid 2nd pass onwards */
u32 exit_addr = addrs[flen];
+ /* List of callee functions - will only be valid for the extra pass */
+ struct bpf_prog **callee = fp->aux->func;
+ u32 callee_cnt = fp->aux->func_cnt, callee_idx = 0;
+
for (i = 0; i < flen; i++) {
u32 code = insn[i].code;
u32 dst_reg = b2p[insn[i].dst_reg];
@@ -746,11 +750,17 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
break;
/*
-* Call kernel helper
+* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
- func = (u8 *) __bpf_call_base + imm;
+ if (insn[i].src_reg == BPF_PSEUDO_CALL && extra_pass)
+ if (callee && callee_idx < callee_cnt)
+ func = (u8 *)
callee[callee_idx++]->bpf_func;
+ else
+ return -EINVAL;
+ else
+ func = (u8 *) __bpf_call_base + imm;
/* Save skb pointer if we need to re-cache skb data */
if ((ctx->seen & SEEN_SKB) &&
@@ -970,6 +980,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32
*image,
return 0;
}
+struct powerpc64_jit_data {
+ struct bpf_binary_header *header;
+ u32 *addrs;
+ u8 *image;
+ u32 proglen;
+ struct codegen_context ctx;
+};
+
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 proglen;
@@ -977,6 +995,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
u8 *image = NULL;
u32 *code_base;
u32 *addrs;
+ struct powerpc64_jit_data *jit_data;
struct codegen_context cgctx;
int pass;
int flen;
@@ -984,6 +1003,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *org_fp = fp;
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
+ bool extra_pass = false;
if (!fp->jit_requested)
return org_fp;
@@ -997,7 +1017,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
fp = tmp_fp;
}
+ jit_data = fp->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ fp = org_fp;
+ goto out;
+ }
+ fp->aux->jit_data = jit_data;
+ }
+
flen = fp->len;
+ addrs = jit_data->addrs;
+ if (addrs) {
+ cgctx = jit_data->ctx;
+ image = jit_data->image;
+ bpf_hdr = jit_data->header;
+ proglen = jit_data->proglen;
+ alloclen = proglen + FUNCTION_DESCR_SIZE;
+ extra_pass = true;
+ goto skip_init_ctx;
+ }
+
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
if (addrs == NULL) {
fp = org_fp;
@@ -1010,10 +1051,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog
*fp)
cgctx.stack_size = round_up(fp->aux->
[RFC][PATCH bpf 1/2] bpf: allow 64-bit offsets for bpf function calls
The imm field of a bpf_insn is a signed 32-bit integer. For
JIT-ed bpf-to-bpf function calls, it stores the offset from
__bpf_call_base to the start of the callee function.
For some architectures, such as powerpc64, it was found that
this offset may be as large as 64 bits because of which this
cannot be accomodated in the imm field without truncation.
To resolve this, we additionally use the aux data within each
bpf_prog associated with the caller functions to store the
addresses of their respective callees.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
kernel/bpf/verifier.c | 39 ++-
1 file changed, 38 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5fb69a85d967..52088b4ca02f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5282,6 +5282,19 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* run last pass of JIT
*/
for (i = 0; i <= env->subprog_cnt; i++) {
+ u32 flen = func[i]->len, callee_cnt = 0;
+ struct bpf_prog **callee;
+
+ /* for now assume that the maximum number of bpf function
+* calls that can be made by a caller must be at most the
+* number of bpf instructions in that function
+*/
+ callee = kzalloc(sizeof(func[i]) * flen, GFP_KERNEL);
+ if (!callee) {
+ err = -ENOMEM;
+ goto out_free;
+ }
+
insn = func[i]->insnsi;
for (j = 0; j < func[i]->len; j++, insn++) {
if (insn->code != (BPF_JMP | BPF_CALL) ||
@@ -5292,6 +5305,26 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
func[subprog]->bpf_func -
__bpf_call_base;
+
+ /* the offset to the callee from __bpf_call_base
+* may be larger than what the 32 bit integer imm
+* can accomodate which will truncate the higher
+* order bits
+*
+* to avoid this, we additionally utilize the aux
+* data of each caller function for storing the
+* addresses of every callee associated with it
+*/
+ callee[callee_cnt++] = func[subprog];
+ }
+
+ /* free up callee list if no function calls were made */
+ if (!callee_cnt) {
+ kfree(callee);
+ callee = NULL;
+ } else {
+ func[i]->aux->func = callee;
+ func[i]->aux->func_cnt = callee_cnt;
}
}
for (i = 0; i <= env->subprog_cnt; i++) {
@@ -5338,8 +5371,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
return 0;
out_free:
for (i = 0; i <= env->subprog_cnt; i++)
- if (func[i])
+ if (func[i]) {
+ /* cleanup callee list */
+ if (func[i]->aux->func)
+ kfree(func[i]->aux->func);
bpf_jit_free(func[i]);
+ }
kfree(func);
/* cleanup main prog to be interpreted */
prog->jit_requested = 0;
--
2.14.3
Re: [RFC PATCH] bpf: Add helpers to read useful task_struct members
Hi Alexei, Naveen, On 11/04/2017 11:01 PM, Naveen N. Rao wrote: > > I think the offsets described in dwarf were incorrect with > CONFIG_GCC_PLUGIN_RANDSTRUCT, but I'll let Sandipan confirm that. > I think that the offsets described in dwarf are probably incorrect when CONFIG_GCC_PLUGIN_RANDSTRUCT is enabled. To verify this, I used perf to attach a probe to try_to_wake_up() which is the also the function to which waker() is attached in the previously mentioned kernel sample. So, if the run the following: # perf probe "try_to_wake_up" "p->pid" # perf record -a -e probe:try_to_wake_up # perf script The value of p->pid is reported as 0. Similarly, if I try to read p->comm, it is reported to be an empty string. The same problem is seen with systemtap as well. Also, if I do a printk with offsetof(struct task_struct, pid) and offsetof(struct task_struct, comm) inside the kernel code and then compare the values with the offsets reported by pahole, they are completely different. - Sandipan
[RFC PATCH] bpf: Add helpers to read useful task_struct members
For added security, the layout of some structures can be
randomized by enabling CONFIG_GCC_PLUGIN_RANDSTRUCT. One
such structure is task_struct. To build BPF programs, we
use Clang which does not support this feature. So, if we
attempt to read a field of a structure with a randomized
layout within a BPF program, we do not get the expected
value because of incorrect offsets. To observe this, it
is not mandatory to have CONFIG_GCC_PLUGIN_RANDSTRUCT
enabled because the structure annotations/members added
for this purpose are enough to cause this. So, all kernel
builds are affected.
For example, considering samples/bpf/offwaketime_kern.c,
if we try to print the values of pid and comm inside the
task_struct passed to waker() by adding the following
lines of code at the appropriate place
char fmt[] = "waker(): p->pid = %u, p->comm = %s\n";
bpf_trace_printk(fmt, sizeof(fmt), _(p->pid), _(p->comm));
it is seen that upon rebuilding and running this sample
followed by inspecting /sys/kernel/debug/tracing/trace,
the output looks like the following
_-=> irqs-off
/ _=> need-resched
| / _---=> hardirq/softirq
|| / _--=> preempt-depth
||| / delay
TASK-PID CPU# TIMESTAMP FUNCTION
| | | | |
-0 [007] d.s. 1883.443594: 0x0001: waker(): p->pid =
0, p->comm =
-0 [018] d.s. 1883.453588: 0x0001: waker(): p->pid =
0, p->comm =
-0 [007] d.s. 1883.463584: 0x0001: waker(): p->pid =
0, p->comm =
-0 [009] d.s. 1883.483586: 0x0001: waker(): p->pid =
0, p->comm =
-0 [005] d.s. 1883.493583: 0x0001: waker(): p->pid =
0, p->comm =
-0 [009] d.s. 1883.503583: 0x0001: waker(): p->pid =
0, p->comm =
-0 [018] d.s. 1883.513578: 0x0001: waker(): p->pid =
0, p->comm =
systemd-journal-3140 [003] d... 1883.627660: 0x0001: waker(): p->pid =
0, p->comm =
systemd-journal-3140 [003] d... 1883.627704: 0x0001: waker(): p->pid =
0, p->comm =
systemd-journal-3140 [003] d... 1883.627723: 0x0001: waker(): p->pid =
0, p->comm =
To avoid this, we add new BPF helpers that read the
correct values for some of the important task_struct
members such as pid, tgid, comm and flags which are
extensively used in BPF-based analysis tools such as
bcc. Since these helpers are built with GCC, they use
the correct offsets when referencing a member.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
include/linux/bpf.h | 3 ++
include/uapi/linux/bpf.h | 13 ++
kernel/bpf/core.c | 3 ++
kernel/bpf/helpers.c | 75 +++
kernel/trace/bpf_trace.c | 6 +++
tools/testing/selftests/bpf/bpf_helpers.h | 9
6 files changed, 109 insertions(+)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index f1af7d63d678..5993a0f5262b 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -418,6 +418,9 @@ extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
extern const struct bpf_func_proto bpf_get_current_comm_proto;
+extern const struct bpf_func_proto bpf_get_task_pid_tgid_proto;
+extern const struct bpf_func_proto bpf_get_task_comm_proto;
+extern const struct bpf_func_proto bpf_get_task_flags_proto;
extern const struct bpf_func_proto bpf_skb_vlan_push_proto;
extern const struct bpf_func_proto bpf_skb_vlan_pop_proto;
extern const struct bpf_func_proto bpf_get_stackid_proto;
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index f90860d1f897..324508d27bd2 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -338,6 +338,16 @@ union bpf_attr {
* @skb: pointer to skb
* Return: classid if != 0
*
+ * u64 bpf_get_task_pid_tgid(struct task_struct *task)
+ * Return: task->tgid << 32 | task->pid
+ *
+ * int bpf_get_task_comm(struct task_struct *task)
+ * Stores task->comm into buf
+ * Return: 0 on success or negative error
+ *
+ * u32 bpf_get_task_flags(struct task_struct *task)
+ * Return: task->flags
+ *
* int bpf_skb_vlan_push(skb, vlan_proto, vlan_tci)
* Return: 0 on success or negative error
*
@@ -602,6 +612,9 @@ union bpf_attr {
FN(get_current_uid_gid),\
FN(get_current_comm), \
FN(get_cgroup_classid), \
+ FN(get_task_pid_tgid), \
+ FN(get_task_comm), \
+ FN(get_task_flags), \
FN(skb_vlan_push),
[PATCH 1/1] bpf: take advantage of stack_depth tracking in powerpc JIT
Take advantage of stack_depth tracking, originally introduced for
x64, in powerpc JIT as well. Round up allocated stack by 16 bytes
to make sure it stays aligned for functions called from JITed bpf
program.
Signed-off-by: Sandipan Das <sandi...@linux.vnet.ibm.com>
---
arch/powerpc/net/bpf_jit64.h | 7 ---
arch/powerpc/net/bpf_jit_comp64.c | 16 ++--
2 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit64.h b/arch/powerpc/net/bpf_jit64.h
index 62fa7589db2b..8bdef7ed28a8 100644
--- a/arch/powerpc/net/bpf_jit64.h
+++ b/arch/powerpc/net/bpf_jit64.h
@@ -23,7 +23,7 @@
* [ nv gpr save area] 8*8 |
* [tail_call_cnt ] 8 |
* [local_tmp_var ] 8 |
- * fp (r31) -->[ ebpf stack space] 512 |
+ * fp (r31) -->[ ebpf stack space] upto 512 |
* [ frame header ] 32/112|
* sp (r1) --->[stack pointer ] --
*/
@@ -32,8 +32,8 @@
#define BPF_PPC_STACK_SAVE (8*8)
/* for bpf JIT code internal usage */
#define BPF_PPC_STACK_LOCALS 16
-/* Ensure this is quadword aligned */
-#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + MAX_BPF_STACK + \
+/* stack frame excluding BPF stack, ensure this is quadword aligned */
+#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \
BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE)
#ifndef __ASSEMBLY__
@@ -103,6 +103,7 @@ struct codegen_context {
*/
unsigned int seen;
unsigned int idx;
+ unsigned int stack_size;
};
#endif /* !__ASSEMBLY__ */
diff --git a/arch/powerpc/net/bpf_jit_comp64.c
b/arch/powerpc/net/bpf_jit_comp64.c
index 6ba5d253e857..a01362c88f6a 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -69,7 +69,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context
*ctx)
static int bpf_jit_stack_local(struct codegen_context *ctx)
{
if (bpf_has_stack_frame(ctx))
- return STACK_FRAME_MIN_SIZE + MAX_BPF_STACK;
+ return STACK_FRAME_MIN_SIZE + ctx->stack_size;
else
return -(BPF_PPC_STACK_SAVE + 16);
}
@@ -82,8 +82,9 @@ static int bpf_jit_stack_tailcallcnt(struct codegen_context
*ctx)
static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
{
if (reg >= BPF_PPC_NVR_MIN && reg < 32)
- return (bpf_has_stack_frame(ctx) ? BPF_PPC_STACKFRAME : 0)
- - (8 * (32 - reg));
+ return (bpf_has_stack_frame(ctx) ?
+ (BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
+ - (8 * (32 - reg));
pr_err("BPF JIT is asking about unknown registers");
BUG();
@@ -134,7 +135,7 @@ static void bpf_jit_build_prologue(u32 *image, struct
codegen_context *ctx)
PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
}
- PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME);
+ PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size));
}
/*
@@ -161,7 +162,7 @@ static void bpf_jit_build_prologue(u32 *image, struct
codegen_context *ctx)
/* Setup frame pointer to point to the bpf stack area */
if (bpf_is_seen_register(ctx, BPF_REG_FP))
PPC_ADDI(b2p[BPF_REG_FP], 1,
- STACK_FRAME_MIN_SIZE + MAX_BPF_STACK);
+ STACK_FRAME_MIN_SIZE + ctx->stack_size);
}
static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context
*ctx)
@@ -183,7 +184,7 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct
codegen_context *ctx
/* Tear down our stack frame */
if (bpf_has_stack_frame(ctx)) {
- PPC_ADDI(1, 1, BPF_PPC_STACKFRAME);
+ PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
if (ctx->seen & SEEN_FUNC) {
PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
PPC_MTLR(0);
@@ -993,6 +994,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
memset(, 0, sizeof(struct codegen_context));
+ /* Make sure that the stack is quadword aligned. */
+ cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
+
/* Scouting faux-generate pass 0 */
if (bpf_jit_build_body(fp, 0, , addrs)) {
/* We hit something illegal or unsupported. */
--
2.13.5
