GCC's -fpatchable-function-entry=5 may emit five one-byte NOPs. Normally that's not safe to patch because some might jump into a later nop. But for the gcc case it's safe because nobody jumps into the nops. Add a %multinop that allows the user opting into patching these sites. This way patching for the gcc instrumentation works.
Assisted-by: omp:gpt-5.6-luna Signed-off-by: Andi Kleen <[email protected]> --- arch/x86/include/asm/uprobes.h | 24 ++++++-- arch/x86/kernel/uprobes.c | 63 +++++++++++++++++--- include/linux/uprobes.h | 2 + kernel/trace/trace_uprobe.c | 58 +++++++++++++----- samples/uprobe-ptwrite/uprobe_ptwrite_test.c | 6 ++ 5 files changed, 126 insertions(+), 27 deletions(-) diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h index e5a668ba5ad6..c46b3fe09025 100644 --- a/arch/x86/include/asm/uprobes.h +++ b/arch/x86/include/asm/uprobes.h @@ -30,12 +30,23 @@ enum { struct uprobe_xol_ops; /* - * Stub block array size. Worst case = 250 B (8 MEM args, rsp bases, fault - * table); 288 leaves 38 B slack. A file-scope static_assert in - * arch/x86/kernel/uprobes.c re-derives the worst case; prepare() also - * enforces it with -E2BIG at runtime. + * Stub block size. Conservative worst case is 298 bytes: 9-byte header, + * one lead fence, eight 21-byte memory forms with one 3-byte fence each, + * a 16-byte original-instruction copy, a 5-byte return jump, alignment, + * and 66 bytes of data/fault metadata. 384 leaves room. A static_assert in + * arch/x86/kernel/uprobes.c checks the bound; prepare() also checks it with + * -E2BIG. */ -#define UPROBE_PTWRITE_STUB_SIZE 288 +#define UPROBE_PTWRITE_STUB_SIZE 384 + + +/* + * Word pacing: insert this many LFENCEs between emitted ptwrite words and + * before the first word, unless UPROBE_PTWRITE_FL_NO_LEAD_PACE is requested. + */ +#define UPROBE_PTWRITE_SERIALIZE_LFENCES 1 /* LFENCEs per word gap */ +/* The encoded LFENCE instruction occupies three bytes. */ +#define UPROBE_PTWRITE_LFENCE_SIZE 3 /* * ptwrite probe state. The stub template (code + data slots) is built @@ -49,6 +60,9 @@ struct uprobe_ptwrite_arch { u8 jmp_off; /* offset of the final jmp's rel32 field */ u8 ndata; /* number of u64 data slots */ u8 orig[MAX_UINSN_BYTES]; /* pristine file bytes, before generic analysis */ + u16 ft_off; /* fault table offset within the block (0 if none) */ + u8 nft; /* number of fault entries */ + bool allow_nop_run; /* accept a five-byte run of 0x90 */ }; /* Per-mm page holding generated ptwrite stub blocks, like tramp_mapping. */ diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 0029b66cd64e..f915dda8bcb4 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -1323,6 +1323,24 @@ static int ptwrite_emit_riprel(u8 *p, s32 disp) return 9; } +static int ptwrite_emit_lfence(u8 *p) +{ + *p++ = 0x0f; + *p++ = 0xae; + *p++ = 0xe8; /* LFENCE */ + return UPROBE_PTWRITE_LFENCE_SIZE; +} + +/* The default pacing: one or more fences per word gap. */ +static int ptwrite_emit_lfences(u8 *p) +{ + int i; + + for (i = 0; i < UPROBE_PTWRITE_SERIALIZE_LFENCES; i++) + p += ptwrite_emit_lfence(p); + return UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE; +} bool arch_uprobe_ptwrite_supported(void) { u32 eax, ebx, ecx, edx; @@ -1411,13 +1429,20 @@ int arch_uprobe_ptwrite_fetch(struct uprobe_ptwrite_arg *a, } /* - * Worst-case stub block: header ptwriteq (9) + max memory args (10 bytes - * each, including a SIB byte) + final jmp (5), rounded up; data adds one - * header slot and one slot per immediate. Keep the bound below the stub size. + * Worst-case paced stub before instruction punning: a 9-byte header, one + * lead fence, one fence after the header, one fence between each argument, + * the largest memory form (10 bytes), and the return jump. Data adds one + * header slot and one slot per immediate. */ -static_assert((((9 + UPROBE_PTWRITE_MAX_ARGS * 10 + 5 + 7) & ~7) + - 8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <= UPROBE_PTWRITE_STUB_SIZE, - "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE"); +static_assert((((9 + UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE + + UPROBE_PTWRITE_MAX_ARGS * + (10 + UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE) + + 5 + 7) & ~7) + + 8 * (1 + UPROBE_PTWRITE_MAX_ARGS)) <= + UPROBE_PTWRITE_STUB_SIZE, + "worst-case ptwrite stub block exceeds UPROBE_PTWRITE_STUB_SIZE"); static bool ptwrite_has_room(const u8 *base, const u8 *p, size_t len) { @@ -1439,6 +1464,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe, unsigned int data_off; unsigned int hdr_off = 0; unsigned int imm_idx = 0, n_imm = 0; + bool paced = false; u64 hdr; int i; @@ -1446,7 +1472,9 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe, return -EINVAL; if (desc->nargs > UPROBE_PTWRITE_MAX_ARGS) return -E2BIG; - if (desc->flags & ~UPROBE_PTWRITE_FL_ALLOW_MEM) + if (desc->flags & ~(UPROBE_PTWRITE_FL_ALLOW_MEM | + UPROBE_PTWRITE_FL_NO_LEAD_PACE | + UPROBE_PTWRITE_FL_ALLOW_NOP_RUN)) return -EINVAL; /* The generic registration path copied these bytes before this hook. */ @@ -1477,9 +1505,22 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe, } } + paced = !(desc->flags & UPROBE_PTWRITE_FL_NO_LEAD_PACE); + if (paced) { + PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE); + p += ptwrite_emit_lfences(p); + } + /* header word emission (disp32 patched below) */ PTW_NEED(9); + hdr_off = p - code; p += ptwrite_emit_riprel(p, 0); + if (paced) { + PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE); + p += ptwrite_emit_lfences(p); + } for (i = 0; i < desc->nargs; i++) { switch (desc->args[i].src) { @@ -1515,9 +1556,14 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe, break; } } + if (paced && i + 1 < desc->nargs) { + PTW_NEED(UPROBE_PTWRITE_SERIALIZE_LFENCES * + UPROBE_PTWRITE_LFENCE_SIZE); + p += ptwrite_emit_lfences(p); + } } - /* final jmp back to probe+5; rel32 patched per-mm at install */ + /* final jmp back to probe+len; rel32 patched per-mm at install */ PTW_NEED(5); *p++ = 0xe9; if (p - code > U8_MAX) @@ -1549,6 +1595,7 @@ int arch_uprobe_ptwrite_prepare(struct arch_uprobe *auprobe, ptw->stub_len = data_off + 8 * (1 + n_imm); ptw->ndata = 1 + n_imm; + ptw->allow_nop_run = desc->flags & UPROBE_PTWRITE_FL_ALLOW_NOP_RUN; return 0; } #undef PTW_NEED diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f6ffb0637991..5f16799c9fb9 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -210,6 +210,8 @@ enum uprobe_ptwrite_src { /* uprobe_ptwrite_desc.flags */ #define UPROBE_PTWRITE_FL_ALLOW_MEM BIT(0) /* SRC_MEM args enabled */ +#define UPROBE_PTWRITE_FL_NO_LEAD_PACE BIT(1) /* don't slow down probes */ +#define UPROBE_PTWRITE_FL_ALLOW_NOP_RUN BIT(2) /* accept five 1-byte NOPs */ struct uprobe_ptwrite_arg { u8 src; /* enum uprobe_ptwrite_src */ diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index ab8ad3f6083d..a226b89e5b9a 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -635,6 +635,8 @@ static int __trace_uprobe_create(int argc, const char **argv) enum probe_print_type ptype; bool is_return = false; bool is_ptwrite = false; + bool is_nopace = false; + bool is_nop_run = false; int i, ret, arg_start = 2; ref_ctr_offset = 0; @@ -660,13 +662,6 @@ static int __trace_uprobe_create(int argc, const char **argv) trlog = trace_probe_log_init("trace_uprobe", argc, argv); - if (argc - 2 > MAX_TRACE_ARGS || - (is_ptwrite && argc - 2 > UPROBE_PTWRITE_MAX_ARGS)) { - trace_probe_log_set_index(2); - trace_probe_log_err(0, TOO_MANY_ARGS); - return -E2BIG; - } - if (is_ptwrite) event = argv[0][3] == ':' && argv[0][4] ? &argv[0][4] : NULL; @@ -728,9 +723,19 @@ static int __trace_uprobe_create(int argc, const char **argv) /* Check if there is %return suffix */ tmp = strchr(arg, '%'); + if (tmp && is_ptwrite && !strcmp(tmp, "%nopace")) { + *tmp = '\0'; + is_nopace = true; + tmp = NULL; + } if (tmp && is_ptwrite) { - trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); - return -EINVAL; + if (!strcmp(tmp, "%multinop")) { + *tmp = '\0'; + is_nop_run = true; + } else { + trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); + return -EINVAL; + } } else if (tmp) { if (!strcmp(tmp, "%return")) { *tmp = '\0'; @@ -747,6 +752,28 @@ static int __trace_uprobe_create(int argc, const char **argv) trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); return ret; } + if (is_ptwrite && arg_start < argc && + !strcmp(argv[arg_start], "%nopace")) { + is_nopace = true; + arg_start++; + } + if (is_ptwrite) { + while (arg_start < argc && !strcmp(argv[arg_start], "%multinop")) { + is_nop_run = true; + arg_start++; + } + if (arg_start < argc && !strcmp(argv[arg_start], "%nopace")) { + is_nopace = true; + arg_start++; + } + } + + if (argc - arg_start > MAX_TRACE_ARGS || + (is_ptwrite && argc - arg_start > UPROBE_PTWRITE_MAX_ARGS)) { + trace_probe_log_set_index(arg_start); + trace_probe_log_err(0, TOO_MANY_ARGS); + return -E2BIG; + } /* setup a probe */ trace_probe_log_set_index(0); @@ -782,8 +809,8 @@ static int __trace_uprobe_create(int argc, const char **argv) kfree(tail); } - argc -= 2; - argv += 2; + argc -= arg_start; + argv += arg_start; tu = alloc_trace_uprobe(group, event, argc, is_return); if (IS_ERR(tu)) { @@ -806,7 +833,7 @@ static int __trace_uprobe_create(int argc, const char **argv) /* parse arguments */ for (i = 0; i < argc; i++) { - trace_probe_log_set_index(i + 2); + trace_probe_log_set_index(i + arg_start); ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx); if (ret) return ret; @@ -814,13 +841,16 @@ static int __trace_uprobe_create(int argc, const char **argv) if (is_ptwrite) { if (!argc) { - trace_probe_log_set_index(2); + trace_probe_log_set_index(arg_start); trace_probe_log_err(0, NO_ARG_BODY); return -EINVAL; /* core rejects desc->nargs == 0 */ } tu->is_ptwrite = true; tu->ptwrite_desc.nargs = argc; - tu->ptwrite_desc.flags = 0; + tu->ptwrite_desc.flags = is_nop_run ? + UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0; + if (is_nopace) + tu->ptwrite_desc.flags |= UPROBE_PTWRITE_FL_NO_LEAD_PACE; for (i = 0; i < argc; i++) { ret = ptwrite_compile_arg(tu, i); if (ret) { diff --git a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c index 55e23ff27dd5..ca28d511b60c 100644 --- a/samples/uprobe-ptwrite/uprobe_ptwrite_test.c +++ b/samples/uprobe-ptwrite/uprobe_ptwrite_test.c @@ -17,6 +17,7 @@ * m<N>[:<disp>][:<size>] = memory arg [reg + disp32], * size 4 (u32 load) or 8 (u64 load, default) * event_id=0x1234 identifier carried in the PTW header word + * allow_nop_run=1 accept five one-byte NOPs at the site */ #include <linux/module.h> #include <linux/uprobes.h> @@ -36,6 +37,10 @@ static ushort event_id = 0x1234; module_param(event_id, ushort, 0444); MODULE_PARM_DESC(event_id, "event id carried in the PTW header word"); +static bool allow_nop_run; +module_param(allow_nop_run, bool, 0444); +MODULE_PARM_DESC(allow_nop_run, "accept five one-byte NOPs at the site"); + static char *args = "r0"; module_param(args, charp, 0444); MODULE_PARM_DESC(args, "comma-separated args: r<N> GPR, i<hex> immediate, m<N>[:disp][:4|8] memory"); @@ -142,6 +147,7 @@ static int __init uprobe_ptwrite_test_init(void) int ret; desc.event_id = event_id; + desc.flags = allow_nop_run ? UPROBE_PTWRITE_FL_ALLOW_NOP_RUN : 0; ret = parse_probe_args(); if (ret) return ret; -- 2.54.0
