The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=e6be6dedeea1e6d2e5206e1e7422e2d556a6da0c
commit e6be6dedeea1e6d2e5206e1e7422e2d556a6da0c Author: Mark Johnston <[email protected]> AuthorDate: 2026-05-20 14:49:41 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2026-05-20 14:49:41 +0000 kinst/arm64: Handle an additional PC-relative instruction "ldr <reg>, <literal>" loads a value from a literal memory address into a register. It's PC-relative and so cannot be directly implemented using the trampoline mechanism. Unfortunately, on arm64 it can't easily be emulated either since the return-to-EL1 handler does not restore callee-saved registers, so like adr/adrp, we simply don't handle it. These instructions are fairly rare in an arm64 kernel. While here, refactor the code so that all instruction decoding is done in one place: introduce an enum type which characterizes the instruction type, add a helper to map instructions to enum values, and store the corresponding enum value in the probe description. Reviewed by: christos MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D56988 --- sys/cddl/dev/kinst/aarch64/kinst_isa.c | 121 +++++++++++++++------------------ sys/cddl/dev/kinst/aarch64/kinst_isa.h | 15 +++- 2 files changed, 69 insertions(+), 67 deletions(-) diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.c b/sys/cddl/dev/kinst/aarch64/kinst_isa.c index 1ccfe20b8dcb..d9a8fd0276f2 100644 --- a/sys/cddl/dev/kinst/aarch64/kinst_isa.c +++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.c @@ -18,6 +18,30 @@ DPCPU_DEFINE_STATIC(struct kinst_cpu_state, kinst_state); +static enum kinst_instr +kinst_instr_type(kinst_patchval_t instr) +{ + if (((instr >> 22) & 0xff) == 0b00100001) + return (KINST_INSTR_LDX); + else if (((instr >> 22) & 0xff) == 0b00100000) + return (KINST_INSTR_STX); + if (((instr >> 24) & 0x1f) == 0b10000) + return (KINST_INSTR_ADR); + else if (((instr >> 26) & 0x3f) == 0b000101) + return (KINST_INSTR_B); + else if (((instr >> 24) & 0xff) == 0b01010100) + return (KINST_INSTR_BCOND); + else if (((instr >> 26) & 0x3f) == 0b100101) + return (KINST_INSTR_BL); + else if (((instr >> 25) & 0x3f) == 0b011010) + return (KINST_INSTR_CBZ); + else if (((instr >> 25) & 0x3f) == 0b011011) + return (KINST_INSTR_TBZ); + else if (((instr >> 24) & 0xbf) == 0b11000) + return (KINST_INSTR_LDR_LITERAL); + return (KINST_INSTR_COMMON); +} + static void kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) { @@ -26,8 +50,8 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) uint8_t cond, reg, bitpos; bool res; - if (((instr >> 24) & 0x1f) == 0b10000) { - /* adr/adrp */ + switch (kp->kp_md.kp_type) { + case KINST_INSTR_ADR: reg = instr & 0x1f; imm = (instr >> 29) & 0x3; imm |= ((instr >> 5) & 0x0007ffff) << 2; @@ -44,14 +68,14 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm; } frame->tf_elr += INSN_SIZE; - } else if (((instr >> 26) & 0x3f) == 0b000101) { - /* b */ + break; + case KINST_INSTR_B: imm = instr & 0x03ffffff; if (imm & 0x0000000002000000) imm |= 0xfffffffffe000000; frame->tf_elr += imm << 2; - } else if (((instr >> 24) & 0xff) == 0b01010100) { - /* b.cond */ + break; + case KINST_INSTR_BCOND: imm = (instr >> 5) & 0x0007ffff; if (imm & 0x0000000000040000) imm |= 0xfffffffffffc0000; @@ -92,15 +116,15 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) frame->tf_elr += imm << 2; else frame->tf_elr += INSN_SIZE; - } else if (((instr >> 26) & 0x3f) == 0b100101) { - /* bl */ + break; + case KINST_INSTR_BL: imm = instr & 0x03ffffff; if (imm & 0x0000000002000000) imm |= 0xfffffffffe000000; frame->tf_lr = frame->tf_elr + INSN_SIZE; frame->tf_elr += imm << 2; - } else if (((instr >> 25) & 0x3f) == 0b011010) { - /* cbnz/cbz */ + break; + case KINST_INSTR_CBZ: cond = (instr >> 24) & 0x1; reg = instr & 0x1f; imm = (instr >> 5) & 0x0007ffff; @@ -114,8 +138,8 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) frame->tf_elr += imm << 2; else frame->tf_elr += INSN_SIZE; - } else if (((instr >> 25) & 0x3f) == 0b011011) { - /* tbnz/tbz */ + break; + case KINST_INSTR_TBZ: cond = (instr >> 24) & 0x1; reg = instr & 0x1f; bitpos = (instr >> 19) & 0x1f; @@ -131,6 +155,9 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) frame->tf_elr += imm << 2; else frame->tf_elr += INSN_SIZE; + break; + default: + __assert_unreachable(); } } @@ -211,7 +238,7 @@ kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); cpu->cpu_dtrace_caller = 0; - if (kp->kp_md.emulate) { + if (kp->kp_md.kp_type != KINST_INSTR_COMMON) { kinst_emulate(frame, kp); } else { ks->state = KINST_PROBE_FIRED; @@ -245,50 +272,6 @@ kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) cpu_icache_sync_range(kp->kp_patchpoint, INSN_SIZE); } -static void -kinst_instr_dissect(struct kinst_probe *kp) -{ - struct kinst_probe_md *kpmd; - kinst_patchval_t instr = kp->kp_savedval; - - kpmd = &kp->kp_md; - kpmd->emulate = false; - - if (((instr >> 24) & 0x1f) == 0b10000) - kpmd->emulate = true; /* adr/adrp */ - else if (((instr >> 26) & 0x3f) == 0b000101) - kpmd->emulate = true; /* b */ - else if (((instr >> 24) & 0xff) == 0b01010100) - kpmd->emulate = true; /* b.cond */ - else if (((instr >> 26) & 0x3f) == 0b100101) - kpmd->emulate = true; /* bl */ - else if (((instr >> 25) & 0x3f) == 0b011010) - kpmd->emulate = true; /* cbnz/cbz */ - else if (((instr >> 25) & 0x3f) == 0b011011) - kpmd->emulate = true; /* tbnz/tbz */ - - if (!kpmd->emulate) - kinst_trampoline_populate(kp); -} - -static bool -kinst_instr_ldx(kinst_patchval_t instr) -{ - if (((instr >> 22) & 0xff) == 0b00100001) - return (true); - - return (false); -} - -static bool -kinst_instr_stx(kinst_patchval_t instr) -{ - if (((instr >> 22) & 0xff) == 0b00100000) - return (true); - - return (false); -} - int kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) @@ -359,6 +342,8 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, ldxstx_block = false; for (n = 0; instr < limit; instr++) { + enum kinst_instr type; + off = (int)((uint8_t *)instr - (uint8_t *)symval->value); /* @@ -366,9 +351,10 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, * breakpoint is placed in a LDX/STX block, we violate the * operation and the loop might fail. */ - if (kinst_instr_ldx(*instr)) + type = kinst_instr_type(*instr); + if (type == KINST_INSTR_LDX) ldxstx_block = true; - else if (kinst_instr_stx(*instr)) { + else if (type == KINST_INSTR_STX) { ldxstx_block = false; continue; } @@ -376,13 +362,14 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, continue; /* - * XXX: Skip ADR and ADRP instructions. The arm64 exception - * handler has a micro-optimization where it doesn't restore - * callee-saved registers when returning from exceptions in - * EL1. This results in a panic when the kinst emulation code - * modifies one of those registers. + * XXX: The arm64 exception handler has a micro-optimization + * where it doesn't restore callee-saved registers when + * returning from exceptions in EL1. As a result, instruction + * emulation doesn't work if a (callee-saved) register is + * modified. Hence, exclude the position-dependent ADR/ADRP and + * LDR <literal> instructions. */ - if (((*instr >> 24) & 0x1f) == 0b10000) + if (type == KINST_INSTR_ADR || type == KINST_INSTR_LDR_LITERAL) continue; if (pd->kpd_off != -1 && off != pd->kpd_off) @@ -408,12 +395,14 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, kp->kp_patchpoint = instr; kp->kp_savedval = *instr; kp->kp_patchval = KINST_PATCHVAL; + kp->kp_md.kp_type = type; if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) { KINST_LOG("cannot allocate trampoline for %p", instr); return (ENOMEM); } + if (kp->kp_md.kp_type == KINST_INSTR_COMMON) + kinst_trampoline_populate(kp); - kinst_instr_dissect(kp); kinst_probe_create(kp, lf); } if (ldxstx_block) diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.h b/sys/cddl/dev/kinst/aarch64/kinst_isa.h index 7e1fd8d123e9..39cf6d49290a 100644 --- a/sys/cddl/dev/kinst/aarch64/kinst_isa.h +++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.h @@ -19,8 +19,21 @@ typedef uint32_t kinst_patchval_t; +enum kinst_instr { + KINST_INSTR_ADR, /* adr/adrp */ + KINST_INSTR_B, + KINST_INSTR_BCOND, + KINST_INSTR_BL, + KINST_INSTR_CBZ, /* cbz/cbnz */ + KINST_INSTR_TBZ, /* tbnz/tbz */ + KINST_INSTR_LDR_LITERAL, + KINST_INSTR_LDX, + KINST_INSTR_STX, + KINST_INSTR_COMMON, +}; + struct kinst_probe_md { - bool emulate; /* emulate in sw */ + enum kinst_instr kp_type; }; #endif /* _KINST_ISA_H_ */
