The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=a6deeaa2fb3b28a61ae56a85f2275f67fcf23262
commit a6deeaa2fb3b28a61ae56a85f2275f67fcf23262 Author: Benjamin Jacobs <[email protected]> AuthorDate: 2026-09-16 13:07:01 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2026-09-21 00:04:43 +0000 dtrace/arm64: Fix copyout and copyoutstr Calling dtrace_copy and dtrace_copystr with the kaddr and uaddr arguments inversed does not work with PAN. Rename them dtrace_copyin_pan and dtrace_copyinstr_pan, respectively, and implement dtrace_copyout_pan and dtrace_copyoutstr_pan. Avoid excessive faulting by checkin DTrace's CPU flags. Implement the trick from OpenSolaris/Illumos of only checking the flags when crossing into a new page, altough more effectively by examining the vaddr instead of the count. Reviewed by: markj MFC after: 3 weeks Differential Revision: https://reviews.freebsd.org/D59449 --- .../opensolaris/uts/common/sys/dtrace_impl.h | 7 +++ sys/cddl/dev/dtrace/aarch64/dtrace_asm.S | 68 ++++++++++++++++++++-- sys/cddl/dev/dtrace/aarch64/dtrace_isa.c | 8 +-- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h index c26956a646ff..88d3d88ddfcb 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h +++ b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h @@ -1302,8 +1302,15 @@ extern int dtrace_attached(void); extern hrtime_t dtrace_gethrestime(void); #endif +#if defined(__aarch64__) +extern void dtrace_copyin_pan(uintptr_t, uintptr_t, size_t); +extern void dtrace_copyinstr_pan(uintptr_t, uintptr_t, size_t); +extern void dtrace_copyout_pan(uintptr_t, uintptr_t, size_t, volatile uint16_t *); +extern void dtrace_copyoutstr_pan(uintptr_t, uintptr_t, size_t, volatile uint16_t *); +#else extern void dtrace_copy(uintptr_t, uintptr_t, size_t); extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); +#endif /* * DTrace Assertions diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S index decca6f79e0c..e27974eecc8f 100644 --- a/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S @@ -104,9 +104,9 @@ END(dtrace_fuword64_nocheck) /* void -dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) +dtrace_copyin_pan(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ -ENTRY(dtrace_copy) +ENTRY(dtrace_copyin_pan) cbz x2, 2f /* If len == 0 then skip loop */ 1: mvn x4, xzr /* Set all bits */ @@ -119,13 +119,13 @@ ENTRY(dtrace_copy) cbnz x2, 1b 2: RET -END(dtrace_copy) +END(dtrace_copyin_pan) /* void -dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size) +dtrace_copyinstr_pan(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ -ENTRY(dtrace_copystr) +ENTRY(dtrace_copyinstr_pan) cbz x2, 2f /* If len == 0 then skip loop */ 1: mvn x4, xzr /* Set all bits */ @@ -139,7 +139,63 @@ ENTRY(dtrace_copystr) cbnz x2, 1b 2: RET -END(dtrace_copystr) +END(dtrace_copyinstr_pan) + +/* +void +dtrace_copyout_pan(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +*/ +ENTRY(dtrace_copyout_pan) + cbz x2, 4f /* If len == 0 then skip loop */ + ldrb w4, [x0], #1 /* Load from kaddr */ + sttrb w4, [x1] /* Store in uaddr or raise data abort */ + b 2f /* Check fault */ +1: + ldrb w4, [x0], #1 /* Load from kaddr */ + sttrb w4, [x1] /* Store in uaddr or raise data abort */ + tst x1, #0xfff /* uaddr & 2^12-1 */ + bne 3f /* If not 4k aligned then skip check */ +2: + ldrh w5, [x3] /* Load cpu flags */ + tst w5, #CPU_DTRACE_BADADDR + bne 4f /* If flags&CPU_DTRACE_BADADDR != 0 then break */ +3: + add x1, x1, #1 /* uaddr++ */ + sub x2, x2, #1 /* len-- */ + cbnz x2, 1b +4: + RET +END(dtrace_copyout_pan) + + +/* +void +dtrace_copyoutstr_pan(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +*/ +ENTRY(dtrace_copyoutstr_pan) + cbz x2, 4f /* If len == 0 then skip loop */ + ldrb w4, [x0], #1 /* Load from kaddr */ + sttrb w4, [x1] /* Store in uaddr or raise data abort */ + b 2f /* Check fault */ +1: + ldrb w4, [x0], #1 /* Load from kaddr */ + sttrb w4, [x1] /* Store in uaddr or raise data abort */ + tst x1, #0xfff /* uaddr & 2^12-1 */ + bne 3f /* If not 4k aligned then skip check */ +2: + ldrh w5, [x3] /* Load cpu flags */ + tst w5, #CPU_DTRACE_BADADDR + bne 4f /* If flags&CPU_DTRACE_BADADDR != 0 then break */ +3: + cbz w4, 4f /* If == 0 then break */ + add x1, x1, #1 /* uaddr++ */ + sub x2, x2, #1 /* len-- */ + cbnz x2, 1b +4: + RET +END(dtrace_copyoutstr_pan) /* uintptr_t diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c index 037d4734659e..80998af3f0f4 100644 --- a/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c @@ -351,7 +351,7 @@ dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, { if (dtrace_copycheck(uaddr, kaddr, size)) - dtrace_copy(uaddr, kaddr, size); + dtrace_copyin_pan(uaddr, kaddr, size); } void @@ -360,7 +360,7 @@ dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, { if (dtrace_copycheck(uaddr, kaddr, size)) - dtrace_copy(kaddr, uaddr, size); + dtrace_copyout_pan(kaddr, uaddr, size, flags); } void @@ -369,7 +369,7 @@ dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, { if (dtrace_copycheck(uaddr, kaddr, size)) - dtrace_copystr(uaddr, kaddr, size, flags); + dtrace_copyinstr_pan(uaddr, kaddr, size); } void @@ -378,7 +378,7 @@ dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, { if (dtrace_copycheck(uaddr, kaddr, size)) - dtrace_copystr(kaddr, uaddr, size, flags); + dtrace_copyoutstr_pan(kaddr, uaddr, size, flags); } uint8_t
