On 1.6.2022. 7:01, Hrvoje Popovski wrote:
> Hi all,
>
> while playing around with TCP Large Receive Offloading for ix I have
> configure httpd and relayd on test box.
> Same second I've start relayd box panic.
> This is latest snapshot and it easely reproduciable..
With WITNESS
r420-1# rcctl -f start relayd
relayd(ok)
WuAvRm_NfINaGu:l t(S0PLx ffNfOTff LdO8W6E2fR8ED2 37O3N0 T,R 0AxP0 E,X
0I,T a1 )0 -
> Stopped at proc_trampoline+0xdc: m
ovl $0,%gs:0x538
TID PID UID PRFLAGS PFLAGS CPU COMMAND
434783 78195 0 0x2 0 4 relayd
416901 1262 89 0x1000012 0 3 relayd
290632 38913 0 0x2 0 2 relayd
239447 37685 0 0x2 0 5 relayd
72623 6837 89 0x1100012 0 0K relayd
*174940 41382 0 0x100003 0 1 ksh
proc_trampoline() at proc_trampoline+0xdc
end of kernel
end trace frame: 0x7f7ffffdd400, count: 14
https://www.openbsd.org/ddb.html describes the minimum info required in
bug reports. Insufficient info makes it difficult to find and fix bugs.
ddb{1}>
ddb{1}> show panic
*cpu0: uvm_fault(0xfffffd862f823730, 0x0, 0, 1) -> e
ddb{1}>
ddb{1}> show reg
rdi 0xffffffff822c0d48 kprintf_mutex
rsi 0x5
rbp 0xffff8000227afea0
rbx 0
rdx 0xc000000000000000
rcx 0x286
rax 0x2a
r8 0
r9 0
r10 0xf417d734fa974b8
r11 0x7ea5978c0be9feb6
r12 0
r13 0
r14 0
r15 0
rip 0xffffffff8118b50c proc_trampoline+0xdc
cs 0x8
rflags 0x246
rsp 0xffff8000227afe20
ss 0
proc_trampoline+0xdc: movl $0,%gs:0x538
ddb{1}>
ddb{1}> show all locks
CPU 1:
exclusive mutex &pmap->pm_mtx r = 0 (0xfffffd862f8226d8)
#0 witness_lock+0x311
#1 mtx_enter_try+0x95
#2 mtx_enter+0x48
#3 pmap_enter+0xf8
#4 uvm_fault_upper+0x1e5
#5 uvm_fault+0xde
#6 upageflttrap+0x62
#7 usertrap+0x129
#8 recall_trap+0x8
Process 37685 (relayd) thread 0xffff80002273f508 (239447)
exclusive rwlock uobjlk r = 0 (0xfffffd8575064088)
#0 witness_lock+0x311
#1 rw_enter+0x292
#2 uvm_fault_lower_lookup+0x41
#3 uvm_fault_lower+0x45
#4 uvm_fault+0x1b3
#5 upageflttrap+0x62
#6 usertrap+0x129
#7 recall_trap+0x8
shared rwlock vmmaplk r = 0 (0xfffffd862f823a28)
#0 witness_lock+0x311
#1 uvmfault_lookup+0x8a
#2 uvm_fault_check+0x32
#3 uvm_fault+0xfb
#4 upageflttrap+0x62
#5 usertrap+0x129
#6 recall_trap+0x8
Process 6837 (relayd) thread 0xffff80002273f268 (72623)
exclusive rwlock pf_lock r = 0 (0xffffffff822ce1f8)
#0 witness_lock+0x311
#1 pfr_add_tables+0x384
#2 pfioctl+0x1daf
#3 VOP_IOCTL+0x5c
#4 vn_ioctl+0x75
#5 sys_ioctl+0x2c4
#6 syscall+0x374
#7 Xsyscall+0x128
exclusive rwlock netlock r = 0 (0xffffffff822adc60)
#0 witness_lock+0x311
#1 pfr_add_tables+0x342
#2 pfioctl+0x1daf
#3 VOP_IOCTL+0x5c
#4 vn_ioctl+0x75
#5 sys_ioctl+0x2c4
#6 syscall+0x374
#7 Xsyscall+0x128
exclusive rwlock pfioctl_rw r = 0 (0xffffffff822ce258)
#0 witness_lock+0x311
#1 pfioctl+0x21e
#2 VOP_IOCTL+0x5c
#3 vn_ioctl+0x75
#4 sys_ioctl+0x2c4
#5 syscall+0x374
#6 Xsyscall+0x128
exclusive kernel_lock &kernel_lock r = 1 (0xffffffff8247f570)
#0 witness_lock+0x311
#1 vn_ioctl+0x3b
#2 sys_ioctl+0x2c4
#3 syscall+0x374
#4 Xsyscall+0x128
Process 41382 (ksh) thread 0xffff80002273f7a8 (174940)
exclusive rwlock amaplk r = 0 (0xfffffd857123cad0)
#0 witness_lock+0x311
#1 uvm_fault_check+0x3f7
#2 uvm_fault+0xfb
#3 upageflttrap+0x62
#4 usertrap+0x129
#5 recall_trap+0x8
shared rwlock vmmaplk r = 0 (0xfffffd857136d758)
#0 witness_lock+0x311
#1 uvmfault_lookup+0x8a
#2 uvm_fault_check+0x32
#3 uvm_fault+0xfb
#4 upageflttrap+0x62
#5 usertrap+0x129
#6 recall_trap+0x8
exclusive mutex &pmap->pm_mtx r = 0 (0xfffffd862f8226d8)
#0 witness_lock+0x311
#1 mtx_enter_try+0x95
#2 mtx_enter+0x48
#3 pmap_enter+0xf8
#4 uvm_fault_upper+0x1e5
#5 uvm_fault+0xde
#6 upageflttrap+0x62
#7 usertrap+0x129
#8 recall_trap+0x8
ddb{1}>
ddb{1}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
11599 104649 1 0 3 0x80 kqread relayd
61284 290693 1 0 2 0x2 relayd
78195 434783 1 0 7 0x2 relayd
51529 52072 1 89 2 0x1000012 relayd
1262 416901 1 89 7 0x1000012 relayd
38913 290632 1 0 7 0x2 relayd
37685 239447 1 0 7 0x2 relayd
59481 105452 1 0 2 0x2 relayd
6837 72623 1 89 7 0x1100012 relayd
87966 8902 20284 0 3 0x100083 nanoslp sleep
20284 235699 1 0 3 0x100089 sigsusp ksh
*41382 174940 1 0 7 0x100003 ksh
82076 137778 1 0 3 0x100098 kqread cron
80217 41671 79610 95 3 0x1100092 kqread smtpd
40924 173448 79610 103 3 0x1100092 kqread smtpd
77851 106396 79610 95 3 0x1100092 kqread smtpd
63568 219944 79610 95 3 0x100092 kqread smtpd
16176 75962 79610 95 3 0x1100092 kqread smtpd
58185 48255 79610 95 3 0x1100092 kqread smtpd
79610 373699 1 0 3 0x100080 kqread smtpd
22645 387795 1 0 3 0x88 kqread sshd
64997 201072 1 0 3 0x100080 kqread ntpd
74645 363110 58860 83 3 0x100092 kqread ntpd
58860 502227 1 83 3 0x1100092 kqread ntpd
67534 337185 26614 74 2 0x1100492 pflogd
26614 470756 1 0 3 0x80 netio pflogd
63648 47392 50034 73 3 0x1100090 kqread syslogd
50034 149575 1 0 3 0x100082 netio syslogd
93597 67785 0 0 3 0x14200 bored smr
34570 333066 0 0 2 0x14200 zerothread
65312 71814 0 0 3 0x14200 aiodoned aiodoned
30704 258711 0 0 3 0x14200 syncer update
94705 283338 0 0 3 0x14200 cleaner cleaner
78117 430341 0 0 3 0x14200 reaper reaper
85715 241190 0 0 3 0x14200 pgdaemon pagedaemon
28589 288770 0 0 3 0x14200 usbtsk usbtask
72661 280819 0 0 3 0x14200 usbatsk usbatsk
12488 427211 0 0 3 0x40014200 acpi0 acpi0
22519 106480 0 0 3 0x40014200 idle5
53533 190417 0 0 3 0x40014200 idle4
55817 201223 0 0 3 0x40014200 idle3
21436 482095 0 0 3 0x40014200 idle2
71474 26978 0 0 3 0x40014200 idle1
96463 27478 0 0 3 0x14200 bored sensors
25454 488957 0 0 3 0x14200 bored softnet
78955 159759 0 0 3 0x14200 bored softnet
15911 49455 0 0 3 0x14200 bored softnet
13330 315223 0 0 3 0x14200 bored softnet
61577 434829 0 0 3 0x14200 bored systqmp
65123 476452 0 0 3 0x14200 bored systq
91797 376440 0 0 3 0x40014200 bored softclock
56907 110962 0 0 3 0x40014200 idle0
1 499940 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{1}>
ddb{5}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
434783 78195 0 0x2 0 4 relayd
416901 1262 89 0x1000012 0 3 relayd
290632 38913 0 0x2 0 2 relayd
*239447 37685 0 0x2 0 5 relayd
72623 6837 89 0x1100012 0 0K relayd
174940 41382 0 0x100003 0 1 ksh
ddb{5}> trace /t 0t434783
__kernel_virt_to_phys() at __kernel_virt_to_phys+0x1eff045
end of kernel
end trace frame: 0x153fe6a6000, count: -1
ddb{5}> trace /t 0t416901
fffffd857256a518(ffffffff8184e9ca,ffff8000227eb630,fffffd857256a518,ffff8000227
eb380,0,ffff80002271f270) at 0xfffffd857256a518
end of kernel
end trace frame: 0x1, count: -1
ddb{5}> trace /t 0t290632
__kernel_virt_to_phys() at __kernel_virt_to_phys+0x1eff045
end of kernel
end trace frame: 0x7f7fffff1b08, count: -1
ddb{5}> trace /t 0t239447
__kernel_virt_to_phys() at __kernel_virt_to_phys+0x1eff045
end of kernel
end trace frame: 0x89928a282d0, count: -1
ddb{5}> trace /t 0t72623
end trace frame: 0x0, count: -1
ddb{5}> trace /t 0t174940
kernel: protection fault trap, code=0
Faulted in DDB; continuing...
ddb{1}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff822a3ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
x86_bus_space_io_read_1(2f8,5) at x86_bus_space_io_read_1+0x15
comcnputc(801,20) at comcnputc+0x7f
cnputc(20) at cnputc+0x37
db_putchar(65) at db_putchar+0x25a
kprintf() at kprintf+0x133b
db_printf(ffffffff81fe6f4a) at db_printf+0x69
fault(ffffffff81f95ea8) at fault+0x8e
kpageflttrap(ffff8000227cc480,0) at kpageflttrap+0x190
kerntrap(ffff8000227cc480) at kerntrap+0x91
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
pf_find_or_create_ruleset(0) at pf_find_or_create_ruleset+0x1c
end trace frame: 0xffff8000227ccb70, count: 0
ddb{0}> mach ddbcpu 1
Stopped at proc_trampoline+0xdc: movl $0,%gs:0x538
proc_trampoline() at proc_trampoline+0xdc
end of kernel
end trace frame: 0x7f7ffffdd400, count: 14
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff8247f368) at __mp_lock+0xa7
syscall(ffff8000227e5220) at syscall+0x278
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7fffff1b58, count: 9
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff8247f368) at __mp_lock+0xa0
kqueue_register(fffffd857256a518,ffff8000227eb380,0,ffff80002271f270) at
kqueue_register+0x63d
sys_kevent(ffff80002271f270,ffff8000227eb640,ffff8000227eb690) at
sys_kevent+0x21a
syscall(ffff8000227eb700) at syscall+0x374
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffd7f90, count: 7
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff8247f368) at __mp_lock+0xb3
syscall(ffff8000227f6ff0) at syscall+0x278
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffd6410, count: 9
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff8247f368) at __mp_lock+0xac
syscall(ffff8000227d99d0) at syscall+0x278
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffe05b0, count: 9
ddb{5}>
