It continues to crash rapidly. Up to now crashed 4 times with different
messages.
You may want to read so report is attached.
Thanks,
Valdrin
________________________________
From: Valdrin MUJA
Sent: Wednesday, July 5, 2023 10:56
To: [email protected] <[email protected]>
Subject: kernel diagnostic assertion "!_kernel_lock_held()" failed
Hello,
My system hanged with these error messages and gone to ddb.
Here you can find the details. Also crash.txt includes more.
OpenBSD TEST1 7.3 GENERIC.MP#1125 amd64
# sysctl kern.version
kern.version=OpenBSD 7.3 (GENERIC.MP) #1125: Sat Mar 25 10:36:29 MDT 2023
[email protected]:/usr/src/sys/arch/amd64/compile/GENERIC.MP
# syspatch -l
001_bgpd
002_bgpd
003_rpki
004_ssl
005_libx11
ddb{3}> show panic
*cpu0: uvm_fault(0xfffffd82067b5440, 0x8, 0, 1) -> e
cpu3: kernel diagnostic assertion "!_kernel_lock_held()" failed: file "/usr/sr
c/sys/kern/kern_fork.c", line 674
ddb{3}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 12883 18843K 18843K 78643K 36588 0
pcb 13 14K 18K 78643K 17 0
rtable 445 61K 73K 78643K 24030 0
ifaddr 233 52K 52K 78643K 1351 0
sysctl 3 1K 1K 78643K 3 0
counters 226 85K 85K 78643K 438 0
ioctlops 0 0K 4K 78643K 42936 0
iov 0 0K 4K 78643K 620 0
mount 8 8K 8K 78643K 9 0
log 0 0K 0K 78643K 1 0
vnodes 1220 77K 79K 78643K 4125 0
UFS quota 1 32K 32K 78643K 1 0
UFS mount 31 69K 69K 78643K 39 0
shm 2 1K 1K 78643K 2 0
VM map 2 1K 1K 78643K 2 0
sem 2 10K 10K 78643K 4 0
dirhash 156 29K 29K 78643K 198 0
ACPI 3556 426K 453K 78643K 13924 0
file desc 30 21K 24K 78643K 20846 0
sigio 1 0K 0K 78643K 2 0
proc 111 93K 101K 78643K 35667 0
MFS node 6 0K 0K 78643K 6 0
NFS srvsock 1 0K 0K 78643K 1 0
NFS daemon 1 16K 16K 78643K 1 0
in_multi 37 2K 2K 78643K 180 0
ether_multi 2 0K 0K 78643K 3 0
ISOFS mount 1 32K 32K 78643K 1 0
MSDOSFS mount 1 16K 16K 78643K 1 0
ttys 31 148K 148K 78643K 31 0
exec 0 0K 1K 78643K 47003 0
pfkey data 0 0K 0K 78643K 2 0
tdb 3 0K 0K 78643K 3 0
pagedep 1 8K 8K 78643K 1 0
inodedep 1 32K 1558K 78643K 16937 0
newblk 1 0K 0K 78643K 1 0
indirdep 0 0K 672K 78643K 1376 0
VM swap 8 1110K 1112K 78643K 10 0
UVM amap 4170 376K 379K 78643K 475831 0
UVM aobj 3 2K 2K 78643K 3 0
USB 11 10K 10K 78643K 13 0
USB device 4 0K 0K 78643K 4 0
USB HC 1 0K 0K 78643K 1 0
memdesc 1 4K 4K 78643K 1 0
crypto data 18 258K 258K 78643K 18 0
NDP 46 0K 0K 78643K 99 0
temp 202 5824K 5890K 78643K 1138487 0
kqueue 79 141K 147K 78643K 2026 0
SYN cache 2 16K 16K 78643K 2 0
ddb{3}> trace
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: -4
ddb{3}>
db_enter() at db_enter+0x10
end trace frame: 0xffff800022ab0d70, count: 0
ddb{3}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa6
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
cnputc(62) at cnputc+0x37
db_putchar(62) at db_putchar+0x2ea
kprintf() at kprintf+0x133b
db_printf(ffffffff8211cc51) at db_printf+0x69
fault(ffffffff820cd05b) at fault+0x8e
kpageflttrap(ffff800022a2cb60,8) at kpageflttrap+0x1cf
kerntrap(ffff800022a2cb60) at kerntrap+0x91
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
end trace frame: 0xffff800022a2cc20, count: 0
ddb{0}> machine ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(ffff800001b79f18) at mtx_enter+0x48
msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61
taskq_thread(ffff800001b79f00) at taskq_thread+0xeb
end trace frame: 0x0, count: 8
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_release_all(ffffffff824bca80) at __mp_release_all+0x21
msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10
b
kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe
kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00
50,ffff8000222643ac) at kqueue_scan+0x108
sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x
371
syscall(ffff8000222644e0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2b22a4b60, count: 5
ddb{2}> machine ddbcpu 3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: 11
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(ffff800001b79f18) at mtx_enter+0x48
msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61
taskq_thread(ffff800001b79f00) at taskq_thread+0xeb
end trace frame: 0x0, count: 8
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_release_all(ffffffff824bca80) at __mp_release_all+0x21
msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10
b
kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe
kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00
50,ffff8000222643ac) at kqueue_scan+0x108
sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x
371
syscall(ffff8000222644e0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2b22a4b60, count: 5
ddb{2}> machine ddbcpu 0t3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: 11
ddb{3}>
P.S. dmesg.txt is also attached..
--
Thanks for reading.
Valdrin
CRASH 01 :
ddb{3}> show panic
*cpu0: uvm_fault(0xfffffd82067b5440, 0x8, 0, 1) -> e
cpu3: kernel diagnostic assertion "!_kernel_lock_held()" failed: file "/usr/sr
c/sys/kern/kern_fork.c", line 674
ddb{3}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
*53409 454121 10474 0 7 0x100000 sh
58907 516606 10474 0 2 0x100000 sh
10474 417896 35677 0 2 0x100002 sh
52042 375502 89267 0 2 0x100002 ps
77764 83714 35677 0 3 0x10008a sigsusp sh
89267 222930 35677 0 3 0x100082 piperd sh
41409 870 35677 0 3 0x82 thrsleep mine-filter
41409 26909 35677 0 3 0x4000082 nanoslp mine-filter
41409 436637 35677 0 3 0x4000082 thrsleep mine-filter
41409 383490 35677 0 3 0x4000082 thrsleep mine-filter
41409 387156 35677 0 3 0x4000082 thrsleep mine-filter
41409 139392 35677 0 3 0x4000082 thrsleep mine-filter
41409 302614 35677 0 3 0x4000082 thrsleep mine-filter
41409 61846 35677 0 3 0x4000082 thrsleep mine-filter
41409 396396 35677 0 3 0x4000082 netio mine-filter
41409 77669 35677 0 3 0x4000082 thrsleep mine-filter
41409 432551 35677 0 3 0x4000082 thrsleep mine-filter
41409 305280 35677 0 3 0x4000082 netio mine-filter
41409 199867 35677 0 3 0x4000082 thrsleep mine-filter
41409 498811 35677 0 3 0x4000082 thrsleep mine-filter
41409 301417 35677 0 3 0x4000082 thrsleep mine-filter
41409 25438 35677 0 3 0x4000082 netio mine-filter
41409 188591 35677 0 3 0x4000082 thrsleep mine-filter
41409 66557 35677 0 3 0x4000082 thrsleep mine-filter
41409 247571 35677 0 3 0x4000082 thrsleep mine-filter
41409 480213 35677 0 3 0x4000082 kqread mine-filter
41409 496842 35677 0 3 0x4000082 thrsleep mine-filter
41409 352012 35677 0 3 0x4000082 thrsleep mine-filter
41409 49963 35677 0 3 0x4000082 thrsleep mine-filter
41409 324956 35677 0 3 0x4000082 thrsleep mine-filter
41409 49414 35677 0 3 0x4000082 netio mine-filter
41409 65635 35677 0 3 0x4000082 thrsleep mine-filter
26392 500122 62485 75 3 0x1100092 kqread bgpd
94740 458157 62485 75 3 0x1100092 kqread bgpd
31711 437095 62485 75 3 0x1100092 kqread bgpd
62485 48056 1 0 7 0 bgpd
38680 483276 34098 0 3 0x83 thrsleep minesh
38680 278236 34098 0 3 0x4000083 thrsleep minesh
38680 127964 34098 0 3 0x4000083 thrsleep minesh
38680 376298 34098 0 3 0x4000083 thrsleep minesh
38680 165768 34098 0 3 0x4000083 thrsleep minesh
38680 370077 34098 0 3 0x4000083 kqread minesh
38680 476528 34098 0 3 0x4000083 thrsleep minesh
38680 223917 34098 0 3 0x4000083 thrsleep minesh
38680 83255 34098 0 3 0x4000083 ttyin minesh
38680 145891 34098 0 3 0x4000083 thrsleep minesh
34098 391627 12941 0 3 0x10008b sigsusp ksh
12941 469320 1 0 3 0x100080 kqread tmux
71476 442894 1 0 3 0x82 thrsleep minesshd73
71476 25570 1 0 3 0x4000082 thrsleep minesshd73
71476 33137 1 0 3 0x4000082 thrsleep minesshd73
71476 14073 1 0 3 0x4000082 kqread minesshd73
71476 102328 1 0 3 0x4000082 thrsleep minesshd73
71476 97369 1 0 3 0x4000082 thrsleep minesshd73
71476 194464 1 0 3 0x4000082 thrsleep minesshd73
21723 459592 30444 67 3 0x90 kqread nginx
30444 230853 1 0 3 0x88 sigsusp nginx
76900 201770 35677 0 3 0x82 thrsleep mine-wan-health
76900 404375 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 174219 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 373529 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 4377 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 491455 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 138577 35677 0 3 0x4000082 thrsleep mine-wan-health
76900 147944 35677 0 3 0x4000082 kqread mine-wan-health
26372 503310 35677 0 3 0x82 thrsleep mine-alarmd
26372 443895 35677 0 3 0x4000082 thrsleep mine-alarmd
26372 464332 35677 0 3 0x4000082 kqread mine-alarmd
26372 134881 35677 0 3 0x4000082 thrsleep mine-alarmd
26372 34808 35677 0 3 0x4000082 thrsleep mine-alarmd
26372 68007 35677 0 3 0x4000082 thrsleep mine-alarmd
26372 304746 35677 0 3 0x4000082 thrsleep mine-alarmd
52103 14397 35677 0 3 0x82 thrsleep mine-core-health
che
52103 349746 35677 0 3 0x4000082 nanoslp mine-core-health
che
52103 412063 35677 0 3 0x4000082 thrsleep mine-core-health
che
52103 475177 35677 0 2 0x4000002 mine-core-health
che
52103 460879 35677 0 3 0x4000082 thrsleep mine-core-health
che
52103 92085 35677 0 3 0x4000082 thrsleep mine-core-health
che
52103 26178 35677 0 3 0x4000082 thrsleep mine-core-health
che
52103 1125 35677 0 3 0x4000082 thrsleep mine-core-health
che
56789 386802 35677 0 3 0x82 kqread mine-bandwidth
56789 265559 35677 0 3 0x4000082 thrsleep mine-bandwidth
56789 184046 35677 0 3 0x4000082 thrsleep mine-bandwidth
56789 67005 35677 0 3 0x4000082 thrsleep mine-bandwidth
56789 19080 35677 0 3 0x4000082 thrsleep mine-bandwidth
56789 470207 35677 0 3 0x4000082 thrsleep mine-bandwidth
54522 193990 35677 0 3 0x82 kqread mine-siem
54522 425862 35677 0 3 0x4000082 thrsleep mine-siem
54522 331269 35677 0 3 0x4000082 thrsleep mine-siem
54522 205830 35677 0 3 0x4000082 thrsleep mine-siem
54522 224655 35677 0 3 0x4000082 thrsleep mine-siem
54522 292926 35677 0 3 0x4000082 thrsleep mine-siem
54522 306662 35677 0 3 0x4000082 thrsleep mine-siem
55971 307381 35677 0 3 0x82 kqread mine-make-pdf
55971 262056 35677 0 3 0x4000082 thrsleep mine-make-pdf
55971 447966 35677 0 3 0x4000082 thrsleep mine-make-pdf
55971 209150 35677 0 3 0x4000082 thrsleep mine-make-pdf
55971 381650 35677 0 3 0x4000082 thrsleep mine-make-pdf
55971 339221 35677 0 3 0x4000082 thrsleep mine-make-pdf
55971 236192 35677 0 3 0x4000082 thrsleep mine-make-pdf
90491 288981 35677 0 3 0x82 thrsleep mine-ips
90491 155316 35677 0 3 0x4000082 thrsleep mine-ips
90491 45012 35677 0 3 0x4000082 thrsleep mine-ips
90491 261871 35677 0 3 0x4000082 thrsleep mine-ips
90491 168569 35677 0 3 0x4000082 thrsleep mine-ips
90491 53098 35677 0 3 0x4000082 kqread mine-ips
90491 244240 35677 0 3 0x4000082 thrsleep mine-ips
29550 86948 1 109 3 0x90 kqread ftp-proxy
44043 101149 35677 0 3 0x82 thrsleep mine-telemetry
44043 298959 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 462054 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 139715 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 122066 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 363261 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 309998 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 437390 35677 0 3 0x4000082 thrsleep mine-telemetry
44043 64490 35677 0 3 0x4000082 kqread mine-telemetry
44043 312984 35677 0 3 0x4000082 thrsleep mine-telemetry
46983 263004 35677 0 3 0x82 thrsleep mine-hotspot
46983 111857 35677 0 3 0x4000082 thrsleep mine-hotspot
46983 491777 35677 0 3 0x4000082 thrsleep mine-hotspot
46983 282887 35677 0 3 0x4000082 kqread mine-hotspot
46983 36728 35677 0 3 0x4000082 thrsleep mine-hotspot
46983 434110 35677 0 3 0x4000082 thrsleep mine-hotspot
46983 170719 35677 0 3 0x4000082 thrsleep mine-hotspot
19670 280057 35677 0 3 0x82 thrsleep mine-sms-server
19670 451318 35677 0 3 0x4000082 thrsleep mine-sms-server
19670 516071 35677 0 3 0x4000082 thrsleep mine-sms-server
19670 355292 35677 0 3 0x4000082 kqread mine-sms-server
19670 385776 35677 0 3 0x4000082 thrsleep mine-sms-server
19670 508543 35677 0 3 0x4000082 thrsleep mine-sms-server
94941 384439 35677 0 3 0x82 kqread mine-actmonL7
94941 237887 35677 0 3 0x4000082 thrsleep mine-actmonL7
94941 276098 35677 0 3 0x4000082 thrsleep mine-actmonL7
94941 183600 35677 0 3 0x4000082 thrsleep mine-actmonL7
94941 5296 35677 0 3 0x4000082 thrsleep mine-actmonL7
94941 142492 35677 0 3 0x4000082 thrsleep mine-actmonL7
94941 430308 35677 0 3 0x4000082 thrsleep mine-actmonL7
81102 128162 35677 0 3 0x82 thrsleep mine-notifier
81102 127888 35677 0 3 0x4000082 thrsleep mine-notifier
81102 94240 35677 0 3 0x4000082 kqread mine-notifier
81102 440499 35677 0 3 0x4000082 thrsleep mine-notifier
81102 241102 35677 0 3 0x4000082 thrsleep mine-notifier
81102 197233 35677 0 3 0x4000082 thrsleep mine-notifier
81102 98579 35677 0 3 0x4000082 thrsleep mine-notifier
81102 181374 35677 0 3 0x4000082 thrsleep mine-notifier
48957 134520 35677 0 3 0x82 kqread mine-flt-helper
48957 360647 35677 0 3 0x4000082 thrsleep mine-flt-helper
48957 438668 35677 0 3 0x4000082 thrsleep mine-flt-helper
48957 26806 35677 0 3 0x4000082 thrsleep mine-flt-helper
48957 353886 35677 0 3 0x4000082 thrsleep mine-flt-helper
48957 272416 35677 0 3 0x4000082 thrsleep mine-flt-helper
48957 162349 35677 0 3 0x4000082 thrsleep mine-flt-helper
91580 311709 35677 0 3 0x82 thrsleep mine-pptmon
91580 132790 35677 0 3 0x4000082 thrsleep mine-pptmon
91580 221615 35677 0 3 0x4000082 kqread mine-pptmon
91580 220177 35677 0 3 0x4000082 thrsleep mine-pptmon
91580 73681 35677 0 3 0x4000082 thrsleep mine-pptmon
91580 124150 35677 0 3 0x4000082 thrsleep mine-pptmon
91580 173766 35677 0 3 0x4000082 thrsleep mine-pptmon
93392 45109 35677 0 3 0x82 thrsleep mine-helper
93392 103893 35677 0 3 0x4000082 thrsleep mine-helper
93392 475620 35677 0 3 0x4000082 thrsleep mine-helper
93392 198294 35677 0 3 0x4000082 thrsleep mine-helper
93392 166481 35677 0 3 0x4000082 thrsleep mine-helper
93392 279701 35677 0 3 0x4000082 thrsleep mine-helper
93392 476293 35677 0 3 0x4000082 kqread mine-helper
93392 96892 35677 0 3 0x4000082 thrsleep mine-helper
93392 192380 35677 0 3 0x4000082 thrsleep mine-helper
77986 107818 35677 0 3 0x82 kqread mine-micro
77986 376224 35677 0 3 0x4000082 thrsleep mine-micro
77986 264495 35677 0 3 0x4000082 thrsleep mine-micro
77986 479876 35677 0 3 0x4000082 thrsleep mine-micro
77986 419795 35677 0 3 0x4000082 thrsleep mine-micro
71642 403694 35677 0 3 0x82 thrsleep mine-macro
71642 384903 35677 0 3 0x4000082 thrsleep mine-macro
71642 425207 35677 0 3 0x4000082 thrsleep mine-macro
71642 441785 35677 0 3 0x4000082 thrsleep mine-macro
71642 187253 35677 0 3 0x4000082 thrsleep mine-macro
71642 78525 35677 0 3 0x4000082 thrsleep mine-macro
71642 452682 35677 0 3 0x4000082 thrsleep mine-macro
71642 66656 35677 0 3 0x4000082 thrsleep mine-macro
71642 422115 35677 0 3 0x4000082 kqread mine-macro
36097 122688 35677 0 3 0x82 thrsleep mine-msad
36097 70580 35677 0 3 0x4000082 thrsleep mine-msad
36097 154493 35677 0 3 0x4000082 thrsleep mine-msad
36097 416620 35677 0 3 0x4000082 thrsleep mine-msad
36097 516865 35677 0 3 0x4000082 kqread mine-msad
36097 425500 35677 0 3 0x4000082 thrsleep mine-msad
36097 129706 35677 0 3 0x4000082 thrsleep mine-msad
69865 459920 88498 73 3 0x1100090 kqread syslogd
88498 401370 1 0 3 0x100082 netio syslogd
94362 140282 37428 0 3 0x82 kqread ttyd
37428 334295 35677 0 3 0x10008a sigsusp sh
76208 316854 24044 91 3 0x92 kqread snmpd_metrics
24044 320340 1 0 3 0x100080 kqread snmpd
26816 44984 1 91 3 0x1100092 kqread snmpd
61151 176999 35677 0 3 0x82 thrsleep mine-today-analy
tic
61151 447874 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
61151 501341 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
61151 474997 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
61151 187885 35677 0 3 0x4000082 kqread mine-today-analy
tic
61151 417546 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
61151 69243 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
61151 217429 35677 0 3 0x4000082 thrsleep mine-today-analy
tic
72620 43920 3032 74 3 0x1100092 bpf pflogd
3032 330435 1 0 3 0x80 netio pflogd
48816 313640 0 0 7 0x14200 wg_crypt
2686 112494 0 0 3 0x14200 bored wg_crypt
53490 33827 0 0 3 0x14200 bored wg_crypt
70796 350782 0 0 3 0x14200 bored wg_crypt
92476 89471 0 0 3 0x14200 bored wg_handshake
55192 272102 0 0 3 0x14200 bored wg_handshake
86545 402133 69249 0 3 0x82 kqread mine-ssl-proxy
86545 54369 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
86545 516582 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
86545 127393 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
86545 102064 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
86545 500515 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
86545 144775 69249 0 3 0x4000082 thrsleep mine-ssl-proxy
69249 185788 35677 0 3 0x10008a sigsusp sh
70450 29936 56975 0 3 0x82 kqread redis-server
70450 192614 56975 0 3 0x4000082 fsleep redis-server
70450 185074 56975 0 3 0x4000082 fsleep redis-server
70450 255500 56975 0 3 0x4000082 fsleep redis-server
56975 314509 35677 0 3 0x10008a sigsusp sh
75982 469991 75440 502 3 0x92 kqread mariadbd
75982 33351 75440 502 3 0x4000092 fsleep mariadbd
75982 330286 75440 502 3 0x4000092 fsleep mariadbd
75982 374476 75440 502 3 0x4000092 fsleep mariadbd
75982 495191 75440 502 3 0x4000092 fsleep mariadbd
75982 488772 75440 502 3 0x4000092 fsleep mariadbd
75982 498925 75440 502 3 0x400009a sigwait mariadbd
75982 294554 75440 502 3 0x4000092 kqread mariadbd
75982 262511 75440 502 3 0x4000092 kqread mariadbd
75982 294730 75440 502 3 0x4000092 kqread mariadbd
75982 288099 75440 502 3 0x4000092 kqread mariadbd
75982 257010 75440 502 3 0x4000092 kqread mariadbd
75982 73761 75440 502 3 0x4000092 kqread mariadbd
75982 129897 75440 502 3 0x4000092 kqread mariadbd
75982 29065 75440 502 3 0x4000092 kqread mariadbd
75982 8357 75440 502 3 0x4000092 kqread mariadbd
75982 128685 75440 502 3 0x4000092 kqread mariadbd
75982 404937 75440 502 3 0x4000092 fsleep mariadbd
75982 521599 75440 502 3 0x4000092 kqread mariadbd
75982 221257 75440 502 3 0x4000092 fsleep mariadbd
75982 16484 75440 502 3 0x4000092 fsleep mariadbd
75982 222624 75440 502 3 0x4000092 fsleep mariadbd
75982 271704 75440 502 3 0x4000092 fsleep mariadbd
75982 308779 75440 502 3 0x4000092 fsleep mariadbd
75982 349801 75440 502 3 0x4000092 fsleep mariadbd
75982 479106 75440 502 3 0x4000092 fsleep mariadbd
75440 365729 1 0 3 0x10008a sigsusp sh
69109 294197 1 0 3 0x100098 kqread cron
90264 120852 1 797 3 0x100082 piperd logger
5635 374561 1 797 3 0x82 thrsleep influxd
5635 212975 1 797 3 0x4000082 thrsleep influxd
5635 336817 1 797 3 0x4000082 thrsleep influxd
5635 170880 1 797 3 0x4000082 kqread influxd
5635 285826 1 797 3 0x4000082 thrsleep influxd
5635 400810 1 797 3 0x4000082 thrsleep influxd
5635 33496 1 797 3 0x4000082 thrsleep influxd
5635 353278 1 797 3 0x4000082 thrsleep influxd
5635 256862 1 797 3 0x4000082 thrsleep influxd
5635 131562 1 797 3 0x4000082 thrsleep influxd
77805 399770 91118 67 3 0x90 netcon php-fpm-7.4
75007 146604 91118 67 3 0x90 netcon php-fpm-7.4
91118 521027 1 0 3 0x80 kqread php-fpm-7.4
76376 346633 78526 0 3 0x83 thrsleep minesh
76376 272425 78526 0 3 0x4000083 thrsleep minesh
76376 493654 78526 0 3 0x4000083 thrsleep minesh
76376 465326 78526 0 3 0x4000083 thrsleep minesh
76376 227926 78526 0 3 0x4000083 ttyin minesh
76376 442413 78526 0 3 0x4000083 thrsleep minesh
76376 278721 78526 0 3 0x4000083 thrsleep minesh
76376 339336 78526 0 3 0x4000083 kqread minesh
76376 284848 78526 0 3 0x4000083 thrsleep minesh
76376 58834 78526 0 3 0x4000083 thrsleep minesh
35677 419724 20831 0 3 0x82 wait mine-core
35677 441780 20831 0 3 0x4000082 nanoslp mine-core
35677 142174 20831 0 3 0x4000082 wait mine-core
35677 478480 20831 0 3 0x4000082 thrsleep mine-core
35677 296393 20831 0 3 0x4000082 thrsleep mine-core
35677 290959 20831 0 3 0x4000082 thrsleep mine-core
35677 295712 20831 0 3 0x4000082 wait mine-core
35677 33782 20831 0 3 0x4000082 wait mine-core
35677 79298 20831 0 3 0x4000082 wait mine-core
35677 120769 20831 0 3 0x4000082 wait mine-core
35677 455767 20831 0 3 0x4000082 wait mine-core
35677 523365 20831 0 7 0x4000002 mine-core
35677 429698 20831 0 3 0x4000082 wait mine-core
35677 101500 20831 0 3 0x4000082 wait mine-core
35677 316524 20831 0 3 0x4000082 wait mine-core
35677 332836 20831 0 3 0x4000082 wait mine-core
35677 22227 20831 0 3 0x4000082 wait mine-core
35677 437092 20831 0 3 0x4000082 wait mine-core
35677 43136 20831 0 3 0x4000082 wait mine-core
35677 33230 20831 0 3 0x4000082 wait mine-core
35677 365898 20831 0 3 0x4000082 wait mine-core
35677 118793 20831 0 3 0x4000082 wait mine-core
35677 92481 20831 0 3 0x4000082 wait mine-core
35677 306937 20831 0 3 0x4000082 wait mine-core
35677 411527 20831 0 3 0x4000082 wait mine-core
35677 25724 20831 0 3 0x4000082 wait mine-core
35677 134272 20831 0 3 0x4000082 wait mine-core
35677 231350 20831 0 3 0x4000082 thrsleep mine-core
35677 158485 20831 0 3 0x4000082 wait mine-core
35677 515481 20831 0 3 0x4000082 wait mine-core
35677 178524 20831 0 3 0x4000082 thrsleep mine-core
35677 330288 20831 0 3 0x4000082 wait mine-core
35677 47033 20831 0 3 0x4000082 thrsleep mine-core
35677 338013 20831 0 3 0x4000082 wait mine-core
35677 345804 20831 0 3 0x4000082 thrsleep mine-core
35677 170128 20831 0 3 0x4000082 thrsleep mine-core
20831 136546 1723 0 3 0x10008a sigsusp sh
1723 78628 1 0 3 0x80 wait tcd
71728 208745 1 0 3 0x80 mfsidl mount_mfs
70059 317080 0 0 3 0x14200 bored srdis
78526 452701 74136 0 3 0x10008b sigsusp sh
90821 194504 1 99 3 0x1100090 kqread sndiod
37723 511980 1 110 3 0x100090 kqread sndiod
96045 488449 1 0 3 0x88 kqread sshd
6907 407668 70926 77 3 0x100092 kqread dhcpleased
85709 156918 70926 77 3 0x100092 kqread dhcpleased
70926 97302 1 0 3 0x80 kqread dhcpleased
87037 274836 24805 115 3 0x100092 kqread slaacd
37205 253965 24805 115 3 0x100092 kqread slaacd
24805 458014 1 0 3 0x100080 kqread slaacd
80910 71151 1 0 3 0x80 mfsidl mount_mfs
97488 272313 1 0 3 0x80 mfsidl mount_mfs
56394 58531 1 0 3 0x80 mfsidl mount_mfs
54474 409239 1 0 3 0x80 mfsidl mount_mfs
74136 260831 1 0 3 0x10008b sigsusp sh
74803 292052 0 0 3 0x14200 bored smr
32967 428425 0 0 2 0x14200 zerothread
85676 459738 0 0 3 0x14200 aiodoned aiodoned
3977 241155 0 0 3 0x14200 syncer update
24566 186529 0 0 3 0x14200 cleaner cleaner
54255 155427 0 0 3 0x14200 reaper reaper
90962 185765 0 0 3 0x14200 pgdaemon pagedaemon
29380 222709 0 0 3 0x14200 mmctsk sdmmc0
32359 180296 0 0 3 0x14200 usbtsk usbtask
48460 11228 0 0 3 0x14200 usbatsk usbatsk
90494 73299 0 0 3 0x40014200 acpi0 acpi0
45562 105376 0 0 3 0x40014200 idle3
89925 447854 0 0 3 0x40014200 idle2
83426 382000 0 0 3 0x40014200 idle1
7693 360036 0 0 3 0x14200 bored sensors
86534 8444 0 0 2 0x14200 softnet
70698 200098 0 0 3 0x14200 bored softnet
1033 449697 0 0 3 0x14200 bored softnet
58177 473601 0 0 2 0x14200 softnet
6117 379028 0 0 3 0x14200 bored systqmp
65246 342284 0 0 3 0x14200 bored systq
89502 121087 0 0 3 0x40014200 bored softclock
69163 404148 0 0 3 0x40014200 idle0
1 57202 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{3}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 12883 18843K 18843K 78643K 36588 0
pcb 13 14K 18K 78643K 17 0
rtable 445 61K 73K 78643K 24030 0
ifaddr 233 52K 52K 78643K 1351 0
sysctl 3 1K 1K 78643K 3 0
counters 226 85K 85K 78643K 438 0
ioctlops 0 0K 4K 78643K 42936 0
iov 0 0K 4K 78643K 620 0
mount 8 8K 8K 78643K 9 0
log 0 0K 0K 78643K 1 0
vnodes 1220 77K 79K 78643K 4125 0
UFS quota 1 32K 32K 78643K 1 0
UFS mount 31 69K 69K 78643K 39 0
shm 2 1K 1K 78643K 2 0
VM map 2 1K 1K 78643K 2 0
sem 2 10K 10K 78643K 4 0
dirhash 156 29K 29K 78643K 198 0
ACPI 3556 426K 453K 78643K 13924 0
file desc 30 21K 24K 78643K 20846 0
sigio 1 0K 0K 78643K 2 0
proc 111 93K 101K 78643K 35667 0
MFS node 6 0K 0K 78643K 6 0
NFS srvsock 1 0K 0K 78643K 1 0
NFS daemon 1 16K 16K 78643K 1 0
in_multi 37 2K 2K 78643K 180 0
ether_multi 2 0K 0K 78643K 3 0
ISOFS mount 1 32K 32K 78643K 1 0
MSDOSFS mount 1 16K 16K 78643K 1 0
ttys 31 148K 148K 78643K 31 0
exec 0 0K 1K 78643K 47003 0
pfkey data 0 0K 0K 78643K 2 0
tdb 3 0K 0K 78643K 3 0
pagedep 1 8K 8K 78643K 1 0
inodedep 1 32K 1558K 78643K 16937 0
newblk 1 0K 0K 78643K 1 0
indirdep 0 0K 672K 78643K 1376 0
VM swap 8 1110K 1112K 78643K 10 0
UVM amap 4170 376K 379K 78643K 475831 0
UVM aobj 3 2K 2K 78643K 3 0
USB 11 10K 10K 78643K 13 0
USB device 4 0K 0K 78643K 4 0
USB HC 1 0K 0K 78643K 1 0
memdesc 1 4K 4K 78643K 1 0
crypto data 18 258K 258K 78643K 18 0
NDP 46 0K 0K 78643K 99 0
temp 202 5824K 5890K 78643K 1138487 0
kqueue 79 141K 147K 78643K 2026 0
SYN cache 2 16K 16K 78643K 2 0
ddb{3}> trace
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: -4
ddb{3}>
db_enter() at db_enter+0x10
end trace frame: 0xffff800022ab0d70, count: 0
ddb{3}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa6
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
cnputc(62) at cnputc+0x37
db_putchar(62) at db_putchar+0x2ea
kprintf() at kprintf+0x133b
db_printf(ffffffff8211cc51) at db_printf+0x69
fault(ffffffff820cd05b) at fault+0x8e
kpageflttrap(ffff800022a2cb60,8) at kpageflttrap+0x1cf
kerntrap(ffff800022a2cb60) at kerntrap+0x91
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
end trace frame: 0xffff800022a2cc20, count: 0
ddb{0}> machine ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(ffff800001b79f18) at mtx_enter+0x48
msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61
taskq_thread(ffff800001b79f00) at taskq_thread+0xeb
end trace frame: 0x0, count: 8
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_release_all(ffffffff824bca80) at __mp_release_all+0x21
msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10
b
kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe
kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00
50,ffff8000222643ac) at kqueue_scan+0x108
sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x
371
syscall(ffff8000222644e0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2b22a4b60, count: 5
ddb{2}> machine ddbcpu 3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: 11
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(ffff800001b79f18) at mtx_enter+0x48
msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61
taskq_thread(ffff800001b79f00) at taskq_thread+0xeb
end trace frame: 0x0, count: 8
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_release_all(ffffffff824bca80) at __mp_release_all+0x21
msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10
b
kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe
kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00
50,ffff8000222643ac) at kqueue_scan+0x108
sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x
371
syscall(ffff8000222644e0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2b22a4b60, count: 5
ddb{2}> machine ddbcpu 0t3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x
25
proc_trampoline_mp() at proc_trampoline_mp+0xcf
end trace frame: 0x0, count: 11
ddb{3}>
----------------------------------------------------------------------------------
CRASH 02 :
ddb{3}> show panic
*cpu3: kernel diagnostic assertion "!ISSET(rt->rt_flags, RTF_UP)" failed: file "
/usr/src/sys/net/route.c", line 496
ddb{3}> trace
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x
25
rtfree(fffffd8275365a90) at rtfree+0x1af
route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413
route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57
sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f
dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew
ritev+0x14d
sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x
d2
syscall(ffff80002254d550) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7699faee55e0, count: -11
ddb{3}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
msleep(ffffffff8253cc44,ffffffff8253cda8,7f,ffffffff820442c7,0) at msleep+0xe0
uvm_pagezero_thread(ffff8000ffff5b40) at uvm_pagezero_thread+0xa6
end trace frame: 0x0, count: 10
ddb{0}> machine ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
reaper(ffff8000ffff8008) at reaper+0x133
end trace frame: 0x0, count: 10
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
rtfree(fffffd8275365c50) at rtfree+0x4c
ip_output(fffffd8002a23200,0,fffffd81e1b055d0,800,0,fffffd81e1b05558,89c3b2da6f
974957) at ip_output+0x218
tcp_output(ffff800001c863c8) at tcp_output+0x168e
tcp_send(fffffd81623d1a28,fffffd8075758600,0,0) at tcp_send+0x84
sosend(fffffd81623d1a28,0,ffff80002252b6b8,0,0,80) at sosend+0x37f
sendit(ffff8000224deb48,9,ffff80002252b820,0,ffff80002252b910) at sendit+0x373
sys_sendmsg(ffff8000224deb48,ffff80002252b8b0,ffff80002252b910) at sys_sendmsg+
0x131
syscall(ffff80002252b980) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7ce33fafee50, count: 2
ddb{2}> machine ddbcpu 3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x
25
rtfree(fffffd8275365a90) at rtfree+0x1af
route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413
route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57
sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f
dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew
ritev+0x14d
sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x
d2
syscall(ffff80002254d550) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7699faee55e0, count: 4
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
msleep(ffffffff8253cc44,ffffffff8253cda8,7f,ffffffff820442c7,0) at msleep+0xe0
uvm_pagezero_thread(ffff8000ffff5b40) at uvm_pagezero_thread+0xa6
end trace frame: 0x0, count: 10
ddb{0}> machine ddbcpu 0t1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
reaper(ffff8000ffff8008) at reaper+0x133
end trace frame: 0x0, count: 10
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
rtfree(fffffd8275365c50) at rtfree+0x4c
ip_output(fffffd8002a23200,0,fffffd81e1b055d0,800,0,fffffd81e1b05558,89c3b2da6f
974957) at ip_output+0x218
tcp_output(ffff800001c863c8) at tcp_output+0x168e
tcp_send(fffffd81623d1a28,fffffd8075758600,0,0) at tcp_send+0x84
sosend(fffffd81623d1a28,0,ffff80002252b6b8,0,0,80) at sosend+0x37f
sendit(ffff8000224deb48,9,ffff80002252b820,0,ffff80002252b910) at sendit+0x373
sys_sendmsg(ffff8000224deb48,ffff80002252b8b0,ffff80002252b910) at sys_sendmsg+
0x131
syscall(ffff80002252b980) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7ce33fafee50, count: 2
ddb{2}> machine ddbcpu 0t3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82067518) at panic+0xbf
__assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x
25
rtfree(fffffd8275365a90) at rtfree+0x1af
route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413
route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57
sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f
dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew
ritev+0x14d
sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x
d2
syscall(ffff80002254d550) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7699faee55e0, count: 4
ddb{3}> machine ddbcpu 0t4
Invalid cpu 4
ddb{3}>
----------------------------------------------------------------------------------
CRASH 03 :
TEST1> uvm_fault(0xfffffd826717bcc0, 0x8, 0, 1) -> e
kernel: page fault trap, code=0
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
TID PID UID PRFLAGS PFLAGS CPU COMMAND
*225335 47125 0 0 0 1 bgpd
231752 78299 73 0x1100010 0 3 syslogd
344909 6421 0 0x14000 0x200 2 wg_handshake
361415 98860 0 0x14000 0x200 0 reaper
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc
route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57
sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f
dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew
ritev+0x14d
sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x
d2
syscall(ffff8000225d58c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7b583d6b8570, count: 4
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{1}> show panic
*cpu1: uvm_fault(0xfffffd826717bcc0, 0x8, 0, 1) -> e
ddb{1}> show malloc
Type InUse MemUse HighUse Limit Requests Type Lim
devbuf 12882 18843K 18843K 78643K 36398 0
pcb 13 14K 18K 78643K 19 0
rtable 473 63K 78K 78643K 3131 0
ifaddr 236 52K 53K 78643K 1429 0
sysctl 3 1K 1K 78643K 3 0
counters 226 85K 85K 78643K 226 0
ioctlops 0 0K 4K 78643K 9602 0
iov 0 0K 2K 78643K 78 0
mount 8 8K 8K 78643K 9 0
log 0 0K 0K 78643K 1 0
vnodes 1220 77K 79K 78643K 3833 0
UFS quota 1 32K 32K 78643K 1 0
UFS mount 31 69K 69K 78643K 39 0
shm 2 1K 1K 78643K 2 0
VM map 2 1K 1K 78643K 2 0
sem 2 10K 10K 78643K 4 0
dirhash 153 28K 28K 78643K 195 0
ACPI 3556 426K 453K 78643K 13924 0
file desc 29 21K 24K 78643K 2807 0
sigio 1 0K 0K 78643K 1 0
proc 101 76K 101K 78643K 6602 0
MFS node 6 0K 0K 78643K 6 0
NFS srvsock 1 0K 0K 78643K 1 0
NFS daemon 1 16K 16K 78643K 1 0
in_multi 37 2K 2K 78643K 48 0
ether_multi 2 0K 0K 78643K 2 0
ISOFS mount 1 32K 32K 78643K 1 0
MSDOSFS mount 1 16K 16K 78643K 1 0
ttys 25 122K 122K 78643K 25 0
exec 0 0K 1K 78643K 6746 0
pfkey data 0 0K 0K 78643K 2 0
tdb 3 0K 0K 78643K 3 0
pagedep 1 8K 8K 78643K 1 0
inodedep 1 32K 1047K 78643K 17614 0
newblk 1 0K 0K 78643K 1 0
indirdep 0 0K 256K 78643K 235 0
VM swap 8 1110K 1112K 78643K 10 0
UVM amap 3693 346K 348K 78643K 69709 0
UVM aobj 3 2K 2K 78643K 3 0
USB 11 10K 10K 78643K 13 0
USB device 4 0K 0K 78643K 4 0
USB HC 1 0K 0K 78643K 1 0
memdesc 1 4K 4K 78643K 1 0
crypto data 18 258K 258K 78643K 18 0
NDP 46 0K 0K 78643K 46 0
temp 200 5824K 5889K 78643K 145594 0
kqueue 70 125K 143K 78643K 288 0
SYN cache 2 16K 16K 78643K 2 0
ddb{1}> trace
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc
route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57
sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f
dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew
ritev+0x14d
sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x
d2
syscall(ffff8000225d58c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7b583d6b8570, count: -11
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
_kernel_lock() at _kernel_lock+0xb2
reaper(ffff8000ffff8da0) at reaper+0x133
end trace frame: 0x0, count: 7
ddb{0}> machine ddbcpu 1
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc
route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57
sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f
dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew
ritev+0x14d
sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x
d2
syscall(ffff8000225d58c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7b583d6b8570, count: 4
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
ifa_ifwithaddr(ffff8000224ed160,0) at ifa_ifwithaddr+0x2e
in_pcbaddrisavail(fffffd82101dc400,ffff8000224ed160,0,ffff8000224fb8e8) at in_p
cbaddrisavail+0xca
udp_output(fffffd82101dc400,fffffd8079b94c00,ffff8000224ed2a0,fffffd8074318e00)
at udp_output+0x23e
sosend(fffffd823147b760,ffff8000224ed2a0,0,fffffd8079b94c00,fffffd8074318e00,0)
at sosend+0x37f
wg_send(ffff800001afa000,ffff8000224ed468,fffffd8079b94c00) at wg_send+0x115
wg_send_buf(ffff800001afa000,ffff8000224ed468,ffff8000224ed4f8,94) at wg_send_b
uf+0x6b
wg_peer_send_buf(ffff80000801d838,ffff8000224ed4f8,94) at wg_peer_send_buf+0x10
7
wg_send_initiation(ffff80000801d838) at wg_send_initiation+0x13d
taskq_thread(ffff800001af9480) at taskq_thread+0x100
end trace frame: 0x0, count: 2
ddb{2}> machine ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_write(fffffd81d35fa3c8,ffff8000226cd530,0) at vn_write+0x3b
dofilewritev(ffff800022297070,e,ffff8000226cd530,0,ffff8000226cd630) at dofilew
ritev+0x14d
sys_writev(ffff800022297070,ffff8000226cd5d0,ffff8000226cd630) at sys_writev+0x
d2
syscall(ffff8000226cd6a0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f592e3e8d20, count: 6
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
_kernel_lock() at _kernel_lock+0xb2
reaper(ffff8000ffff8da0) at reaper+0x133
end trace frame: 0x0, count: 7
ddb{0}> machine ddbcpu 0t1
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc
route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57
sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f
dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew
ritev+0x14d
sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x
d2
syscall(ffff8000225d58c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7b583d6b8570, count: 4
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
ifa_ifwithaddr(ffff8000224ed160,0) at ifa_ifwithaddr+0x2e
in_pcbaddrisavail(fffffd82101dc400,ffff8000224ed160,0,ffff8000224fb8e8) at in_p
cbaddrisavail+0xca
udp_output(fffffd82101dc400,fffffd8079b94c00,ffff8000224ed2a0,fffffd8074318e00)
at udp_output+0x23e
sosend(fffffd823147b760,ffff8000224ed2a0,0,fffffd8079b94c00,fffffd8074318e00,0)
at sosend+0x37f
wg_send(ffff800001afa000,ffff8000224ed468,fffffd8079b94c00) at wg_send+0x115
wg_send_buf(ffff800001afa000,ffff8000224ed468,ffff8000224ed4f8,94) at wg_send_b
uf+0x6b
wg_peer_send_buf(ffff80000801d838,ffff8000224ed4f8,94) at wg_peer_send_buf+0x10
7
wg_send_initiation(ffff80000801d838) at wg_send_initiation+0x13d
taskq_thread(ffff800001af9480) at taskq_thread+0x100
end trace frame: 0x0, count: 2
ddb{2}> machine ddbcpu 0t3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_write(fffffd81d35fa3c8,ffff8000226cd530,0) at vn_write+0x3b
dofilewritev(ffff800022297070,e,ffff8000226cd530,0,ffff8000226cd630) at dofilew
ritev+0x14d
sys_writev(ffff800022297070,ffff8000226cd5d0,ffff8000226cd630) at sys_writev+0x
d2
syscall(ffff8000226cd6a0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f592e3e8d20, count: 6
ddb{3}> machine ddbcpu 0t4
Invalid cpu 4
ddb{3}>
----------------------------------------------------------------------------------
CRASH 04 :
SPOKE1> uvm_fault(0xfffffd81d5995878, 0x8, 0, 1) -> e
kernel: page fault trap, code=0
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
TID PID UID PRFLAGS PFLAGS CPU COMMAND
448769 98731 0 0x100002 0 3 sh
350289 69698 73 0x1100010 0 0 syslogd
*114462 84824 0 0 0 1 bgpd
256495 50081 0 0x14000 0x200 2 wg_handshake
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc
route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57
sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f
dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew
ritev+0x14d
sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x
d2
syscall(ffff8000224dd530) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7c334ef822e0, count: 4
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{1}> show panic
*cpu1: uvm_fault(0xfffffd81d5995878, 0x8, 0, 1) -> e
ddb{1}> trace
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc
route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57
sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f
dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew
ritev+0x14d
sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x
d2
syscall(ffff8000224dd530) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7c334ef822e0, count: -11
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa6
vn_write(fffffd82120ca790,ffff80002268db60,0) at vn_write+0x3b
dofilewritev(ffff8000225658f0,12,ffff80002268db60,0,ffff80002268dc60) at dofile
writev+0x14d
sys_writev(ffff8000225658f0,ffff80002268dc00,ffff80002268dc60) at sys_writev+0x
d2
syscall(ffff80002268dcd0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x70d107adf4d0, count: 6
ddb{0}> machine ddbcpu 1
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc
route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57
sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f
dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew
ritev+0x14d
sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x
d2
syscall(ffff8000224dd530) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7c334ef822e0, count: 4
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter_try(ffff8000000c6c98) at mtx_enter_try+0x17
mtx_enter(ffff8000000c6c98) at mtx_enter+0x35
msleep(ffff8000000c6c80,ffff8000000c6c98,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff8000000c6c80,ffff8000224e3440) at taskq_next_work+0x61
taskq_thread(ffff8000000c6c80) at taskq_thread+0xeb
end trace frame: 0x0, count: 7
ddb{2}> machine ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
syscall(ffff800022af5ce0) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x77c660727b70, count: 9
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa6
vn_write(fffffd82120ca790,ffff80002268db60,0) at vn_write+0x3b
dofilewritev(ffff8000225658f0,12,ffff80002268db60,0,ffff80002268dc60) at dofile
writev+0x14d
sys_writev(ffff8000225658f0,ffff80002268dc00,ffff80002268dc60) at sys_writev+0x
d2
syscall(ffff80002268dcd0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x70d107adf4d0, count: 6
ddb{0}> machine ddbcpu 0t1
Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax
srp_get_locked(8) at srp_get_locked+0x11
rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_
reprio+0x175
rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc
route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57
sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f
dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew
ritev+0x14d
sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x
d2
syscall(ffff8000224dd530) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7c334ef822e0, count: 4
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter_try(ffff8000000c6c98) at mtx_enter_try+0x17
mtx_enter(ffff8000000c6c98) at mtx_enter+0x35
msleep(ffff8000000c6c80,ffff8000000c6c98,20,ffffffff82099fe5,0) at msleep+0x12a
taskq_next_work(ffff8000000c6c80,ffff8000224e3440) at taskq_next_work+0x61
taskq_thread(ffff8000000c6c80) at taskq_thread+0xeb
end trace frame: 0x0, count: 7
ddb{2}> machine ddbcpu 0t3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
syscall(ffff800022af5ce0) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x77c660727b70, count: 9
ddb{3}> machine ddbcpu 0t4
Invalid cpu 4
ddb{3}>
