Hi People,
Kernel enters panic again. The ddb output is below;
ddb{0}> show panic
malloc: out of space in kmem_map
ddb{0}>
ddb{1}> machine ddbcpu 0t0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff820e6ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffffffff820e6ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{0}> trace
x86_ipi_db(ffffffff820e6ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffffffff820e6ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{0}> machine ddbcpu 0t1
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff81e3e7a8) at panic+0x12a
malloc(70800,27,9) at malloc+0x795
fdcopy(ffff800025fb3fe8) at fdcopy+0x14c
process_new(ffff800026069ce0,ffff800025fb3fe8,1) at process_new+0x107
fork1(ffff8000260a4d28,1,ffffffff8116d450,0,ffff80002617abe0,0) at fork1+0x23b
syscall(ffff80002617ac50) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2f5ef6090, count: 7
ddb{1}> show panic
malloc: out of space in kmem_map
ddb{1}> trace
db_enter() at db_enter+0x10
panic(ffffffff81e3e7a8) at panic+0x12a
malloc(70800,27,9) at malloc+0x795
fdcopy(ffff800025fb3fe8) at fdcopy+0x14c
process_new(ffff800026069ce0,ffff800025fb3fe8,1) at process_new+0x107
fork1(ffff8000260a4d28,1,ffffffff8116d450,0,ffff80002617abe0,0) at fork1+0x23b
syscall(ffff80002617ac50) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2f5ef6090, count: -8
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d863ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_acquire_count(ffffffff82204c90,1) at __mp_acquire_count+0x86
tsleep(fffffd8d3f848dd0,118,ffffffff81e48974,49) at tsleep+0xef
kqueue_sleep(fffffd8d3f848dd0,ffff800026635fb0) at kqueue_sleep+0xb1
kqueue_scan(ffff800026635ec0,8,ffff800026635dc0,ffff800026635fb0,ffff8000265cca
80,ffff800026635ffc) at kqueue_scan+0xfb
sys_kevent(ffff8000265cca80,ffff800026636060,ffff8000266360c0) at sys_kevent+0x
3b6
syscall(ffff800026636130) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x27ca68d10, count: 5
ddb{2}> trace
x86_ipi_db(ffff80001d863ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_acquire_count(ffffffff82204c90,1) at __mp_acquire_count+0x86
tsleep(fffffd8d3f848dd0,118,ffffffff81e48974,49) at tsleep+0xef
kqueue_sleep(fffffd8d3f848dd0,ffff800026635fb0) at kqueue_sleep+0xb1
kqueue_scan(ffff800026635ec0,8,ffff800026635dc0,ffff800026635fb0,ffff8000265cca
80,ffff800026635ffc) at kqueue_scan+0xfb
sys_kevent(ffff8000265cca80,ffff800026636060,ffff8000266360c0) at sys_kevent+0x
3b6
syscall(ffff800026636130) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x27ca68d10, count: -10
ddb{2}> machine ddbcpu 0t3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d86cff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
msleep_nsec(ffff800000038080,ffff800000038098,20,ffffffff81e19a6f,fffffffffffff
fff) at msleep_nsec+0xe6
taskq_next_work(ffff800000038080,ffff800025c15dc0) at taskq_next_work+0x61
taskq_thread(ffff800000038080) at taskq_thread+0xa6
end trace frame: 0x0, count: 9
ddb{3}> trace
x86_ipi_db(ffff80001d86cff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
msleep_nsec(ffff800000038080,ffff800000038098,20,ffffffff81e19a6f,fffffffffffff
fff) at msleep_nsec+0xe6
taskq_next_work(ffff800000038080,ffff800025c15dc0) at taskq_next_work+0x61
taskq_thread(ffff800000038080) at taskq_thread+0xa6
end trace frame: 0x0, count: -6
ddb{3}> machine ddbcpu 0t4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d875ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_acquire_count(ffffffff82204c90,2) at __mp_acquire_count+0x92
mi_switch() at mi_switch+0x241
sleep_finish(ffff8000260896c0,1) at sleep_finish+0x2ce
tsleep(fffffd8e894517e0,118,ffffffff81e48974,43) at tsleep+0xb2
kqueue_sleep(fffffd8e894517e0,ffff800026089a50) at kqueue_sleep+0xb1
kqueue_scan(ffff800026089960,8,ffff800026089860,ffff800026089a50,ffff8000ffff05
40,ffff800026089a9c) at kqueue_scan+0xfb
sys_kevent(ffff8000ffff0540,ffff800026089b00,ffff800026089b60) at sys_kevent+0x
3b6
syscall(ffff800026089bd0) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2886c2ca0, count: 3
ddb{4}> trace
x86_ipi_db(ffff80001d875ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_acquire_count(ffffffff82204c90,2) at __mp_acquire_count+0x92
mi_switch() at mi_switch+0x241
sleep_finish(ffff8000260896c0,1) at sleep_finish+0x2ce
tsleep(fffffd8e894517e0,118,ffffffff81e48974,43) at tsleep+0xb2
kqueue_sleep(fffffd8e894517e0,ffff800026089a50) at kqueue_sleep+0xb1
kqueue_scan(ffff800026089960,8,ffff800026089860,ffff800026089a50,ffff8000ffff05
40,ffff800026089a9c) at kqueue_scan+0xfb
sys_kevent(ffff8000ffff0540,ffff800026089b00,ffff800026089b60) at sys_kevent+0x
3b6
syscall(ffff800026089bd0) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2886c2ca0, count: -12
ddb{4}> machine ddbcpu 0t5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d87eff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_ioctl(fffffd8c005e3968,8004667e,ffff800025da4c38,ffff8000fffe6fc0) at vn_ioc
tl+0x3b
sys_fcntl(ffff8000fffe6fc0,ffff800025da4cc0,ffff800025da4d20) at sys_fcntl+0x58
3
syscall(ffff800025da4d90) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2166817b0, count: 7
ddb{5}> trace
x86_ipi_db(ffff80001d87eff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_ioctl(fffffd8c005e3968,8004667e,ffff800025da4c38,ffff8000fffe6fc0) at vn_ioc
tl+0x3b
sys_fcntl(ffff8000fffe6fc0,ffff800025da4cc0,ffff800025da4d20) at sys_fcntl+0x58
3
syscall(ffff800025da4d90) at syscall+0x389
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x2166817b0, count: -8
ddb{5}> machine ddbcpu 0t6
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d887ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d887ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{6}> trace
x86_ipi_db(ffff80001d887ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d887ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{6}> machine ddbcpu 0t7
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d890ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d890ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{7}> trace
x86_ipi_db(ffff80001d890ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d890ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{7}> machine ddbcpu 0t8
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d899ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d899ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{8}> trace
x86_ipi_db(ffff80001d899ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d899ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{8}> machine ddbcpu 0t9
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8a2ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8a2ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{9}> trace
x86_ipi_db(ffff80001d8a2ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8a2ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{9}> machine ddbcpu 0t10
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8abff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8abff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{10}> trace
x86_ipi_db(ffff80001d8abff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8abff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{10}> machine ddbcpu 0t11
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8b4ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8b4ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{11}> trace
x86_ipi_db(ffff80001d8b4ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8b4ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{11}> machine ddbcpu 0t12
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8bdff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8bdff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{12}> trace
x86_ipi_db(ffff80001d8bdff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8bdff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{12}> machine ddbcpu 0t13
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8c6ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8c6ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{13}> trace
x86_ipi_db(ffff80001d8c6ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8c6ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{13}> machine ddbcpu 0t14
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8cfff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8cfff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{14}> trace
x86_ipi_db(ffff80001d8cfff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8cfff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{14}> machine ddbcpu 0t15
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8d8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8d8ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{15}> trace
x86_ipi_db(ffff80001d8d8ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8d8ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{15}> machine ddbcpu 0t16
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8e1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8e1ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{16}> trace
x86_ipi_db(ffff80001d8e1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8e1ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{16}> machine ddbcpu 0t17
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8eaff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8eaff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{17}> trace
x86_ipi_db(ffff80001d8eaff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8eaff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{17}> machine ddbcpu 0t18
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8f3ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8f3ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{18}> trace
x86_ipi_db(ffff80001d8f3ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8f3ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{18}> machine ddbcpu 0t19
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80001d8fcff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8fcff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{19}> trace
x86_ipi_db(ffff80001d8fcff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff80001d8fcff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
________________________________
From: owner-m...@openbsd.org <owner-m...@openbsd.org> on behalf of Stuart
Henderson <stu.li...@spacehopper.org>
Sent: Tuesday, September 14, 2021 10:41 PM
To: m...@openbsd.org <m...@openbsd.org>
Subject: Re: Kernel ddb trace output
On 2021-09-14, Edward Crawler <edwardcraw...@outlook.com> wrote:
> Hi People,
>
> I have a OpenBSD 6.9 device which I'm using as firewall. The system crashes
> frequently -almost once a day-
> Also, there is lots of "pmap_unwire" errors in dmesg. Can it be a reason for
> system crash?
> What would be the reason of these crashes?
- What is printed when it enters DDB? (I guess this is on serial console;
please connect it before it crashes so you can see)
- Please include "ps" and "ps /o"
> ddb{9}> machine ddbcpu 10
^^ ^^
> Stopped at x86_ipi_db+0x12: leave
> ddb{16}> trace
^^
the prompt shows the cpu number in decimal, but the input to commands
(including "machine ddbcpu") is read in hex. therefore if you have cpu10
and higher, you either need to use hex, or specify that the number is
in decimal, i.e. "mach ddbcpu 0t10"
> tun_dev_write(2807,ffff80002657fe88,11,0) at tun_dev_write+0x18f
an outline of what you've got running on the system would be helpful too.
When you've collected this, please send it along with the information from your
first mail (so it's all self-contained and useful to someone without reference
to the earlier misc@ post) to bugs@openbsd.org.
