OpenBSD 6.6/amd64 kernel crash: softdep_deallocate_dependencies: unrecovered I/O error
Hi Misc,
This is just a report. My lovely OpenBSD firewall crashed lately too much. So
here kernel debugger output.
I was using softdep in every partition. Now i removed every softdep from fstab.
I think using softdep under /usr partition was the source of my fault:
# cat /etc/fstab.old
6d79d3e0564cde27.b none swap sw
6d79d3e0564cde27.a / ffs rw 1 1
6d79d3e0564cde27.g /log ffs rw,softdep,nodev,nosuid 1 2
6d79d3e0564cde27.d /tmp ffs rw,softdep,nodev,nosuid 1 2
6d79d3e0564cde27.f /usr ffs rw,softdep,wxallowed,nodev 1 2
6d79d3e0564cde27.e /var ffs rw,softdep,nodev,nosuid 1 2
Here ddb outputs:
ddb{0}> show panic
softdep_deallocate_dependencies: unrecovered I/O error
ddb{0}> trace
db_enter() at db_enter+0x10
panic() at panic+0x128
softdep_deallocate_dependencies(fd83e1c6af00) at softdep_deallocate_depende
ncies+0x49
brelse(fd83e1c6af00) at brelse+0xdc
sd_buf_done(fd83e1eb4548) at sd_buf_done+0x124
scsi_done(fd83e1eb4548) at scsi_done+0x24
ahci_port_intr(80277d00,8000) at ahci_port_intr+0xa88
ahci_ata_cmd_timeout(802a0f00) at ahci_ata_cmd_timeout+0x226
softclock(0) at softclock+0x142
softintr_dispatch(0) at softintr_dispatch+0xf2
Xsoftclock(0,40,1388,0,40,81f3c6f8) at Xsoftclock+0x1f
acpicpu_idle() at acpicpu_idle+0x1d2
sched_idle(81f3bff0) at sched_idle+0x225
end trace frame: 0x0, count: -13
ddb{0}> machine ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi(c,800022008ff0,80002202bff0,0,0,8000cef0) at X
resume_lapic_ipi+0x23
__mp_acquire_count(81fbd988,1) at __mp_acquire_count+0x82
mi_switch() at mi_switch+0x243
sleep_finish(8000335e7eb8,1) at sleep_finish+0x84
tsleep(fd83f0d36b60,118,81c9866f,bb9) at tsleep+0xcb
kqueue_scan(fd83f0d36b60,40,48097867800,8000335e8270,8000cef0,f
fff8000335e82b8) at kqueue_scan+0x113
sys_kevent(8000cef0,8000335e8320,8000335e8380) at sys_kevent+0x
2a9
syscall(8000335e83f0) at syscall+0x389
Xsyscall(6,48,7f7f1f80,48,0,48097867800) at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7f1f40, count: 4
ddb{1}> trace
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi(c,800022008ff0,80002202bff0,0,0,8000cef0) at X
resume_lapic_ipi+0x23
__mp_acquire_count(81fbd988,1) at __mp_acquire_count+0x82
mi_switch() at mi_switch+0x243
sleep_finish(8000335e7eb8,1) at sleep_finish+0x84
tsleep(fd83f0d36b60,118,81c9866f,bb9) at tsleep+0xcb
kqueue_scan(fd83f0d36b60,40,48097867800,8000335e8270,8000cef0,f
fff8000335e82b8) at kqueue_scan+0x113
sys_kevent(8000cef0,8000335e8320,8000335e8380) at sys_kevent+0x
2a9
syscall(8000335e83f0) at syscall+0x389
Xsyscall(6,48,7f7f1f80,48,0,48097867800) at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7f1f40, count: -11
ddb{1}>
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
end trace frame: 0x8000335e7d40, count: 0
ddb{1}>
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
end trace frame: 0x8000335e7d40, count: 0
ddb{1}>
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
end trace frame: 0x8000335e7d40, count: 0
ddb{1}>
x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
end trace frame: 0x8000335e7d40, count: 0
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(800022019ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi(d,800022019ff0,8000223e8000,0,0,8000223e80d8) at X
resume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa2
timeout_del_barrier(8000223e80d8) at timeout_del_barrier+0xa2
msleep(81efbe70,81efbe90,20,81c70ac3,0) at msleep+0xf5
taskq_next_work(81efbe70,800022409010) at taskq_next_work+0x38
taskq_thread(81efbe70) at taskq_thread+0x6f
end trace frame: 0x0, count: 7
ddb{2}> trace
x86_ipi_db(800022019ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi(d,800022019ff0,8000223e8000,0,0,8000223e80d8) at X
resume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xa2
timeout_del_barrier(8000223e80d8) at timeout_del_barrier+0xa2
msleep(81efbe70,81efbe90,20,81c70ac3,0) at msleep+0xf5
taskq_next_work(81efbe70,800022409010) at taskq_next_work+0x38
taskq_thread(81efbe70) at taskq_thread+0x6f
end trace frame: 0x0, count: -8
ddb{2}> machine ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(800022022ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi(c,800022022ff0,81f3bff0,0,0,8000fffeb8f0) at X
resume_lapic_ipi+0x23
__mp_acquire_count(81fbd988,1) at __mp_acquire_count+0x82
Re: OpenBSD 6.6/amd64 kernel crash: softdep_deallocate_dependencies: unrecovered I/O error
On Sun, Jun 07, 2020 at 02:18:50PM +0200, Antonius Bekasi wrote:
> Hi Misc,
>
>
>
> This is just a report. My lovely OpenBSD firewall crashed lately too much. So
> here kernel debugger output.
>
> I was using softdep in every partition. Now i removed every softdep from
> fstab. I think using softdep under /usr partition was the source of my fault:
>
>
>
> # cat /etc/fstab.old
>
> 6d79d3e0564cde27.b none swap sw
>
> 6d79d3e0564cde27.a / ffs rw 1 1
>
> 6d79d3e0564cde27.g /log ffs rw,softdep,nodev,nosuid 1 2
>
> 6d79d3e0564cde27.d /tmp ffs rw,softdep,nodev,nosuid 1 2
>
> 6d79d3e0564cde27.f /usr ffs rw,softdep,wxallowed,nodev 1 2
>
> 6d79d3e0564cde27.e /var ffs rw,softdep,nodev,nosuid 1 2
>
>
>
>
> Here ddb outputs:
>
>
>
> ddb{0}> show panic
>
> softdep_deallocate_dependencies: unrecovered I/O error
This is a clear sign of hardware failing.
-Otto
>
> ddb{0}> trace
>
> db_enter() at db_enter+0x10
>
> panic() at panic+0x128
>
> softdep_deallocate_dependencies(fd83e1c6af00) at
> softdep_deallocate_depende
>
> ncies+0x49
>
> brelse(fd83e1c6af00) at brelse+0xdc
>
> sd_buf_done(fd83e1eb4548) at sd_buf_done+0x124
>
> scsi_done(fd83e1eb4548) at scsi_done+0x24
>
> ahci_port_intr(80277d00,8000) at ahci_port_intr+0xa88
>
> ahci_ata_cmd_timeout(802a0f00) at ahci_ata_cmd_timeout+0x226
>
> softclock(0) at softclock+0x142
>
> softintr_dispatch(0) at softintr_dispatch+0xf2
>
> Xsoftclock(0,40,1388,0,40,81f3c6f8) at Xsoftclock+0x1f
>
> acpicpu_idle() at acpicpu_idle+0x1d2
>
> sched_idle(81f3bff0) at sched_idle+0x225
>
> end trace frame: 0x0, count: -13
>
> ddb{0}> machine ddbcpu 1
>
> Stopped at x86_ipi_db+0x12: leave
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> x86_ipi_handler() at x86_ipi_handler+0x80
>
> Xresume_lapic_ipi(c,800022008ff0,80002202bff0,0,0,8000cef0)
> at X
>
> resume_lapic_ipi+0x23
>
> __mp_acquire_count(81fbd988,1) at __mp_acquire_count+0x82
>
> mi_switch() at mi_switch+0x243
>
> sleep_finish(8000335e7eb8,1) at sleep_finish+0x84
>
> tsleep(fd83f0d36b60,118,81c9866f,bb9) at tsleep+0xcb
>
> kqueue_scan(fd83f0d36b60,40,48097867800,8000335e8270,8000cef0,f
>
> fff8000335e82b8) at kqueue_scan+0x113
>
> sys_kevent(8000cef0,8000335e8320,8000335e8380) at
> sys_kevent+0x
>
> 2a9
>
> syscall(8000335e83f0) at syscall+0x389
>
> Xsyscall(6,48,7f7f1f80,48,0,48097867800) at Xsyscall+0x128
>
> end of kernel
>
> end trace frame: 0x7f7f1f40, count: 4
>
> ddb{1}> trace
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> x86_ipi_handler() at x86_ipi_handler+0x80
>
> Xresume_lapic_ipi(c,800022008ff0,80002202bff0,0,0,8000cef0)
> at X
>
> resume_lapic_ipi+0x23
>
> __mp_acquire_count(81fbd988,1) at __mp_acquire_count+0x82
>
> mi_switch() at mi_switch+0x243
>
> sleep_finish(8000335e7eb8,1) at sleep_finish+0x84
>
> tsleep(fd83f0d36b60,118,81c9866f,bb9) at tsleep+0xcb
>
> kqueue_scan(fd83f0d36b60,40,48097867800,8000335e8270,8000cef0,f
>
> fff8000335e82b8) at kqueue_scan+0x113
>
> sys_kevent(8000cef0,8000335e8320,8000335e8380) at
> sys_kevent+0x
>
> 2a9
>
> syscall(8000335e83f0) at syscall+0x389
>
> Xsyscall(6,48,7f7f1f80,48,0,48097867800) at Xsyscall+0x128
>
> end of kernel
>
> end trace frame: 0x7f7f1f40, count: -11
>
> ddb{1}>
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> end trace frame: 0x8000335e7d40, count: 0
>
> ddb{1}>
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> end trace frame: 0x8000335e7d40, count: 0
>
> ddb{1}>
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> end trace frame: 0x8000335e7d40, count: 0
>
> ddb{1}>
>
> x86_ipi_db(800022008ff0) at x86_ipi_db+0x12
>
> end trace frame: 0x8000335e7d40, count: 0
>
> ddb{1}> machine ddbcpu 2
>
> Stopped at x86_ipi_db+0x12: leave
>
> x86_ipi_db(800022019ff0) at x86_ipi_db+0x12
>
> x86_ipi_handler() at x86_ipi_handler+0x80
>
> Xresume_lapic_ipi(d,800022019ff0,8000223e8000,0,0,8000223e80d8)
> at X
>
> resume_lapic_ipi+0x23
>
> _kernel_lock() at _kernel_lock+0xa2
>
> timeout_del_barrier(8000223e80d8) at timeout_del_barrier+0xa2
>
> msleep(81efbe70,81efbe90,20,81c70ac3,0) at msleep+0xf5
>
> taskq_next_work(81efbe70,800022409010) at taskq_next_work+0x38
>
> taskq_thread(81efbe70) at taskq_thread+0x6f
>
> end trace frame: 0x0, count: 7
>
> ddb{2}> trace
>
> x86_ipi_db(800022019ff0) at x86_ipi_db+0x12
>
> x86_ipi_handler() at x86_ipi_handler+0x80
>
> Xresume_lapic_ipi(d,800022019ff0,8000223e8000,0,0,8000223e80d8)
> at X
>
> resume_lapic_ipi+0x23
>
> _kernel_lock() at _kernel_lock+0xa2
>
> timeout_del_barrier(8000223e80d8) at timeout_del_barrier+0xa2
>
>
