On 14.05.2014 - 15:48:27, Philip Guenther wrote:
> On Wed, 14 May 2014, Matthias Pitzl wrote:
> > On our server running with OpenBSD 5.5 calling netstat sometimes lead to
> > an uvm_fault in sysctl_file(). I can trigger the problem by running some
> > of our tests which create/delete a lot of files, open/close sockets etc.
> > and call netstat -anv.
> >
> > Attached file contains dmesg, trace and ps output.
> ...
> > ddb{3}> trace
> > sysctl_file() at sysctl_file+0x180
> > kern_sysctl() at kern_sysctl+0x686
>
> You don't happen to have the output of "show reg", do you?
Hi,
This time with show registers/trace output per CPU.
Hope this will help to nail down the bug.
-- Matthias
ddb{2}> show panic
the kernel did not panic
ddb{2}> trace
sysctl_file() at sysctl_file+0x180
kern_sysctl() at kern_sysctl+0x686
sys___sysctl() at sys___sysctl+0x1f4
syscall() at syscall+0x24f
--- syscall (number 202) ---
end of kernel
end trace frame: 0x1b5a4c8b3760, count: -4
0x1b5a3ee4574a:
ddb{2}> ps
PID PPID PGRP UID S FLAGS WAIT COMMAND
*17378 29432 29432 2433 7 0x33 netstat
4860 17533 17533 2433 7 0x3 bash
17533 8908 17533 2433 3 0x83 nanosleep vim
5430 1399 29432 2433 3 0x81 select perl
1399 29432 29432 2433 3 0x83 select perl
29432 29489 29432 2433 3 0x81 piperd perl
29489 1699 19910 2433 3 0x83 wait perl
1699 21025 19910 2433 3 0x83 wait perl
30750 1676 30750 2315 3 0x83 ttyin bash
11674 24848 11674 2433 3 0x83 ttyin bash
18395 24916 18395 2445 3 0x83 ttyin less
4160 1676 4160 2315 3 0x83 ttyin bash
1676 3706 3706 2315 3 0x90 select sshd
3706 24920 3706 0 3 0x92 poll sshd
11902 9786 11902 2433 3 0x83 select vim
31435 0 0 0 3 0x4200 nfskqpw nfskqpoll
8276 21025 19910 2433 3 0x83 kqread tail
21025 12863 19910 2433 3 0x8b pause sh
30738 15589 30738 2445 3 0x483 select vim
24916 32518 24916 2445 3 0x83 wait bash
32518 16671 16671 2445 3 0x90 select sshd
16671 24920 16671 0 3 0x92 poll sshd
2719 6794 2719 2242 4 0x8000403 vim
6794 32269 6794 2242 3 0x83 ttyin bash
32269 12552 12552 2242 3 0x90 select sshd
12552 24920 12552 0 3 0x92 poll sshd
22467 22195 25950 2242 3 0x83 ttyin less
22195 25480 22195 2242 3 0x83 wait bash
25480 32071 32071 2242 3 0x90 select sshd
32071 24920 32071 0 3 0x92 poll sshd
22904 10223 22904 2433 3 0x83 select vim
10223 11167 10223 2433 3 0x83 wait bash
11167 11732 11732 2433 3 0x90 select sshd
11732 24920 11732 0 3 0x92 poll sshd
15631 15721 15631 2136 3 0x83 ttyin bash
15721 17290 17290 2136 3 0x90 select sshd
17290 24920 17290 0 3 0x92 poll sshd
1057 9593 1057 2136 3 0x83 ttyin bash
9593 27084 27084 2136 3 0x90 select sshd
27084 24920 27084 0 3 0x92 poll sshd
13809 29132 13809 2482 3 0x83 ttyin less
31210 26802 31210 2482 3 0x83 select vim
3060 2365 3060 2482 3 0x83 select vim
17634 31054 17634 2482 3 0x83 select vim
17786 27470 17786 2482 3 0x83 ttyin bash
27470 5329 5329 2482 3 0x90 select sshd
5329 24920 5329 0 3 0x92 poll sshd
4327 5635 4327 2136 3 0x83 ttyin bash
5635 1 5635 2136 3 0x80 select screen
2347 15547 2347 2482 3 0x83 select vim
9080 5015 9080 2433 3 0x8b pause screen
5015 18889 5015 2433 3 0x83 wait bash
18889 20562 20562 2433 3 0x90 select sshd
20562 24920 20562 0 3 0x92 poll sshd
27613 9732 27613 2136 3 0x83 ttyin bash
9732 26955 26955 2136 3 0x90 select sshd
26955 24920 26955 0 3 0x92 poll sshd
4088 16947 4088 2275 3 0x83 ttyin bash
16947 8716 8716 2275 3 0x90 select sshd
8716 24920 8716 0 3 0x92 poll sshd
26802 6878 26802 2482 3 0x83 wait bash
6878 4281 4281 2482 3 0x90 select sshd
4281 24920 4281 0 3 0x92 poll sshd
2365 27016 2365 2482 3 0x83 wait bash
27016 7063 7063 2482 3 0x90 select sshd
7063 24920 7063 0 3 0x92 poll sshd
23452 29737 23452 2482 3 0x83 ttyin bash
29737 29898 29898 2482 3 0x90 select sshd
29898 24920 29898 0 3 0x92 poll sshd
31054 10653 31054 2482 3 0x83 wait bash
15547 22594 15547 2482 3 0x83 wait bash
5196 20131 5196 2482 3 0x83 ttyin bash
29132 30080 29132 2482 3 0x83 wait bash
10653 20890 20890 2482 3 0x90 select sshd
22594 1470 1470 2482 3 0x90 select sshd
30080 20360 20360 2482 3 0x90 select sshd
20131 17406 17406 2482 3 0x90 select sshd
17406 24920 17406 0 3 0x92 poll sshd
1470 24920 1470 0 3 0x92 poll sshd
20360 24920 20360 0 3 0x92 poll sshd
20890 24920 20890 0 3 0x92 poll sshd
25518 19093 25518 2136 3 0x83 ttyin bash
19093 12559 12559 2136 3 0x90 select sshd
12559 24920 12559 0 3 0x92 poll sshd
6577 11567 6577 2136 3 0x83 ttyin bash
11567 30058 30058 2136 3 0x90 select sshd
30058 24920 30058 0 3 0x92 poll sshd
18531 9575 18531 2136 3 0x83 ttyin bash
9575 2748 2748 2136 3 0x90 select sshd
2748 24920 2748 0 3 0x92 poll sshd
15589 32474 15589 2445 3 0x83 wait bash
32474 16674 16674 2445 3 0x90 select sshd
16674 24920 16674 0 3 0x92 poll sshd
8908 24848 8908 2433 3 0x83 wait bash
9786 24848 9786 2433 3 0x83 wait bash
12863 24848 12863 2433 3 0x83 wait bash
24848 1 24848 2433 3 0x80 select screen
5564 14844 5564 2433 3 0x83 ttyin bash
14844 29900 29900 2433 3 0x90 select sshd
29900 24920 29900 0 3 0x92 poll sshd
7684 7118 7684 2136 3 0x83 ttyin bash
7118 10158 10158 2136 3 0x90 select sshd
10158 24920 10158 0 3 0x92 poll sshd
20826 20005 20826 2275 3 0x83 ttyin bash
20005 30357 30357 2275 3 0x90 select sshd
30357 24920 30357 0 3 0x92 poll sshd
24981 21663 24981 2445 3 0x83 ttyin bash
21663 12151 12151 2445 3 0x90 select sshd
12151 24920 12151 0 3 0x92 poll sshd
11775 27502 11775 2445 3 0x83 ttyin bash
27502 19310 19310 2445 3 0x90 select sshd
19310 24920 19310 0 3 0x92 poll sshd
16014 1 16014 0 3 0x83 ttyin getty
15931 1 15931 0 3 0x83 ttyin getty
25779 1 25779 0 3 0x83 ttyin getty
26180 1 26180 0 3 0x83 ttyin getty
24512 1 24512 0 3 0x83 ttyin getty
28106 1 28106 0 3 0x83 ttyin getty
6452 1 6452 0 3 0x80 select cron
4880 1 4880 0 3 0x80 nanosleep sensorsd
14070 1 14070 99 3 0x90 poll sndiod
6439 15545 15545 67 3 0x90 netcon httpd
14121 15545 15545 67 3 0x90 netcon httpd
16873 15545 15545 67 3 0x90 netcon httpd
7125 15545 15545 67 3 0x90 netcon httpd
701 15545 15545 67 3 0x90 netcon httpd
15545 1 15545 0 3 0x80 select httpd
2715 1 2715 0 3 0xb0 select sendmail
9631 2691 2691 90 3 0x90 kqread ospf6d
6974 2691 2691 90 3 0x90 kqread ospf6d
2691 1 2691 0 3 0x80 kqread ospf6d
10656 21894 21894 85 3 0x90 kqread ospfd
9075 21894 21894 85 3 0x90 kqread ospfd
21894 1 21894 0 3 0x80 kqread ospfd
24920 1 24920 0 3 0x80 select sshd
30407 0 0 0 3 0x4280 nfsidl nfsio
23298 0 0 0 3 0x4280 nfsidl nfsio
28539 0 0 0 3 0x4280 nfsidl nfsio
8002 0 0 0 3 0x4280 nfsidl nfsio
29786 1 29786 0 3 0x80 poll ntpd
2593 30988 2593 83 3 0x90 poll ntpd
30988 1 30988 83 3 0x90 poll ntpd
14289 23043 23043 73 7 0x90 syslogd
23043 1 23043 0 3 0x80 netio syslogd
17511 1 17511 0 3 0x80 mfsidl mount_mfs
1554 0 0 0 3 0x4200 aiodoned aiodoned
9938 0 0 0 3 0x4200 syncer update
940 0 0 0 3 0x4200 cleaner cleaner
27214 0 0 0 3 0x4200 reaper reaper
4483 0 0 0 3 0x4200 pgdaemon pagedaemon
14933 0 0 0 3 0x4200 bored crypto
14251 0 0 0 3 0x4200 pftm pfpurge
2736 0 0 0 3 0x4200 usbtsk usbtask
6115 0 0 0 3 0x4200 usbatsk usbatsk
5758 0 0 0 3 0x40004200 acpi0 acpi0
8281 0 0 0 3 0x40004200 idle3
19275 0 0 0 3 0x40004200 idle2
20802 0 0 0 3 0x40004200 idle1
2020 0 0 0 3 0x4200 bored sensors
16006 0 0 0 3 0x4200 bored systq
24195 0 0 0 3 0x4200 bored syswq
18540 0 0 0 7 0x40004200 idle0
1 0 1 0 3 0x82 wait init
0 -1 0 0 3 0x200 scheduler swapper
ddb{1}> machine ddbcpu 0
Stopped at Debugger+0x5: leave
ddb{0}> trace
Debugger() at Debugger+0x5
x86_ipi_handler() at x86_ipi_handler+0x64
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x1b
--- interrupt ---
Bad frame pointer: 0xffff80002219adf0
end trace frame: 0xffff80002219adf0, count: -3
__mp_lock+0x55:
ddb{0}> show registers
ds 0
es 0x259b
fs 0xace0
gs 0x10ac
rdi 0xffffffff81a63f00 cpu_info_primary
rsi 0x286
rbp 0xffff80002219acd8
rbx 0x7
rdx 0xffff8000001a8001
rcx 0x7
rax 0xffffff7f
r8 0
r9 0
r10 0
r11 0xffffffff8133f450 x86_bus_space_mem_read_4
r12 0
r13 0xffffffff81a63f00 cpu_info_primary
r14 0xfffffe81bf5f0248
r15 0
rip 0xffffffff81342cf5 Debugger+0x5
cs 0x8
rflags 0x246
rsp 0xffff80002219acd8
ss 0x10
Debugger+0x5: leave
ddb{0}> machine ddbcpu 1
Stopped at Debugger+0x5: leave
ddb{1}> trace
Debugger() at Debugger+0x5
x86_ipi_handler() at x86_ipi_handler+0x64
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x1b
--- interrupt ---
Bad frame pointer: 0xffff80002239be90
end trace frame: 0xffff80002239be90, count: -3
__mp_lock+0x55:
ddb{1}> show registers
ds 0xbd90
es 0x51f3
fs 0xbd30
gs 0xffff
rdi 0xffff8000001a2800
rsi 0x400
rbp 0xffff80002239bd78
rbx 0x7
rdx 0xffff8000001a8001
rcx 0x7
rax 0xffffff7f
r8 0x100
r9 0xa
r10 0x6c14
r11 0
r12 0
r13 0xffff8000001a2800
r14 0xfffffe81bbfcfb68
r15 0xffff800022397000
rip 0xffffffff81342cf5 Debugger+0x5
cs 0x8
rflags 0x246
rsp 0xffff80002239bd78
ss 0
Debugger+0x5: leave
ddb{1}> machine ddbcpu 2
Stopped at sysctl_file+0x180: cmpq $0,0x18(%rbx)
ddb{2}> trace
sysctl_file() at sysctl_file+0x180
kern_sysctl() at kern_sysctl+0x686
sys___sysctl() at sys___sysctl+0x1f4
syscall() at syscall+0x24f
--- syscall (number 202) ---
end of kernel
end trace frame: 0x1b5a4c8b3760, count: -4
0x1b5a3ee4574a:
ddb{2}> show registers
ds 0xa800
es 0
fs 0
gs 0
rdi 0x1b5a4664b310
rsi 0xffff800000c4b650
rbp 0xffff8000222eac30
rbx 0xffffffffdeafbead
rdx 0x250
rcx 0
rax 0x250
r8 0x7f7fffffc000
r9 0xffff8000222ea808
r10 0
r11 0xffffffff8133e0a0 copy_fault
r12 0x1b5a4664b310
r13 0
r14 0xffff8000222eae28
r15 0x3c200 acpi_pdirpa+0x2aca0
rip 0xffffffff811942a0 sysctl_file+0x180
cs 0x8
rflags 0x10282 mp_pdirpa+0x19b
rsp 0xffff8000222eab80
ss 0x10
sysctl_file+0x180: cmpq $0,0x18(%rbx)
ddb{2}> machine ddbcpu 3
Stopped at Debugger+0x5: leave
ddb{3}> trace
Debugger() at Debugger+0x5
x86_ipi_handler() at x86_ipi_handler+0x64
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x1b
--- interrupt ---
Bad frame pointer: 0xffff800022215c78
end trace frame: 0xffff800022215c78, count: -3
__mp_lock+0x5e:
ddb{3}> show registers
ds 0x5b70
es 0xa000
fs 0x4
gs 0x2000
rdi 0xffff8000001a8800
rsi 0x1
rbp 0xffff800022215b58
rbx 0x7
rdx 0xffff8000001a8001
rcx 0x7
rax 0xffffff7f
r8 0
r9 0x1
r10 0x44b82fa09b5a53
r11 0xffffffff8133f450 x86_bus_space_mem_read_4
r12 0
r13 0xffff8000001a8800
r14 0xffff8000001a8810
r15 0x1
rip 0xffffffff81342cf5 Debugger+0x5
cs 0x8
rflags 0x246
rsp 0xffff800022215b58
ss 0x10
Debugger+0x5: leave
