[email protected] (Stefan Kempf), 2016.01.28 (Thu) 06:48 (CET):
> Stuart Henderson wrote:
> > On 2016/01/27 20:10, Stefan Kempf wrote:
> > > So what I suspect to happen is that:
> > > - userland does a syscall
> > > - something goes wrong in the kernel, causing it to call
> > > sigexit(SIGILL), terminating the process
> > > - and the offending instruction you see in the core dump
> > > is the 'syscall' instruction.
> >
> > If this is the case, perhaps ktrace will give clues.
>
> Let's give it a try.
>
> Marcus, can you run this as root, please?
> ktrace /sbin/ping some.domain
>
> Or whatever way you invoked ping that made it crash.
>
> And send us the output of kdump -f ktrace.out?
Thanks for the advise:
# ktrace /sbin/ping 192.168.188.189
PING 192.168.188.189 (192.168.188.189): 56 data bytes
64 bytes from 192.168.188.189: icmp_seq=0 ttl=255 time=3.286 ms
Illegal instruction
# kdump -f ./ktrace.out
31378 EMUL "native"
31378 ktrace RET ktrace 0
31378 ktrace CALL
execve(0x7f7ffffc1fff,0x7f7ffffc1f20,0x7f7ffffc1f38)
31378 ktrace NAMI "/sbin/ping"
31378 ktrace ARGS
[0] = "/sbin/ping"
[1] = "192.168.188.189"
31378 ping RET execve 0
31378 ping CALL mprotect(0x15a413e2f000,0x2000,0x1<PROT_READ>)
31378 ping RET mprotect 0
31378 ping CALL kbind(0,0x7f7ffffd8a5a,0)
31378 ping RET kbind 0
31378 ping CALL sysctl(6.7<hw.pagesize>,0x15a413f38c60,
0x7f7ffffd88c0,0,0)
31378 ping RET sysctl 0
31378 ping CALL mmap(0,0x1000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23805231312896/0x15a6965b3000
31378 ping CALL mprotect(0x15a6965b3000,0x1000,0x1<PROT_READ>)
31378 ping RET mprotect 0
31378 ping CALL socket(AF_INET,0x3<SOCK_RAW>,0x1)
31378 ping RET socket 3
31378 ping CALL getuid()
31378 ping RET getuid 0<"root">
31378 ping CALL setresuid(0<"root">,0<"root">,0<"root">)
31378 ping RET setresuid 0
31378 ping CALL getentropy(0x7f7ffffd81d0,40)
31378 ping RET getentropy 0
31378 ping CALL mmap(0,0x450,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23804690755584/0x15a67622f000
31378 ping CALL minherit(0x15a67622f000,0x450,MAP_INHERIT_ZERO)
31378 ping RET minherit 0
31378 ping CALL readlink(0x15a413c2ac78,0x7f7ffffd81a0,63)
31378 ping NAMI "/etc/malloc.conf"
31378 ping RET readlink -1 errno 2 No such file or directory
31378 ping CALL issetugid()
31378 ping RET issetugid 1
31378 ping CALL mmap(0,0x4000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23806166503424/0x15a6ce191000
31378 ping CALL mprotect(0x15a6ce191000,0x1000,0<PROT_NONE>)
31378 ping RET mprotect 0
31378 ping CALL mprotect(0x15a6ce194000,0x1000,0<PROT_NONE>)
31378 ping RET mprotect 0
31378 ping CALL mmap(0,0x2000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23805153824768/0x15a691bcd000
31378 ping CALL mprotect(0x15a413f35000,0x1000,0x1<PROT_READ>)
31378 ping RET mprotect 0
31378 ping CALL mmap(0,0x1000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23805802340352/0x15a6b8646000
31378 ping CALL mmap(0,0x1000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23805649330176/0x15a6af45a000
31378 ping CALL getpid()
31378 ping RET getpid 31378/0x7a92
31378 ping CALL getsockopt(3,SOL_SOCKET,SO_SNDBUF,0x7f7ffffd87c0,
0x7f7ffffd87b8)
31378 ping RET getsockopt 0
31378 ping CALL setsockopt(3,SOL_SOCKET,SO_RCVBUF,
0x15a413f30238,4)
31378 ping RET setsockopt 0
31378 ping CALL mprotect(0x15a6965b3000,0x1000,0x3<PROT_READ|
PROT_WRITE>)
31378 ping RET mprotect 0
31378 ping CALL mprotect(0x15a6965b3000,0x1000,0x1<PROT_READ>)
31378 ping RET mprotect 0
31378 ping CALL fstat(1,0x7f7ffffd7640)
31378 ping STRU struct stat { dev=1040, ino=338283,
mode=crw--w---- , nlink=1, uid=1000<"asfer">,
gid=4<"tty">, rdev=1280, atime=1453968739
<"Jan 28 09:12:19 2016">.870031187,
mtime=1453968739
<"Jan 28 09:12:19 2016">.870031187,
ctime=1453968739
<"Jan 28 09:12:19 2016">.870031187, size=0,
blocks=0, blksize=65536, flags=0x0,
gen=0x88d85db9 }
31378 ping RET fstat 0
31378 ping CALL mmap(0,0x10000,0x3<PROT_READ|PROT_WRITE>,0x1002
<MAP_PRIVATE|MAP_ANON>,-1,0)
31378 ping RET mmap 23803532505088/0x15a631197000
31378 ping CALL fcntl(1,F_ISATTY)
31378 ping RET fcntl 1
31378 ping CALL write(1,0x15a631197000,0x36)
31378 ping GIO fd 1 wrote 54 bytes
"PING 192.168.188.189 (192.168.188.189): 56 data bytes
"
31378 ping RET write 54/0x36
31378 ping CALL pledge(0x15a413c2a1c1,0)
31378 ping STRU pledge request="stdio inet dns"
31378 ping RET pledge 0
31378 ping CALL sigaction(SIGINT,0x7f7ffffd8230,0x7f7ffffd8220)
31378 ping STRU struct sigaction { handler=0x15a413b03050,
mask=0<>, flags=0x2<SA_RESTART> }
31378 ping STRU struct sigaction { handler=SIG_DFL, mask=0<>,
flags=0<> }
31378 ping RET sigaction 0
31378 ping CALL sigaction(SIGINFO,0x7f7ffffd8230,0x7f7ffffd8220)
31378 ping STRU struct sigaction { handler=0x15a413b03050,
mask=0<>, flags=0x2<SA_RESTART> }
31378 ping STRU struct sigaction { handler=SIG_DFL, mask=0<>,
flags=0x12<SA_RESTART|SA_NODEFER> }
31378 ping RET sigaction 0
31378 ping CALL sigaction(SIGALRM,0x7f7ffffd8230,0x7f7ffffd8220)
31378 ping STRU struct sigaction { handler=0x15a413b03050,
mask=0<>, flags=0x2<SA_RESTART> }
31378 ping STRU struct sigaction { handler=SIG_DFL, mask=0<>,
flags=0<> }
31378 ping RET sigaction 0
31378 ping CALL setitimer(ITIMER_REAL,0x7f7ffffd8740,0)
31378 ping RET setitimer 0
31378 ping CALL clock_gettime(CLOCK_MONOTONIC,0x7f7ffffd81c0)
31378 ping STRU struct timespec { 871<"Jan 1 01:14:31
1970">.350784414 }
31378 ping RET clock_gettime 0
31378 ping CALL sendto(3,0x15a413f38eb4,0x40,0,0x15a413f4af10,
0x10)
31378 ping STRU struct sockaddr { AF_INET, 192.168.188.189:0 }
31378 ping GIO fd 3 wrote 64 bytes
"\b\0\M^Pc\M^Rz\0\0C\^O{\^W=\M-'N\M-eC\M^OX\^Wf\M^I\M-v\M^]\f4\
M-*\^X\M-P~\M^XR\^X\^Y\^Z\^[\^\\^]\^^\^_!"#$%&'()*+,-.\
/01234567"
31378 ping RET sendto 64/0x40
31378 ping CALL poll(0x7f7ffffd8790,1,INFTIM)
31378 ping RET poll 1
31378 ping CALL recvmsg(3,0x7f7ffffd86e0,0)
31378 ping GIO fd 3 read 84 bytes
"E\0\0T\a\^Z\0\0\M^?\^A\M-:\^Z\M-@\M-(\M-<\M-=\M-@\M-(\M-<e\0\0\
M^Xc\M^Rz\0\0C\^O{\^W=\M-'N\M-eC\M^OX\^Wf\M^I\M-v\M^]\\f4\M-*\
^X\M-P~\M^XR\^X\^Y\^Z\^[\^\\^]\^^\^_!"#$%&'()*+,-./01234567"
31378 ping STRU struct sockaddr { AF_INET, 192.168.188.189:0 }
31378 ping STRU struct msghdr { name=0x7f7ffffd8780, namelen=16,
iov=0x7f7ffffd8760, iovlen=1,
control=0x7f7ffffd82d0, controllen=0, flags=0 }
31378 ping STRU struct iovec { base=0x15a6b8646f54, len=108 }
31378 ping RET recvmsg 84/0x54
31378 ping CALL clock_gettime(CLOCK_MONOTONIC,0x7f7ffffd8200)
31378 ping STRU struct timespec { 871<"Jan 1 01:14:31
1970">.354070662 }
31378 ping RET clock_gettime 0
31378 ping CALL write(1,0x15a631197000,0x40)
31378 ping GIO fd 1 wrote 64 bytes
"64 bytes from 192.168.188.189: icmp_seq=0 ttl=255 time=3.286 ms
"
31378 ping RET write 64/0x40
31378 ping CALL poll(0x7f7ffffd8790,1,INFTIM)
31378 ping PSIG SIGALRM caught handler=0x15a413b03050 mask=0<>
# gdb -q /usr/sbin/sshd /sshd.core
(no debugging symbols found)
Core was generated by `sshd'.
Program terminated with signal 4, Illegal instruction.
(no debugging symbols found)
Loaded symbols for /usr/sbin/sshd
Reading symbols from /usr/lib/libutil.so.12.1...done.
Loaded symbols for /usr/lib/libutil.so.12.1
Reading symbols from /usr/lib/libcrypto.so.37.0...done.
Loaded symbols for /usr/lib/libcrypto.so.37.0
Reading symbols from /usr/lib/libz.so.5.0...done.
Loaded symbols for /usr/lib/libz.so.5.0
Reading symbols from /usr/lib/libc.so.84.2...done.
Loaded symbols for /usr/lib/libc.so.84.2
Reading symbols from /usr/libexec/ld.so...done.
Loaded symbols for /usr/libexec/ld.so
#0 0x00000d9b0d57d52a in select () at <stdin>:2
2 <stdin>: No such file or directory.
in <stdin>
(gdb) bt
#0 0x00000d9b0d57d52a in select () at <stdin>:2
#1 0x00000d990c00de91 in sshd_hostkey_sign () from /usr/sbin/sshd
#2 0x00000d990c00b4a1 in ?? () from /usr/sbin/sshd
#3 0x0000000000000000 in ?? ()
Current language: auto; currently asm
(gdb)
Thanks for looking, Marcus
