Hello Fernando Toledo, Dear Maintainer,
On Wed, 16 Jan 2019 01:22:31 -0300 Fernando Toledo <ftol...@docksud.com.ar> wrote: > > I need help to know how to get more info/debug. Maybe you can install a core file collector. On stable or testing I would propose systemd-coredump, but that is not available in jessie. The package corekeeper sounds similar and might put after a crash a core file somewhere below /var/crash. That might provide some more information when opened with gdb: gdb -q /usr/sbin/asterisk /var/crash/<corefile> backtrace The other way would be to install gdb and attach that to the live process. This maybe in a detachable session like in tmux. tmux gdb -q -ex 'set width' -ex 'set pagination off' -ex 'cont' -ex 'backtrace' -ex 'generate-core-file ~/asterisk.core' -ex 'detach' -ex 'quit' --pid $(pidof asterisk) But without the debug information that might not be sufficient. In that case rebuilding the packages with debug information may be needed. See pointer for that in [1]. > > i found that my asterisk crash in dmesg on stretch: > > > > [1329393.987739] asterisk[27758]: segfault at 10 ip 00007facf79d5233 sp > > 00007facc14428d0 error 4 in libiksemel.so.3.1.1[7facf79ce000+d000] That line may point to below line in libiksemel-1.4 in src/stream.c:552. Kind regards, Bernhard [1] https://wiki.debian.org/HowToGetABacktrace (gdb) disassemble /m iks_send_raw Dump of assembler code for function iks_send_raw: ... 0x00007ffff7bd41f7 <+55>: js 0x7ffff7bd421b <iks_send_raw+91> 549 } else 550 #endif 551 { 552 ret = data->trans->send (data->sock, xmlstr, strlen (xmlstr)); 0x00007ffff7bd4220 <+96>: callq 0x7ffff7bcfef0 <strlen@plt> 0x00007ffff7bd4225 <+101>: mov 0x10(%rbx),%rcx 0x00007ffff7bd4229 <+105>: mov 0x50(%rbx),%rdi 0x00007ffff7bd422d <+109>: mov %rax,%rdx 0x00007ffff7bd4230 <+112>: mov %rbp,%rsi 0x00007ffff7bd4233 <+115>: callq *0x10(%rcx) <<<<<<<<<<<<<< here $rcx seems to contain 0 -> data->trans == NULL ? 553 if (ret) return ret; 0x00007ffff7bd4236 <+118>: test %eax,%eax 0x00007ffff7bd4238 <+120>: jne 0x7ffff7bd421b <iks_send_raw+91> 554 } 555 if (data->logHook) data->logHook (data->user_data, xmlstr, strlen (xmlstr), 0);
[1329393.987739] asterisk[27758]: segfault at 10 ip 00007facf79d5233 sp 00007facc14428d0 error 4 in libiksemel.so.3.1.1[7facf79ce000+d000] https://stackoverflow.com/questions/2549214/interpreting-segfault-messages "error 4" == 0b100 /* * Page fault error code bits: * * bit 0 == 0: no page found 1: protection fault * bit 1 == 0: read access 1: write access * bit 2 == 0: kernel-mode access 1: user-mode access * bit 3 == 1: use of reserved bit detected * bit 4 == 1: fault was an instruction fetch */ --> 0: no page found 0: read access 1: user-mode access ############ # Jessie amd64 qemu VM 2019-01-16 apt update apt dist-upgrade apt install corekeeper gdb binutils asterisk # objdump --disassemble /usr/lib/x86_64-linux-gnu/libiksemel.so.3 | grep 233 7233: ff 51 10 callq *0x10(%rcx) --> likely -> offset of 0x10 matches also the sefault line. 9233: 89 f1 mov %esi,%ecx --> unlikely root@debian:~# objdump --disassemble /usr/lib/x86_64-linux-gnu/libiksemel.so.3 | grep 7233: -B 40 -A 4 00000000000071c0 <iks_send_raw>: 71c0: 41 54 push %r12 71c2: 55 push %rbp 71c3: 48 89 f5 mov %rsi,%rbp 71c6: 53 push %rbx 71c7: e8 a4 be ff ff callq 3070 <iks_user_data@plt> 71cc: f6 40 58 04 testb $0x4,0x58(%rax) 71d0: 48 89 c3 mov %rax,%rbx 71d3: 48 89 ef mov %rbp,%rdi 71d6: 74 48 je 7220 <iks_send_raw+0x60> 71d8: e8 13 bd ff ff callq 2ef0 <strlen@plt> 71dd: 48 8b 7b 70 mov 0x70(%rbx),%rdi 71e1: 48 89 c2 mov %rax,%rdx 71e4: 48 89 ee mov %rbp,%rsi 71e7: e8 f4 be ff ff callq 30e0 <gnutls_record_send@plt> 71ec: 48 89 c2 mov %rax,%rdx 71ef: b8 07 00 00 00 mov $0x7,%eax 71f4: 48 85 d2 test %rdx,%rdx 71f7: 78 22 js 721b <iks_send_raw+0x5b> 71f9: 4c 8b 63 38 mov 0x38(%rbx),%r12 71fd: 4d 85 e4 test %r12,%r12 7200: 74 17 je 7219 <iks_send_raw+0x59> 7202: 48 89 ef mov %rbp,%rdi 7205: e8 e6 bc ff ff callq 2ef0 <strlen@plt> 720a: 48 8b 7b 20 mov 0x20(%rbx),%rdi 720e: 31 c9 xor %ecx,%ecx 7210: 48 89 c2 mov %rax,%rdx 7213: 48 89 ee mov %rbp,%rsi 7216: 41 ff d4 callq *%r12 7219: 31 c0 xor %eax,%eax 721b: 5b pop %rbx 721c: 5d pop %rbp 721d: 41 5c pop %r12 721f: c3 retq 7220: e8 cb bc ff ff callq 2ef0 <strlen@plt> 7225: 48 8b 4b 10 mov 0x10(%rbx),%rcx 7229: 48 8b 7b 50 mov 0x50(%rbx),%rdi 722d: 48 89 c2 mov %rax,%rdx 7230: 48 89 ee mov %rbp,%rsi 7233: ff 51 10 callq *0x10(%rcx) 7236: 85 c0 test %eax,%eax 7238: 75 e1 jne 721b <iks_send_raw+0x5b> 723a: 4c 8b 63 38 mov 0x38(%rbx),%r12 723e: 4d 85 e4 test %r12,%r12 . apt install dpkg-dev devscripts apt-get build-dep libiksemel3 mkdir source/libiksemel3/orig -p cd source/libiksemel3/orig apt-get source libiksemel3 cd cd source/libiksemel3 cp orig/ try1 -a cd try1/libiksemel-1.4/ dpkg-buildpackage find -iname "libiksemel.so.3*" -type f| xargs file ./debian/tmp/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, not stripped ./debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, stripped ./src/.libs/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, not stripped cmp debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1 /usr/lib/x86_64-linux-gnu/libiksemel.so.3 debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1 /usr/lib/x86_64-linux-gnu/libiksemel.so.3 differieren: Byte 473, Zeile 1 # not yet reproducible? different compiler? objdump --disassemble src/.libs/libiksemel.so.3.1.1 | grep "<iks_send_raw>" -A40 00000000000071c0 <iks_send_raw>: 71c0: 41 54 push %r12 71c2: 55 push %rbp 71c3: 48 89 f5 mov %rsi,%rbp 71c6: 53 push %rbx 71c7: e8 a4 be ff ff callq 3070 <iks_user_data@plt> 71cc: f6 40 58 04 testb $0x4,0x58(%rax) 71d0: 48 89 c3 mov %rax,%rbx 71d3: 48 89 ef mov %rbp,%rdi 71d6: 74 48 je 7220 <iks_send_raw+0x60> 71d8: e8 13 bd ff ff callq 2ef0 <strlen@plt> 71dd: 48 8b 7b 70 mov 0x70(%rbx),%rdi 71e1: 48 89 c2 mov %rax,%rdx 71e4: 48 89 ee mov %rbp,%rsi 71e7: e8 f4 be ff ff callq 30e0 <gnutls_record_send@plt> 71ec: 48 89 c2 mov %rax,%rdx 71ef: b8 07 00 00 00 mov $0x7,%eax 71f4: 48 85 d2 test %rdx,%rdx 71f7: 78 22 js 721b <iks_send_raw+0x5b> 71f9: 4c 8b 63 38 mov 0x38(%rbx),%r12 71fd: 4d 85 e4 test %r12,%r12 7200: 74 17 je 7219 <iks_send_raw+0x59> 7202: 48 89 ef mov %rbp,%rdi 7205: e8 e6 bc ff ff callq 2ef0 <strlen@plt> 720a: 48 8b 7b 20 mov 0x20(%rbx),%rdi 720e: 31 c9 xor %ecx,%ecx 7210: 48 89 c2 mov %rax,%rdx 7213: 48 89 ee mov %rbp,%rsi 7216: 41 ff d4 callq *%r12 7219: 31 c0 xor %eax,%eax 721b: 5b pop %rbx 721c: 5d pop %rbp 721d: 41 5c pop %r12 721f: c3 retq 7220: e8 cb bc ff ff callq 2ef0 <strlen@plt> 7225: 48 8b 4b 10 mov 0x10(%rbx),%rcx 7229: 48 8b 7b 50 mov 0x50(%rbx),%rdi 722d: 48 89 c2 mov %rax,%rdx 7230: 48 89 ee mov %rbp,%rsi 7233: ff 51 10 callq *0x10(%rcx) 7236: 85 c0 test %eax,%eax . -> This function looks quite similar, even the offsets LD_PRELOAD=/home/benutzer/source/libiksemel3/try1/libiksemel-1.4/src/.libs/libiksemel.so.3.1.1 gdb -q -ex 'set width 0' -ex 'set pagination off' --args /usr/sbin/asterisk Reading symbols from /usr/sbin/asterisk...(no debugging symbols found)...done. (gdb) info target Symbols from "/usr/sbin/asterisk". Local exec file: `/usr/sbin/asterisk', file type elf64-x86-64. Entry point: 0x31e43 0x0000000000000238 - 0x0000000000000254 is .interp ... 0x00000000003f2e40 - 0x000000000042ccc0 is .bss (gdb) b *0x31e43 Breakpoint 1 at 0x31e43 (gdb) run Starting program: /usr/sbin/asterisk Warning: Cannot insert breakpoint 1. Cannot access memory at address 0x31e43 (gdb) dele 1 (gdb) info target Symbols from "/usr/sbin/asterisk". Unix child process: Using the running image of child process 15000. While running this, GDB does not access memory from... Local exec file: `/usr/sbin/asterisk', file type elf64-x86-64. Entry point: 0x555555585e43 0x0000555555554238 - 0x0000555555554254 is .interp ... 0x00007ffff7ffe000 - 0x00007ffff7ffe1b0 is .bss in /lib64/ld-linux-x86-64.so.2 (gdb) b *0x555555585e43 Breakpoint 2 at 0x555555585e43 (gdb) cont Continuing. [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Breakpoint 2, 0x0000555555585e43 in ?? () (gdb) info share From To Syms Read Shared Object Library 0x00007ffff7ddcb00 0x00007ffff7df5370 Yes /lib64/ld-linux-x86-64.so.2 No linux-vdso.so.1 0x00007ffff7bd0460 0x00007ffff7bd731b Yes /home/benutzer/source/libiksemel3/try1/libiksemel-1.4/src/.libs/libiksemel.so.3.1.1 0x00007ffff78414a0 0x00007ffff796d943 Yes /lib/x86_64-linux-gnu/libc.so.6 ... 0x00007ffff43358b0 0x00007ffff433a1bc Yes (*) /usr/lib/x86_64-linux-gnu/libffi.so.6 (*): Shared library is missing debugging information. (gdb) disassemble /m iks_send_raw Dump of assembler code for function iks_send_raw: 542 { 0x00007ffff7bd41c0 <+0>: push %r12 0x00007ffff7bd41c2 <+2>: push %rbp 0x00007ffff7bd41c3 <+3>: mov %rsi,%rbp 0x00007ffff7bd41c6 <+6>: push %rbx 543 struct stream_data *data = iks_user_data (prs); 0x00007ffff7bd41c7 <+7>: callq 0x7ffff7bd0070 <iks_user_data@plt> 0x00007ffff7bd41d0 <+16>: mov %rax,%rbx 544 int ret; 545 546 #ifdef HAVE_GNUTLS 547 if (data->flags & SF_SECURE) { 0x00007ffff7bd41cc <+12>: testb $0x4,0x58(%rax) 0x00007ffff7bd41d6 <+22>: je 0x7ffff7bd4220 <iks_send_raw+96> 548 if (gnutls_record_send (data->sess, xmlstr, strlen (xmlstr)) < 0) return IKS_NET_RWERR; 0x00007ffff7bd41d3 <+19>: mov %rbp,%rdi 0x00007ffff7bd41d8 <+24>: callq 0x7ffff7bcfef0 <strlen@plt> 0x00007ffff7bd41dd <+29>: mov 0x70(%rbx),%rdi 0x00007ffff7bd41e1 <+33>: mov %rax,%rdx 0x00007ffff7bd41e4 <+36>: mov %rbp,%rsi 0x00007ffff7bd41e7 <+39>: callq 0x7ffff7bd00e0 <gnutls_record_send@plt> 0x00007ffff7bd41ec <+44>: mov %rax,%rdx 0x00007ffff7bd41ef <+47>: mov $0x7,%eax 0x00007ffff7bd41f4 <+52>: test %rdx,%rdx 0x00007ffff7bd41f7 <+55>: js 0x7ffff7bd421b <iks_send_raw+91> 549 } else 550 #endif 551 { 552 ret = data->trans->send (data->sock, xmlstr, strlen (xmlstr)); 0x00007ffff7bd4220 <+96>: callq 0x7ffff7bcfef0 <strlen@plt> 0x00007ffff7bd4225 <+101>: mov 0x10(%rbx),%rcx 0x00007ffff7bd4229 <+105>: mov 0x50(%rbx),%rdi 0x00007ffff7bd422d <+109>: mov %rax,%rdx 0x00007ffff7bd4230 <+112>: mov %rbp,%rsi 0x00007ffff7bd4233 <+115>: callq *0x10(%rcx) <<<<<<<<<<<<<< here $rcx seems to contain 0 -> data->trans == NULL ? 553 if (ret) return ret; 0x00007ffff7bd4236 <+118>: test %eax,%eax 0x00007ffff7bd4238 <+120>: jne 0x7ffff7bd421b <iks_send_raw+91> 554 } 555 if (data->logHook) data->logHook (data->user_data, xmlstr, strlen (xmlstr), 0); 0x00007ffff7bd41f9 <+57>: mov 0x38(%rbx),%r12 0x00007ffff7bd41fd <+61>: test %r12,%r12 0x00007ffff7bd4200 <+64>: je 0x7ffff7bd4219 <iks_send_raw+89> 0x00007ffff7bd4202 <+66>: mov %rbp,%rdi 0x00007ffff7bd4205 <+69>: callq 0x7ffff7bcfef0 <strlen@plt> 0x00007ffff7bd420a <+74>: mov 0x20(%rbx),%rdi 0x00007ffff7bd420e <+78>: xor %ecx,%ecx 0x00007ffff7bd4210 <+80>: mov %rax,%rdx 0x00007ffff7bd4213 <+83>: mov %rbp,%rsi 0x00007ffff7bd4216 <+86>: callq *%r12 0x00007ffff7bd423a <+122>: mov 0x38(%rbx),%r12 0x00007ffff7bd423e <+126>: test %r12,%r12 0x00007ffff7bd4241 <+129>: jne 0x7ffff7bd4202 <iks_send_raw+66> 0x00007ffff7bd4243 <+131>: jmp 0x7ffff7bd4219 <iks_send_raw+89> 0x00007ffff7bd4245: data32 nopw %cs:0x0(%rax,%rax,1) 556 return IKS_OK; 0x00007ffff7bd4219 <+89>: xor %eax,%eax 557 } 0x00007ffff7bd421b <+91>: pop %rbx 0x00007ffff7bd421c <+92>: pop %rbp 0x00007ffff7bd421d <+93>: pop %r12 0x00007ffff7bd421f <+95>: retq End of assembler dump.