On Wed, Feb 24, 2016 at 10:54 AM, Robin Lee <robinlee.s...@gmail.com> wrote:
> Hi all, > > We have a slightly costumed xapi based on xapi-0.2 build 6786 from > XenServer 6.5SP1. Xapi is built with OCaml 3.12 binary > (ocaml-3.12.1.ocamlspotter-unknown.x86_64) from XenServer binpkg. > > We met random core dumps of xenopsd at the function > 'st_masterlock_acquire'. All the segfaulted threads have the same stacks. > > Any hint for us to deal with this issue? > > Following are some gdb output: > > Core was generated by `/opt/xensource/libexec/xenopsd -daemon -config > /etc/xenopsd.conf'. > Program terminated with signal 11, Segmentation fault. > #0 0x0000000000623eaa in st_masterlock_acquire () > > (gdb) info threads > 13 Thread 0x7f3e49ce4940 (LWP 11185) 0x00007f3e4d2fdccb in accept () > from /lib64/libpthread.so.0 > 12 Thread 0x7f3e472d1940 (LWP 974) 0x00007f3e4d2fb019 in > pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 > 11 Thread 0x7f3e494e3940 (LWP 11186) 0x00007f3e4cbde162 in select () > from /lib64/libc.so.6 > 10 Thread 0x7f3e4a4e5940 (LWP 11184) 0x00007f3e4d2fdccb in accept () > from /lib64/libpthread.so.0 > 9 Thread 0x7f3e48ce2940 (LWP 11187) 0x00007f3e4d2fd654 in > __lll_lock_wait () from /lib64/libpthread.so.0 > 8 Thread 0x7f3e450d2940 (LWP 973) 0x00007f3e4d2fb019 in > pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 > 7 Thread 0x7f3e4dd826e0 (LWP 11177) 0x00007f3e4cbde162 in select () > from /lib64/libc.so.6 > 6 Thread 0x7f3e4b4e7940 (LWP 11182) 0x00007f3e4cbf90b0 in __read_chk > (fd=<value optimized out>, buf=0x7f3e4b4e2dd0, nbytes=16, buflen=<value > optimized out>) at read_chk.c:34 > 5 Thread 0x7f3e4ace6940 (LWP 11183) 0x00007f3e4cbde162 in select () > from /lib64/libc.so.6 > 4 Thread 0x7f3e47ce0940 (LWP 11190) 0x00007f3e4d2fb019 in > pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 > 3 Thread 0x7f3e3fce0940 (LWP 11189) 0x00007f3e4d2fd654 in > __lll_lock_wait () from /lib64/libpthread.so.0 > 2 Thread 0x7f3e4c4e9940 (LWP 11180) 0x00007f3e4d2fb019 in > pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 > * 1 Thread 0x7f3e484e1940 (LWP 11188) 0x0000000000623eaa in > st_masterlock_acquire () > > (gdb) info sharedlibrary > From To Syms Read Shared Object Library > 0x00007f3e4d952510 0x00007f3e4d96b6d8 Yes (*) > /usr/lib64/libxenctrl.so.4.4 > 0x00007f3e4d71b760 0x00007f3e4d73e108 Yes (*) > /usr/lib64/libxenguest.so.4.4 > 0x00007f3e4d50e220 0x00007f3e4d511d98 Yes (*) /lib64/librt.so.1 > 0x00007f3e4d2f5210 0x00007f3e4d3003e8 Yes (*) /lib64/libpthread.so.0 > 0x00007f3e4d070e60 0x00007f3e4d0b1028 Yes (*) /lib64/libm.so.6 > 0x00007f3e4ce69e10 0x00007f3e4ce6aa08 Yes (*) /lib64/libdl.so.2 > 0x00007f3e4cb2d7e0 0x00007f3e4cc1b418 Yes (*) /lib64/libc.so.6 > 0x00007f3e4c901610 0x00007f3e4c90d678 Yes (*) /usr/lib64/libbz2.so.1 > 0x00007f3e4c6ede90 0x00007f3e4c6f8958 Yes (*) /lib64/libz.so.1 > 0x00007f3e4db73a70 0x00007f3e4db89a6e Yes (*) > /lib64/ld-linux-x86-64.so.2 > 0x00007f3e472d3e50 0x00007f3e472dd018 Yes (*) /lib64/libgcc_s.so.1 > (*): Shared library is missing debugging information. > > (gdb) info registers > rax 0x10 16 > rbx 0x9d79c0 10320320 > rcx 0x7f3e4cbf90b0 139905552322736 > rdx 0x10 16 > rsi 0x7f3e47ce2090 139905469390992 > rdi 0x9d79c0 10320320 > rbp 0x9d79c0 0x9d79c0 > rsp 0x7f3e47ce2000 0x7f3e47ce2000 > r8 0x9d79f0 10320368 > r9 0x2bb4 11188 > r10 0x65786966746f68 28561366733582184 > r11 0x246 582 > r12 0x1 1 > r13 0x21 33 > r14 0x7f3e47ce6180 139905469407616 > r15 0x7f3e4c6a1920 139905546721568 > rip 0x623eaa 0x623eaa <st_masterlock_acquire+10> > eflags 0x10202 [ IF RF ] > cs 0xe033 57395 > ss 0xe02b 57387 > ds 0x0 0 > es 0x0 0 > fs 0x0 0 > gs 0x0 0 > fctrl 0x0 0 > fstat 0x0 0 > ftag 0x0 0 > fiseg 0x0 0 > fioff 0x0 0 > foseg 0x0 0 > fooff 0x0 0 > fop 0x0 0 > mxcsr 0x0 [ ] > > (gdb) disassemble > Dump of assembler code for function st_masterlock_acquire: > 0x0000000000623ea0 <st_masterlock_acquire+0>: push %r12 > 0x0000000000623ea2 <st_masterlock_acquire+2>: push %rbp > 0x0000000000623ea3 <st_masterlock_acquire+3>: push %rbx > 0x0000000000623ea4 <st_masterlock_acquire+4>: mov %rdi,%rbp > 0x0000000000623ea7 <st_masterlock_acquire+7>: mov %rdi,%rbx > 0x0000000000623eaa <st_masterlock_acquire+10>: callq 0x43d1d8 > <pthread_mutex_lock@plt> > 0x0000000000623eaf <st_masterlock_acquire+15>: cmpl $0x0,0x28(%rbp) > 0x0000000000623eb3 <st_masterlock_acquire+19>: je 0x623edc > <st_masterlock_acquire+60> > 0x0000000000623eb5 <st_masterlock_acquire+21>: lea 0x30(%rbp),%r12 > 0x0000000000623eb9 <st_masterlock_acquire+25>: mov 0x2c(%rbx),%eax > 0x0000000000623ebc <st_masterlock_acquire+28>: add $0x1,%eax > 0x0000000000623ebf <st_masterlock_acquire+31>: mov %eax,0x2c(%rbx) > 0x0000000000623ec2 <st_masterlock_acquire+34>: mov %rbp,%rsi > 0x0000000000623ec5 <st_masterlock_acquire+37>: mov %r12,%rdi > 0x0000000000623ec8 <st_masterlock_acquire+40>: callq 0x43cfa8 > <pthread_cond_wait@plt> > 0x0000000000623ecd <st_masterlock_acquire+45>: mov 0x2c(%rbx),%eax > 0x0000000000623ed0 <st_masterlock_acquire+48>: sub $0x1,%eax > 0x0000000000623ed3 <st_masterlock_acquire+51>: mov %eax,0x2c(%rbx) > 0x0000000000623ed6 <st_masterlock_acquire+54>: cmpl $0x0,0x28(%rbx) > 0x0000000000623eda <st_masterlock_acquire+58>: jne 0x623eb9 > <st_masterlock_acquire+25> > 0x0000000000623edc <st_masterlock_acquire+60>: movl $0x1,0x28(%rbx) > 0x0000000000623ee3 <st_masterlock_acquire+67>: mov %rbp,%rdi > 0x0000000000623ee6 <st_masterlock_acquire+70>: callq 0x43d318 > <pthread_mutex_unlock@plt> > 0x0000000000623eeb <st_masterlock_acquire+75>: pop %rbx > 0x0000000000623eec <st_masterlock_acquire+76>: pop %rbp > 0x0000000000623eed <st_masterlock_acquire+77>: pop %r12 > 0x0000000000623eef <st_masterlock_acquire+79>: retq > End of assembler dump. > > (gdb) thread apply all bt > > Thread 13 (Thread 0x7f3e49ce4940 (LWP 11185)): > #0 0x00007f3e4d2fdccb in accept () from /lib64/libpthread.so.0 > #1 0x0000000000624e4f in unix_accept () > #2 0x000000000063a304 in caml_c_call () > #3 0x00000000016f0018 in ?? () > #4 0x00000000005e8bae in camlThread__fun_1103 () > #5 0x00007f3e49ce3fe0 in ?? () > #6 0x00000000005e8b4c in camlThread__fun_1103 () > #7 0x00000000009d79c0 in thread_next_ident () > #8 0x000000000063a35a in caml_start_program () > #9 0x0000000000000000 in ?? () > > Thread 12 (Thread 0x7f3e472d1940 (LWP 974)): > #0 0x00007f3e4d2fb019 in pthread_cond_wait@@GLIBC_2.3.2 () from > /lib64/libpthread.so.0 > #1 0x0000000000623be0 in caml_condition_wait () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e4c66a250 in ?? () > #4 0x0000000001e2a2e0 in ?? () > #5 0x00007f3e4c680c10 in ?? () > #6 0x00000000005d5a06 in camlThreadext__execute_1035 () > #7 0x00007f3e472d0e90 in ?? () > #8 0x00000000005d59d1 in camlThreadext__execute_1035 () > #9 0x00007f3e4c67fd60 in ?? () > #10 0x00000000016faf00 in ?? () > #11 0x00000000016e2d88 in ?? () > #12 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #13 0x00007f3e472d0ec0 in ?? () > #14 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #15 0x00007f3e4c680c10 in ?? () > #16 0x0000000001e2a2b8 in ?? () > #17 0x0000000000000001 in ?? () > #18 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #19 0x00007f3e472d0ef0 in ?? () > #20 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #21 0x00007f3e4c680cf0 in ?? () > #22 0x00000000009a2090 in camlDebug__18 () > #23 0x0000000000630216 in caml_string_notequal () > #24 0x000000000044f926 in camlXenops_interface__process_5023 () > #25 0x00007f3e472d0f90 in ?? () > #26 0x000000000044b833 in camlXenops_interface__process_5023 () > #27 0x00007f3e4c680d20 in ?? () > #28 0x00007f3e4c684160 in ?? () > #29 0x00007f3e4c680d30 in ?? () > #30 0x00000000005e5861 in camlJsonrpc__call_of_string_1310 () > #31 0x00007f3e4c680da8 in ?? () > #32 0x00007f3e4c6879c0 in ?? () > #33 0x00000000016f03f8 in ?? () > #34 0x000000000087b258 in camlXenopsd__69 () > #35 0x00007f3e4c68f5e8 in ?? () > #36 0x000000000044129d in camlXenopsd__binary_handler_1581 () > #37 0x00000000000000bb in ?? () > #38 0x00007f3e4c689d08 in ?? () > #39 0x00000000009b65c8 in camlJsonrpc__3 () > #40 0x000000000087b258 in camlXenopsd__69 () > #41 0x00000000016f03f8 in ?? () > #42 0x00000000009b67c8 in camlJsonrpc__24 () > #43 0x0000000000000023 in ?? () > #44 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #45 0x00007f3e472d0fc0 in ?? () > #46 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #47 0x0000000000000001 in ?? () > #48 0x0000000001e2a298 in ?? () > #49 0x0000000000000000 in ?? () > > Thread 11 (Thread 0x7f3e494e3940 (LWP 11186)): > #0 0x00007f3e4cbde162 in select () from /lib64/libc.so.6 > #1 0x00000000006276dd in unix_select () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e494e2f40 in ?? () > #4 0x00000000005d5800 in camlThreadext__fun_1798 () > #5 0x0000000001e2a388 in ?? () > #6 0x000000000063a391 in caml_start_program () > #7 0x0000000000000019 in ?? () > #8 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #9 0x00007f3e494e2f90 in ?? () > #10 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #11 0x00007f3e4c67c618 in ?? () > #12 0x0000000001e2a358 in ?? () > #13 0x00007f3e4c67c170 in ?? () > #14 0x0000000000483d5b in camlUpdates__main_loop_1260 () > #15 0x00007f3e41635cd8 in ?? () > #16 0x00007f3e4c67c650 in ?? () > #17 0x00000000016f0060 in ?? () > #18 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #19 0x00007f3e494e2fc0 in ?? () > #20 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #21 0x0000000000000000 in ?? () > > Thread 10 (Thread 0x7f3e4a4e5940 (LWP 11184)): > #0 0x00007f3e4d2fdccb in accept () from /lib64/libpthread.so.0 > #1 0x0000000000624e4f in unix_accept () > #2 0x000000000063a304 in caml_c_call () > #3 0x00000000016effd0 in ?? () > #4 0x00000000005e8bae in camlThread__fun_1103 () > #5 0x00007f3e4a4e4fe0 in ?? () > #6 0x00000000005e8b4c in camlThread__fun_1103 () > #7 0x00000000009d79c0 in thread_next_ident () > #8 0x000000000063a35a in caml_start_program () > #9 0x0000000000000000 in ?? () > > Thread 9 (Thread 0x7f3e48ce2940 (LWP 11187)): > #0 0x00007f3e4d2fd654 in __lll_lock_wait () from /lib64/libpthread.so.0 > #1 0x00007f3e4d2f8f4a in _L_lock_1034 () from /lib64/libpthread.so.0 > #2 0x00007f3e4d2f8e0c in __pthread_mutex_lock (mutex=0x1646e00) at > pthread_mutex_lock.c:61 > #3 0x0000000000624d41 in caml_mutex_lock () > #4 0x000000000063a304 in caml_c_call () > #5 0x00007f3e41ca6850 in ?? () > #6 0x00000000016fadc8 in ?? () > #7 0x00007f3e48ce1f90 in ?? () > #8 0x00000000004e76f9 in camlXenops_server__fun_7488 () > #9 0x00007f3e4c53b438 in ?? () > #10 0x00000000016f0330 in ?? () > #11 0x00007f3e41ab04e0 in ?? () > #12 0x00007f3e41ab0e50 in ?? () > #13 0x00007f3e4c5411d8 in ?? () > #14 0x00007f3e41ab0e78 in ?? () > #15 0x00007f3e4d2fa217 in _L_unlock_766 () from /lib64/libpthread.so.0 > #16 0x00007f3e4d2fa17e in __pthread_mutex_unlock_usercnt (mutex=0x0) at > pthread_mutex_unlock.c:52 > #17 __pthread_mutex_unlock (mutex=0x0) at pthread_mutex_unlock.c:289 > #18 0x0000000000623eeb in st_masterlock_acquire () > #19 0x0000000000000003 in ?? () > #20 0x0000000000000000 in ?? () > > Thread 8 (Thread 0x7f3e450d2940 (LWP 973)): > #0 0x00007f3e4d2fb019 in pthread_cond_wait@@GLIBC_2.3.2 () from > /lib64/libpthread.so.0 > #1 0x0000000000623be0 in caml_condition_wait () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e4c66b918 in ?? () > #4 0x0000000001e2a238 in ?? () > #5 0x00007f3e4c68b5d0 in ?? () > #6 0x00000000005d5a06 in camlThreadext__execute_1035 () > #7 0x00007f3e450d1e90 in ?? () > #8 0x00000000005d59d1 in camlThreadext__execute_1035 () > #9 0x00007f3e4c68b158 in ?? () > #10 0x00000000016faf00 in ?? () > #11 0x00000000016e2d88 in ?? () > #12 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #13 0x00007f3e450d1ec0 in ?? () > #14 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #15 0x00007f3e4c68b248 in ?? () > #16 0x0000000001e311e0 in ?? () > #17 0x0000000000000001 in ?? () > #18 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #19 0x00007f3e450d1ef0 in ?? () > #20 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #21 0x00007f3e4c68b328 in ?? () > #22 0x00000000009a2090 in camlDebug__18 () > #23 0x0000000000630216 in caml_string_notequal () > #24 0x000000000044f926 in camlXenops_interface__process_5023 () > #25 0x00007f3e450d1f90 in ?? () > #26 0x000000000044b833 in camlXenops_interface__process_5023 () > #27 0x00007f3e4c68b358 in ?? () > #28 0x00007f3e4c68e258 in ?? () > #29 0x0000000000000001 in ?? () > #30 0x00000000005e5861 in camlJsonrpc__call_of_string_1310 () > #31 0x00007f3e4c68b3d0 in ?? () > #32 0x00007f3e4c692240 in ?? () > #33 0x00000000016f03f8 in ?? () > #34 0x000000000087b258 in camlXenopsd__69 () > #35 0x00007f3e4c69de08 in ?? () > #36 0x000000000044129d in camlXenopsd__binary_handler_1581 () > #37 0x00000000000000c1 in ?? () > #38 0x00007f3e4c694588 in ?? () > #39 0x00000000009b65c8 in camlJsonrpc__3 () > #40 0x000000000087b258 in camlXenopsd__69 () > #41 0x00000000016f03f8 in ?? () > #42 0x00000000009b67c8 in camlJsonrpc__24 () > #43 0x0000000000000021 in ?? () > #44 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #45 0x00007f3e450d1fc0 in ?? () > #46 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #47 0x0000000000000001 in ?? () > #48 0x0000000001e311c0 in ?? () > #49 0x0000000000000000 in ?? () > > Thread 7 (Thread 0x7f3e4dd826e0 (LWP 11177)): > #0 0x00007f3e4cbde162 in select () from /lib64/libc.so.6 > #1 0x00000000006276dd in unix_select () > #2 0x000000000063a304 in caml_c_call () > #3 0x0000000000000001 in ?? () > #4 0x0000000000442b5d in camlXenopsd__entry () > #5 0x00007fff6609f560 in ?? () > #6 0x0000000000442b15 in camlXenopsd__entry () > #7 0x00007f3e4c6b2c80 in ?? () > #8 0x000000000000000b in ?? () > #9 0x0000000000000009 in ?? () > #10 0x000000000043dd79 in caml_program () > #11 0x00000000003d7d60 in ?? () > #12 0x000000000063a35a in caml_start_program () > #13 0x0000000000000000 in ?? () > > Thread 6 (Thread 0x7f3e4b4e7940 (LWP 11182)): > #0 0x00007f3e4cbf90b0 in __read_chk (fd=<value optimized out>, > buf=0x7f3e4b4e2dd0, nbytes=16, buflen=<value optimized out>) at > read_chk.c:34 > #1 0x00000000006272c9 in unix_read () > #2 0x000000000063a304 in caml_c_call () > #3 0x0000000000000021 in ?? () > #4 0x000000000052bbd6 in camlXenbus__Xb__read_fd_1120 () > #5 0x00007f3e4c544720 in ?? () > #6 0x000000000052bef1 in camlXenbus__Xb__input_1161 () > #7 0x0000000000000021 in ?? () > #8 0x00007f3e41ab0e20 in ?? () > #9 0x00007f3e41ab0e00 in ?? () > #10 0x00000000009ba1a0 in camlUuid__22 () > #11 0x00000000005d5a22 in camlThreadext__execute_1035 () > #12 0x00000000016ed590 in ?? () > #13 0x00007f3e4b4e6ed0 in ?? () > #14 0x0000000000528af5 in camlXenstore__Xsraw__pkt_recv_1214 () > #15 0x00000000016ed820 in ?? () > #16 0x00007f3e41ab0fa0 in ?? () > #17 0x0000000000000001 in ?? () > #18 0x0000000000528d5d in camlXenstore__Xsraw__read_watchevent_1233 () > #19 0x00000000016ed820 in ?? () > #20 0x00000000004b51b0 in camlXenstore_watch__fun_1555 () > #21 0x00000000016efea8 in ?? () > #22 0x00000000016efe70 in ?? () > #23 0x00000000016efe60 in ?? () > #24 0x00000000016efe50 in ?? () > #25 0x000000000164cad0 in ?? () > #26 0x00000000016efdb8 in ?? () > #27 0x00007f3e4b4e6f00 in ?? () > #28 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #29 0x00007f3e4b4e6f00 in ?? () > #30 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #31 0x00007f3e4b4e6f30 in ?? () > #32 0x00000000016efd98 in ?? () > #33 0x00007f3e4c660608 in ?? () > #34 0x000000000052ca53 in camlXenctrl__with_intf_1174 () > #35 0x00007f3e4b4e6f30 in ?? () > #36 0x000000000052ca25 in camlXenctrl__with_intf_1174 () > #37 0x00007f3e4c660630 in ?? () > #38 0x000000000164cad0 in ?? () > #39 0x0000000000000001 in ?? () > #40 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #41 0x00007f3e4b4e6f60 in ?? () > #42 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #43 0x00007f3e4c6af1c8 in ?? () > #44 0x00000000009a2090 in camlDebug__18 () > #45 0x0000000000000000 in ?? () > > Thread 5 (Thread 0x7f3e4ace6940 (LWP 11183)): > #0 0x00007f3e4cbde162 in select () from /lib64/libc.so.6 > #1 0x00000000006276dd in unix_select () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e4ace5f60 in ?? () > #4 0x000000000052e717 in camlServer_io__establish_server_1373 () > #5 0x00000000005d5a22 in camlThreadext__execute_1035 () > #6 0x000000000063a391 in caml_start_program () > #7 0x00007f3e4ace5f90 in ?? () > #8 0x0000000000623d31 in caml_mutex_unlock () > #9 0x0000000000000009 in ?? () > #10 0x00000000016eff70 in ?? () > #11 0x000000000052e45a in camlServer_io__fun_1455 () > #12 0x00007f3e4ace5f90 in ?? () > #13 0x000000000052e389 in camlServer_io__fun_1455 () > #14 0x00007f3e4c662d00 in ?? () > #15 0x00007f3e4c65fda8 in ?? () > #16 0x00007f3e4d2fa217 in _L_unlock_766 () from /lib64/libpthread.so.0 > #17 0x00007f3e4d2fa17e in __pthread_mutex_unlock_usercnt > (mutex=0x7f3e4ace5d90) at pthread_mutex_unlock.c:52 > #18 __pthread_mutex_unlock (mutex=0x7f3e4ace5d90) at > pthread_mutex_unlock.c:289 > #19 0x0000000000623eeb in st_masterlock_acquire () > #20 0x0000000000000003 in ?? () > #21 0x0000000000000000 in ?? () > > Thread 4 (Thread 0x7f3e47ce0940 (LWP 11190)): > #0 0x00007f3e4d2fb019 in pthread_cond_wait@@GLIBC_2.3.2 () from > /lib64/libpthread.so.0 > #1 0x0000000000623be0 in caml_condition_wait () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e47cdff90 in ?? () > #4 0x0000000000624d60 in caml_mutex_lock () > #5 0x00000000005d59d1 in camlThreadext__execute_1035 () > #6 0x00007f3e4260b918 in ?? () > #7 0x00000000016fadc8 in ?? () > #8 0x00007f3e47cdff90 in ?? () > #9 0x00000000004e76f9 in camlXenops_server__fun_7488 () > #10 0x00007f3e4c512470 in ?? () > #11 0x00000000016efcc8 in ?? () > #12 0x00007f3e425a55e0 in ?? () > #13 0x00007f3e425a5758 in ?? () > #14 0x00007f3e4c517440 in ?? () > #15 0x00007f3e425a56a8 in ?? () > #16 0x0000000000000000 in ?? () > > Thread 3 (Thread 0x7f3e3fce0940 (LWP 11189)): > #0 0x00007f3e4d2fd654 in __lll_lock_wait () from /lib64/libpthread.so.0 > #1 0x00007f3e4d2f8f4a in _L_lock_1034 () from /lib64/libpthread.so.0 > #2 0x00007f3e4d2f8e0c in __pthread_mutex_lock (mutex=0x1646e00) at > pthread_mutex_lock.c:61 > #3 0x0000000000624d41 in caml_mutex_lock () > #4 0x000000000063a304 in caml_c_call () > #5 0x00007f3e426e6220 in ?? () > #6 0x00000000016fadc8 in ?? () > #7 0x00007f3e3fcdff90 in ?? () > #8 0x00000000004e76f9 in camlXenops_server__fun_7488 () > #9 0x00007f3e4c646fc0 in ?? () > #10 0x00000000016efc88 in ?? () > #11 0x00007f3e425a55e0 in ?? () > #12 0x00007f3e4260b5c0 in ?? () > #13 0x00007f3e425d9e70 in ?? () > #14 0x00007f3e425d9e10 in ?? () > #15 0x00007f3e4d2fa217 in _L_unlock_766 () from /lib64/libpthread.so.0 > #16 0x00007f3e4d2fa17e in __pthread_mutex_unlock_usercnt (mutex=0x0) at > pthread_mutex_unlock.c:52 > #17 __pthread_mutex_unlock (mutex=0x0) at pthread_mutex_unlock.c:289 > #18 0x0000000000623eeb in st_masterlock_acquire () > #19 0x0000000000000003 in ?? () > #20 0x0000000000000000 in ?? () > > Thread 2 (Thread 0x7f3e4c4e9940 (LWP 11180)): > #0 0x00007f3e4d2fb019 in pthread_cond_wait@@GLIBC_2.3.2 () from > /lib64/libpthread.so.0 > #1 0x0000000000623be0 in caml_condition_wait () > #2 0x000000000063a304 in caml_c_call () > #3 0x00007f3e4c53c770 in ?? () > #4 0x00007f3e41ab0f50 in ?? () > #5 0x00000000005d5a22 in camlThreadext__execute_1035 () > #6 0x00000000005d5a06 in camlThreadext__execute_1035 () > #7 0x00007f3e4c4e8f00 in ?? () > #8 0x00000000005d59d1 in camlThreadext__execute_1035 () > #9 0x00007f3e4c53da00 in ?? () > #10 0x00000000016fad80 in ?? () > #11 0x00000000016e2d88 in ?? () > #12 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #13 0x00007f3e4c4e8f60 in ?? () > #14 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #15 0x00007f3e4c53daf0 in ?? () > #16 0x00007f3e41ab0f28 in ?? () > #17 0x00000000005d5a22 in camlThreadext__execute_1035 () > #18 0x00000000004fd2f6 in camlXenops_server__fun_8769 () > #19 0x00000000016f1930 in ?? () > #20 0x00000000016efd48 in ?? () > #21 0x000000000090d660 in camlXenops_server__169 () > #22 0x0000000000000121 in ?? () > #23 0x00000000016efd38 in ?? () > #24 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #25 0x00007f3e4c4e8f90 in ?? () > #26 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #27 0x00007f3e4c6b22c8 in ?? () > #28 0x00000000009a2090 in camlDebug__18 () > #29 0x00007f3e4d2fa217 in _L_unlock_766 () from /lib64/libpthread.so.0 > #30 0x00007f3e4d2fa17e in __pthread_mutex_unlock_usercnt (mutex=0x83) at > pthread_mutex_unlock.c:52 > #31 __pthread_mutex_unlock (mutex=0x83) at pthread_mutex_unlock.c:289 > #32 0x0000000000623eeb in st_masterlock_acquire () > #33 0x0000000000000003 in ?? () > #34 0x0000000000000000 in ?? () > > Thread 1 (Thread 0x7f3e484e1940 (LWP 11188)): > #0 0x0000000000623eaa in st_masterlock_acquire () > #1 0x0000000000624b1c in caml_thread_leave_blocking_section () > #2 0x000000000062aebc in caml_leave_blocking_section () > #3 0x00000000006272d0 in unix_read () > #4 0x000000000063a304 in caml_c_call () > #5 0x0000000000000021 in ?? () > #6 0x000000000052bbd6 in camlXenbus__Xb__read_fd_1120 () > #7 0x0000000000000021 in ?? () > #8 0x000000000052bef1 in camlXenbus__Xb__input_1161 () > #9 0x0000000000000021 in ?? () > #10 0x00007f3e4c6a1948 in ?? () > #11 0x00007f3e4c6a1928 in ?? () > #12 0x000000000052be54 in camlXenbus__Xb__output_1155 () > #13 0x0000000000000067 in ?? () > #14 0x00007f3e4c6cf138 in ?? () > #15 0x00007f3e4c6cf138 in ?? () > #16 0x0000000000528af5 in camlXenstore__Xsraw__pkt_recv_1214 () > #17 0x00007f3e4c6cf0f0 in ?? () > #18 0x00007f3e4c6a1958 in ?? () > #19 0x00007f3e4c6cf138 in ?? () > #20 0x0000000000528db5 in camlXenstore__Xsraw__sync_recv_1237 () > #21 0x00007f3e4c6cf0f0 in ?? () > #22 0x0000000000000003 in ?? () > #23 0x00007f3e4c6a1dd0 in ?? () > #24 0x000000000052934d in camlXenstore__Xsraw__directory_1259 () > #25 0x0000000000000001 in ?? () > #26 0x00007f3e4c6a1dd0 in ?? () > #27 0x00007f3e4c6cf0f0 in ?? () > #28 0x0000000000503097 in camlXenops_server_xen__ls_lR_6028 () > #29 0x00007f3e47ce6560 in ?? () > #30 0x0000000000503047 in camlXenops_server_xen__ls_lR_6028 () > #31 0x00007f3e4c6cf018 in ?? () > #32 0x00007f3e4c6a1e38 in ?? () > #33 0x00007f3e4c6bf9d8 in ?? () > #34 0x00007f3e4c6bf3a0 in ?? () > #35 0x00007f3e4c6a3860 in ?? () > #36 0x00000000005f3fc5 in camlList__map_1062 () > #37 0x00007f3e4c6a3808 in ?? () > #38 0x0000000000000001 in ?? () > #39 0x00007f3e4c6a3150 in ?? () > #40 0x00000000005f3fd8 in camlList__map_1062 () > #41 0x00007f3e4c6a3808 in ?? () > #42 0x00007f3e4c6a3848 in ?? () > #43 0x00007f3e4c6a24d8 in ?? () > #44 0x00000000005030f9 in camlXenops_server_xen__ls_lR_6028 () > #45 0x00007f3e4c6a3830 in ?? () > #46 0x00007f3e4c6a3fc0 in ?? () > #47 0x00007f3e4c6bf9d8 in ?? () > #48 0x00007f3e4c6bf3a0 in ?? () > #49 0x00007f3e4c6b1dd0 in ?? () > #50 0x00000000005f3fc5 in camlList__map_1062 () > #51 0x00007f3e4c6b1c28 in ?? () > #52 0x00007f3e4c6b1d88 in ?? () > #53 0x00007f3e4c6a50e8 in ?? () > #54 0x00000000005f3fd8 in camlList__map_1062 () > #55 0x00007f3e4c6b1c28 in ?? () > #56 0x00007f3e4c6b1d70 in ?? () > #57 0x00007f3e4c6a4610 in ?? () > #58 0x00000000005f3fd8 in camlList__map_1062 () > #59 0x00007f3e4c6b1c28 in ?? () > #60 0x00007f3e4c6b1d58 in ?? () > #61 0x00007f3e4c6a56e8 in ?? () > #62 0x00000000005f3fd8 in camlList__map_1062 () > #63 0x00007f3e4c6b1c28 in ?? () > #64 0x00007f3e4c6b1d40 in ?? () > #65 0x00007f3e4c6a6ab8 in ?? () > #66 0x00000000005f3fd8 in camlList__map_1062 () > #67 0x00007f3e4c6b1c28 in ?? () > #68 0x00007f3e4c6b1d28 in ?? () > #69 0x00007f3e4c6a7d48 in ?? () > #70 0x00000000005f3fd8 in camlList__map_1062 () > #71 0x00007f3e4c6b1c28 in ?? () > #72 0x00007f3e4c6b1d10 in ?? () > #73 0x00007f3e4c6a8e60 in ?? () > #74 0x00000000005f3fd8 in camlList__map_1062 () > #75 0x00007f3e4c6b1c28 in ?? () > #76 0x00007f3e4c6b1cf8 in ?? () > #77 0x00007f3e4c6a9fb8 in ?? () > #78 0x00000000005f3fd8 in camlList__map_1062 () > #79 0x00007f3e4c6b1c28 in ?? () > #80 0x00007f3e4c6b1ce0 in ?? () > #81 0x00007f3e4c6ab1e8 in ?? () > #82 0x00000000005f3fd8 in camlList__map_1062 () > #83 0x00007f3e4c6b1c28 in ?? () > #84 0x00007f3e4c6b1cc8 in ?? () > #85 0x00007f3e4c6ac418 in ?? () > #86 0x00000000005f3fd8 in camlList__map_1062 () > #87 0x00007f3e4c6b1c28 in ?? () > #88 0x00007f3e4c6b1cb0 in ?? () > #89 0x00007f3e4c6ad6a8 in ?? () > #90 0x00000000005f3fd8 in camlList__map_1062 () > #91 0x00007f3e4c6b1c28 in ?? () > #92 0x00007f3e4c6b1c98 in ?? () > #93 0x00007f3e4c6ae800 in ?? () > #94 0x00000000005f3fd8 in camlList__map_1062 () > #95 0x00007f3e4c6b1c28 in ?? () > #96 0x00007f3e4c6b1c80 in ?? () > #97 0x00007f3e4c6af958 in ?? () > #98 0x00000000005f3fd8 in camlList__map_1062 () > #99 0x00007f3e4c6b1c28 in ?? () > #100 0x00007f3e4c6b1c68 in ?? () > #101 0x00007f3e4c6b0ab0 in ?? () > #102 0x00000000005030f9 in camlXenops_server_xen__ls_lR_6028 () > #103 0x00007f3e4c6b1c50 in ?? () > #104 0x00007f3e4c6b2e48 in ?? () > #105 0x00007f3e4c6bf9d8 in ?? () > #106 0x00007f3e4c6bf3a0 in ?? () > #107 0x00007f3e4c6bd640 in ?? () > #108 0x00000000005f3fc5 in camlList__map_1062 () > #109 0x00007f3e4c6bd548 in ?? () > #110 0x00007f3e4c6bd5b8 in ?? () > #111 0x00007f3e4c6baa88 in ?? () > #112 0x00000000005f3fd8 in camlList__map_1062 () > #113 0x00007f3e4c6bd548 in ?? () > #114 0x00007f3e4c6bd5a0 in ?? () > #115 0x00007f3e4c6b33d0 in ?? () > #116 0x00000000005f3fd8 in camlList__map_1062 () > #117 0x00007f3e4c6bd548 in ?? () > #118 0x00007f3e4c6bd588 in ?? () > #119 0x00007f3e4c6bb098 in ?? () > #120 0x00000000005030f9 in camlXenops_server_xen__ls_lR_6028 () > #121 0x00007f3e4c6bd570 in ?? () > #122 0x00007f3e4c6bde68 in ?? () > #123 0x00007f3e4c6bf9d8 in ?? () > #124 0x00007f3e4c6bf3a0 in ?? () > #125 0x0000000000924710 in camlXenops_server_xen__575 () > #126 0x00000000005f3fc5 in camlList__map_1062 () > #127 0x00007f3e4c6bf378 in ?? () > #128 0x00000000009246c0 in camlXenops_server_xen__575 () > #129 0x00007f3e4c6bedb0 in ?? () > #130 0x00000000005f3fd8 in camlList__map_1062 () > #131 0x00007f3e4c6bf378 in ?? () > #132 0x00000000009246a8 in camlXenops_server_xen__575 () > #133 0x00007f3e4c6be3a8 in ?? () > #134 0x00000000005086d9 in camlXenops_server_xen__fun_9041 () > #135 0x00000000009c2490 in camlList__29 () > #136 0x00000000009c24b0 in camlList__30 () > #137 0x00007f3e4c6bf9d8 in ?? () > #138 0x0000000000000001 in ?? () > #139 0x0000000000000001 in ?? () > #140 0x00007f3e4c6c15d0 in ?? () > #141 0x00007f3e4c6c1588 in ?? () > #142 0x00007f3e4c6c1588 in ?? () > #143 0x00007f3e4c6c0810 in ?? () > #144 0x0000000000000001 in ?? () > #145 0x00007f3e4c6cf058 in ?? () > #146 0x00007f3e4c6cee68 in ?? () > #147 0x00007f3e431c9ab0 in ?? () > #148 0x00007f3e4c6bfa90 in ?? () > #149 0x00007f3e4c6ceb40 in ?? () > #150 0x00007f3e4c6cf230 in ?? () > #151 0x00007f3e47ce6590 in ?? () > #152 0x00000000005d42d0 in camlPervasiveext__finally_1031 () > #153 0x00007f3e47ce6590 in ?? () > #154 0x00000000005d4281 in camlPervasiveext__finally_1031 () > #155 0x00007f3e47ce6620 in ?? () > #156 0x00007f3e4c6cedb0 in ?? () > #157 0x00007f3e4c6cf1e8 in ?? () > #158 0x000000000052ca53 in camlXenctrl__with_intf_1174 () > #159 0x00007f3e47ce6620 in ?? () > #160 0x000000000052ca25 in camlXenctrl__with_intf_1174 () > #161 0x00007f3e4c6cf210 in ?? () > #162 0x00007f3e431c9ab0 in ?? () > #163 0x00000000016f0378 in ?? () > #164 0x00000000004e9aa3 in camlXenops_server__one_4818 () > #165 0x00000000009a22c8 in camlDebug__35 () > #166 0x00000000016efc38 in ?? () > #167 0x0000000001e2a530 in ?? () > #168 0x00007f3e4c66c6e8 in ?? () > #169 0x000000000000000f in ?? () > #170 0x0000000000000017 in ?? () > #171 0x0000000000000000 in ?? () > > -robin > The rsp value at segfault is 4k aligned and the last call that the process attempted to call is 'callq'. So it looks like a stack overflow. And maybe this issue is related: http://caml.inria.fr/mantis/view.php?id=5064 -robin
_______________________________________________ Xen-api mailing list Xen-api@lists.xen.org http://lists.xen.org/cgi-bin/mailman/listinfo/xen-api
