[
https://issues.apache.org/jira/browse/AMQCPP-624?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Karthik Kashyap updated AMQCPP-624:
-----------------------------------
Description:
We use ActiveMQ-CPP for inter-component messaging within a voice recorder
software. When doing some long term load tests for a few hours, we encountered
a random crash which pointed to ActiveMQ-CPP with the below backtrace on GDB as
listed below:
[Thread debugging using libthread_db enabled] Using host libthread_db library
"/lib64/libthread_db.so.1". Missing separate debuginfo for
/root/mediarecorder_ext/target/linux/lib/libaudiofile.so.1 Try: yum
--enablerepo='*debug*' install
/usr/lib/debug/.build-id/3f/ab3a9b600e5d5156497eac8d35217947a324aa.debug Core
was generated by `/root/amr/bin/tdaemon --daemon -f wav'. Program terminated
with signal 6, Aborted. #0 0x00007f18f0e64989 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install apr-1.4.8-3.el7.x86_64
glibc-2.17-55.el7.x86_64 keyutils-libs-1.5.8-3.el7.x86_64
krb5-libs-1.11.3-49.el7.x86_64 libcom_err-1.42.9-4.el7.x86_64
libgcc-4.8.2-16.el7.x86_64 libselinux-2.2.2-6.el7.x86_64
libstdc++-4.8.2-16.el7.x86_64 libuuid-2.23.2-16.el7.x86_64
nss-softokn-freebl-3.15.4-2.el7.x86_64 openssl-libs-1.0.1e-34.el7.x86_64
pcre-8.32-12.el7.x86_64 xz-libs-5.1.2-8alpha.el7.x86_64 (gdb) bt full #0
0x00007f18f0e64989 in raise () from /lib64/libc.so.6 No symbol table info
available.+
+#1 0x00007f18f0e66098 in abort () from /lib64/libc.so.6 No symbol table info
available.+
+#2 0x00007f18f8fad305 in Poco::SignalHandler::handleSignal (sig=11) at
/root/poco/latest/poco-1.6.1-all/Foundation/src/SignalHandler.cpp:96 jb =
<optimized out>+
+#3 <signal handler called> No symbol table info available.+
+#4 0x00007f18f0f78886 in __memcpy_ssse3_back () from /lib64/libc.so.6 No
symbol table info available._+
+_#5 0x00007f18f17c65b4 in std::string::append(std::string const&) () from
/lib64/libstdc_+_+.so.6 No symbol table info available._
_#6 0x00007f18f3a47053 in decaf::util::concurrent::Mutex::Mutex
(this=0x7f1859ffabc0) at decaf/util/concurrent/Mutex.cpp:50 idStr = "10474"
this = 0x7f1888000be0_
_#7 0x00007f18f3a50340 in AbstractCollection (__vtt_parm=<optimized out>,
this=0x7f1859ffabb8, __in_chrg=<optimized out>) at
./decaf/util/AbstractCollection.h:65 No locals._
_#8 AbstractList (__vtt_parm=<optimized out>, this=0x7f1859ffabb0,
__in_chrg=<optimized out>) at ./decaf/util/AbstractList.h:341 No locals._
_#9 AbstractSequentialList (__vtt_parm=0x7f18f4069ce8 <VTT for
decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>+8>,
this=0x7f1859ffabb0, __in_chrg=<optimized out>) at
./decaf/util/AbstractSequentialList.h:59 No locals.
#10
decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>::LinkedList
(this=this@entry=0x7f1859ffabb0, __in_chrg=<optimized out>,
__vtt_parm=<optimized out>) at ./decaf/util/LinkedList.h:91 No locals.
#11 0x00007f18f3a5366f in
decaf::util::concurrent::ExecutorKernel::WorkerKiller::run
(this=0x7f187c00aad0) at decaf/util/concurrent/ThreadPoolExecutor.cpp:248
toDeleteList = warning: can't find linker symbol for virtual table for
`decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
value warning: found `construction vtable for
decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>-in-decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
instead
{<decaf::util::AbstractSequentialList<decaf::util::concurrent::ExecutorKernel::Worker*>>
=
{<decaf::util::AbstractList<decaf::util::concurrent::ExecutorKernel::Worker*>>
= {<decaf::util::List<decaf::util::concurrent::ExecutorKernel::Worker*>> =
{<decaf::util::Collection<decaf::util::concurrent::ExecutorKernel::Worker*>> =
<invalid address>, <No data fields>}
,
<decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>>
= {mutex = {<decaf::util::concurrent::Synchronizable> =
{ _vptr.Synchronizable = 0x7f18f4064db0 <vtable for
decaf::util::concurrent::Mutex+16>}
, properties = 0x0}}, modCount = 0}, <No data fields>},
<decaf::util::Deque<decaf::util::concurrent::ExecutorKernel::Worker*>> =
{<decaf::util::Queue<decaf::util::concurrent::ExecutorKernel::Worker*>---Type
<return> to continue, or q <return> to quit--- > =
{<No data fields>}
, <No data fields>}, listSize = 147917545, head = \{value = 0x7f1859ffaca0,
prev = 0x7f187c009a80, next = 0x7f18f3a46840
<decaf::util::concurrent::Lock::~Lock()+160>}, tail = \{value = 0x7f187c009aa8,
prev = 0x7f1859ffac80, next = 0x7f1888014768}} #12 0x00007f18f3a41abd in
decaf::util::TimerImpl::run (this=0x7f187c009a80) at decaf/util/Timer.cpp:156
task = {<decaf::util::concurrent::atomic::AtomicRefCounter> =
{ _vptr.AtomicRefCounter = 0x7f18f40170d0 <vtable for
decaf::lang::Pointer<decaf::util::TimerTask,
decaf::util::concurrent::atomic::AtomicRefCounter>+16>, counter =
0x7f187c00ab40}
, value = 0x7f187c00aad0, onDelete = 0x7f18f38c7fb0
<decaf::lang::Pointer<decaf::util::TimerTask,
decaf::util::concurrent::atomic::AtomicRefCounter>::onDeleteFunc(decaf::util::TimerTask*)>}
#13 0x00007f18f39bd3d2 in (anonymous namespace)::runCallback
(arg=0x7f187c009b40) at decaf/internal/util/concurrent/Threading.cpp:268 thread
= 0x7f187c009b40 #14 0x00007f18f39bdc0c in (anonymous
namespace)::threadEntryMethod (arg=0x7f187c009b40) at
decaf/internal/util/concurrent/Threading.cpp:256 thread = 0x7f187c009b40 #15
0x00007f18f5862df3 in start_thread () from /lib64/libpthread.so.0 No symbol
table info available. #16 0x00007f18f0f253dd in clone () from /lib64/libc.so.6
No symbol table info available.
was:
We use ActiveMQ-CPP for inter-component messaging within a voice recorder
software. When doing some long term load tests, we encountered a random crash
which pointed to ActiveMQ-CPP with the below backtrace on GDB:
[Thread debugging using libthread_db enabled] Using host libthread_db library
"/lib64/libthread_db.so.1". Missing separate debuginfo for
/root/mediarecorder_ext/target/linux/lib/libaudiofile.so.1 Try: yum
--enablerepo='*debug*' install
/usr/lib/debug/.build-id/3f/ab3a9b600e5d5156497eac8d35217947a324aa.debug Core
was generated by `/root/amr/bin/tdaemon --daemon -f wav'. Program terminated
with signal 6, Aborted. #0 0x00007f18f0e64989 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install apr-1.4.8-3.el7.x86_64
glibc-2.17-55.el7.x86_64 keyutils-libs-1.5.8-3.el7.x86_64
krb5-libs-1.11.3-49.el7.x86_64 libcom_err-1.42.9-4.el7.x86_64
libgcc-4.8.2-16.el7.x86_64 libselinux-2.2.2-6.el7.x86_64
libstdc++-4.8.2-16.el7.x86_64 libuuid-2.23.2-16.el7.x86_64
nss-softokn-freebl-3.15.4-2.el7.x86_64 openssl-libs-1.0.1e-34.el7.x86_64
pcre-8.32-12.el7.x86_64 xz-libs-5.1.2-8alpha.el7.x86_64 (gdb) bt full #0
0x00007f18f0e64989 in raise () from /lib64/libc.so.6 No symbol table info
available. #1 0x00007f18f0e66098 in abort () from /lib64/libc.so.6 No symbol
table info available. #2 0x00007f18f8fad305 in
Poco::SignalHandler::handleSignal (sig=11) at
/root/poco/latest/poco-1.6.1-all/Foundation/src/SignalHandler.cpp:96 jb =
<optimized out> #3 <signal handler called> No symbol table info available. #4
0x00007f18f0f78886 in __memcpy_ssse3_back () from /lib64/libc.so.6 No symbol
table info available. #5 0x00007f18f17c65b4 in std::string::append(std::string
const&) () from /lib64/libstdc++.so.6 No symbol table info available. #6
0x00007f18f3a47053 in decaf::util::concurrent::Mutex::Mutex
(this=0x7f1859ffabc0) at decaf/util/concurrent/Mutex.cpp:50 idStr = "10474"
this = 0x7f1888000be0 #7 0x00007f18f3a50340 in AbstractCollection
(__vtt_parm=<optimized out>, this=0x7f1859ffabb8, __in_chrg=<optimized out>) at
./decaf/util/AbstractCollection.h:65 No locals. #8 AbstractList
(__vtt_parm=<optimized out>, this=0x7f1859ffabb0, __in_chrg=<optimized out>) at
./decaf/util/AbstractList.h:341 No locals. #9 AbstractSequentialList
(__vtt_parm=0x7f18f4069ce8 <VTT for
decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>+8>,
this=0x7f1859ffabb0, __in_chrg=<optimized out>) at
./decaf/util/AbstractSequentialList.h:59 No locals. #10
decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>::LinkedList
(this=this@entry=0x7f1859ffabb0, __in_chrg=<optimized out>,
__vtt_parm=<optimized out>) at ./decaf/util/LinkedList.h:91 No locals. #11
0x00007f18f3a5366f in
decaf::util::concurrent::ExecutorKernel::WorkerKiller::run
(this=0x7f187c00aad0) at decaf/util/concurrent/ThreadPoolExecutor.cpp:248
toDeleteList = warning: can't find linker symbol for virtual table for
`decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
value warning: found `construction vtable for
decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>-in-decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
instead
\{<decaf::util::AbstractSequentialList<decaf::util::concurrent::ExecutorKernel::Worker*>>
=
{<decaf::util::AbstractList<decaf::util::concurrent::ExecutorKernel::Worker*>>
= {<decaf::util::List<decaf::util::concurrent::ExecutorKernel::Worker*>> =
{<decaf::util::Collection<decaf::util::concurrent::ExecutorKernel::Worker*>> =
<invalid address>, <No data fields>},
<decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>>
= \{mutex = {<decaf::util::concurrent::Synchronizable> = {
_vptr.Synchronizable = 0x7f18f4064db0 <vtable for
decaf::util::concurrent::Mutex+16>}, properties = 0x0}}, modCount = 0}, <No
data fields>},
<decaf::util::Deque<decaf::util::concurrent::ExecutorKernel::Worker*>> =
\{<decaf::util::Queue<decaf::util::concurrent::ExecutorKernel::Worker*>---Type
<return> to continue, or q <return> to quit--- > = {<No data fields>}, <No data
fields>}, listSize = 147917545, head = \{value = 0x7f1859ffaca0, prev =
0x7f187c009a80, next = 0x7f18f3a46840
<decaf::util::concurrent::Lock::~Lock()+160>}, tail = \{value = 0x7f187c009aa8,
prev = 0x7f1859ffac80, next = 0x7f1888014768}} #12 0x00007f18f3a41abd in
decaf::util::TimerImpl::run (this=0x7f187c009a80) at decaf/util/Timer.cpp:156
task = \{<decaf::util::concurrent::atomic::AtomicRefCounter> = {
_vptr.AtomicRefCounter = 0x7f18f40170d0 <vtable for
decaf::lang::Pointer<decaf::util::TimerTask,
decaf::util::concurrent::atomic::AtomicRefCounter>+16>, counter =
0x7f187c00ab40}, value = 0x7f187c00aad0, onDelete = 0x7f18f38c7fb0
<decaf::lang::Pointer<decaf::util::TimerTask,
decaf::util::concurrent::atomic::AtomicRefCounter>::onDeleteFunc(decaf::util::TimerTask*)>}
#13 0x00007f18f39bd3d2 in (anonymous namespace)::runCallback
(arg=0x7f187c009b40) at decaf/internal/util/concurrent/Threading.cpp:268 thread
= 0x7f187c009b40 #14 0x00007f18f39bdc0c in (anonymous
namespace)::threadEntryMethod (arg=0x7f187c009b40) at
decaf/internal/util/concurrent/Threading.cpp:256 thread = 0x7f187c009b40 #15
0x00007f18f5862df3 in start_thread () from /lib64/libpthread.so.0 No symbol
table info available. #16 0x00007f18f0f253dd in clone () from /lib64/libc.so.6
No symbol table info available.
> Random crash with ActiveMQ-CMS when running the system long term
> ----------------------------------------------------------------
>
> Key: AMQCPP-624
> URL: https://issues.apache.org/jira/browse/AMQCPP-624
> Project: ActiveMQ C++ Client
> Issue Type: Bug
> Environment: RHEL-7 linux OS and ActiveMQ-CPP version 3.9.4.
> Reporter: Karthik Kashyap
> Assignee: Timothy Bish
> Priority: Major
>
> We use ActiveMQ-CPP for inter-component messaging within a voice recorder
> software. When doing some long term load tests for a few hours, we
> encountered a random crash which pointed to ActiveMQ-CPP with the below
> backtrace on GDB as listed below:
> [Thread debugging using libthread_db enabled] Using host libthread_db library
> "/lib64/libthread_db.so.1". Missing separate debuginfo for
> /root/mediarecorder_ext/target/linux/lib/libaudiofile.so.1 Try: yum
> --enablerepo='*debug*' install
> /usr/lib/debug/.build-id/3f/ab3a9b600e5d5156497eac8d35217947a324aa.debug Core
> was generated by `/root/amr/bin/tdaemon --daemon -f wav'. Program terminated
> with signal 6, Aborted. #0 0x00007f18f0e64989 in raise () from
> /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install
> apr-1.4.8-3.el7.x86_64 glibc-2.17-55.el7.x86_64
> keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.11.3-49.el7.x86_64
> libcom_err-1.42.9-4.el7.x86_64 libgcc-4.8.2-16.el7.x86_64
> libselinux-2.2.2-6.el7.x86_64 libstdc++-4.8.2-16.el7.x86_64
> libuuid-2.23.2-16.el7.x86_64 nss-softokn-freebl-3.15.4-2.el7.x86_64
> openssl-libs-1.0.1e-34.el7.x86_64 pcre-8.32-12.el7.x86_64
> xz-libs-5.1.2-8alpha.el7.x86_64 (gdb) bt full #0 0x00007f18f0e64989 in raise
> () from /lib64/libc.so.6 No symbol table info available.+
> +#1 0x00007f18f0e66098 in abort () from /lib64/libc.so.6 No symbol table info
> available.+
> +#2 0x00007f18f8fad305 in Poco::SignalHandler::handleSignal (sig=11) at
> /root/poco/latest/poco-1.6.1-all/Foundation/src/SignalHandler.cpp:96 jb =
> <optimized out>+
> +#3 <signal handler called> No symbol table info available.+
> +#4 0x00007f18f0f78886 in __memcpy_ssse3_back () from /lib64/libc.so.6 No
> symbol table info available._+
> +_#5 0x00007f18f17c65b4 in std::string::append(std::string const&) () from
> /lib64/libstdc_+_+.so.6 No symbol table info available._
> _#6 0x00007f18f3a47053 in decaf::util::concurrent::Mutex::Mutex
> (this=0x7f1859ffabc0) at decaf/util/concurrent/Mutex.cpp:50 idStr = "10474"
> this = 0x7f1888000be0_
> _#7 0x00007f18f3a50340 in AbstractCollection (__vtt_parm=<optimized out>,
> this=0x7f1859ffabb8, __in_chrg=<optimized out>) at
> ./decaf/util/AbstractCollection.h:65 No locals._
> _#8 AbstractList (__vtt_parm=<optimized out>, this=0x7f1859ffabb0,
> __in_chrg=<optimized out>) at ./decaf/util/AbstractList.h:341 No locals._
> _#9 AbstractSequentialList (__vtt_parm=0x7f18f4069ce8 <VTT for
> decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>+8>,
> this=0x7f1859ffabb0, __in_chrg=<optimized out>) at
> ./decaf/util/AbstractSequentialList.h:59 No locals.
> #10
> decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>::LinkedList
> (this=this@entry=0x7f1859ffabb0, __in_chrg=<optimized out>,
> __vtt_parm=<optimized out>) at ./decaf/util/LinkedList.h:91 No locals.
> #11 0x00007f18f3a5366f in
> decaf::util::concurrent::ExecutorKernel::WorkerKiller::run
> (this=0x7f187c00aad0) at decaf/util/concurrent/ThreadPoolExecutor.cpp:248
> toDeleteList = warning: can't find linker symbol for virtual table for
> `decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
> value warning: found `construction vtable for
> decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>-in-decaf::util::LinkedList<decaf::util::concurrent::ExecutorKernel::Worker*>'
> instead
> {<decaf::util::AbstractSequentialList<decaf::util::concurrent::ExecutorKernel::Worker*>>
> =
> {<decaf::util::AbstractList<decaf::util::concurrent::ExecutorKernel::Worker*>>
> = {<decaf::util::List<decaf::util::concurrent::ExecutorKernel::Worker*>> =
> {<decaf::util::Collection<decaf::util::concurrent::ExecutorKernel::Worker*>>
> = <invalid address>, <No data fields>}
> ,
> <decaf::util::AbstractCollection<decaf::util::concurrent::ExecutorKernel::Worker*>>
> = {mutex = {<decaf::util::concurrent::Synchronizable> =
> { _vptr.Synchronizable = 0x7f18f4064db0 <vtable for
> decaf::util::concurrent::Mutex+16>}
> , properties = 0x0}}, modCount = 0}, <No data fields>},
> <decaf::util::Deque<decaf::util::concurrent::ExecutorKernel::Worker*>> =
> {<decaf::util::Queue<decaf::util::concurrent::ExecutorKernel::Worker*>---Type
> <return> to continue, or q <return> to quit--- > =
> {<No data fields>}
> , <No data fields>}, listSize = 147917545, head = \{value = 0x7f1859ffaca0,
> prev = 0x7f187c009a80, next = 0x7f18f3a46840
> <decaf::util::concurrent::Lock::~Lock()+160>}, tail = \{value =
> 0x7f187c009aa8, prev = 0x7f1859ffac80, next = 0x7f1888014768}} #12
> 0x00007f18f3a41abd in decaf::util::TimerImpl::run (this=0x7f187c009a80) at
> decaf/util/Timer.cpp:156 task =
> {<decaf::util::concurrent::atomic::AtomicRefCounter> =
> { _vptr.AtomicRefCounter = 0x7f18f40170d0 <vtable for
> decaf::lang::Pointer<decaf::util::TimerTask,
> decaf::util::concurrent::atomic::AtomicRefCounter>+16>, counter =
> 0x7f187c00ab40}
> , value = 0x7f187c00aad0, onDelete = 0x7f18f38c7fb0
> <decaf::lang::Pointer<decaf::util::TimerTask,
> decaf::util::concurrent::atomic::AtomicRefCounter>::onDeleteFunc(decaf::util::TimerTask*)>}
> #13 0x00007f18f39bd3d2 in (anonymous namespace)::runCallback
> (arg=0x7f187c009b40) at decaf/internal/util/concurrent/Threading.cpp:268
> thread = 0x7f187c009b40 #14 0x00007f18f39bdc0c in (anonymous
> namespace)::threadEntryMethod (arg=0x7f187c009b40) at
> decaf/internal/util/concurrent/Threading.cpp:256 thread = 0x7f187c009b40 #15
> 0x00007f18f5862df3 in start_thread () from /lib64/libpthread.so.0 No symbol
> table info available. #16 0x00007f18f0f253dd in clone () from
> /lib64/libc.so.6 No symbol table info available.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
