[
https://issues.apache.org/jira/browse/KUDU-1528?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15418435#comment-15418435
]
Mike Percy commented on KUDU-1528:
----------------------------------
Also saw this on a Jenkins run, triggered by TSAN on ksck_remote-test, at
http://dist-test.cloudera.org/job?job_id=jenkins-slave.1470973192.3972
{code}
WARNING: ThreadSanitizer: data race (pid=172)
Write of size 8 at 0x7d4c0023c8f8 by main thread (mutexes: write M156764,
write M148683):
#0 operator delete(void*)
/home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:77
(ksck_remote-test+0x0000004ba559)
#1 kudu::DefaultDeleter<kudu::ThreadPool>::operator()(kudu::ThreadPool*)
const /home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/gscoped_ptr.h:145:5
(libksck.so+0x0000000678a6)
#2 kudu::internal::gscoped_ptr_impl<kudu::ThreadPool,
kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr_impl()
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/gscoped_ptr.h:228:7
(libksck.so+0x000000067869)
#3 gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool>
>::~gscoped_ptr()
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/gscoped_ptr.h:318:7
(libksck.so+0x000000064649)
#4 kudu::consensus::RaftConsensus::~RaftConsensus()
/home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/raft_consensus.cc:244:1
(libconsensus.so+0x0000000d3591)
#5 kudu::consensus::RaftConsensus::~RaftConsensus()
/home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/raft_consensus.cc:242:33
(libconsensus.so+0x0000000d3699)
#6 kudu::RefCountedThreadSafe<kudu::consensus::Consensus,
kudu::DefaultRefCountedThreadSafeTraits<kudu::consensus::Consensus>
>::DeleteInternal(kudu::consensus::Consensus const*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:151:44
(libmaster.so+0x00000014b80a)
#7
kudu::DefaultRefCountedThreadSafeTraits<kudu::consensus::Consensus>::Destruct(kudu::consensus::Consensus
const*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:114:5
(libmaster.so+0x00000014b7c9)
#8 kudu::RefCountedThreadSafe<kudu::consensus::Consensus,
kudu::DefaultRefCountedThreadSafeTraits<kudu::consensus::Consensus>
>::Release() const
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:142:7
(libmaster.so+0x00000014b799)
#9
scoped_refptr<kudu::consensus::Consensus>::operator=(kudu::consensus::Consensus*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:284:7
(libtablet.so+0x00000022c45d)
#10
scoped_refptr<kudu::consensus::Consensus>::reset(kudu::consensus::Consensus*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:321:11
(libtablet.so+0x000000229680)
#11 kudu::tablet::TabletPeer::Shutdown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tablet/tablet_peer.cc:262:5
(libtablet.so+0x0000002260ac)
#12 kudu::tserver::TSTabletManager::Shutdown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/ts_tablet_manager.cc:733:5
(libtserver.so+0x000000158577)
#13 kudu::tserver::TabletServer::Shutdown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server.cc:141:5
(libtserver.so+0x00000012fc99)
#14 kudu::tserver::MiniTabletServer::Shutdown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/mini_tablet_server.cc:97:5
(libtserver.so+0x0000000faa66)
#15 kudu::MiniCluster::Shutdown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/integration-tests/mini_cluster.cc:184:5
(libintegration-tests.so+0x0000000a2267)
#16 kudu::tools::RemoteKsckTest::TearDown()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tools/ksck_remote-test.cc:109:7
(ksck_remote-test+0x0000004c1675)
#17 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-3/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x0000000442f9)
#18 void
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-3/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x0000000442f9)
#19 main
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/test_main.cc:48:13
(libkudu_test_main.so+0x000000001c65)
Previous read of size 8 at 0x7d4c0023c8f8 by thread T103:
#0 scoped_refptr<kudu::Histogram>::operator kudu::Histogram*
scoped_refptr<kudu::Histogram>::*() const
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:269:38
(libkrpc.so+0x0000000f74a9)
#1 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:241:7
(libkudu_util.so+0x0000002476b9)
#2 kudu::ThreadPool::SubmitFunc(boost::function<void ()> const&)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:182:10
(libkudu_util.so+0x000000247384)
#3 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()> const&)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:178:10
(libkudu_util.so+0x0000002472b2)
#4 kudu::consensus::Peer::ProcessResponse()
/home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/consensus_peers.cc:270:14
(libconsensus.so+0x00000009c57e)
#5 boost::_mfi::mf0<void,
kudu::consensus::Peer>::operator()(kudu::consensus::Peer*) const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
(libconsensus.so+0x0000000a2546)
#6 void boost::_bi::list1<boost::_bi::value<kudu::consensus::Peer*>
>::operator()<boost::_mfi::mf0<void, kudu::consensus::Peer>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::consensus::Peer>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:259:9
(libconsensus.so+0x0000000a24aa)
#7 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::consensus::Peer>,
boost::_bi::list1<boost::_bi::value<kudu::consensus::Peer*> > >::operator()()
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libconsensus.so+0x0000000a2453)
#8
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::consensus::Peer>,
boost::_bi::list1<boost::_bi::value<kudu::consensus::Peer*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libconsensus.so+0x0000000a2279)
#9 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000dfb01)
#10 kudu::rpc::OutboundCall::CallCallback()
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/outbound_call.cc:189:5
(libkrpc.so+0x0000000dd5c6)
#11
kudu::rpc::OutboundCall::SetResponse(gscoped_ptr<kudu::rpc::CallResponse,
kudu::DefaultDeleter<kudu::rpc::CallResponse> >)
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/outbound_call.cc:221:5
(libkrpc.so+0x0000000dd7d7)
#12
kudu::rpc::Connection::HandleCallResponse(gscoped_ptr<kudu::rpc::InboundTransfer,
kudu::DefaultDeleter<kudu::rpc::InboundTransfer> >)
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/connection.cc:534:3
(libkrpc.so+0x0000000e7678)
#13 kudu::rpc::Connection::ReadHandler(ev::io&, int)
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/connection.cc:470:7
(libkrpc.so+0x0000000e6fbc)
#14 void ev::base<ev_io, ev::io>::method_thunk<kudu::rpc::Connection,
&kudu::rpc::Connection::ReadHandler>(ev_loop*, ev_io*, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/ev++.h:479:7
(libkrpc.so+0x0000000ef68a)
#15 ev_invoke_pending
/home/jenkins-slave/workspace/kudu-1/thirdparty/libev-4.20/ev.c:3155
(libev.so.4+0x000000007605)
#16 kudu::rpc::ReactorThread::RunThread()
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:306:3
(libkrpc.so+0x000000106c63)
#17 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x00000010eef6)
#18 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x00000010ee5a)
#19 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x00000010ee03)
#20
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x00000010ec29)
#21 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000dfb01)
#22 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
(libkudu_util.so+0x00000023e09f)
Mutex M156764 (0x7d4800257e20) created at:
#0 __tsan_atomic32_compare_exchange_strong
/home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:779
(ksck_remote-test+0x00000047fb27)
#1 base::subtle::Acquire_CompareAndSwap(int volatile*, int, int)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/atomicops-internals-tsan.h:85:3
(libksck.so+0x00000006b767)
#2 base::SpinLock::Lock()
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/spinlock.h:73:9
(libksck.so+0x00000006b6e0)
#3 kudu::simple_spinlock::lock()
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/locks.h:45:5
(libksck.so+0x00000006b6a9)
#4
std::lock_guard<kudu::simple_spinlock>::lock_guard(kudu::simple_spinlock&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:377:9
(libksck.so+0x00000006b398)
#5 kudu::tablet::TabletPeer::Start(kudu::consensus::ConsensusBootstrapInfo
const&)
/home/jenkins-slave/workspace/kudu-3/src/kudu/tablet/tablet_peer.cc:187:36
(libtablet.so+0x000000225765)
#6
kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletMetadata>
const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&)
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/ts_tablet_manager.cc:678:9
(libtserver.so+0x000000154948)
#7 boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
scoped_refptr<kudu::tablet::TabletMetadata> const&,
scoped_refptr<kudu::tserver::TransitionInProgressDeleter>
const&>::operator()(kudu::tserver::TSTabletManager*,
scoped_refptr<kudu::tablet::TabletMetadata> const&,
scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:280:29
(libtserver.so+0x00000015f2e7)
#8 void
boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> >
>::operator()<boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
scoped_refptr<kudu::tablet::TabletMetadata> const&,
scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void,
kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata>
const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>&,
boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:398:9
(libtserver.so+0x00000015f233)
#9 boost::_bi::bind_t<void, boost::_mfi::mf2<void,
kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata>
const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > >
>::operator()()
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libtserver.so+0x00000015f1b3)
#10
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
scoped_refptr<kudu::tablet::TabletMetadata> const&,
scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > >
>, void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libtserver.so+0x00000015ef71)
#11 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000dfb01)
#12 kudu::FunctionRunnable::Run()
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:48:5
(libkudu_util.so+0x0000002497fd)
#13 kudu::ThreadPool::DispatchThread(bool)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:343:7
(libkudu_util.so+0x000000247c4f)
#14 boost::_mfi::mf1<void, kudu::ThreadPool,
bool>::operator()(kudu::ThreadPool*, bool) const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:165:29
(libkudu_util.so+0x00000024bf5e)
#15 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool,
bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void,
kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:319:9
(libkudu_util.so+0x00000024bebb)
#16 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
boost::_bi::value<bool> > >::operator()()
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libkudu_util.so+0x00000024be43)
#17
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool>
> >, void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libkudu_util.so+0x00000024bc41)
#18 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000dfb01)
#19 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
(libkudu_util.so+0x00000023e09f)
Mutex M148683 (0x7d4800257e1c) created at:
#0 __tsan_atomic32_compare_exchange_strong
/home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:779
(ksck_remote-test+0x00000047fb27)
#1 base::subtle::Acquire_CompareAndSwap(int volatile*, int, int)
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/atomicops-internals-tsan.h:85:3
(libksck.so+0x00000006b767)
#2 base::SpinLock::Lock()
/home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/spinlock.h:73:9
(libksck.so+0x00000006b6e0)
#3 kudu::simple_spinlock::lock()
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/locks.h:45:5
(libksck.so+0x00000006b6a9)
#4
std::lock_guard<kudu::simple_spinlock>::lock_guard(kudu::simple_spinlock&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:377:9
(libksck.so+0x00000006b398)
#5 kudu::tablet::TabletPeer::state() const
/home/jenkins-slave/workspace/kudu-3/src/kudu/tablet/tablet_peer.h:148:38
(libtserver.so+0x000000128e89)
#6 kudu::tserver::TSTabletManager::GetNumLiveTablets() const
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/ts_tablet_manager.cc:817:35
(libtserver.so+0x000000158a68)
#7 kudu::tserver::Heartbeater::Thread::DoHeartbeat()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/heartbeater.cc:384:28
(libtserver.so+0x0000000f016d)
#8 kudu::tserver::Heartbeater::Thread::RunThread()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/heartbeater.cc:453:16
(libtserver.so+0x0000000f0af0)
#9 boost::_mfi::mf0<void,
kudu::tserver::Heartbeater::Thread>::operator()(kudu::tserver::Heartbeater::Thread*)
const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
(libtserver.so+0x0000000f7b36)
#10 void
boost::_bi::list1<boost::_bi::value<kudu::tserver::Heartbeater::Thread*>
>::operator()<boost::_mfi::mf0<void, kudu::tserver::Heartbeater::Thread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::tserver::Heartbeater::Thread>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:259:9
(libtserver.so+0x0000000f7a9a)
#11 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::tserver::Heartbeater::Thread>,
boost::_bi::list1<boost::_bi::value<kudu::tserver::Heartbeater::Thread*> >
>::operator()()
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libtserver.so+0x0000000f7a23)
#12
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::tserver::Heartbeater::Thread>,
boost::_bi::list1<boost::_bi::value<kudu::tserver::Heartbeater::Thread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libtserver.so+0x0000000f77f9)
#13 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000dfb01)
#14 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
(libkudu_util.so+0x00000023e09f)
Thread T103 'rpc reactor-151' (tid=1513, running) created by main thread at:
#0 pthread_create
/home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954
(ksck_remote-test+0x00000045d583)
#1 kudu::Thread::StartThread(std::string const&, std::string const&,
boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:509:15
(libkudu_util.so+0x00000023d927)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(),
kudu::rpc::ReactorThread*>(std::string const&, std::string const&, void
(kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&,
scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.h:156:12
(libkrpc.so+0x00000010aa57)
#3 kudu::rpc::ReactorThread::Init()
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:108:10
(libkrpc.so+0x000000106af1)
#4 kudu::rpc::Reactor::Init()
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:488:10
(libkrpc.so+0x000000109c0e)
#5 kudu::rpc::Messenger::Init()
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:266:5
(libkrpc.so+0x0000000fa00d)
#6 kudu::rpc::MessengerBuilder::Build(kudu::rpc::Messenger**)
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:106:3
(libkrpc.so+0x0000000f9f10)
#7
kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
/home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:113:3
(libkrpc.so+0x0000000fa0af)
#8 kudu::server::ServerBase::Init()
/home/jenkins-slave/workspace/kudu-3/src/kudu/server/server_base.cc:179:3
(libserver_process.so+0x0000000786a0)
#9 kudu::tserver::TabletServer::Init()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server.cc:90:3
(libtserver.so+0x000000130055)
#10 kudu::tserver::MiniTabletServer::Start()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/mini_tablet_server.cc:78:3
(libtserver.so+0x0000000fa82a)
#11 kudu::MiniCluster::AddTabletServer()
/home/jenkins-slave/workspace/kudu-3/src/kudu/integration-tests/mini_cluster.cc:177:3
(libintegration-tests.so+0x0000000a1d44)
#12 kudu::MiniCluster::Start()
/home/jenkins-slave/workspace/kudu-3/src/kudu/integration-tests/mini_cluster.cc:93:5
(libintegration-tests.so+0x0000000a0d6a)
#13 kudu::tools::RemoteKsckTest::SetUp()
/home/jenkins-slave/workspace/kudu-3/src/kudu/tools/ksck_remote-test.cc:82:5
(ksck_remote-test+0x0000004c0aae)
#14 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-3/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x0000000442f9)
#15 void
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-3/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x0000000442f9)
#16 main
/home/jenkins-slave/workspace/kudu-3/src/kudu/util/test_main.cc:48:13
(libkudu_test_main.so+0x000000001c65)
SUMMARY: ThreadSanitizer: data race
/home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:77
in operator delete(void*)
{code}
> heap-use-after-free in Peer::ProcessResponse while deleting a tablet
> ---------------------------------------------------------------------
>
> Key: KUDU-1528
> URL: https://issues.apache.org/jira/browse/KUDU-1528
> Project: Kudu
> Issue Type: Bug
> Components: consensus
> Affects Versions: 0.9.1
> Reporter: Adar Dembo
> Priority: Critical
>
> Observed this in a [master-stress-test
> failure|http://104.196.14.100/job/kudu-gerrit/2314/BUILD_TYPE=ASAN]. It
> appears that we deleted a tablet while we were processing the response to
> ConsensusService::UpdateConsensus().
> {noformat}
> ==721==ERROR: AddressSanitizer: heap-use-after-free on address 0x6140001b8fb0
> at pc 0x7f54968ac684 bp 0x7f5487e485b0 sp 0x7f5487e485a8
> READ of size 8 at 0x6140001b8fb0 thread T6 (rpc reactor-728)
> #0 0x7f54968ac683 in scoped_refptr<kudu::Histogram>::operator
> kudu::Histogram* scoped_refptr<kudu::Histogram>::*() const
> /home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:269:38
> #1 0x7f5491297e5b in
> kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:241:7
> #2 0x7f54912978cd in kudu::ThreadPool::SubmitFunc(boost::function<void
> ()> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:182:10
> #3 0x7f54912976ef in kudu::ThreadPool::SubmitClosure(kudu::Callback<void
> ()> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:178:10
> #4 0x7f5497f3929c in kudu::consensus::Peer::ProcessResponse()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/consensus_peers.cc:263:14
> #5 0x7f5497f440ad in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::consensus::Peer>,
> boost::_bi::list1<boost::_bi::value<kudu::consensus::Peer*> > >::operator()()
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> #6 0x7f549688135e in boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> #7 0x7f549687ca1e in kudu::rpc::OutboundCall::CallCallback()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/outbound_call.cc:189:5
> #8 0x7f549687ce01 in
> kudu::rpc::OutboundCall::SetResponse(gscoped_ptr<kudu::rpc::CallResponse,
> kudu::DefaultDeleter<kudu::rpc::CallResponse> >)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/outbound_call.cc:221:5
> #9 0x7f549688ea1f in
> kudu::rpc::Connection::HandleCallResponse(gscoped_ptr<kudu::rpc::InboundTransfer,
> kudu::DefaultDeleter<kudu::rpc::InboundTransfer> >)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/connection.cc:534:3
> #10 0x7f549688ded2 in kudu::rpc::Connection::ReadHandler(ev::io&, int)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/connection.cc:470:7
> #11 0x7f5496184605 in ev_invoke_pending
> /home/jenkins-slave/workspace/kudu-2/thirdparty/libev-4.20/ev.c:3155
> #12 0x7f54961854f7 in ev_run
> /home/jenkins-slave/workspace/kudu-2/thirdparty/libev-4.20/ev.c:3555
> #13 0x7f54968c6cca in kudu::rpc::ReactorThread::RunThread()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:306:3
> #14 0x7f54968d69dd in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
> >::operator()()
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> #15 0x7f549688135e in boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> #16 0x7f5491285dd6 in kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
> #17 0x7f5493bcb181 in start_thread
> /build/eglibc-3GlaMS/eglibc-2.19/nptl/pthread_create.c:312
> #18 0x7f548e6fe47c in clone
> /build/eglibc-3GlaMS/eglibc-2.19/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111
> 0x6140001b8fb0 is located 368 bytes inside of 416-byte region
> [0x6140001b8e40,0x6140001b8fe0)
> freed by thread T31 (rpc worker-753) here:
> #0 0x4f54d0 in operator delete(void*)
> /home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/asan/asan_new_delete.cc:94
> #1 0x7f5497f9e40d in kudu::consensus::RaftConsensus::~RaftConsensus()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/raft_consensus.cc:244:1
> #2 0x7f5497f9e551 in kudu::consensus::RaftConsensus::~RaftConsensus()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/raft_consensus.cc:242:33
> #3 0x7f5498adf6e8 in
> scoped_refptr<kudu::consensus::Consensus>::operator=(kudu::consensus::Consensus*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/gutil/ref_counted.h:284:7
> #4 0x7f5498ad3d41 in kudu::tablet::TabletPeer::Shutdown()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tablet/tablet_peer.cc:255:5
> #5 0x7f549adedd6d in
> kudu::tserver::TSTabletManager::DeleteTablet(std::string const&,
> kudu::tablet::TabletDataState, boost::optional<long> const&,
> boost::optional<kudu::tserver::TabletServerErrorPB_Code>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/ts_tablet_manager.cc:525:3
> #6 0x7f549adacdbd in
> kudu::tserver::TabletServiceAdminImpl::DeleteTablet(kudu::tserver::DeleteTabletRequestPB
> const*, kudu::tserver::DeleteTabletResponsePB*, kudu::rpc::RpcContext*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_service.cc:664:14
> #7 0x7f549707daae in std::_Function_handler<void
> (google::protobuf::Message const*, google::protobuf::Message*,
> kudu::rpc::RpcContext*),
> kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity>
> const&)::$_1>::_M_invoke(std::_Any_data const&, google::protobuf::Message
> const*, google::protobuf::Message*, kudu::rpc::RpcContext*)
> /usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8/functional:2071:2
> #8 0x7f549691f220 in std::function<void (google::protobuf::Message
> const*, google::protobuf::Message*,
> kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*,
> google::protobuf::Message*, kudu::rpc::RpcContext*) const
> /usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8/functional:2471:14
> #9 0x7f549691e913 in
> kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/service_if.cc:94:3
> #10 0x7f5496921599 in kudu::rpc::ServicePool::RunThread()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/service_pool.cc:206:5
> #11 0x7f549692533d in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >
> >::operator()()
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> #12 0x7f549688135e in boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> #13 0x7f5491285dd6 in kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
> #14 0x7f5493bcb181 in start_thread
> /build/eglibc-3GlaMS/eglibc-2.19/nptl/pthread_create.c:312
> previously allocated by thread T121 (tablet-bootstra) here:
> #0 0x4f4e90 in operator new(unsigned long)
> /home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/asan/asan_new_delete.cc:62
> #1 0x7f5491295c3e in
> kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool,
> kudu::DefaultDeleter<kudu::ThreadPool> >*) const
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:96:15
> #2 0x7f5497f9d2f0 in
> kudu::consensus::RaftConsensus::Create(kudu::consensus::ConsensusOptions
> const&, gscoped_ptr<kudu::consensus::ConsensusMetadata,
> kudu::DefaultDeleter<kudu::consensus::ConsensusMetadata> >,
> kudu::consensus::RaftPeerPB const&, scoped_refptr<kudu::MetricEntity> const&,
> scoped_refptr<kudu::server::Clock> const&,
> kudu::consensus::ReplicaTransactionFactory*,
> std::shared_ptr<kudu::rpc::Messenger> const&, scoped_refptr<kudu::log::Log>
> const&, std::shared_ptr<kudu::MemTracker> const&, kudu::Callback<void
> (std::string const&)> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/consensus/raft_consensus.cc:172:3
> #3 0x7f5498ad244d in
> kudu::tablet::TabletPeer::Init(std::shared_ptr<kudu::tablet::Tablet> const&,
> scoped_refptr<kudu::server::Clock> const&,
> std::shared_ptr<kudu::rpc::Messenger> const&, scoped_refptr<kudu::log::Log>
> const&, scoped_refptr<kudu::MetricEntity> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tablet/tablet_peer.cc:156:18
> #4 0x7f549ade847e in
> kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/ts_tablet_manager.cc:659:10
> #5 0x7f549adfac7d in boost::_bi::bind_t<void, boost::_mfi::mf2<void,
> kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
> boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
> boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
> boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> >
> > >::operator()()
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> #6 0x7f549688135e in boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> #7 0x7f5491298bfb in kudu::ThreadPool::DispatchThread(bool)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/threadpool.cc:343:7
> #8 0x7f54912a00dd in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> #9 0x7f549688135e in boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-3/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> #10 0x7f5491285dd6 in kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:586:3
> #11 0x7f5493bcb181 in start_thread
> /build/eglibc-3GlaMS/eglibc-2.19/nptl/pthread_create.c:312
> Thread T6 (rpc reactor-728) created by T0 here:
> #0 0x4301cd in __interceptor_pthread_create
> /home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:238
> #1 0x7f54912852bb in kudu::Thread::StartThread(std::string const&,
> std::string const&, boost::function<void ()> const&, unsigned long,
> scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:509:15
> #2 0x7f54968ce906 in kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.h:156:12
> #3 0x7f54968c69e6 in kudu::rpc::ReactorThread::Init()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:108:10
> #4 0x7f54968cca15 in kudu::rpc::Reactor::Init()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/reactor.cc:488:10
> #5 0x7f54968b0691 in kudu::rpc::Messenger::Init()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:266:5
> #6 0x7f54968b03e9 in
> kudu::rpc::MessengerBuilder::Build(kudu::rpc::Messenger**)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:106:3
> #7 0x7f54968b083f in
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/messenger.cc:113:3
> #8 0x7f54998b380d in kudu::server::ServerBase::Init()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/server/server_base.cc:178:3
> #9 0x7f549ada533a in kudu::tserver::TabletServer::Init()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server.cc:90:3
> #10 0x4f80cf in kudu::tserver::TabletServerMain(int, char**)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server_main.cc:55:3
> #11 0x4f7d34 in main
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server_main.cc:72:10
> #12 0x7f548e625ec4 in __libc_start_main
> /build/eglibc-3GlaMS/eglibc-2.19/csu/libc-start.c:287
> Thread T31 (rpc worker-753) created by T0 here:
> #0 0x4301cd in __interceptor_pthread_create
> /home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:238
> #1 0x7f54912852bb in kudu::Thread::StartThread(std::string const&,
> std::string const&, boost::function<void ()> const&, unsigned long,
> scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:509:15
> #2 0x7f54969236e6 in kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ServicePool::*)(), kudu::rpc::ServicePool*>(std::string const&,
> std::string const&, void (kudu::rpc::ServicePool::* const&)(),
> kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.h:156:12
> #3 0x7f5496920ffc in kudu::rpc::ServicePool::Init(int)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/rpc/service_pool.cc:80:5
> #4 0x7f54998ab7e4 in
> kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf,
> kudu::DefaultDeleter<kudu::rpc::ServiceIf> >)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/server/rpc_server.cc:122:3
> #5 0x7f54998b4bc3 in
> kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf,
> kudu::DefaultDeleter<kudu::rpc::ServiceIf> >)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/server/server_base.cc:239:10
> #6 0x7f549ada5984 in kudu::tserver::TabletServer::Start()
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server.cc:120:3
> #7 0x4f8133 in kudu::tserver::TabletServerMain(int, char**)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server_main.cc:58:3
> #8 0x4f7d34 in main
> /home/jenkins-slave/workspace/kudu-3/src/kudu/tserver/tablet_server_main.cc:72:10
> #9 0x7f548e625ec4 in __libc_start_main
> /build/eglibc-3GlaMS/eglibc-2.19/csu/libc-start.c:287
> Thread T121 (tablet-bootstra) created by T46 (rpc worker-768) here:
> #0 0x4301cd in __interceptor_pthread_create
> /home/jenkins-slave/workspace/kudu-3/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:238
> #1 0x7f54912852bb in kudu::Thread::StartThread(std::string const&,
> std::string const&, boost::function<void ()> const&, unsigned long,
> scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-3/src/kudu/util/thread.cc:509:15
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
