[
https://issues.apache.org/jira/browse/KUDU-1264?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dinesh Bhat reassigned KUDU-1264:
---------------------------------
Assignee: Dinesh Bhat
> TSAN data race between Tablet Bootstrap and ListTablets
> -------------------------------------------------------
>
> Key: KUDU-1264
> URL: https://issues.apache.org/jira/browse/KUDU-1264
> Project: Kudu
> Issue Type: Bug
> Reporter: Mike Percy
> Assignee: Dinesh Bhat
>
> This came up in a dist-test run of a TSAN build on
> TabletReplacementITest.TestRemoteBoostrapWithPendingConfigChangeCommits as
> part of KUDU-1233. I am somewhat skeptical that it's a real bug, but it could
> just be somehow misreported by TSAN and the real error is elsewhere.
> {noformat}
> WARNING: ThreadSanitizer: data race (pid=20565)
> Write of size 8 at 0x7d1400027e88 by thread T124:
> #0 operator delete(void*) <null> (kudu-tserver+0x0000000494e6)
> #1 std::string::assign(std::string const&) <null>
> (libstdc++.so.6+0x0000000bb4e8)
> #2
> kudu::tablet::TabletBootstrap::Bootstrap(std::tr1::shared_ptr<kudu::tablet::Tablet>*,
> scoped_refptr<kudu::log::Log>*, kudu::consensus::ConsensusBootstrapInfo*)
> /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:439
> (libtablet.so+0x000000240bd8)
> #3
> kudu::tablet::BootstrapTablet(scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::server::Clock> const&,
> std::tr1::shared_ptr<kudu::MemTracker> const&, kudu::MetricRegistry*,
> kudu::tablet::TabletStatusListener*,
> std::tr1::shared_ptr<kudu::tablet::Tablet>*, scoped_refptr<kudu::log::Log>*,
> scoped_refptr<kudu::log::LogAnchorRegistry> const&,
> kudu::consensus::ConsensusBootstrapInfo*)
> /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:376
> (libtablet.so+0x000000240952)
> #4
> kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&)
> /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:607
> (libtserver.so+0x000000161d4e)
> #5 boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
> scoped_refptr<kudu::tablet::TabletMetadata> const&,
> scoped_refptr<kudu::tserver::TransitionInProgressDeleter>
> const&>::operator()(kudu::tserver::TSTabletManager*,
> scoped_refptr<kudu::tablet::TabletMetadata> const&,
> scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) const
> /usr/include/boost/bind/mem_fn_template.hpp:280 (libtserver.so+0x00000016e05e)
> #6 void
> boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
> boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
> boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> >
> >::operator()<boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
> scoped_refptr<kudu::tablet::TabletMetadata> const&,
> scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void,
> kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>&,
> boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:392
> (libtserver.so+0x00000016dfb3)
> #7 boost::_bi::bind_t<void, boost::_mfi::mf2<void,
> kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata>
> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
> boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
> boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
> boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> >
> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20
> (libtserver.so+0x00000016df33)
> #8
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf2<void, kudu::tserver::TSTabletManager,
> scoped_refptr<kudu::tablet::TabletMetadata> const&,
> scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>,
> boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>,
> boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >,
> boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> >
> > >, void>::invoke(boost::detail::function::function_buffer&)
> /usr/include/boost/function/function_template.hpp:153
> (libtserver.so+0x00000016dcf1)
> #9 boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:766
> (libkrpc.so+0x000000096a61)
> #10 kudu::FunctionRunnable::Run()
> /home/todd/git/kudu/src/kudu/util/threadpool.cc:46
> (libkudu_util.so+0x00000021099d)
> #11 kudu::ThreadPool::DispatchThread(bool)
> /home/todd/git/kudu/src/kudu/util/threadpool.cc:317
> (libkudu_util.so+0x00000020f4b6)
> #12 boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>::operator()(kudu::ThreadPool*, bool) const
> /usr/include/boost/bind/mem_fn_template.hpp:165
> (libkudu_util.so+0x000000212845)
> #13 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
> /usr/include/boost/bind/bind.hpp:313 (libkudu_util.so+0x0000002127ab)
> #14 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /usr/include/boost/bind/bind_template.hpp:20 (libkudu_util.so+0x000000212733)
> #15
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /usr/include/boost/function/function_template.hpp:153
> (libkudu_util.so+0x000000212541)
> #16 boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:766
> (libkrpc.so+0x000000096a61)
> #17 kudu::Thread::SuperviseThread(void*)
> /home/todd/git/kudu/src/kudu/util/thread.cc:503
> (libkudu_util.so+0x0000002052b9)
> Previous read of size 8 at 0x7d1400027e88 by thread T12:
> #0 memcpy <null> (kudu-tserver+0x0000000509fb)
> #1 std::string::data() const
> /home/todd/git/kudu/thirdparty/protobuf-2.6.1/src/google/protobuf/io/coded_stream.cc:640
> (libprotobuf.so.9+0x000000072a4c)
> #2 unsigned char*
> google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tablet::TabletStatusPB>(int,
> kudu::tablet::TabletStatusPB const&, unsigned char*)
> /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793
> (libtserver_proto.so+0x00000002eb70)
> #3 unsigned char*
> google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB>(int,
> kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB const&, unsigned
> char*)
> /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793
> (libtserver_proto.so+0x00000002faf9)
> #4
> kudu::rpc::serialization::SerializeMessage(google::protobuf::MessageLite
> const&, kudu::faststring*, int, bool)
> /home/todd/git/kudu/src/kudu/rpc/serialization.cc:65
> (libkrpc.so+0x0000000d8841)
> #5
> kudu::rpc::InboundCall::SerializeResponseBuffer(google::protobuf::MessageLite
> const&, bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:144
> (libkrpc.so+0x0000000a7fb2)
> #6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&,
> bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:116
> (libkrpc.so+0x0000000a7754)
> #7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite
> const&) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:79
> (libkrpc.so+0x0000000a7665)
> #8 kudu::rpc::RpcContext::RespondSuccess()
> /home/todd/git/kudu/src/kudu/rpc/rpc_context.cc:51 (libkrpc.so+0x0000000c727d)
> #9
> kudu::tserver::TabletServiceImpl::ListTablets(kudu::tserver::ListTabletsRequestPB
> const*, kudu::tserver::ListTabletsResponsePB*, kudu::rpc::RpcContext*)
> /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:1078
> (libtserver.so+0x00000014d325)
> #10 kudu::tserver::TabletServerServiceIf::Handle(kudu::rpc::InboundCall*)
> /home/todd/git/kudu/src/kudu/tserver/tserver_service.service.cc:122
> (libtserver_service_proto.so+0x000000015a56)
> #11 kudu::rpc::ServicePool::RunThread()
> /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173
> (libkrpc.so+0x0000000dac85)
> #12 boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
> /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d)
> #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
> /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a)
> #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >
> >::operator()() /usr/include/boost/bind/bind_template.hpp:20
> (libkrpc.so+0x0000000dceb3)
> #15
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /usr/include/boost/function/function_template.hpp:153
> (libkrpc.so+0x0000000dccd9)
> #16 boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:766
> (libkrpc.so+0x000000096a61)
> #17 kudu::Thread::SuperviseThread(void*)
> /home/todd/git/kudu/src/kudu/util/thread.cc:503
> (libkudu_util.so+0x0000002052b9)
> Thread T124 'tablet-bootstra' (tid=20737, running) created by thread T64 at:
> #0 pthread_create <null> (kudu-tserver+0x00000004e7f3)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, scoped_refptr<kudu::Thread>*)
> /home/todd/git/kudu/src/kudu/util/thread.cc:431
> (libkudu_util.so+0x000000204d86)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::string const&, std::string const&, void
> (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&,
> scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:143
> (libkudu_util.so+0x00000021067b)
> #3 kudu::ThreadPool::CreateThreadUnlocked()
> /home/todd/git/kudu/src/kudu/util/threadpool.cc:344
> (libkudu_util.so+0x00000020e983)
> #4 kudu::ThreadPool::Submit(std::tr1::shared_ptr<kudu::Runnable> const&)
> /home/todd/git/kudu/src/kudu/util/threadpool.cc:192
> (libkudu_util.so+0x00000020ee42)
> #5 kudu::ThreadPool::SubmitFunc(boost::function<void ()> const&)
> /home/todd/git/kudu/src/kudu/util/threadpool.cc:163
> (libkudu_util.so+0x00000020ec55)
> #6
> kudu::tserver::TSTabletManager::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB
> const&) /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:444
> (libtserver.so+0x000000163e3c)
> #7
> kudu::tserver::ConsensusServiceImpl::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB
> const*, kudu::consensus::StartRemoteBootstrapResponsePB*,
> kudu::rpc::RpcContext*)
> /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:951
> (libtserver.so+0x00000014959b)
> #8 kudu::consensus::ConsensusServiceIf::Handle(kudu::rpc::InboundCall*)
> /home/todd/git/kudu/src/kudu/consensus/consensus.service.cc:192
> (libconsensus_proto.so+0x0000000680cf)
> #9 kudu::rpc::ServicePool::RunThread()
> /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173
> (libkrpc.so+0x0000000dac85)
> #10 boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
> /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d)
> #11 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
> /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a)
> #12 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >
> >::operator()() /usr/include/boost/bind/bind_template.hpp:20
> (libkrpc.so+0x0000000dceb3)
> #13
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /usr/include/boost/function/function_template.hpp:153
> (libkrpc.so+0x0000000dccd9)
> #14 boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:766
> (libkrpc.so+0x000000096a61)
> #15 kudu::Thread::SuperviseThread(void*)
> /home/todd/git/kudu/src/kudu/util/thread.cc:503
> (libkudu_util.so+0x0000002052b9)
> Thread T12 'rpc worker-2058' (tid=20580, running) created by main thread at:
> #0 pthread_create <null> (kudu-tserver+0x00000004e7f3)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()> const&, scoped_refptr<kudu::Thread>*)
> /home/todd/git/kudu/src/kudu/util/thread.cc:431
> (libkudu_util.so+0x000000204d86)
> #2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::*)(),
> kudu::rpc::ServicePool*>(std::string const&, std::string const&, void
> (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&,
> scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:137
> (libkrpc.so+0x0000000dbbeb)
> #3 kudu::rpc::ServicePool::Init(int)
> /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:77
> (libkrpc.so+0x0000000da994)
> #4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf,
> kudu::DefaultDeleter<kudu::rpc::ServiceIf> >)
> /home/todd/git/kudu/src/kudu/server/rpc_server.cc:122
> (libserver_process.so+0x00000007f2c6)
> #5
> kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf,
> kudu::DefaultDeleter<kudu::rpc::ServiceIf> >)
> /home/todd/git/kudu/src/kudu/server/server_base.cc:237
> (libserver_process.so+0x000000086160)
> #6 kudu::tserver::TabletServer::Start()
> /home/todd/git/kudu/src/kudu/tserver/tablet_server.cc:117
> (libtserver.so+0x00000013ee50)
> #7 kudu::tserver::TabletServerMain(int, char**)
> /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:55
> (kudu-tserver+0x0000000a3bcc)
> #8 main /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:69
> (kudu-tserver+0x0000000a39ae)
> SUMMARY: ThreadSanitizer: data race
> (/tmp/run_tha_testcGwYR3/build/fastdebug/kudu-tserver+0x494e6) in operator
> delete(void*)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
