[ https://issues.apache.org/jira/browse/KUDU-1264?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Dinesh Bhat resolved KUDU-1264. ------------------------------- Resolution: Duplicate Fix Version/s: 1.0.0 > TSAN data race between Tablet Bootstrap and ListTablets > ------------------------------------------------------- > > Key: KUDU-1264 > URL: https://issues.apache.org/jira/browse/KUDU-1264 > Project: Kudu > Issue Type: Bug > Reporter: Mike Percy > Assignee: Dinesh Bhat > Fix For: 1.0.0 > > > This came up in a dist-test run of a TSAN build on > TabletReplacementITest.TestRemoteBoostrapWithPendingConfigChangeCommits as > part of KUDU-1233. I am somewhat skeptical that it's a real bug, but it could > just be somehow misreported by TSAN and the real error is elsewhere. > {noformat} > WARNING: ThreadSanitizer: data race (pid=20565) > Write of size 8 at 0x7d1400027e88 by thread T124: > #0 operator delete(void*) <null> (kudu-tserver+0x0000000494e6) > #1 std::string::assign(std::string const&) <null> > (libstdc++.so.6+0x0000000bb4e8) > #2 > kudu::tablet::TabletBootstrap::Bootstrap(std::tr1::shared_ptr<kudu::tablet::Tablet>*, > scoped_refptr<kudu::log::Log>*, kudu::consensus::ConsensusBootstrapInfo*) > /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:439 > (libtablet.so+0x000000240bd8) > #3 > kudu::tablet::BootstrapTablet(scoped_refptr<kudu::tablet::TabletMetadata> > const&, scoped_refptr<kudu::server::Clock> const&, > std::tr1::shared_ptr<kudu::MemTracker> const&, kudu::MetricRegistry*, > kudu::tablet::TabletStatusListener*, > std::tr1::shared_ptr<kudu::tablet::Tablet>*, scoped_refptr<kudu::log::Log>*, > scoped_refptr<kudu::log::LogAnchorRegistry> const&, > kudu::consensus::ConsensusBootstrapInfo*) > /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:376 > (libtablet.so+0x000000240952) > #4 > kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletMetadata> > const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) > /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:607 > (libtserver.so+0x000000161d4e) > #5 boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, > scoped_refptr<kudu::tablet::TabletMetadata> const&, > scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > const&>::operator()(kudu::tserver::TSTabletManager*, > scoped_refptr<kudu::tablet::TabletMetadata> const&, > scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) const > /usr/include/boost/bind/mem_fn_template.hpp:280 (libtserver.so+0x00000016e05e) > #6 void > boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, > boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, > boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > > >::operator()<boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, > scoped_refptr<kudu::tablet::TabletMetadata> const&, > scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void, > kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> > const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>&, > boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:392 > (libtserver.so+0x00000016dfb3) > #7 boost::_bi::bind_t<void, boost::_mfi::mf2<void, > kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> > const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, > boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, > boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, > boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 > (libtserver.so+0x00000016df33) > #8 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, > scoped_refptr<kudu::tablet::TabletMetadata> const&, > scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, > boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, > boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, > boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > > > >, void>::invoke(boost::detail::function::function_buffer&) > /usr/include/boost/function/function_template.hpp:153 > (libtserver.so+0x00000016dcf1) > #9 boost::function0<void>::operator()() const > /usr/include/boost/function/function_template.hpp:766 > (libkrpc.so+0x000000096a61) > #10 kudu::FunctionRunnable::Run() > /home/todd/git/kudu/src/kudu/util/threadpool.cc:46 > (libkudu_util.so+0x00000021099d) > #11 kudu::ThreadPool::DispatchThread(bool) > /home/todd/git/kudu/src/kudu/util/threadpool.cc:317 > (libkudu_util.so+0x00000020f4b6) > #12 boost::_mfi::mf1<void, kudu::ThreadPool, > bool>::operator()(kudu::ThreadPool*, bool) const > /usr/include/boost/bind/mem_fn_template.hpp:165 > (libkudu_util.so+0x000000212845) > #13 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, > boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, > kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, > boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) > /usr/include/boost/bind/bind.hpp:313 (libkudu_util.so+0x0000002127ab) > #14 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, > bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, > boost::_bi::value<bool> > >::operator()() > /usr/include/boost/bind/bind_template.hpp:20 (libkudu_util.so+0x000000212733) > #15 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf1<void, kudu::ThreadPool, bool>, > boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, > boost::_bi::value<bool> > >, > void>::invoke(boost::detail::function::function_buffer&) > /usr/include/boost/function/function_template.hpp:153 > (libkudu_util.so+0x000000212541) > #16 boost::function0<void>::operator()() const > /usr/include/boost/function/function_template.hpp:766 > (libkrpc.so+0x000000096a61) > #17 kudu::Thread::SuperviseThread(void*) > /home/todd/git/kudu/src/kudu/util/thread.cc:503 > (libkudu_util.so+0x0000002052b9) > Previous read of size 8 at 0x7d1400027e88 by thread T12: > #0 memcpy <null> (kudu-tserver+0x0000000509fb) > #1 std::string::data() const > /home/todd/git/kudu/thirdparty/protobuf-2.6.1/src/google/protobuf/io/coded_stream.cc:640 > (libprotobuf.so.9+0x000000072a4c) > #2 unsigned char* > google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tablet::TabletStatusPB>(int, > kudu::tablet::TabletStatusPB const&, unsigned char*) > /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793 > (libtserver_proto.so+0x00000002eb70) > #3 unsigned char* > google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB>(int, > kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB const&, unsigned > char*) > /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793 > (libtserver_proto.so+0x00000002faf9) > #4 > kudu::rpc::serialization::SerializeMessage(google::protobuf::MessageLite > const&, kudu::faststring*, int, bool) > /home/todd/git/kudu/src/kudu/rpc/serialization.cc:65 > (libkrpc.so+0x0000000d8841) > #5 > kudu::rpc::InboundCall::SerializeResponseBuffer(google::protobuf::MessageLite > const&, bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:144 > (libkrpc.so+0x0000000a7fb2) > #6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&, > bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:116 > (libkrpc.so+0x0000000a7754) > #7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite > const&) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:79 > (libkrpc.so+0x0000000a7665) > #8 kudu::rpc::RpcContext::RespondSuccess() > /home/todd/git/kudu/src/kudu/rpc/rpc_context.cc:51 (libkrpc.so+0x0000000c727d) > #9 > kudu::tserver::TabletServiceImpl::ListTablets(kudu::tserver::ListTabletsRequestPB > const*, kudu::tserver::ListTabletsResponsePB*, kudu::rpc::RpcContext*) > /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:1078 > (libtserver.so+0x00000014d325) > #10 kudu::tserver::TabletServerServiceIf::Handle(kudu::rpc::InboundCall*) > /home/todd/git/kudu/src/kudu/tserver/tserver_service.service.cc:122 > (libtserver_service_proto.so+0x000000015a56) > #11 kudu::rpc::ServicePool::RunThread() > /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173 > (libkrpc.so+0x0000000dac85) > #12 boost::_mfi::mf0<void, > kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const > /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d) > #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, > kudu::rpc::ServicePool>&, boost::_bi::list0&, int) > /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a) > #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, > kudu::rpc::ServicePool>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 > (libkrpc.so+0x0000000dceb3) > #15 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf0<void, kudu::rpc::ServicePool>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, > void>::invoke(boost::detail::function::function_buffer&) > /usr/include/boost/function/function_template.hpp:153 > (libkrpc.so+0x0000000dccd9) > #16 boost::function0<void>::operator()() const > /usr/include/boost/function/function_template.hpp:766 > (libkrpc.so+0x000000096a61) > #17 kudu::Thread::SuperviseThread(void*) > /home/todd/git/kudu/src/kudu/util/thread.cc:503 > (libkudu_util.so+0x0000002052b9) > Thread T124 'tablet-bootstra' (tid=20737, running) created by thread T64 at: > #0 pthread_create <null> (kudu-tserver+0x00000004e7f3) > #1 kudu::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, scoped_refptr<kudu::Thread>*) > /home/todd/git/kudu/src/kudu/util/thread.cc:431 > (libkudu_util.so+0x000000204d86) > #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), > kudu::ThreadPool*, bool>(std::string const&, std::string const&, void > (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, > scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:143 > (libkudu_util.so+0x00000021067b) > #3 kudu::ThreadPool::CreateThreadUnlocked() > /home/todd/git/kudu/src/kudu/util/threadpool.cc:344 > (libkudu_util.so+0x00000020e983) > #4 kudu::ThreadPool::Submit(std::tr1::shared_ptr<kudu::Runnable> const&) > /home/todd/git/kudu/src/kudu/util/threadpool.cc:192 > (libkudu_util.so+0x00000020ee42) > #5 kudu::ThreadPool::SubmitFunc(boost::function<void ()> const&) > /home/todd/git/kudu/src/kudu/util/threadpool.cc:163 > (libkudu_util.so+0x00000020ec55) > #6 > kudu::tserver::TSTabletManager::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB > const&) /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:444 > (libtserver.so+0x000000163e3c) > #7 > kudu::tserver::ConsensusServiceImpl::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB > const*, kudu::consensus::StartRemoteBootstrapResponsePB*, > kudu::rpc::RpcContext*) > /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:951 > (libtserver.so+0x00000014959b) > #8 kudu::consensus::ConsensusServiceIf::Handle(kudu::rpc::InboundCall*) > /home/todd/git/kudu/src/kudu/consensus/consensus.service.cc:192 > (libconsensus_proto.so+0x0000000680cf) > #9 kudu::rpc::ServicePool::RunThread() > /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173 > (libkrpc.so+0x0000000dac85) > #10 boost::_mfi::mf0<void, > kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const > /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d) > #11 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, > kudu::rpc::ServicePool>&, boost::_bi::list0&, int) > /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a) > #12 boost::_bi::bind_t<void, boost::_mfi::mf0<void, > kudu::rpc::ServicePool>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 > (libkrpc.so+0x0000000dceb3) > #13 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf0<void, kudu::rpc::ServicePool>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, > void>::invoke(boost::detail::function::function_buffer&) > /usr/include/boost/function/function_template.hpp:153 > (libkrpc.so+0x0000000dccd9) > #14 boost::function0<void>::operator()() const > /usr/include/boost/function/function_template.hpp:766 > (libkrpc.so+0x000000096a61) > #15 kudu::Thread::SuperviseThread(void*) > /home/todd/git/kudu/src/kudu/util/thread.cc:503 > (libkudu_util.so+0x0000002052b9) > Thread T12 'rpc worker-2058' (tid=20580, running) created by main thread at: > #0 pthread_create <null> (kudu-tserver+0x00000004e7f3) > #1 kudu::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, scoped_refptr<kudu::Thread>*) > /home/todd/git/kudu/src/kudu/util/thread.cc:431 > (libkudu_util.so+0x000000204d86) > #2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::*)(), > kudu::rpc::ServicePool*>(std::string const&, std::string const&, void > (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&, > scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:137 > (libkrpc.so+0x0000000dbbeb) > #3 kudu::rpc::ServicePool::Init(int) > /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:77 > (libkrpc.so+0x0000000da994) > #4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, > kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) > /home/todd/git/kudu/src/kudu/server/rpc_server.cc:122 > (libserver_process.so+0x00000007f2c6) > #5 > kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, > kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) > /home/todd/git/kudu/src/kudu/server/server_base.cc:237 > (libserver_process.so+0x000000086160) > #6 kudu::tserver::TabletServer::Start() > /home/todd/git/kudu/src/kudu/tserver/tablet_server.cc:117 > (libtserver.so+0x00000013ee50) > #7 kudu::tserver::TabletServerMain(int, char**) > /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:55 > (kudu-tserver+0x0000000a3bcc) > #8 main /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:69 > (kudu-tserver+0x0000000a39ae) > SUMMARY: ThreadSanitizer: data race > (/tmp/run_tha_testcGwYR3/build/fastdebug/kudu-tserver+0x494e6) in operator > delete(void*) > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)