Hao Hao created KUDU-2480:
-----------------------------
Summary: tsan failure of master-stress-test
Key: KUDU-2480
URL: https://issues.apache.org/jira/browse/KUDU-2480
Project: Kudu
Issue Type: Test
Reporter: Hao Hao
Attachments: master-stress-test.txt
master-stress-test recently has been very flaky(~24%). One of the failure log
{noformat}WARNING: ThreadSanitizer: data race (pid=26513)
Read of size 8 at 0x7ffb5e5b88b8 by thread T65:
#0 kudu::Status::operator=(kudu::Status const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/status.h:469:7
(libmaster.so+0x10bd00)
#1 kudu::Synchronizer::StatusCB(kudu::Status const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/async_util.h:44:8
(libmaster.so+0x10bc40)
#2 kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status
const&)>::Run(kudu::Synchronizer*, kudu::Status const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:192:12
(libmaster.so+0x10c708)
#3 kudu::internal::InvokeHelper<false, void,
kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status
const&)>, void ()(kudu::Synchronizer*, kudu::Status
const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
(kudu::Synchronizer::*)(kudu::Status const&)>, kudu::Synchronizer*,
kudu::Status const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:889:14
(libmaster.so+0x10c5e8)
#4 kudu::internal::Invoker<1,
kudu::internal::BindState<kudu::internal::RunnableAdapter<void
(kudu::Synchronizer::*)(kudu::Status const&)>, void ()(kudu::Synchronizer*,
kudu::Status const&), void
()(kudu::internal::UnretainedWrapper<kudu::Synchronizer>)>, void
()(kudu::Synchronizer*, kudu::Status
const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1118:12
(libmaster.so+0x10c51a)
#5 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&)
const
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:436:12
(libmaster.so+0x10b831)
#6 kudu::master::HmsNotificationLogListenerTask::RunLoop()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/hms_notification_log_listener.cc:136:10
(libmaster.so+0x108e0a)
#7 boost::_mfi::mf0<void,
kudu::master::HmsNotificationLogListenerTask>::operator()(kudu::master::HmsNotificationLogListenerTask*)
const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
(libmaster.so+0x110ea9)
#8 void
boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
>::operator()<boost::_mfi::mf0<void,
kudu::master::HmsNotificationLogListenerTask>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::master::HmsNotificationLogListenerTask>&, boost::_bi::list0&, int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
(libmaster.so+0x110dfa)
#9 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::master::HmsNotificationLogListenerTask>,
boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
> >::operator()()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
(libmaster.so+0x110d83)
#10
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>,
boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
> >, void>::invoke(boost::detail::function::function_buffer&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
(libmaster.so+0x110b79)
#11 boost::function0<void>::operator()() const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0xb64b1)
#12 kudu::Thread::SuperviseThread(void*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3
(libkudu_util.so+0x1bd8b4)
Previous write of size 8 at 0x7ffb5e5b88b8 by thread T24 (mutexes: read M1468):
#0
boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*>
>::init(boost::intrusive::list_node<void*>* const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/circular_list_algorithms.hpp:72:22
(libkrpc.so+0x99c92)
#1
boost::intrusive::generic_hook<boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*>
>, boost::intrusive::dft_tag, (boost::intrusive::link_mode_type)1,
(boost::intrusive::base_hook_type)1>::generic_hook()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/detail/generic_hook.hpp:174:10
(libkrpc.so+0xc4669)
#2 boost::intrusive::list_base_hook<void, void, void>::list_base_hook()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/list_hook.hpp:83:7
(libkrpc.so+0xc2049)
#3 kudu::rpc::ReactorTask::ReactorTask()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:262:14
(libkrpc.so+0xbd5fb)
#4
kudu::rpc::QueueTransferTask::QueueTransferTask(gscoped_ptr<kudu::rpc::OutboundTransfer,
kudu::DefaultDeleter<kudu::rpc::OutboundTransfer> >, kudu::rpc::Connection*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:432:3
(libkrpc.so+0x98ea4)
#5
kudu::rpc::Connection::QueueResponseForCall(gscoped_ptr<kudu::rpc::InboundCall,
kudu::DefaultDeleter<kudu::rpc::InboundCall> >)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:474:33
(libkrpc.so+0x94d79)
#6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&, bool)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:165:10
(libkrpc.so+0xa11b9)
#7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite
const&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:109:3
(libkrpc.so+0xa0fd5)
#8 kudu::rpc::RpcContext::RespondSuccess()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/rpc_context.cc:77:12
(libkrpc.so+0xd5b35)
#9
kudu::master::MasterServiceImpl::AlterTable(kudu::master::AlterTableRequestPB
const*, kudu::master::AlterTableResponsePB*, kudu::rpc::RpcContext*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_service.cc:350:8
(libmaster.so+0x128903)
#10
kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker>
const&)::$_13::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/master/master.service.cc:259:13
(libmaster_proto.so+0xe2544)
#11
_ZNSt3__18__invokeIRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE4$_13JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
(libmaster_proto.so+0xe24d1)
#12
_ZNSt3__128__invoke_void_return_wrapperIvE6__callIJRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS3_12MetricEntityEERKS6_INS3_3rpc13ResultTrackerEEE4$_13PKN6google8protobuf7MessageEPSK_PNSB_10RpcContextEEEEvDpOT_
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:349
(libmaster_proto.so+0xe24d1)
#13
std::__1::__function::__func<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13,
std::__1::allocator<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13>, void
()(google::protobuf::Message const*, google::protobuf::Message*,
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&,
google::protobuf::Message*&&, kudu::rpc::RpcContext*&&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
(libmaster_proto.so+0xe23e4)
#14 std::__1::function<void ()(google::protobuf::Message const*,
google::protobuf::Message*,
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
(libkrpc.so+0xefce9)
#15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:145:3
(libkrpc.so+0xef632)
#16 kudu::rpc::ServicePool::RunThread()
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:225:15
(libkrpc.so+0xf0b8c)
#17 boost::_mfi::mf0<void,
kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0xf2a69)
#18 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0xf29ba)
#19 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()()
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0xf2943)
#20
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0xf2739)
#21 boost::function0<void>::operator()() const
/data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0xb64b1)
#22 kudu::Thread::SuperviseThread(void*)
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3
(libkudu_util.so+0x1bd8b4)
Location is stack of thread T24.
{noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
