[
https://issues.apache.org/jira/browse/KUDU-1624?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mike Percy updated KUDU-1624:
-----------------------------
Attachment: rpc_stub-test.txt.gz
Attaching a log from another instance of what might be a related problem.
Snippet:
{code}
[ RUN ] RpcStubTest.TestDumpCallsInFlight
I1207 15:31:10.207124 3749 rpc_stub-test.cc:485] client messenger:
outbound_connections {
remote_ip: "0.0.0.0:58661"
state: NEGOTIATING
calls_in_flight {
header {
call_id: 1
remote_method {
service_name: "kudu.rpc_test.CalculatorService"
method_name: "Sleep"
}
}
micros_elapsed: 1925
}
}
==================
WARNING: ThreadSanitizer: data race (pid=3749)
Read of size 8 at 0x7d1c0001e248 by thread T5:
#0 std::__1::vector<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace>
>, std::__1::allocator<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace> >
> >::empty() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/vector:646:23
(libkudu_util.so+0x00000018fca2)
#1 kudu::Trace::MetricsToJSON(kudu::JsonWriter*) const
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:218
(libkudu_util.so+0x00000018fca2)
#2 kudu::Trace::MetricsAsJSON() const
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:201:3
(libkudu_util.so+0x00000018f984)
#3 kudu::Trace::Dump(std::__1::basic_ostream<char,
std::__1::char_traits<char> >*, int) const
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:185:28
(libkudu_util.so+0x00000018f597)
#4 kudu::Trace::DumpToString(int) const
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:194:3
(libkudu_util.so+0x00000018f7d5)
#5 kudu::rpc::InboundCall::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::RpcCallInProgressPB*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/inbound_call.cc:210:36
(libkrpc.so+0x00000008af0f)
#6 kudu::rpc::Connection::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::RpcConnectionPB*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/connection.cc:734:10
(libkrpc.so+0x000000081b22)
#7
kudu::rpc::ReactorThread::DumpRunningRpcs(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::DumpRunningRpcsResponsePB*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:173:5
(libkrpc.so+0x000000099495)
#8 boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>::operator()(kudu::rpc::ReactorThread*,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280:29
(libkrpc.so+0x0000000a2bff)
#9 kudu::Status
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*>
>::operator()<kudu::Status, boost::_mfi::mf2<kudu::Status,
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list0>(boost::_bi::type<kudu::Status>,
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>&, boost::_bi::list0&, long)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:388:16
(libkrpc.so+0x0000000a2b3f)
#10 boost::_bi::bind_t<kudu::Status, boost::_mfi::mf2<kudu::Status,
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >::operator()()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x0000000a2a99)
#11
boost::detail::function::function_obj_invoker0<boost::_bi::bind_t<kudu::Status,
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >,
kudu::Status>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:138:18
(libkrpc.so+0x0000000a2878)
#12 boost::function0<kudu::Status>::operator()() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x00000009f1a9)
#13 kudu::rpc::RunFunctionTask::Run(kudu::rpc::ReactorThread*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:548:15
(libkrpc.so+0x00000009ef89)
#14 kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:209:10
(libkrpc.so+0x0000000996a4)
#15 void ev::base<ev_async,
ev::async>::method_thunk<kudu::rpc::ReactorThread,
&kudu::rpc::ReactorThread::AsyncHandler>(ev_loop*, ev_async*, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:479:7
(libkrpc.so+0x00000009f523)
#16 ev_invoke_pending
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3155:11
(libev.so.4+0x0000000097ec)
#17 ev_run
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3555:7
(libev.so.4+0x00000000a8fd)
#18 ev::loop_ref::run(int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:211:7
(libkrpc.so+0x00000009d038)
#19 kudu::rpc::ReactorThread::RunThread()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:316:9
(libkrpc.so+0x00000009872c)
#20 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x00000009fbc6)
#21 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x00000009fb2a)
#22 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x00000009fac3)
#23
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x00000009f8e9)
#24 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x000000079791)
#25 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000184974)
Previous write of size 8 at 0x7d1c0001e248 by thread T9 (mutexes: write
M13861):
#0
_ZNSt3__14swapIPNS_4pairI11StringPiece13scoped_refptrIN4kudu5TraceEEEEEENS_9enable_ifIXaasr21is_move_constructibleIT_EE5valuesr18is_move_assignableISA_EE5valueEvE4typeERSA_SD_
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/type_traits:4422:9
(libkudu_util.so+0x000000192500)
#1 std::__1::vector<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace>
>, std::__1::allocator<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace> >
>
>::__swap_out_circular_buffer(std::__1::__split_buffer<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> >, std::__1::allocator<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> > >&>&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/vector:893
(libkudu_util.so+0x000000192500)
#2 void std::__1::vector<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> >, std::__1::allocator<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> > > >::__emplace_back_slow_path<StringPiece&,
scoped_refptr<kudu::Trace>&>(StringPiece&, scoped_refptr<kudu::Trace>&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/vector:1629:5
(libkudu_util.so+0x0000001922e7)
#3 void std::__1::vector<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> >, std::__1::allocator<std::__1::pair<StringPiece,
scoped_refptr<kudu::Trace> > > >::emplace_back<StringPiece&,
scoped_refptr<kudu::Trace>&>(StringPiece&, scoped_refptr<kudu::Trace>&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/vector:1648:9
(libkudu_util.so+0x00000018ff30)
#4 kudu::Trace::AddChildTrace(StringPiece, kudu::Trace*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:247
(libkudu_util.so+0x00000018ff30)
#5 kudu::rpc::CalculatorService::DoSleep(kudu::rpc_test::SleepRequestPB
const*, kudu::rpc::RpcContext*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:295:30
(rpc_stub-test+0x0000004e281f)
#6 kudu::rpc::CalculatorService::Sleep(kudu::rpc_test::SleepRequestPB
const*, kudu::rpc_test::SleepResponsePB*, kudu::rpc::RpcContext*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:221:5
(rpc_stub-test+0x0000004e1ee9)
#7
kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker>
const&)::$_1::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/home/jenkins-slave/workspace/kudu-1/build/tsan/src/kudu/rpc/rtest.service.cc:94:13
(librtest_krpc.so+0x000000035704)
#8
_ZNSt3__18__invokeIRZN4kudu8rpc_test19CalculatorServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE3$_1JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/type_traits:4287:1
(librtest_krpc.so+0x0000000356a1)
#9 void
std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&,
google::protobuf::Message const*, google::protobuf::Message*,
kudu::rpc::RpcContext*>(kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&,
google::protobuf::Message const*&&, google::protobuf::Message*&&,
kudu::rpc::RpcContext*&&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/__functional_base:359
(librtest_krpc.so+0x0000000356a1)
#10
std::__1::__function::__func<kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1,
std::__1::allocator<kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void
(google::protobuf::Message const*, google::protobuf::Message*,
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&,
google::protobuf::Message*&&, kudu::rpc::RpcContext*&&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/functional:1535:12
(librtest_krpc.so+0x0000000355c4)
#11 std::__1::function<void (google::protobuf::Message const*,
google::protobuf::Message*,
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/functional:1898:12
(libkrpc.so+0x0000000c9179)
#12 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/service_if.cc:129:5
(libkrpc.so+0x0000000c8d8f)
#13 kudu::rpc::ServicePool::RunThread()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/service_pool.cc:206:15
(libkrpc.so+0x0000000ca084)
#14 boost::_mfi::mf0<void,
kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x0000000cbb86)
#15 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x0000000cbaea)
#16 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x0000000cba83)
#17
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x0000000cb8a9)
#18 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x000000079791)
#19 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000184974)
Location is heap block of size 104 at 0x7d1c0001e220 allocated by thread T5:
#0 operator new(unsigned long)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:41
(rpc_stub-test+0x0000004cec33)
#1 kudu::rpc::InboundCall::InboundCall(kudu::rpc::Connection*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/inbound_call.cc:48:12
(libkrpc.so+0x0000000892e7)
#2
kudu::rpc::Connection::HandleIncomingCall(gscoped_ptr<kudu::rpc::InboundTransfer,
kudu::DefaultDeleter<kudu::rpc::InboundTransfer> >)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/connection.cc:497:37
(libkrpc.so+0x0000000804d7)
#3 kudu::rpc::Connection::ReadHandler(ev::io&, int)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/connection.cc:478:7
(libkrpc.so+0x00000007fb94)
#4 void ev::base<ev_io, ev::io>::method_thunk<kudu::rpc::Connection,
&kudu::rpc::Connection::ReadHandler>(ev_loop*, ev_io*, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:479:7
(libkrpc.so+0x0000000865ca)
#5 ev_invoke_pending
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3155:11
(libev.so.4+0x0000000097ec)
#6 ev_run
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3555:7
(libev.so.4+0x00000000a8fd)
#7 ev::loop_ref::run(int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:211:7
(libkrpc.so+0x00000009d038)
#8 kudu::rpc::ReactorThread::RunThread()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:316:9
(libkrpc.so+0x00000009872c)
#9 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x00000009fbc6)
#10 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x00000009fb2a)
#11 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x00000009fac3)
#12
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x00000009f8e9)
#13 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x000000079791)
#14 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000184974)
Mutex M13861 (0x7d1c0001e230) created at:
#0 __tsan_atomic32_compare_exchange_strong
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:756
(rpc_stub-test+0x000000490867)
#1 base::subtle::Acquire_CompareAndSwap(int volatile*, int, int)
/home/jenkins-slave/workspace/kudu-1/src/kudu/gutil/atomicops-internals-tsan.h:85:3
(rpc_stub-test+0x0000004e3947)
#2 base::SpinLock::Lock()
/home/jenkins-slave/workspace/kudu-1/src/kudu/gutil/spinlock.h:73:9
(rpc_stub-test+0x0000004e38c0)
#3 kudu::simple_spinlock::lock()
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/locks.h:45:8
(rpc_stub-test+0x0000004e3889)
#4
std::__1::lock_guard<kudu::simple_spinlock>::lock_guard(kudu::simple_spinlock&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/__mutex_base:108:27
(libkudu_util.so+0x00000018ed5b)
#5 kudu::Trace::AddEntry(kudu::TraceEntry*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:110
(libkudu_util.so+0x00000018ed5b)
#6 kudu::Trace::SubstituteAndTrace(char const*, int, StringPiece,
strings::internal::SubstituteArg const&, strings::internal::SubstituteArg
const&, strings::internal::SubstituteArg const&,
strings::internal::SubstituteArg const&, strings::internal::SubstituteArg
const&, strings::internal::SubstituteArg const&,
strings::internal::SubstituteArg const&, strings::internal::SubstituteArg
const&, strings::internal::SubstituteArg const&,
strings::internal::SubstituteArg const&)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/trace.cc:95:3
(libkudu_util.so+0x00000018ec6e)
#7
kudu::rpc::ServicePool::QueueInboundCall(gscoped_ptr<kudu::rpc::InboundCall,
kudu::DefaultDeleter<kudu::rpc::InboundCall> >)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/service_pool.cc:143:3
(libkrpc.so+0x0000000cab1b)
#8
kudu::rpc::Messenger::QueueInboundCall(gscoped_ptr<kudu::rpc::InboundCall,
kudu::DefaultDeleter<kudu::rpc::InboundCall> >)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/messenger.cc:262:3
(libkrpc.so+0x00000008ef4d)
#9
kudu::rpc::Connection::HandleIncomingCall(gscoped_ptr<kudu::rpc::InboundTransfer,
kudu::DefaultDeleter<kudu::rpc::InboundTransfer> >)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/connection.cc:515:44
(libkrpc.so+0x0000000805e2)
#10 kudu::rpc::Connection::ReadHandler(ev::io&, int)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/connection.cc:478:7
(libkrpc.so+0x00000007fb94)
#11 void ev::base<ev_io, ev::io>::method_thunk<kudu::rpc::Connection,
&kudu::rpc::Connection::ReadHandler>(ev_loop*, ev_io*, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:479:7
(libkrpc.so+0x0000000865ca)
#12 ev_invoke_pending
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3155:11
(libev.so.4+0x0000000097ec)
#13 ev_run
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/libev-4.20/ev.c:3555:7
(libev.so.4+0x00000000a8fd)
#14 ev::loop_ref::run(int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/ev++.h:211:7
(libkrpc.so+0x00000009d038)
#15 kudu::rpc::ReactorThread::RunThread()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:316:9
(libkrpc.so+0x00000009872c)
#16 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x00000009fbc6)
#17 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x00000009fb2a)
#18 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x00000009fac3)
#19
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x00000009f8e9)
#20 boost::function0<void>::operator()() const
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x000000079791)
#21 kudu::Thread::SuperviseThread(void*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000184974)
Thread T5 'rpc reactor-396' (tid=3965, running) created by main thread at:
#0 pthread_create
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902
(rpc_stub-test+0x0000004475b6)
#1 kudu::Thread::StartThread(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
long, scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:511:15
(libkudu_util.so+0x00000018420c)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(),
kudu::rpc::ReactorThread*>(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::*
const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:158:12
(libkrpc.so+0x00000009ca97)
#3 kudu::rpc::ReactorThread::Init()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:114:10
(libkrpc.so+0x000000098582)
#4 kudu::rpc::Reactor::Init()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/reactor.cc:504:18
(libkrpc.so+0x00000009bd83)
#5 kudu::rpc::Messenger::Init()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/messenger.cc:312:5
(libkrpc.so+0x00000008e092)
#6
kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/messenger.cc:135:35
(libkrpc.so+0x00000008dd68)
#7 kudu::rpc::RpcTestBase::CreateMessenger(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:437:5
(rpc_stub-test+0x0000004e0905)
#8 void
kudu::rpc::RpcTestBase::DoStartTestServer<kudu::rpc::CalculatorService>(kudu::Sockaddr*,
bool) /home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:541:25
(rpc_stub-test+0x0000004e0cdc)
#9
kudu::rpc::RpcTestBase::StartTestServerWithGeneratedCode(kudu::Sockaddr*, bool)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:514:5
(rpc_stub-test+0x0000004e067a)
#10 kudu::rpc::RpcStubTest::SetUp()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc_stub-test.cc:57:5
(rpc_stub-test+0x0000004dbb89)
#11 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x000000048243)
#12 void
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x000000048243)
#13 testing::Test::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2146:3
(libgmock.so+0x00000002cddd)
#14 testing::TestInfo::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2326:11
(libgmock.so+0x00000002dea7)
#15 testing::TestCase::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2444:28
(libgmock.so+0x00000002eaf8)
#16 testing::internal::UnitTestImpl::RunAllTests()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:4315:43
(libgmock.so+0x000000038f51)
#17 bool
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x000000048df3)
#18 bool
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x000000048df3)
#19 testing::UnitTest::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:3926:10
(libgmock.so+0x000000038988)
#20 RUN_ALL_TESTS()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/gtest/gtest.h:2288:46
(libkudu_test_main.so+0x000000002e9b)
#21 main
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:83:13
(libkudu_test_main.so+0x00000000291f)
Thread T9 'rpc worker-3971' (tid=3971, running) created by main thread at:
#0 pthread_create
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902
(rpc_stub-test+0x0000004475b6)
#1 kudu::Thread::StartThread(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
long, scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:511:15
(libkudu_util.so+0x00000018420c)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::*)(),
kudu::rpc::ServicePool*>(std::__1::basic_string<char,
std::__1::char_traits<char>, std::__1::allocator<char> > const&,
std::__1::basic_string<char, std::__1::char_traits<char>,
std::__1::allocator<char> > const&, void (kudu::rpc::ServicePool::* const&)(),
kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*)
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:158:12
(libkrpc.so+0x0000000cafd7)
#3 kudu::rpc::ServicePool::Init(int)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/service_pool.cc:80:5
(libkrpc.so+0x0000000c9c47)
#4 void
kudu::rpc::RpcTestBase::DoStartTestServer<kudu::rpc::CalculatorService>(kudu::Sockaddr*,
bool) /home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:554:5
(rpc_stub-test+0x0000004e1393)
#5
kudu::rpc::RpcTestBase::StartTestServerWithGeneratedCode(kudu::Sockaddr*, bool)
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc-test-base.h:514:5
(rpc_stub-test+0x0000004e067a)
#6 kudu::rpc::RpcStubTest::SetUp()
/home/jenkins-slave/workspace/kudu-1/src/kudu/rpc/rpc_stub-test.cc:57:5
(rpc_stub-test+0x0000004dbb89)
#7 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x000000048243)
#8 void
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x000000048243)
#9 testing::Test::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2146:3
(libgmock.so+0x00000002cddd)
#10 testing::TestInfo::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2326:11
(libgmock.so+0x00000002dea7)
#11 testing::TestCase::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2444:28
(libgmock.so+0x00000002eaf8)
#12 testing::internal::UnitTestImpl::RunAllTests()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:4315:43
(libgmock.so+0x000000038f51)
#13 bool
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2078:10
(libgmock.so+0x000000048df3)
#14 bool
testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
bool>(testing::internal::UnitTestImpl*, bool
(testing::internal::UnitTestImpl::*)(), char const*)
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:2114
(libgmock.so+0x000000048df3)
#15 testing::UnitTest::Run()
/home/jenkins-slave/workspace/kudu-1/thirdparty/src/gmock-1.7.0/gtest/src/gtest.cc:3926:10
(libgmock.so+0x000000038988)
#16 RUN_ALL_TESTS()
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/gtest/gtest.h:2288:46
(libkudu_test_main.so+0x000000002e9b)
#17 main
/home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:83:13
(libkudu_test_main.so+0x00000000291f)
SUMMARY: ThreadSanitizer: data race
/home/jenkins-slave/workspace/kudu-1/thirdparty/installed/tsan/include/c++/v1/vector:646:23
in std::__1::vector<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace> >,
std::__1::allocator<std::__1::pair<StringPiece, scoped_refptr<kudu::Trace> > >
>::empty() const
{code}
> Possible data races in util/trace
> ---------------------------------
>
> Key: KUDU-1624
> URL: https://issues.apache.org/jira/browse/KUDU-1624
> Project: Kudu
> Issue Type: Bug
> Components: util
> Affects Versions: 1.0.0
> Reporter: Adar Dembo
> Attachments: rpc_stub-test.txt, rpc_stub-test.txt.gz
>
>
> I'm running clang 3.9 against a Kudu source tree using libcpp, so it's
> possible this is partly due to differences in how libstdc++ implements
> std::vector. Or maybe it's real.
> At first glance, it looks like the problem is that
> Trace::MetricsToJSON(JsonWriter*) doesn't hold lock_ while accessing
> child_traces_. It would need to some refactoring to avoid taking lock_
> recursively, though, since it's a recursive function.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
