[
https://issues.apache.org/jira/browse/KUDU-1624?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15497584#comment-15497584
]
Adar Dembo commented on KUDU-1624:
----------------------------------
Just checked the flaky test dashboard and found the same issue already flagged
by TSAN. See
http://dist-test.cloudera.org:8080/diagnose?key=59d33142-7bc9-11e6-8428-0242ac110002:
{noformat}
RpcStubTest.TestDumpCallsInFlight: WARNING: ThreadSanitizer: data race
(pid=22095) Write of size 8 at 0x7d200000faa8 by thread T28 (mutexes: write
M47854):
#0
_ZNSt6vectorISt4pairI11StringPiece13scoped_refptrIN4kudu5TraceEEESaIS6_EE19_M_emplace_back_auxIJRS1_RS5_EEEvDpOT_
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/vector.tcc:441:25
(libkudu_util.so+0x00000025a9f5)
#1
_ZNSt6vectorISt4pairI11StringPiece13scoped_refptrIN4kudu5TraceEEESaIS6_EE12emplace_backIJRS1_RS5_EEEvDpOT_
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/vector.tcc:101:4
(libkudu_util.so+0x000000255f59)
#2 kudu::Trace::AddChildTrace(StringPiece, kudu::Trace*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:247:3
(libkudu_util.so+0x000000254f04)
#3 kudu::rpc::CalculatorService::DoSleep(kudu::rpc_test::SleepRequestPB
const*, kudu::rpc::RpcContext*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/rpc-test-base.h:290:7
(rpc_stub-test+0x0000004f090f)
#4 kudu::rpc::CalculatorService::Sleep(kudu::rpc_test::SleepRequestPB
const*, kudu::rpc_test::SleepResponsePB*, kudu::rpc::RpcContext*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/rpc-test-base.h:215:5
(rpc_stub-test+0x0000004effab)
#5
kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker>
const&)::$_1::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/data1/jenkins-workspace/kudu-workspace/build/tsan/src/kudu/rpc/rtest.service.cc:94:7
(librtest_krpc.so+0x00000003bfb4)
#6 std::_Function_handler<void ()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*),
kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
const&, scoped_refptr<kudu::rpc::ResultTracker>
const&)::$_1>::_M_invoke(std::_Any_data const&, google::protobuf::Message
const*, google::protobuf::Message*, kudu::rpc::RpcContext*)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/functional:2039:2
(librtest_krpc.so+0x00000003bdce)
#7 std::function<void ()(google::protobuf::Message const*,
google::protobuf::Message*,
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*,
google::protobuf::Message*, kudu::rpc::RpcContext*) const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/functional:2439:14
(libkrpc.so+0x0000001499d8)
#8 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/service_if.cc:129:5
(libkrpc.so+0x00000014937d)
#9 kudu::rpc::ServicePool::RunThread()
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/service_pool.cc:206:5
(libkrpc.so+0x00000014b2fd)
#10 boost::_mfi::mf0<void,
kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x00000014d396)
#11 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x00000014d2fa)
#12 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()()
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x00000014d2a3)
#13
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x00000014d0c9)
#14 boost::function0<void>::operator()() const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000e48c1)
#15 kudu::Thread::SuperviseThread(void*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000244a5f)
Previous read of size 8 at 0x7d200000faa8 by thread T20:
#0 std::vector<std::pair<StringPiece, scoped_refptr<kudu::Trace> >,
std::allocator<std::pair<StringPiece, scoped_refptr<kudu::Trace> > > >::begin()
const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/stl_vector.h:557:45
(libkudu_util.so+0x000000255d3c)
#1 std::vector<std::pair<StringPiece, scoped_refptr<kudu::Trace> >,
std::allocator<std::pair<StringPiece, scoped_refptr<kudu::Trace> > > >::empty()
const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/stl_vector.h:744:16
(libkudu_util.so+0x000000255cec)
#2 kudu::Trace::MetricsToJSON(kudu::JsonWriter*) const
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:218:8
(libkudu_util.so+0x000000254cb0)
#3 kudu::Trace::MetricsAsJSON() const
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:201:3
(libkudu_util.so+0x000000254a75)
#4 kudu::Trace::Dump(std::ostream*, int) const
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:185:28
(libkudu_util.so+0x000000254889)
#5 kudu::Trace::DumpToString(int) const
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:194:3
(libkudu_util.so+0x0000002549d5)
#6 kudu::rpc::InboundCall::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::RpcCallInProgressPB*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/inbound_call.cc:210:28
(libkrpc.so+0x0000000fab7f)
#7 kudu::rpc::Connection::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::RpcConnectionPB*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/connection.cc:699:7
(libkrpc.so+0x0000000edabd)
#8
kudu::rpc::ReactorThread::DumpRunningRpcs(kudu::rpc::DumpRunningRpcsRequestPB
const&, kudu::rpc::DumpRunningRpcsResponsePB*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:167:5
(libkrpc.so+0x00000010d07b)
#9 boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>::operator()(kudu::rpc::ReactorThread*,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*) const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:280:29
(libkrpc.so+0x00000011900f)
#10 kudu::Status
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*>
>::operator()<kudu::Status, boost::_mfi::mf2<kudu::Status,
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list0>(boost::_bi::type<kudu::Status>,
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>&, boost::_bi::list0&, long)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:388:16
(libkrpc.so+0x000000118f4f)
#11 boost::_bi::bind_t<kudu::Status, boost::_mfi::mf2<kudu::Status,
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >::operator()()
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x000000118e9f)
#12
boost::detail::function::function_obj_invoker0<boost::_bi::bind_t<kudu::Status,
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread,
kudu::rpc::DumpRunningRpcsRequestPB const&,
kudu::rpc::DumpRunningRpcsResponsePB*>,
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>,
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>,
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >,
kudu::Status>::invoke(boost::detail::function::function_buffer&)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:138:18
(libkrpc.so+0x000000118c88)
#13 boost::function0<kudu::Status>::operator()() const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x000000113959)
#14 kudu::rpc::RunFunctionTask::Run(kudu::rpc::ReactorThread*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:532:15
(libkrpc.so+0x000000113739)
#15 kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int)
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:199:5
(libkrpc.so+0x00000010d2a4)
#16 void ev::base<ev_async,
ev::async>::method_thunk<kudu::rpc::ReactorThread,
&(kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int))>(ev_loop*,
ev_async*, int)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/ev++.h:479:7
(libkrpc.so+0x000000114223)
#17 ev_invoke_pending
/data1/jenkins-workspace/kudu-workspace/thirdparty/libev-4.20/ev.c:3155
(libev.so.4+0x0000000074fc)
#18 kudu::rpc::ReactorThread::RunThread()
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:306:3
(libkrpc.so+0x00000010c403)
#19 boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
(libkrpc.so+0x000000114896)
#20 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:259:9
(libkrpc.so+0x0000001147fa)
#21 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
>::operator()()
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
(libkrpc.so+0x0000001147a3)
#22
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:159:11
(libkrpc.so+0x0000001145c9)
#23 boost::function0<void>::operator()() const
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
(libkrpc.so+0x0000000e48c1)
#24 kudu::Thread::SuperviseThread(void*)
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/thread.cc:588:3
(libkudu_util.so+0x000000244a5f)
{noformat}
> Possible data races in util/trace
> ---------------------------------
>
> Key: KUDU-1624
> URL: https://issues.apache.org/jira/browse/KUDU-1624
> Project: Kudu
> Issue Type: Bug
> Components: util
> Affects Versions: 1.0.0
> Reporter: Adar Dembo
> Attachments: rpc_stub-test.txt
>
>
> I'm running clang 3.9 against a Kudu source tree using libcpp, so it's
> possible this is partly due to differences in how libstdc++ implements
> std::vector. Or maybe it's real.
> At first glance, it looks like the problem is that
> Trace::MetricsToJSON(JsonWriter*) doesn't hold lock_ while accessing
> child_traces_. It would need to some refactoring to avoid taking lock_
> recursively, though, since it's a recursive function.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
