[ 
https://issues.apache.org/jira/browse/KUDU-1624?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15497584#comment-15497584
 ] 

Adar Dembo commented on KUDU-1624:
----------------------------------

Just checked the flaky test dashboard and found the same issue already flagged 
by TSAN. See 
http://dist-test.cloudera.org:8080/diagnose?key=59d33142-7bc9-11e6-8428-0242ac110002:

{noformat}
RpcStubTest.TestDumpCallsInFlight: WARNING: ThreadSanitizer: data race 
(pid=22095)  Write of size 8 at 0x7d200000faa8 by thread T28 (mutexes: write 
M47854):
    #0 
_ZNSt6vectorISt4pairI11StringPiece13scoped_refptrIN4kudu5TraceEEESaIS6_EE19_M_emplace_back_auxIJRS1_RS5_EEEvDpOT_
 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/vector.tcc:441:25
 (libkudu_util.so+0x00000025a9f5)
    #1 
_ZNSt6vectorISt4pairI11StringPiece13scoped_refptrIN4kudu5TraceEEESaIS6_EE12emplace_backIJRS1_RS5_EEEvDpOT_
 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/vector.tcc:101:4
 (libkudu_util.so+0x000000255f59)
    #2 kudu::Trace::AddChildTrace(StringPiece, kudu::Trace*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:247:3 
(libkudu_util.so+0x000000254f04)
    #3 kudu::rpc::CalculatorService::DoSleep(kudu::rpc_test::SleepRequestPB 
const*, kudu::rpc::RpcContext*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/rpc-test-base.h:290:7 
(rpc_stub-test+0x0000004f090f)
    #4 kudu::rpc::CalculatorService::Sleep(kudu::rpc_test::SleepRequestPB 
const*, kudu::rpc_test::SleepResponsePB*, kudu::rpc::RpcContext*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/rpc-test-base.h:215:5 
(rpc_stub-test+0x0000004effab)
    #5 
kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
 const&, scoped_refptr<kudu::rpc::ResultTracker> 
const&)::$_1::operator()(google::protobuf::Message const*, 
google::protobuf::Message*, kudu::rpc::RpcContext*) const 
/data1/jenkins-workspace/kudu-workspace/build/tsan/src/kudu/rpc/rtest.service.cc:94:7
 (librtest_krpc.so+0x00000003bfb4)
    #6 std::_Function_handler<void ()(google::protobuf::Message const*, 
google::protobuf::Message*, kudu::rpc::RpcContext*), 
kudu::rpc_test::CalculatorServiceIf::CalculatorServiceIf(scoped_refptr<kudu::MetricEntity>
 const&, scoped_refptr<kudu::rpc::ResultTracker> 
const&)::$_1>::_M_invoke(std::_Any_data const&, google::protobuf::Message 
const*, google::protobuf::Message*, kudu::rpc::RpcContext*) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/functional:2039:2
 (librtest_krpc.so+0x00000003bdce)
    #7 std::function<void ()(google::protobuf::Message const*, 
google::protobuf::Message*, 
kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, 
google::protobuf::Message*, kudu::rpc::RpcContext*) const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/functional:2439:14
 (libkrpc.so+0x0000001499d8)
    #8 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/service_if.cc:129:5 
(libkrpc.so+0x00000014937d)
    #9 kudu::rpc::ServicePool::RunThread() 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/service_pool.cc:206:5 
(libkrpc.so+0x00000014b2fd)
    #10 boost::_mfi::mf0<void, 
kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
 (libkrpc.so+0x00000014d396)
    #11 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> 
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, 
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, 
kudu::rpc::ServicePool>&, boost::_bi::list0&, int) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:259:9
 (libkrpc.so+0x00000014d2fa)
    #12 boost::_bi::bind_t<void, boost::_mfi::mf0<void, 
kudu::rpc::ServicePool>, 
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
 (libkrpc.so+0x00000014d2a3)
    #13 
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, 
boost::_mfi::mf0<void, kudu::rpc::ServicePool>, 
boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, 
void>::invoke(boost::detail::function::function_buffer&) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:159:11
 (libkrpc.so+0x00000014d0c9)
    #14 boost::function0<void>::operator()() const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
 (libkrpc.so+0x0000000e48c1)
    #15 kudu::Thread::SuperviseThread(void*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/thread.cc:588:3 
(libkudu_util.so+0x000000244a5f)

  Previous read of size 8 at 0x7d200000faa8 by thread T20:
    #0 std::vector<std::pair<StringPiece, scoped_refptr<kudu::Trace> >, 
std::allocator<std::pair<StringPiece, scoped_refptr<kudu::Trace> > > >::begin() 
const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/stl_vector.h:557:45
 (libkudu_util.so+0x000000255d3c)
    #1 std::vector<std::pair<StringPiece, scoped_refptr<kudu::Trace> >, 
std::allocator<std::pair<StringPiece, scoped_refptr<kudu::Trace> > > >::empty() 
const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/stl_vector.h:744:16
 (libkudu_util.so+0x000000255cec)
    #2 kudu::Trace::MetricsToJSON(kudu::JsonWriter*) const 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:218:8 
(libkudu_util.so+0x000000254cb0)
    #3 kudu::Trace::MetricsAsJSON() const 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:201:3 
(libkudu_util.so+0x000000254a75)
    #4 kudu::Trace::Dump(std::ostream*, int) const 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:185:28 
(libkudu_util.so+0x000000254889)
    #5 kudu::Trace::DumpToString(int) const 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/trace.cc:194:3 
(libkudu_util.so+0x0000002549d5)
    #6 kudu::rpc::InboundCall::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB 
const&, kudu::rpc::RpcCallInProgressPB*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/inbound_call.cc:210:28 
(libkrpc.so+0x0000000fab7f)
    #7 kudu::rpc::Connection::DumpPB(kudu::rpc::DumpRunningRpcsRequestPB 
const&, kudu::rpc::RpcConnectionPB*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/connection.cc:699:7 
(libkrpc.so+0x0000000edabd)
    #8 
kudu::rpc::ReactorThread::DumpRunningRpcs(kudu::rpc::DumpRunningRpcsRequestPB 
const&, kudu::rpc::DumpRunningRpcsResponsePB*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:167:5 
(libkrpc.so+0x00000010d07b)
    #9 boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread, 
kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*>::operator()(kudu::rpc::ReactorThread*, 
kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*) const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:280:29
 (libkrpc.so+0x00000011900f)
    #10 kudu::Status 
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>, 
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>, 
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> 
>::operator()<kudu::Status, boost::_mfi::mf2<kudu::Status, 
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*>, 
boost::_bi::list0>(boost::_bi::type<kudu::Status>, 
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread, 
kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*>&, boost::_bi::list0&, long) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:388:16
 (libkrpc.so+0x000000118f4f)
    #11 boost::_bi::bind_t<kudu::Status, boost::_mfi::mf2<kudu::Status, 
kudu::rpc::ReactorThread, kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*>, 
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>, 
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>, 
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >::operator()() 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
 (libkrpc.so+0x000000118e9f)
    #12 
boost::detail::function::function_obj_invoker0<boost::_bi::bind_t<kudu::Status, 
boost::_mfi::mf2<kudu::Status, kudu::rpc::ReactorThread, 
kudu::rpc::DumpRunningRpcsRequestPB const&, 
kudu::rpc::DumpRunningRpcsResponsePB*>, 
boost::_bi::list3<boost::_bi::value<kudu::rpc::ReactorThread*>, 
boost::reference_wrapper<kudu::rpc::DumpRunningRpcsRequestPB const>, 
boost::_bi::value<kudu::rpc::DumpRunningRpcsResponsePB*> > >, 
kudu::Status>::invoke(boost::detail::function::function_buffer&) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:138:18
 (libkrpc.so+0x000000118c88)
    #13 boost::function0<kudu::Status>::operator()() const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
 (libkrpc.so+0x000000113959)
    #14 kudu::rpc::RunFunctionTask::Run(kudu::rpc::ReactorThread*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:532:15 
(libkrpc.so+0x000000113739)
    #15 kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:199:5 
(libkrpc.so+0x00000010d2a4)
    #16 void ev::base<ev_async, 
ev::async>::method_thunk<kudu::rpc::ReactorThread, 
&(kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int))>(ev_loop*, 
ev_async*, int) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/ev++.h:479:7
 (libkrpc.so+0x000000114223)
    #17 ev_invoke_pending 
/data1/jenkins-workspace/kudu-workspace/thirdparty/libev-4.20/ev.c:3155 
(libev.so.4+0x0000000074fc)
    #18 kudu::rpc::ReactorThread::RunThread() 
/data1/jenkins-workspace/kudu-workspace/src/kudu/rpc/reactor.cc:306:3 
(libkrpc.so+0x00000010c403)
    #19 boost::_mfi::mf0<void, 
kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
 (libkrpc.so+0x000000114896)
    #20 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> 
>::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, 
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, 
kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:259:9
 (libkrpc.so+0x0000001147fa)
    #21 boost::_bi::bind_t<void, boost::_mfi::mf0<void, 
kudu::rpc::ReactorThread>, 
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > 
>::operator()() 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
 (libkrpc.so+0x0000001147a3)
    #22 
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, 
boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, 
boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, 
void>::invoke(boost::detail::function::function_buffer&) 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:159:11
 (libkrpc.so+0x0000001145c9)
    #23 boost::function0<void>::operator()() const 
/data1/jenkins-workspace/kudu-workspace/thirdparty/installed/include/boost/function/function_template.hpp:770:14
 (libkrpc.so+0x0000000e48c1)
    #24 kudu::Thread::SuperviseThread(void*) 
/data1/jenkins-workspace/kudu-workspace/src/kudu/util/thread.cc:588:3 
(libkudu_util.so+0x000000244a5f)
{noformat}

> Possible data races in util/trace
> ---------------------------------
>
>                 Key: KUDU-1624
>                 URL: https://issues.apache.org/jira/browse/KUDU-1624
>             Project: Kudu
>          Issue Type: Bug
>          Components: util
>    Affects Versions: 1.0.0
>            Reporter: Adar Dembo
>         Attachments: rpc_stub-test.txt
>
>
> I'm running clang 3.9 against a Kudu source tree using libcpp, so it's 
> possible this is partly due to differences in how libstdc++ implements 
> std::vector. Or maybe it's real.
> At first glance, it looks like the problem is that 
> Trace::MetricsToJSON(JsonWriter*) doesn't hold lock_ while accessing 
> child_traces_. It would need to some refactoring to avoid taking lock_ 
> recursively, though, since it's a recursive function.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Reply via email to