[
https://issues.apache.org/jira/browse/KUDU-1515?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15364915#comment-15364915
]
Todd Lipcon commented on KUDU-1515:
-----------------------------------
Filed a TSAN bug at https://github.com/google/sanitizers/issues/696
> TSAN data race in kudu::ErrnoToCString
> --------------------------------------
>
> Key: KUDU-1515
> URL: https://issues.apache.org/jira/browse/KUDU-1515
> Project: Kudu
> Issue Type: Bug
> Components: util
> Affects Versions: 0.9.1
> Reporter: Adar Dembo
>
> Spotted this while running a new stress test for multi-master code.
> {noformat}
> ==================
> WARNING: ThreadSanitizer: data race (pid=31784)
> Write of size 1 at 0x003ae0b563b5 by thread T7:
> #0 strerror_r
> /data/1/adar/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:2906
> (master-stress-test+0x000000458645)
> #1 kudu::ErrnoToCString(int, char*, unsigned long)
> /data/1/adar/kudu/src/kudu/util/errno.cc:39:15
> (libkudu_util.so+0x0000001ec7ca)
> #2 kudu::ErrnoToString(int) /data/1/adar/kudu/src/kudu/util/errno.h:29:3
> (libkudu_util.so+0x00000027f373)
> #3 kudu::Socket::Connect(kudu::Sockaddr const&)
> /data/1/adar/kudu/src/kudu/util/net/socket.cc:364:33
> (libkudu_util.so+0x000000282993)
> #4 kudu::rpc::ReactorThread::StartConnect(kudu::Socket*, kudu::Sockaddr
> const&, bool*) /data/1/adar/kudu/src/kudu/rpc/reactor.cc:398:16
> (libkrpc.so+0x0000001264d5)
> #5
> kudu::rpc::ReactorThread::FindOrStartConnection(kudu::rpc::ConnectionId
> const&, scoped_refptr<kudu::rpc::Connection>*)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:331:3 (libkrpc.so+0x000000124c81)
> #6
> kudu::rpc::ReactorThread::AssignOutboundCall(std::shared_ptr<kudu::rpc::OutboundCall>
> const&) /data/1/adar/kudu/src/kudu/rpc/reactor.cc:218:14
> (libkrpc.so+0x00000012460a)
> #7 kudu::rpc::AssignOutboundCallTask::Run(kudu::rpc::ReactorThread*)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:607:5 (libkrpc.so+0x00000012f389)
> #8 kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:199:5 (libkrpc.so+0x000000123718)
> #9 void ev::base<ev_async,
> ev::async>::method_thunk<kudu::rpc::ReactorThread,
> &(kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int))>(ev_loop*,
> ev_async*, int) /data/1/adar/kudu/thirdparty/installed/include/ev++.h:479:7
> (libkrpc.so+0x00000013040b)
> #10 ev_invoke_pending /data/1/adar/kudu/thirdparty/libev-4.20/ev.c:3155
> (libev.so.4+0x0000000074fc)
> #11 kudu::rpc::ReactorThread::RunThread()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:306:3 (libkrpc.so+0x00000012192c)
> #12 boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
> (libkrpc.so+0x00000013108b)
> #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/bind.hpp:259:9
> (libkrpc.so+0x000000130f68)
> #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
> >::operator()()
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> (libkrpc.so+0x000000130eca)
> #15
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /data/1/adar/kudu/thirdparty/installed/include/boost/function/function_template.hpp:159:11
> (libkrpc.so+0x000000130b60)
> #16 boost::function0<void>::operator()() const
> /data/1/adar/kudu/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0x0000000daf5a)
> #17 kudu::Thread::SuperviseThread(void*)
> /data/1/adar/kudu/src/kudu/util/thread.cc:586:3
> (libkudu_util.so+0x0000002c2849)
> Previous write of size 1 at 0x003ae0b563b5 by thread T5:
> #0 strerror_r
> /data/1/adar/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:2906
> (master-stress-test+0x000000458645)
> #1 kudu::ErrnoToCString(int, char*, unsigned long)
> /data/1/adar/kudu/src/kudu/util/errno.cc:39:15
> (libkudu_util.so+0x0000001ec7ca)
> #2 kudu::ErrnoToString(int) /data/1/adar/kudu/src/kudu/util/errno.h:29:3
> (libkudu_util.so+0x00000027f373)
> #3 kudu::Socket::Connect(kudu::Sockaddr const&)
> /data/1/adar/kudu/src/kudu/util/net/socket.cc:364:33
> (libkudu_util.so+0x000000282993)
> #4 kudu::rpc::ReactorThread::StartConnect(kudu::Socket*, kudu::Sockaddr
> const&, bool*) /data/1/adar/kudu/src/kudu/rpc/reactor.cc:398:16
> (libkrpc.so+0x0000001264d5)
> #5
> kudu::rpc::ReactorThread::FindOrStartConnection(kudu::rpc::ConnectionId
> const&, scoped_refptr<kudu::rpc::Connection>*)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:331:3 (libkrpc.so+0x000000124c81)
> #6
> kudu::rpc::ReactorThread::AssignOutboundCall(std::shared_ptr<kudu::rpc::OutboundCall>
> const&) /data/1/adar/kudu/src/kudu/rpc/reactor.cc:218:14
> (libkrpc.so+0x00000012460a)
> #7 kudu::rpc::AssignOutboundCallTask::Run(kudu::rpc::ReactorThread*)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:607:5 (libkrpc.so+0x00000012f389)
> #8 kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int)
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:199:5 (libkrpc.so+0x000000123718)
> #9 void ev::base<ev_async,
> ev::async>::method_thunk<kudu::rpc::ReactorThread,
> &(kudu::rpc::ReactorThread::AsyncHandler(ev::async&, int))>(ev_loop*,
> ev_async*, int) /data/1/adar/kudu/thirdparty/installed/include/ev++.h:479:7
> (libkrpc.so+0x00000013040b)
> #10 ev_invoke_pending /data/1/adar/kudu/thirdparty/libev-4.20/ev.c:3155
> (libev.so.4+0x0000000074fc)
> #11 kudu::rpc::ReactorThread::RunThread()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:306:3 (libkrpc.so+0x00000012192c)
> #12 boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/mem_fn_template.hpp:49:29
> (libkrpc.so+0x00000013108b)
> #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/bind.hpp:259:9
> (libkrpc.so+0x000000130f68)
> #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
> >::operator()()
> /data/1/adar/kudu/thirdparty/installed/include/boost/bind/bind.hpp:1222:16
> (libkrpc.so+0x000000130eca)
> #15
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /data/1/adar/kudu/thirdparty/installed/include/boost/function/function_template.hpp:159:11
> (libkrpc.so+0x000000130b60)
> #16 boost::function0<void>::operator()() const
> /data/1/adar/kudu/thirdparty/installed/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0x0000000daf5a)
> #17 kudu::Thread::SuperviseThread(void*)
> /data/1/adar/kudu/src/kudu/util/thread.cc:586:3
> (libkudu_util.so+0x0000002c2849)
> Location is global '??' at 0x000000000000 (libc.so.6+0x003ae0b563b5)
> Thread T7 'rpc reactor-323' (tid=32331, running) created by main thread at:
> #0 pthread_create
> /data/1/adar/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954
> (master-stress-test+0x000000463e83)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()()> const&, unsigned long,
> scoped_refptr<kudu::Thread>*)
> /data/1/adar/kudu/src/kudu/util/thread.cc:509:15
> (libkudu_util.so+0x0000002c1bb0)
> #2 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /data/1/adar/kudu/src/kudu/util/thread.h:156:12 (libkrpc.so+0x0000001295ea)
> #3 kudu::rpc::ReactorThread::Init()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:108:10 (libkrpc.so+0x0000001216d9)
> #4 kudu::rpc::Reactor::Init()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:488:10 (libkrpc.so+0x0000001279ca)
> #5 kudu::rpc::Messenger::Init()
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:266:5 (libkrpc.so+0x00000010a2df)
> #6 kudu::rpc::MessengerBuilder::Build(kudu::rpc::Messenger**)
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:106:3 (libkrpc.so+0x00000010a0f9)
> #7
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:113:3 (libkrpc.so+0x00000010a42c)
> #8
> kudu::client::KuduClientBuilder::Build(std::tr1::shared_ptr<kudu::client::KuduClient>*)
> /data/1/adar/kudu/src/kudu/client/client.cc:224:3
> (libkudu_client.so+0x00000012ccaa)
> #9
> kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder&,
> std::tr1::shared_ptr<kudu::client::KuduClient>*)
> /data/1/adar/kudu/src/kudu/integration-tests/external_mini_cluster.cc:476:10
> (libintegration-tests.so+0x00000009d6a1)
> #10 kudu::MasterStressTest::SetUp()
> /data/1/adar/kudu/src/kudu/integration-tests/master-stress-test.cc:119:5
> (master-stress-test+0x0000004cba77)
> #11 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data/1/adar/kudu/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10
> (libgmock.so+0x000000044939)
> #12 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data/1/adar/kudu/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
> (libgmock.so+0x000000044939)
> #13 main /data/1/adar/kudu/src/kudu/util/test_main.cc:48:13
> (libkudu_test_main.so+0x000000001a04)
> Thread T5 'rpc reactor-323' (tid=32329, running) created by main thread at:
> #0 pthread_create
> /data/1/adar/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954
> (master-stress-test+0x000000463e83)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&,
> boost::function<void ()()> const&, unsigned long,
> scoped_refptr<kudu::Thread>*)
> /data/1/adar/kudu/src/kudu/util/thread.cc:509:15
> (libkudu_util.so+0x0000002c1bb0)
> #2 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::string
> const&, std::string const&, void (kudu::rpc::ReactorThread::* const&)(),
> kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /data/1/adar/kudu/src/kudu/util/thread.h:156:12 (libkrpc.so+0x0000001295ea)
> #3 kudu::rpc::ReactorThread::Init()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:108:10 (libkrpc.so+0x0000001216d9)
> #4 kudu::rpc::Reactor::Init()
> /data/1/adar/kudu/src/kudu/rpc/reactor.cc:488:10 (libkrpc.so+0x0000001279ca)
> #5 kudu::rpc::Messenger::Init()
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:266:5 (libkrpc.so+0x00000010a2df)
> #6 kudu::rpc::MessengerBuilder::Build(kudu::rpc::Messenger**)
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:106:3 (libkrpc.so+0x00000010a0f9)
> #7
> kudu::rpc::MessengerBuilder::Build(std::shared_ptr<kudu::rpc::Messenger>*)
> /data/1/adar/kudu/src/kudu/rpc/messenger.cc:113:3 (libkrpc.so+0x00000010a42c)
> #8
> kudu::client::KuduClientBuilder::Build(std::tr1::shared_ptr<kudu::client::KuduClient>*)
> /data/1/adar/kudu/src/kudu/client/client.cc:224:3
> (libkudu_client.so+0x00000012ccaa)
> #9
> kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder&,
> std::tr1::shared_ptr<kudu::client::KuduClient>*)
> /data/1/adar/kudu/src/kudu/integration-tests/external_mini_cluster.cc:476:10
> (libintegration-tests.so+0x00000009d6a1)
> #10 kudu::MasterStressTest::SetUp()
> /data/1/adar/kudu/src/kudu/integration-tests/master-stress-test.cc:119:5
> (master-stress-test+0x0000004cba77)
> #11 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data/1/adar/kudu/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10
> (libgmock.so+0x000000044939)
> #12 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data/1/adar/kudu/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
> (libgmock.so+0x000000044939)
> #13 main /data/1/adar/kudu/src/kudu/util/test_main.cc:48:13
> (libkudu_test_main.so+0x000000001a04)
> SUMMARY: ThreadSanitizer: data race
> /data/1/adar/kudu/src/kudu/util/errno.cc:39:15 in kudu::ErrnoToCString(int,
> char*, unsigned long)
> ==================
> {noformat}
> My suspicion is that we're running with the GNU version of strerror_r; the
> manpage says:
> {noformat}
> The XSI-compliant strerror_r() is preferred for portable applications.
> It returns the error
> string in the user-supplied buffer buf of length buflen.
> The GNU-specific strerror_r() returns a pointer to a string containing
> the error message. This
> may be either a pointer to a string that the function stores in buf,
> or a pointer to some
> (immutable) static string (in which case buf is unused). If the
> function stores a string in
> buf, then at most buflen bytes are stored (the string may be truncated
> if buflen is too small
> and errnum is unknown). The string always includes a terminating null
> byte ('\0').
> {noformat}
> That mention of a static string seemed suspicious, but it's true that it's
> immutable, and the TSAN warning mentions two writes, so it may be something
> else entirely.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
