Laszlo Gaal created IMPALA-14476:
------------------------------------
Summary: Data race in TSAN build in during dataload in
be/src/kudu/util/status.cc
Key: IMPALA-14476
URL: https://issues.apache.org/jira/browse/IMPALA-14476
Project: IMPALA
Issue Type: Bug
Components: Backend
Affects Versions: Impala 5.0.0
Reporter: Laszlo Gaal
Assignee: Riza Suminto
Since the toolchain was bumped to pick up Kudu's array column feature
(KUDU-1261), Impala's TSAN builds on the master branch consistently break
during dataload with a data race detected by TSAN. Impalad.ERROR reports:
{code}
WARNING: ThreadSanitizer: data race (pid=59148)
Read of size 1 at 0x7b1c00426c54 by thread T280:
#0 kudu::Status::code() const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.h:483:55
(impalad+0x27d2eea)
#1 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:17
(impalad+0x27e89a1)
#2 kudu::rpc::Connection::Shutdown(kudu::Status const&,
std::unique_ptr<kudu::rpc::ErrorStatusPB,
std::default_delete<kudu::rpc::ErrorStatusPB> >)
/mnt/source/kudu/kudu-16689973a/src/kudu/rpc/connection.cc:153:44
(libkudu_client.so.0+0x20aefd)
Previous write of size 1 at 0x7b1c00426c54 by thread T255:
#0 kudu::Status::Status(kudu::Status::Code, kudu::Slice const&, kudu::Slice
const&, short)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:34:13
(impalad+0x27e8584)
#1 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:10
(impalad+0x27e89d2)
#2
kudu::rpc::Negotiation::RunNegotiation(scoped_refptr<kudu::rpc::Connection>
const&, kudu::TriStateFlag, kudu::TriStateFlag, bool, kudu::MonoTime)
/mnt/source/kudu/kudu-16689973a/src/kudu/rpc/negotiation.cc:325:26
(libkudu_client.so.0+0x2180e9)
Location is heap block of size 105 at 0x7b1c00426c50 allocated by thread T255:
#0 operator new[](unsigned long) <null> (impalad+0x229c50a)
#1 kudu::Status::Status(kudu::Status::Code, kudu::Slice const&, kudu::Slice
const&, short)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:32:17
(impalad+0x27e856a)
#2 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:10
(impalad+0x27e89d2)
#3
kudu::rpc::Negotiation::RunNegotiation(scoped_refptr<kudu::rpc::Connection>
const&, kudu::TriStateFlag, kudu::TriStateFlag, bool, kudu::MonoTime)
/mnt/source/kudu/kudu-16689973a/src/kudu/rpc/negotiation.cc:325:26
(libkudu_client.so.0+0x2180e9)
Thread T280 'rpc reactor-121' (tid=121625, running) created by thread T277 at:
#0 pthread_create <null> (impalad+0x223e7d3)
#1 kudu::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*)
/mnt/source/kudu/kudu-16689973a/src/kudu/util/thread.cc:643:29
(libkudu_client.so.0+0x3a985f)
#2
impala::ExecEnv::GetKuduClient(std::vector<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > > > const&,
std::tr1::shared_ptr<kudu::client::KuduClient>*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/exec-env.cc:732:5
(impalad+0x2965d1e)
#3 impala::KuduPartitionExpr::Init(impala::RowDescriptor const&, bool,
impala::FragmentState*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/kudu-partition-expr.cc:53:3
(impalad+0x3a54a5f)
#4 impala::ScalarExpr::Create(impala::TExpr const&, impala::RowDescriptor
const&, impala::FragmentState*, impala::ObjectPool*, impala::ScalarExpr**)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:91:25
(impalad+0x39fbe4a)
#5 impala::ScalarExpr::Create(std::vector<impala::TExpr,
std::allocator<impala::TExpr> > const&, impala::RowDescriptor const&,
impala::FragmentState*, impala::ObjectPool*, std::vector<impala::ScalarExpr*,
std::allocator<impala::ScalarExpr*> >*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:107:5
(impalad+0x39fc6e3)
#6 impala::ScalarExpr::Create(std::vector<impala::TExpr,
std::allocator<impala::TExpr> > const&, impala::RowDescriptor const&,
impala::FragmentState*, std::vector<impala::ScalarExpr*,
std::allocator<impala::ScalarExpr*> >*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:121:10
(impalad+0x39fc8b4)
#7 impala::KrpcDataStreamSenderConfig::Init(impala::TDataSink const&,
impala::RowDescriptor const*, impala::FragmentState*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:93:5
(impalad+0x2ad62b0)
#8 impala::DataSinkConfig::CreateConfig(impala::TDataSink const&,
impala::RowDescriptor const*, impala::FragmentState*, impala::DataSinkConfig**)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exec/data-sink.cc:148:3
(impalad+0x324bce4)
#9 impala::FragmentState::Init()
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/fragment-state.cc:87:3
(impalad+0x2ab9d6e)
#10
impala::FragmentState::CreateFragmentStateMap(impala::TExecPlanFragmentInfo
const&, impala::ExecQueryFInstancesRequestPB const&, impala::QueryState*,
std::unordered_map<int, impala::FragmentState*, std::hash<int>,
std::equal_to<int>, std::allocator<std::pair<int const, impala::FragmentState*>
> >&)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/fragment-state.cc:80:5
(impalad+0x2ab9aaf)
#11 impala::QueryState::StartFInstances()
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/query-state.cc:884:29
(impalad+0x29e997c)
#12 impala::QueryExecMgr::ExecuteQueryHelper(impala::QueryState*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/query-exec-mgr.cc:170:7
(impalad+0x29d6de9)
#13 boost::_mfi::mf1<void, impala::QueryExecMgr,
impala::QueryState*>::operator()(impala::QueryExecMgr*, impala::QueryState*)
const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:165:29
(impalad+0x29df75d)
#14 void boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> >::operator()<boost::_mfi::mf1<void,
impala::QueryExecMgr, impala::QueryState*>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void,
impala::QueryExecMgr, impala::QueryState*>&, boost::_bi::list0&, int)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:319:9
(impalad+0x29df6aa)
#15 boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::QueryExecMgr,
impala::QueryState*>,
boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> > >::operator()()
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
(impalad+0x29df613)
#16
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf1<void, impala::QueryExecMgr, impala::QueryState*>,
boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:158:11
(impalad+0x29df351)
#17 boost::function0<void>::operator()() const
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763:14
(impalad+0x296d231)
#18 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/util/thread.cc:361:3
(impalad+0x3153e65)
#19 void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>
const&, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>
const&, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:531:9
(impalad+0x315d097)
#20 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
(impalad+0x315cfb3)
#21 boost::detail::thread_data<boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>
const&, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run()
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/thread/detail/thread.hpp:120:17
(impalad+0x315c760)
#22 thread_proxy
/mnt/source/boost/boost_1_74_0-p1/libs/thread/src/pthread/thread.cpp:179:37
(impalad+0x3fdc0d6)
Thread T255 'client-negotiat' (tid=121628, running) created by thread T280 at:
#0 pthread_create <null> (impalad+0x223e7d3)
#1 kudu::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*)
/mnt/source/kudu/kudu-16689973a/src/kudu/util/thread.cc:643:29
(libkudu_client.so.0+0x3a985f)
SUMMARY: ThreadSanitizer: data race
/data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.h:483:55
in kudu::Status::code() const
{code}
This kills Impala, which then aborts the complete dataload process.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
