[
https://issues.apache.org/jira/browse/IMPALA-14476?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Riza Suminto resolved IMPALA-14476.
-----------------------------------
Fix Version/s: Impala 5.0.0
Target Version: Impala 5.0.0
Resolution: Fixed
Solved with a workaround.
> Data race in TSAN build in during dataload in be/src/kudu/util/status.cc
> ------------------------------------------------------------------------
>
> Key: IMPALA-14476
> URL: https://issues.apache.org/jira/browse/IMPALA-14476
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 5.0.0
> Reporter: Laszlo Gaal
> Assignee: Riza Suminto
> Priority: Blocker
> Fix For: Impala 5.0.0
>
>
> Since the toolchain was bumped to pick up Kudu's array column feature
> (KUDU-1261), Impala's TSAN builds on the master branch consistently break
> during dataload with a data race detected by TSAN. Impalad.ERROR reports:
> {code}
> WARNING: ThreadSanitizer: data race (pid=59148)
> Read of size 1 at 0x7b1c00426c54 by thread T280:
> #0 kudu::Status::code() const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.h:483:55
> (impalad+0x27d2eea)
> #1 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:17
> (impalad+0x27e89a1)
> #2 kudu::rpc::Connection::Shutdown(kudu::Status const&,
> std::unique_ptr<kudu::rpc::ErrorStatusPB,
> std::default_delete<kudu::rpc::ErrorStatusPB> >)
> /mnt/source/kudu/kudu-16689973a/src/kudu/rpc/connection.cc:153:44
> (libkudu_client.so.0+0x20aefd)
> Previous write of size 1 at 0x7b1c00426c54 by thread T255:
> #0 kudu::Status::Status(kudu::Status::Code, kudu::Slice const&,
> kudu::Slice const&, short)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:34:13
> (impalad+0x27e8584)
> #1 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:10
> (impalad+0x27e89d2)
> #2
> kudu::rpc::Negotiation::RunNegotiation(scoped_refptr<kudu::rpc::Connection>
> const&, kudu::TriStateFlag, kudu::TriStateFlag, bool, kudu::MonoTime)
> /mnt/source/kudu/kudu-16689973a/src/kudu/rpc/negotiation.cc:325:26
> (libkudu_client.so.0+0x2180e9)
> Location is heap block of size 105 at 0x7b1c00426c50 allocated by thread
> T255:
> #0 operator new[](unsigned long) <null> (impalad+0x229c50a)
> #1 kudu::Status::Status(kudu::Status::Code, kudu::Slice const&,
> kudu::Slice const&, short)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:32:17
> (impalad+0x27e856a)
> #2 kudu::Status::CloneAndPrepend(kudu::Slice const&) const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.cc:159:10
> (impalad+0x27e89d2)
> #3
> kudu::rpc::Negotiation::RunNegotiation(scoped_refptr<kudu::rpc::Connection>
> const&, kudu::TriStateFlag, kudu::TriStateFlag, bool, kudu::MonoTime)
> /mnt/source/kudu/kudu-16689973a/src/kudu/rpc/negotiation.cc:325:26
> (libkudu_client.so.0+0x2180e9)
> Thread T280 'rpc reactor-121' (tid=121625, running) created by thread T277
> at:
> #0 pthread_create <null> (impalad+0x223e7d3)
> #1 kudu::Thread::StartThread(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> >,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> >, std::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*)
> /mnt/source/kudu/kudu-16689973a/src/kudu/util/thread.cc:643:29
> (libkudu_client.so.0+0x3a985f)
> #2
> impala::ExecEnv::GetKuduClient(std::vector<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> >,
> std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > > > const&,
> std::tr1::shared_ptr<kudu::client::KuduClient>*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/exec-env.cc:732:5
> (impalad+0x2965d1e)
> #3 impala::KuduPartitionExpr::Init(impala::RowDescriptor const&, bool,
> impala::FragmentState*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/kudu-partition-expr.cc:53:3
> (impalad+0x3a54a5f)
> #4 impala::ScalarExpr::Create(impala::TExpr const&, impala::RowDescriptor
> const&, impala::FragmentState*, impala::ObjectPool*, impala::ScalarExpr**)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:91:25
> (impalad+0x39fbe4a)
> #5 impala::ScalarExpr::Create(std::vector<impala::TExpr,
> std::allocator<impala::TExpr> > const&, impala::RowDescriptor const&,
> impala::FragmentState*, impala::ObjectPool*, std::vector<impala::ScalarExpr*,
> std::allocator<impala::ScalarExpr*> >*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:107:5
> (impalad+0x39fc6e3)
> #6 impala::ScalarExpr::Create(std::vector<impala::TExpr,
> std::allocator<impala::TExpr> > const&, impala::RowDescriptor const&,
> impala::FragmentState*, std::vector<impala::ScalarExpr*,
> std::allocator<impala::ScalarExpr*> >*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exprs/scalar-expr.cc:121:10
> (impalad+0x39fc8b4)
> #7 impala::KrpcDataStreamSenderConfig::Init(impala::TDataSink const&,
> impala::RowDescriptor const*, impala::FragmentState*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:93:5
> (impalad+0x2ad62b0)
> #8 impala::DataSinkConfig::CreateConfig(impala::TDataSink const&,
> impala::RowDescriptor const*, impala::FragmentState*,
> impala::DataSinkConfig**)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/exec/data-sink.cc:148:3
> (impalad+0x324bce4)
> #9 impala::FragmentState::Init()
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/fragment-state.cc:87:3
> (impalad+0x2ab9d6e)
> #10
> impala::FragmentState::CreateFragmentStateMap(impala::TExecPlanFragmentInfo
> const&, impala::ExecQueryFInstancesRequestPB const&, impala::QueryState*,
> std::unordered_map<int, impala::FragmentState*, std::hash<int>,
> std::equal_to<int>, std::allocator<std::pair<int const,
> impala::FragmentState*> > >&)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/fragment-state.cc:80:5
> (impalad+0x2ab9aaf)
> #11 impala::QueryState::StartFInstances()
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/query-state.cc:884:29
> (impalad+0x29e997c)
> #12 impala::QueryExecMgr::ExecuteQueryHelper(impala::QueryState*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/runtime/query-exec-mgr.cc:170:7
> (impalad+0x29d6de9)
> #13 boost::_mfi::mf1<void, impala::QueryExecMgr,
> impala::QueryState*>::operator()(impala::QueryExecMgr*, impala::QueryState*)
> const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:165:29
> (impalad+0x29df75d)
> #14 void boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
> boost::_bi::value<impala::QueryState*> >::operator()<boost::_mfi::mf1<void,
> impala::QueryExecMgr, impala::QueryState*>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void,
> impala::QueryExecMgr, impala::QueryState*>&, boost::_bi::list0&, int)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:319:9
> (impalad+0x29df6aa)
> #15 boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::QueryExecMgr,
> impala::QueryState*>,
> boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
> boost::_bi::value<impala::QueryState*> > >::operator()()
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
> (impalad+0x29df613)
> #16
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, impala::QueryExecMgr, impala::QueryState*>,
> boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
> boost::_bi::value<impala::QueryState*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:158:11
> (impalad+0x29df351)
> #17 boost::function0<void>::operator()() const
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763:14
> (impalad+0x296d231)
> #18 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()> const&, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/util/thread.cc:361:3
> (impalad+0x3153e65)
> #19 void
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
> >::operator()<void (*)(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()> const&, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*),
> boost::_bi::list0>(boost::_bi::type<void>, void
> (*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > const&, std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&, boost::function<void
> ()> const&, impala::ThreadDebugInfo const*, impala::Promise<long,
> (impala::PromiseMode)0>*), boost::_bi::list0&, int)
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:531:9
> (impalad+0x315d097)
> #20 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()> const&, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*),
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
> >::operator()()
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
> (impalad+0x315cfb3)
> #21 boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > const&, std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&, boost::function<void
> ()> const&, impala::ThreadDebugInfo const*, impala::Promise<long,
> (impala::PromiseMode)0>*),
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >
> >::run()
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/thread/detail/thread.hpp:120:17
> (impalad+0x315c760)
> #22 thread_proxy
> /mnt/source/boost/boost_1_74_0-p1/libs/thread/src/pthread/thread.cpp:179:37
> (impalad+0x3fdc0d6)
> Thread T255 'client-negotiat' (tid=121628, running) created by thread T280
> at:
> #0 pthread_create <null> (impalad+0x223e7d3)
> #1 kudu::Thread::StartThread(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> >,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> >, std::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*)
> /mnt/source/kudu/kudu-16689973a/src/kudu/util/thread.cc:643:29
> (libkudu_client.so.0+0x3a985f)
> SUMMARY: ThreadSanitizer: data race
> /data/jenkins/workspace/impala-cdw-master-staging-core-tsan/repos/Impala/be/src/kudu/util/status.h:483:55
> in kudu::Status::code() const
> {code}
> This kills Impala, which then aborts the complete dataload process.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
