Andrew Sherman created IMPALA-12411:
---------------------------------------
Summary: TSAN ThreadSanitizer: data race during expr-test teardown
Key: IMPALA-12411
URL: https://issues.apache.org/jira/browse/IMPALA-12411
Project: IMPALA
Issue Type: Bug
Components: be
Affects Versions: Impala 4.3.0
Reporter: Andrew Sherman
Attachments: expr-test-tsan-failure.log
The racing threads are
{code:java}
20:14:05 Read of size 8 at 0x00000a8d3348 by main thread:
20:14:05 #0 std::vector<impala::TJvmMemoryPool,
std::allocator<impala::TJvmMemoryPool> >::~vector()
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/stl_vector.h:680:54
(unifiedbetests+0x3fcd9b9)
20:14:05 #1
impala::TGetJvmMemoryMetricsResponse::~TGetJvmMemoryMetricsResponse()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/generated-sources/gen-cpp/Frontend_types.cpp:4158:1
(unifiedbetests+0x3fc1397)
20:14:05 #2 impala::JvmMetricCache::~JvmMetricCache()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/memory-metrics.h:170:7
(unifiedbetests+0x4b2989d)
20:14:05 #3 at_exit_wrapper(void*)
/mnt/source/llvm/llvm-5.0.1.src-p7/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:361:31
(unifiedbetests+0x21b3554)
{code}
and
{code:java}
20:14:05 Previous write of size 8 at 0x00000a8d3348 by thread T586:
20:14:05 #0 std::vector<impala::TJvmMemoryPool,
std::allocator<impala::TJvmMemoryPool>
>::_M_erase_at_end(impala::TJvmMemoryPool*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/stl_vector.h:1798:30
(unifiedbetests+0x4afabcc)
20:14:05 #1 std::vector<impala::TJvmMemoryPool,
std::allocator<impala::TJvmMemoryPool> >::clear()
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/stl_vector.h:1499:9
(unifiedbetests+0x4afa4b4)
20:14:05 #2 unsigned int
impala::TGetJvmMemoryMetricsResponse::read<apache::thrift::protocol::TProtocol>(apache::thrift::protocol::TProtocol*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/generated-sources/gen-cpp/Frontend_types.tcc:5673:32
(unifiedbetests+0x4afa21d)
20:14:05 #3 impala::Status
impala::DeserializeThriftMsg<impala::TGetJvmMemoryMetricsResponse>(unsigned
char const*, unsigned int*, bool, impala::TGetJvmMemoryMetricsResponse*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/rpc/thrift-util.h:136:23
(unifiedbetests+0x4af9da6)
20:14:05 #4 impala::Status
impala::DeserializeThriftMsg<impala::TGetJvmMemoryMetricsResponse>(JNIEnv_*,
_jbyteArray*, impala::TGetJvmMemoryMetricsResponse*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/rpc/jni-thrift-util.h:61:3
(unifiedbetests+0x4af9c62)
20:14:05 #5 impala::Status
impala::JniCall::ObjectToResult<impala::TGetJvmMemoryMetricsResponse>(_jobject*,
impala::TGetJvmMemoryMetricsResponse*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/jni-util.h:493:3
(unifiedbetests+0x4af9b24)
20:14:05 #6 impala::Status
impala::JniCall::Call<impala::TGetJvmMemoryMetricsResponse>(impala::TGetJvmMemoryMetricsResponse*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/jni-util.h:486:3
(unifiedbetests+0x4af92a6)
20:14:05 #7
impala::JniUtil::GetJvmMemoryMetrics(impala::TGetJvmMemoryMetricsResponse*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/jni-util.cc:299:72
(unifiedbetests+0x4af89a3)
20:14:05 #8 impala::JvmMetricCache::GrabMetricsIfNecessary()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/memory-metrics.cc:294:19
(unifiedbetests+0x4b2780d)
20:14:05 #9 impala::JvmMetricCache::GetCounterMetric(long
(*)(impala::TGetJvmMemoryMetricsResponse const&))
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/memory-metrics.cc:305:3
(unifiedbetests+0x4b27711)
20:14:05 #10 impala::JvmMemoryCounterMetric::GetValue()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/memory-metrics.cc:270:41
(unifiedbetests+0x4b276bf)
20:14:05 #11 impala::QueryState::Init(impala::ExecQueryFInstancesRequestPB
const*, impala::TExecPlanFragmentInfo const&)::$_3::operator()() const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-state.cc:185:50
(unifiedbetests+0x455bc15)
20:14:05 #12
boost::detail::function::function_obj_invoker0<impala::QueryState::Init(impala::ExecQueryFInstancesRequestPB
const*, impala::TExecPlanFragmentInfo const&)::$_3,
long>::invoke(boost::detail::function::function_buffer&)
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:137:18
(unifiedbetests+0x455b9d9)
20:14:05 #13 boost::function0<long>::operator()() const
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763:14
(unifiedbetests+0x4b57bf1)
20:14:05 #14 impala::RuntimeProfile::DerivedCounter::value() const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile-counters.h:384:12
(unifiedbetests+0x4b8b38d)
20:14:05 #15
impala::RuntimeProfile::ToThriftSubclass(std::vector<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
impala::RuntimeProfileBase::Counter const*>,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
impala::RuntimeProfileBase::Counter const*> > >&, impala::TRuntimeProfileNode*)
const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile.cc:1783:35
(unifiedbetests+0x4b7737b)
20:14:05 #16
impala::RuntimeProfileBase::ToThriftHelper(impala::TRuntimeProfileNode*) const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile.cc:1772:3
(unifiedbetests+0x4b77252)
20:14:05 #17
impala::RuntimeProfileBase::ToThriftHelper(std::vector<impala::TRuntimeProfileNode,
std::allocator<impala::TRuntimeProfileNode> >*) const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile.cc:1732:25
(unifiedbetests+0x4b76e60)
20:14:05 #18
impala::RuntimeProfileBase::ToThrift(impala::TRuntimeProfileTree*) const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile.cc:1711:3
(unifiedbetests+0x4b76d4f)
20:14:05 #19 impala::RuntimeProfile::ToThrift(impala::TRuntimeProfileTree*)
const
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/runtime-profile.cc:1716:23
(unifiedbetests+0x4b76f20)
20:14:05 #20 impala::QueryState::ConstructReport(bool,
impala::ReportExecStatusRequestPB*, impala::TRuntimeProfileForest*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-state.cc:529:18
(unifiedbetests+0x455804b)
20:14:05 #21 impala::QueryState::ReportExecStatus()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-state.cc:639:3
(unifiedbetests+0x455698a)
20:14:05 #22 impala::QueryState::UpdateBackendExecState()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-state.cc:465:40
(unifiedbetests+0x4556673)
20:14:05 #23 impala::QueryState::MonitorFInstances()
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-state.cc:917:3
(unifiedbetests+0x455a08f)
20:14:05 #24 impala::QueryExecMgr::ExecuteQueryHelper(impala::QueryState*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/runtime/query-exec-mgr.cc:162:42
(unifiedbetests+0x4546de5)
20:14:05 #25 boost::_mfi::mf1<void, impala::QueryExecMgr,
impala::QueryState*>::operator()(impala::QueryExecMgr*, impala::QueryState*)
const
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:165:29
(unifiedbetests+0x454e66d)
20:14:05 #26 void
boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> >::operator()<boost::_mfi::mf1<void,
impala::QueryExecMgr, impala::QueryState*>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void,
impala::QueryExecMgr, impala::QueryState*>&, boost::_bi::list0&, int)
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:319:9
(unifiedbetests+0x454e5ba)
20:14:05 #27 boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::QueryExecMgr, impala::QueryState*>,
boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> > >::operator()()
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
(unifiedbetests+0x454e523)
20:14:05 #28
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf1<void, impala::QueryExecMgr, impala::QueryState*>,
boost::_bi::list2<boost::_bi::value<impala::QueryExecMgr*>,
boost::_bi::value<impala::QueryState*> > >,
void>::invoke(boost::detail::function::function_buffer&)
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:158:11
(unifiedbetests+0x454e261)
20:14:05 #29 boost::function0<void>::operator()() const
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763:14
(unifiedbetests+0x44b7371)
20:14:05 #30
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/data/jenkins/workspace/impala-cdw-master-core-tsan/repos/Impala/be/src/util/thread.cc:360:3
(unifiedbetests+0x4bbaa88)
20:14:05 #31 void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:531:9
(unifiedbetests+0x4bc345c)
20:14:05 #32 boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
(unifiedbetests+0x4bc3373)
20:14:05 #33 boost::detail::thread_data<boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run()
/data/jenkins/workspace/impala-cdw-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/thread/detail/thread.hpp:120:17
(unifiedbetests+0x4bc3060)
20:14:05 #34 thread_proxy <null> (unifiedbetests+0x4e0b6b6)
{code}
All the test seemed to run OK, which, plus the presence of at_exit_wrapper in
one thread suggests to me that this is happening during the exit of expr-test
(unifiedbetests).
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org
For additional commands, e-mail: issues-all-h...@impala.apache.org
