[
https://issues.apache.org/jira/browse/IMPALA-13213?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Joe McDonnell updated IMPALA-13213:
-----------------------------------
Priority: Critical (was: Blocker)
> impalad crash in Coordinator::HandleExecStateTransition() on ARM during
> exhaustive-release run
> ----------------------------------------------------------------------------------------------
>
> Key: IMPALA-13213
> URL: https://issues.apache.org/jira/browse/IMPALA-13213
> Project: IMPALA
> Issue Type: Bug
> Affects Versions: Impala 4.5.0
> Reporter: Laszlo Gaal
> Priority: Critical
> Labels: broken-build, flaky
>
> The same test run was hit by multiple instance of IMPALA-12336 as well, so
> the crash might just be a consequence of the memory exhaustion.
> The memory tracker log showed {{AvailableMemory}} going as low as ~40 MB on a
> test runner with 64GB of physical memory.
> The log bundle reports the following stack trace from the core dump:
> {code}
> #0 0x0000ffffb4936274 in raise () from /lib64/libc.so.6
> [Current thread is 1 (Thread 0xfffc0fa3ed80 (LWP 248075))]
> To enable execution of this file add
> add-auto-load-safe-path
> /data0/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/lib64/libstdc++.so.6.0.28-gdb.py
> line to your configuration file "/var/lib/jenkins/.config/gdb/gdbinit".
> To completely disable this security protection add
> set auto-load safe-path /
> line to your configuration file "/var/lib/jenkins/.config/gdb/gdbinit".
> For more information about this security protection see the
> "Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
> info "(gdb)Auto-loading safe path"
> #0 0x0000ffffb4936274 in raise () from /lib64/libc.so.6
> #1 0x0000ffffb4920a2c in abort () from /lib64/libc.so.6
> #2 0x0000ffffb4c29024 in __gnu_cxx::__verbose_terminate_handler () at
> ../../../../gcc-10.4.0/libstdc++-v3/libsupc++/vterminate.cc:50
> #3 0x0000ffffb4c26b8c in __cxxabiv1::__terminate (handler=<optimized out>)
> at ../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
> #4 0x0000ffffb4c26bf0 in std::terminate () at
> ../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
> #5 0x0000ffffb4c26ee4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x425fd940,
> tinfo=0x4bb3298 <typeinfo for boost::wrapexcept<std::out_of_range>>,
> dest=0x13bd220 <boost::wrapexcept<std::out_of_range>::~wrapexcept()>) at
> ../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_throw.cc:95
> #6 0x00000000021ab900 in boost::throw_exception<std::out_of_range> (e=...)
> at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/throw_exception.hpp:165
> #7 boost::unordered::unordered_map<impala::Coordinator::ExecState, char
> const*, boost::hash<impala::Coordinator::ExecState>,
> std::equal_to<impala::Coordinator::ExecState>,
> std::allocator<std::pair<impala::Coordinator::ExecState const, char const*> >
> >::at (this=this@entry=0x4c5e350
> <impala::Coordinator::HandleExecStateTransition(impala::Coordinator::ExecState,
> impala::Coordinator::ExecState)::exec_state_to_event>, k=@0xfffc0fa3d62c:
> impala::Coordinator::ExecState::CANCELLED) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/unordered/unordered_map.hpp:1900
> #8 0x00000000021a28dc in impala::Coordinator::HandleExecStateTransition
> (this=this@entry=0x31bc4000,
> old_state=old_state@entry=impala::Coordinator::ExecState::EXECUTING,
> new_state=<optimized out>,
> new_state@entry=impala::Coordinator::ExecState::CANCELLED) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:865
> #9 0x00000000021a305c in impala::Coordinator::SetNonErrorTerminalState
> (this=this@entry=0x31bc4000,
> state=state@entry=impala::Coordinator::ExecState::CANCELLED) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:798
> #10 0x00000000021a3228 in impala::Coordinator::Cancel (this=0x31bc4000,
> wait_until_finalized=true) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:1066
> #11 0x000000000189fff4 in impala::ClientRequestState::Cancel
> (this=this@entry=0x302000000, cause=cause@entry=0xfffc0fa3dd08,
> wait_until_finalized=wait_until_finalized@entry=true) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/client-request-state.cc:1523
> #12 0x00000000018a012c in impala::ClientRequestState::Finalize
> (this=0x302000000, cause=cause@entry=0xfffc0fa3dd08) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/client-request-state.cc:1066
> #13 0x00000000021cb118 in impala::QueryDriver::Finalize (this=0x41a9f270,
> query_handle=query_handle@entry=0xfffc0fa3daa0,
> check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08)
> at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/query-driver.h:50
> #14 0x00000000016ded88 in impala::ImpalaServer::UnregisterQuery
> (this=this@entry=0x325ef400, query_id=...,
> check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08)
> at
> /data0/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/include/c++/10.4.0/bits/shared_ptr_base.h:1329
> #15 0x00000000016df3bc in impala::ImpalaServer::UnregisterQueryDiscardResult
> (this=this@entry=0x325ef400, query_id=...,
> check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08)
> at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:1584
> #16 0x00000000016e0394 in impala::ImpalaServer::CloseSessionInternal
> (this=this@entry=0x325ef400, session_id=..., secret=...,
> ignore_if_absent=ignore_if_absent@entry=true) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:1835
> #17 0x00000000016e1390 in impala::ImpalaServer::ConnectionEnd
> (this=0x325ef400, connection_context=...) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:2603
> #18 0x00000000012bfd84 in
> impala::ThriftServer::ThriftServerEventProcessor::deleteContext
> (this=0x325cc780, context=<optimized out>, input=..., output=...) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/rpc/thrift-server.cc:256
> #19 0x00000000012d5ccc in
> apache::thrift::server::TAcceptQueueServer::Task::run (this=0x1e07929c0) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/rpc/TAcceptQueueServer.cpp:125
> #20 0x00000000012c87b8 in boost::_mfi::mf2<void, impala::ThriftThread,
> std::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<std::thread::id, (impala::PromiseMode)0>*>::operator()
> (this=0xd4cc8570, this=0xd4cc8570, a2=<optimized out>, a1=..., p=0x8b3f6be0)
> at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:280
> #21 boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
> boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
> boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*>
> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread,
> std::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<std::thread::id, (impala::PromiseMode)0>*>,
> boost::_bi::list0> (a=<synthetic pointer>..., f=..., this=0xd4cc8580) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:398
> #22 boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread,
> std::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<std::thread::id, (impala::PromiseMode)0>*>,
> boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
> boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
> boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*>
> > >::operator() (this=0xd4cc8570) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294
> #23
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf2<void, impala::ThriftThread,
> std::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<std::thread::id, (impala::PromiseMode)0>*>,
> boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
> boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
> boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*>
> > >, void>::invoke (function_obj_ptr=...) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:158
> #24 0x0000000001abb448 in boost::function0<void>::operator()
> (this=0xfffc0fa3e500) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763
> #25 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*) (name=..., category=...,
> functor=..., parent_thread_info=<optimized out>,
> thread_started=0xffff1563dca0) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/util/thread.cc:360
> #26 0x0000000001abc700 in
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
> >::operator()<void (*)(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*),
> boost::_bi::list0>(boost::_bi::type<void>, void
> (*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > const&, std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&, boost::function<void
> ()>, impala::ThreadDebugInfo const*, impala::Promise<long,
> (impala::PromiseMode)0>*), boost::_bi::list0&, int) (a=<synthetic
> pointer>..., f=@0x7a68ad88: 0x1abb1c0
> <impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*)>, this=0x7a68ad90) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:531
> #27 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&,
> std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long, (impala::PromiseMode)0>*),
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
> >::operator()() (this=0x7a68ad88) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294
> #28 boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > const&, std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > const&, boost::function<void
> ()>, impala::ThreadDebugInfo const*, impala::Promise<long,
> (impala::PromiseMode)0>*),
> boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
> std::char_traits<char>, std::allocator<char> > >,
> boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
> std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >
> >::run() (this=0x7a68ac40) at
> /data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/thread/detail/thread.hpp:120
> #29 0x0000000002306538 in thread_proxy ()
> #30 0x0000ffffb69478b8 in start_thread () from /lib64/libpthread.so.0
> #31 0x0000ffffb4923afc in thread_start () from /lib64/libc.so.6
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
