[
https://issues.apache.org/jira/browse/IMPALA-10338?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17253680#comment-17253680
]
Tim Armstrong commented on IMPALA-10338:
----------------------------------------
I caught a use after free when looping this test.
{noformat}
./buildall.sh -asan -skiptests -notests -ninja -noclean && while impala-py.test
tests/custom_cluster/test_admission_controller.py -k test_queue_reasons_slots;
do date; done
{noformat}
from impalad.ERROR:
{noformat}
==5769==ERROR: AddressSanitizer: heap-use-after-free on address 0x60c0004576c0
at pc 0x000001b0491e bp 0x7f9b89c451b0 sp 0x7f9b89c44960
READ of size 117 at 0x60c0004576c0 thread T339
#0 0x1b0491d in __interceptor_memcpy.part.40
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:738
#1 0x1bde953 in void std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*,
char*, std::forward_iterator_tag)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/basic_string.tcc:225:6
#2 0x7f9e931f31de in void std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >::_M_construct_aux<char*>(char*,
char*, std::__false_type)
/mnt/source/gcc/build-7.5.0/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.h:236
#3 0x7f9e931f31de in void std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*,
char*)
/mnt/source/gcc/build-7.5.0/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.h:255
#4 0x7f9e931f31de in std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char>
>::basic_string(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&)
/mnt/source/gcc/build-7.5.0/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.h:440
#5 0x2d047a4 in
impala::RuntimeProfileBase::AddInfoString(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&)
/home/tarmstrong/Impala/incubator-impala/be/src/util/runtime-profile.cc:1025:37
#6 0x2753ec9 in
impala::AdmissionController::SubmitForAdmission(impala::AdmissionController::AdmissionRequest
const&, impala::Promise<impala::AdmissionOutcome, (impala::PromiseMode)1>*,
std::unique_ptr<impala::QuerySchedulePB,
std::default_delete<impala::QuerySchedulePB> >*, bool*,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>*)
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:1236:28
#7 0x37a3fd1 in
impala::LocalAdmissionControlClient::SubmitForAdmission(impala::AdmissionController::AdmissionRequest
const&, impala::RuntimeProfile::EventSequence*,
std::unique_ptr<impala::QuerySchedulePB,
std::default_delete<impala::QuerySchedulePB> >*)
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/local-admission-control-client.cc:40:67
#8 0x2a2c26a in impala::ClientRequestState::FinishExecQueryOrDmlRequest()
/home/tarmstrong/Impala/incubator-impala/be/src/service/client-request-state.cc:578:52
#9 0x2a42e8b in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::ClientRequestState>,
boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#10 0x23d0566 in boost::function0<void>::operator()() const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#11 0x2d92bd9 in
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:360:3
#12 0x2d9e018 in void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9
#13 0x2d9de6b in boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#14 0x45c4d71 in thread_proxy
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c4d71)
#15 0x7f9e95d716b9 in start_thread
(/lib/x86_64-linux-gnu/libpthread.so.0+0x76b9)
#16 0x7f9e928f041c in clone
/build/glibc-LK5gWL/glibc-2.23/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:109
0x60c0004576c0 is located 0 bytes inside of 118-byte region
[0x60c0004576c0,0x60c000457736)
freed by thread T284 here:
#0 0x1bda530 in operator delete(void*)
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_new_delete.cc:137
#1 0x274e83b in
impala::AdmissionController::HasAvailableSlots(impala::ScheduleState const&,
impala::TPoolConfig const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >*, bool&)
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:894:7
#2 0x274edde in
impala::AdmissionController::CanAdmitRequest(impala::ScheduleState const&,
impala::TPoolConfig const&, bool, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >*,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>*, bool&)
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:932:11
#3 0x2755cb7 in
impala::AdmissionController::FindGroupToAdmitOrReject(std::shared_ptr<impala::ClusterMembershipMgr::Snapshot
const>, impala::TPoolConfig const&, bool,
impala::AdmissionController::PoolStats*,
impala::AdmissionController::QueueNode*, bool&)
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:1708:9
#4 0x2748687 in impala::AdmissionController::DequeueLoop()
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:1816:17
#5 0x27899db in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::AdmissionController>,
boost::_bi::list1<boost::_bi::value<impala::AdmissionController*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#6 0x23d0566 in boost::function0<void>::operator()() const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#7 0x2d92bd9 in
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:360:3
#8 0x2d9e018 in void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9
#9 0x2d9de6b in boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#10 0x45c4d71 in thread_proxy
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c4d71)
previously allocated by thread T284 here:
#0 0x1bd97c0 in operator new(unsigned long)
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_new_delete.cc:92
#1 0x7f9e931f107c in std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >::reserve(unsigned long)
/mnt/source/gcc/build-7.5.0/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.tcc:293
Thread T339 created by T334 here:
#0 0x1afee90 in __interceptor_pthread_create
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_interceptors.cc:317
#1 0x45c3dc8 in boost::thread::start_thread_noexcept()
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c3dc8)
#2 0x2d96b36 in boost::thread::thread<void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long,
(impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*,
impala::Promise<long, (impala::PromiseMode)0>*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13
#3 0x2d92037 in
impala::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:317:13
#4 0x2a3bb87 in impala::Status impala::Thread::Create<void
(impala::ClientRequestState::*)(),
impala::ClientRequestState*>(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, void (impala::ClientRequestState::* const&)(),
impala::ClientRequestState* const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.h:81:12
#5 0x2a2411b in
impala::ClientRequestState::ExecAsyncQueryOrDmlRequest(impala::TQueryExecRequest
const&)
/home/tarmstrong/Impala/incubator-impala/be/src/service/client-request-state.cc:569:43
#6 0x2a215e7 in impala::ClientRequestState::Exec()
/home/tarmstrong/Impala/incubator-impala/be/src/service/client-request-state.cc:231:47
#7 0x296ef4e in impala::ImpalaServer::ExecuteInternal(impala::TQueryCtx
const&, std::shared_ptr<impala::ImpalaServer::SessionState>, bool*,
impala::QueryHandle*)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impala-server.cc:1161:60
#8 0x296e545 in impala::ImpalaServer::Execute(impala::TQueryCtx*,
std::shared_ptr<impala::ImpalaServer::SessionState>, impala::QueryHandle*)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impala-server.cc:1094:19
#9 0x2a545f1 in impala::ImpalaServer::query(beeswax::QueryHandle&,
beeswax::Query const&)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impala-beeswax-server.cc:72:33
#10 0x3460bad in beeswax::BeeswaxServiceProcessor::process_query(int,
apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*,
void*)
/home/tarmstrong/Impala/incubator-impala/be/generated-sources/gen-cpp/BeeswaxService.cpp:3227:13
#11 0x34605b7 in
beeswax::BeeswaxServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*,
apache::thrift::protocol::TProtocol*, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, int, void*)
/home/tarmstrong/Impala/incubator-impala/be/generated-sources/gen-cpp/BeeswaxService.cpp:3200:3
#12 0x3401cbe in
impala::ImpalaServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*,
apache::thrift::protocol::TProtocol*, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, int, void*)
/home/tarmstrong/Impala/incubator-impala/be/generated-sources/gen-cpp/ImpalaService.cpp:1824:48
#13 0x1be29c6 in
apache::thrift::TDispatchProcessor::process(boost::shared_ptr<apache::thrift::protocol::TProtocol>,
boost::shared_ptr<apache::thrift::protocol::TProtocol>, void*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/thrift-0.9.3-p8/include/thrift/TDispatchProcessor.h:121:12
#14 0x24bc3a3 in apache::thrift::server::TAcceptQueueServer::Task::run()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/TAcceptQueueServer.cpp:84:26
#15 0x24a920d in
impala::ThriftThread::RunRunnable(boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/thrift-thread.cc:74:13
#16 0x24abf4a in boost::_mfi::mf2<void, impala::ThriftThread,
boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long,
(impala::PromiseMode)0>*>::operator()(impala::ThriftThread*,
boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*) const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:280:16
#17 0x24abd52 in void
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<unsigned long, (impala::PromiseMode)0>*>
>::operator()<boost::_mfi::mf2<void, impala::ThriftThread,
boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void,
impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*>&, boost::_bi::list0&,
int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:398:9
#18 0x24abbeb in boost::_bi::bind_t<void, boost::_mfi::mf2<void,
impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*>,
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<unsigned long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#19 0x23d0566 in boost::function0<void>::operator()() const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#20 0x2d92bd9 in
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:360:3
#21 0x2d9e018 in void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9
#22 0x2d9de6b in boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#23 0x45c4d71 in thread_proxy
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c4d71)
Thread T334 created by T321 here:
#0 0x1afee90 in __interceptor_pthread_create
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_interceptors.cc:317
#1 0x45c3dc8 in boost::thread::start_thread_noexcept()
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c3dc8)
#2 0x2d96b36 in boost::thread::thread<void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long,
(impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*,
impala::Promise<long, (impala::PromiseMode)0>*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13
#3 0x2d92037 in
impala::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:317:13
#4 0x24a9ae9 in impala::Status
impala::Thread::Create<boost::_bi::bind_t<void, boost::_mfi::mf2<void,
impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*>,
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<unsigned long, (impala::PromiseMode)0>*> > >
>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
> const&, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, boost::_bi::bind_t<void, boost::_mfi::mf2<void,
impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<unsigned long, (impala::PromiseMode)0>*>,
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<unsigned long, (impala::PromiseMode)0>*> > >
const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*,
bool) /home/tarmstrong/Impala/incubator-impala/be/src/util/thread.h:74:12
#5 0x24a8d02 in impala::ThriftThread::start()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/thrift-thread.cc:35:19
#6 0x24b5491 in
apache::thrift::server::TAcceptQueueServer::SetupConnection(boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>)
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/TAcceptQueueServer.cpp:292:13
#7 0x24b9056 in
apache::thrift::server::TAcceptQueueServer::serve()::$_0::operator()(int,
boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry> const&) const
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/TAcceptQueueServer.cpp:323:15
#8 0x24b8dbf in
boost::detail::function::void_function_obj_invoker2<apache::thrift::server::TAcceptQueueServer::serve()::$_0,
void, int, boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
const&>::invoke(boost::detail::function::function_buffer&, int,
boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry> const&)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11
#9 0x24c3b65 in boost::function2<void, int,
boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
const&>::operator()(int,
boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry> const&) const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#10 0x24c21af in
impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>::WorkerThread(int)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread-pool.h:166:9
#11 0x24c2b6b in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>, int>,
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>*>, boost::_bi::value<int> > >::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#12 0x23d0566 in boost::function0<void>::operator()() const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#13 0x2d92bd9 in
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:360:3
#14 0x2d9e018 in void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9
#15 0x2d9de6b in boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#16 0x45c4d71 in thread_proxy
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c4d71)
Thread T321 created by T317 here:
#0 0x1afee90 in __interceptor_pthread_create
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_interceptors.cc:317
#1 0x45c3dc8 in boost::thread::start_thread_noexcept()
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c3dc8)
#2 0x2d96b36 in boost::thread::thread<void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long,
(impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*,
impala::Promise<long, (impala::PromiseMode)0>*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13
#3 0x2d92037 in
impala::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:317:13
#4 0x24c1c23 in impala::Status
impala::Thread::Create<boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>, int>,
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>*>, boost::_bi::value<int> > > >(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>, int>,
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>*>, boost::_bi::value<int> > > const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.h:74:12
#5 0x24ba609 in
impala::ThreadPool<boost::shared_ptr<apache::thrift::server::TAcceptQueueEntry>
>::Init()
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread-pool.h:73:23
#6 0x24b69c2 in apache::thrift::server::TAcceptQueueServer::serve()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/TAcceptQueueServer.cpp:326:41
#7 0x2497440 in
impala::ThriftServer::ThriftServerEventProcessor::Supervise()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/thrift-server.cc:152:30
#8 0x24a399b in boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::ThriftServer::ThriftServerEventProcessor>,
boost::_bi::list1<boost::_bi::value<impala::ThriftServer::ThriftServerEventProcessor*>
> >::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#9 0x23d0566 in boost::function0<void>::operator()() const
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14
#10 0x2d92bd9 in
impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:360:3
#11 0x2d9e018 in void
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9
#12 0x2d9de6b in boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()()
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16
#13 0x45c4d71 in thread_proxy
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c4d71)
Thread T317 created by T0 here:
#0 0x1afee90 in __interceptor_pthread_create
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_interceptors.cc:317
#1 0x45c3dc8 in boost::thread::start_thread_noexcept()
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c3dc8)
#2 0x2d96b36 in boost::thread::thread<void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long,
(impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*,
impala::Promise<long, (impala::PromiseMode)0>*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13
#3 0x2d92037 in
impala::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:317:13
#4 0x249c237 in impala::Status impala::Thread::Create<void
(impala::ThriftServer::ThriftServerEventProcessor::*)(),
impala::ThriftServer::ThriftServerEventProcessor*>(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, void (impala::ThriftServer::ThriftServerEventProcessor::* const&)(),
impala::ThriftServer::ThriftServerEventProcessor* const&,
std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.h:81:12
#5 0x2496994 in
impala::ThriftServer::ThriftServerEventProcessor::StartAndWaitForServer()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/thrift-server.cc:116:43
#6 0x249a518 in impala::ThriftServer::Start()
/home/tarmstrong/Impala/incubator-impala/be/src/rpc/thrift-server.cc:452:60
#7 0x2983d4e in impala::ImpalaServer::Start(int, int, int)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impala-server.cc:2831:62
#8 0x29550c3 in ImpaladMain(int, char**)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impalad-main.cc:87:22
#9 0x1bddc88 in main
/home/tarmstrong/Impala/incubator-impala/be/src/service/daemon-main.cc:37:12
#10 0x7f9e9280982f in __libc_start_main
/build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291
Thread T284 created by T0 here:
#0 0x1afee90 in __interceptor_pthread_create
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/asan_interceptors.cc:317
#1 0x45c3dc8 in boost::thread::start_thread_noexcept()
(/home/tarmstrong/Impala/incubator-impala/be/build/debug/service/impalad+0x45c3dc8)
#2 0x2d96b36 in boost::thread::thread<void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long,
(impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>
>, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*,
impala::Promise<long, (impala::PromiseMode)0>*)
/opt/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13
#3 0x2d92037 in
impala::Thread::StartThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.cc:317:13
#4 0x276fcd7 in impala::Status impala::Thread::Create<void
(impala::AdmissionController::*)(),
impala::AdmissionController*>(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, void (impala::AdmissionController::* const&)(),
impala::AdmissionController* const&, std::unique_ptr<impala::Thread,
std::default_delete<impala::Thread> >*, bool)
/home/tarmstrong/Impala/incubator-impala/be/src/util/thread.h:81:12
#5 0x27475f5 in impala::AdmissionController::Init()
/home/tarmstrong/Impala/incubator-impala/be/src/scheduling/admission-controller.cc:641:43
#6 0x2517b03 in impala::ExecEnv::Init()
/home/tarmstrong/Impala/incubator-impala/be/src/runtime/exec-env.cc:478:66
#7 0x2954d52 in ImpaladMain(int, char**)
/home/tarmstrong/Impala/incubator-impala/be/src/service/impalad-main.cc:71:52
#8 0x1bddc88 in main
/home/tarmstrong/Impala/incubator-impala/be/src/service/daemon-main.cc:37:12
#9 0x7f9e9280982f in __libc_start_main
/build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291
SUMMARY: AddressSanitizer: heap-use-after-free
/mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:738
in __interceptor_memcpy.part.40
Shadow bytes around the buggy address:
0x0c1880082e80: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082e90: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082ea0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082eb0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082ec0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x0c1880082ed0: fa fa fa fa fa fa fa fa[fd]fd fd fd fd fd fd fd
0x0c1880082ee0: fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa
0x0c1880082ef0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082f00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082f10: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1880082f20: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
Addressable: 00
Partially addressable: 01 02 03 04 05 06 07
Heap left redzone: fa
Freed heap region: fd
Stack left redzone: f1
Stack mid redzone: f2
Stack right redzone: f3
Stack after return: f5
Stack use after scope: f8
Global redzone: f9
Global init order: f6
Poisoned by user: f7
Container overflow: fc
Array cookie: ac
Intra object redzone: bb
ASan internal: fe
Left alloca redzone: ca
Right alloca redzone: cb
==5769==ABORTING
{noformat}
> TestAdmissionController.test_queue_reasons_slots flaky because of
> slow/hanging fragment
> ---------------------------------------------------------------------------------------
>
> Key: IMPALA-10338
> URL: https://issues.apache.org/jira/browse/IMPALA-10338
> Project: IMPALA
> Issue Type: Bug
> Components: Infrastructure
> Affects Versions: Impala 4.0
> Reporter: Tim Armstrong
> Assignee: Tim Armstrong
> Priority: Critical
> Labels: broken-build, flaky, hang
> Attachments: failure-output.txt, impalad.ERROR, impalad.INFO
>
>
> This is on an s3 debug build, commit 5a00a4c06f8ec40a8867dcbc036cf5bb47b8a3be
> {noformat}
> custom_cluster.test_admission_controller.TestAdmissionController.test_queue_reasons_slots
> (from pytest)
> Failing for the past 1 build (Since Failed#673 )
> Took 1 min 58 sec.
> add description
> Error Message
> Timeout: query 504b3a2511f3cd0e:e27bec6b00000000 did not reach one of the
> expected states [4], last known state 3
> Stacktrace
> custom_cluster/test_admission_controller.py:967: in test_queue_reasons_slots
> TIMEOUT_S, config_options={"mt_dop": 4})
> custom_cluster/test_admission_controller.py:277: in
> _execute_and_collect_profiles
> state = self.wait_for_any_state(handle, expected_states, timeout_s)
> common/impala_test_suite.py:1081: in wait_for_any_state
> actual_state))
> E Timeout: query 504b3a2511f3cd0e:e27bec6b00000000 did not reach one of the
> expected states [4], last known state 3
> {noformat}
> Those numbers are beeswax QUeryStates:
> {noformat}
> enum QueryState {
> CREATED = 0
> INITIALIZED = 1
> COMPILED = 2
> RUNNING = 3
> FINISHED = 4
> EXCEPTION = 5
> }
> {noformat}
> I.e. it appears to have run for > 60 seconds.
> The timing of that query:
> {noformat}
> I1116 13:25:08.449323 32665 impala-server.cc:1242]
> 504b3a2511f3cd0e:e27bec6b00000000] Registered query
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> session_id=874c2c9eaf2ad730:0004c1bb0ba7b4a7
> I1116 13:25:08.449626 32665 Frontend.java:1532]
> 504b3a2511f3cd0e:e27bec6b00000000] Analyzing query: select
> min(ss_wholesale_cost) from tpcds_parquet.store_sales db: default
> ...
> I1116 13:25:08.567667 367 admission-controller.cc:1532]
> 504b3a2511f3cd0e:e27bec6b00000000] Scheduling query
> 504b3a2511f3cd0e:e27bec6b00000000 with membership version 2
> I1116 13:25:08.567767 367 admission-controller.cc:1590]
> 504b3a2511f3cd0e:e27bec6b00000000] Scheduling for executor group:
> default-pool-group1 with 3 executors
> I1116 13:25:08.643026 367 admission-controller.cc:1640]
> 504b3a2511f3cd0e:e27bec6b00000000] Trying to admit query to pool default-pool
> in executor group default-pool-group1 (3 executors)
> ...
> I1116 13:25:49.184185 32432 admission-controller.cc:1811] Admitting from
> queue: query=504b3a2511f3cd0e:e27bec6b00000000
> I1116 13:25:49.184196 32432 admission-controller.cc:1903] For Query
> 504b3a2511f3cd0e:e27bec6b00000000 per_backend_mem_limit set to: -1.00 B
> per_backend_m
> em_to_admit set to: 114.02 MB coord_backend_mem_limit set to: -1.00 B
> coord_backend_mem_to_admit set to: 114.02 MB
> I1116 13:25:49.184350 367 admission-controller.cc:1288]
> 504b3a2511f3cd0e:e27bec6b00000000] Admitted queued query
> id=504b3a2511f3cd0e:e27bec6b00000000
> I1116 13:25:49.184370 367 admission-controller.cc:1289]
> 504b3a2511f3cd0e:e27bec6b00000000] Final: agg_num_running=1,
> agg_num_queued=0, agg_mem_reserved
> =17.25 KB, local_host(local_mem_admitted=342.05 MB, num_admitted_running=1,
> num_queued=0, backend_mem_reserved=17.25 KB, topN_query_stats: queries=[0a42
> 32658ea48bc5:c015646900000000, 5b470ebea7782154:ea22bdb000000000,
> 554d88d8f812e22d:efbaf75200000000, aa4a301189a4d144:fdc17ff700000000],
> total_mem_consum
> ed=17.25 KB, fraction_of_pool_total_mem=1; pool_level_stats: num_running=4,
> min=0, max=17.25 KB, pool_total_mem=17.25 KB, average_per_query=4.31 KB)
> I1116 13:25:49.185214 367 impala-server.cc:2062]
> 504b3a2511f3cd0e:e27bec6b00000000] Registering query locations
> I1116 13:25:49.185261 367 coordinator.cc:149]
> 504b3a2511f3cd0e:e27bec6b00000000] Exec()
> query_id=504b3a2511f3cd0e:e27bec6b00000000 stmt=select min(ss_w
> holesale_cost) from tpcds_parquet.store_sales
> I1116 13:25:49.186172 367 coordinator.cc:473]
> 504b3a2511f3cd0e:e27bec6b00000000] starting execution on 3 backends for
> query_id=504b3a2511f3cd0e:e27bec6
> b00000000
> I1116 13:25:49.189028 32071 control-service.cc:142]
> 504b3a2511f3cd0e:e27bec6b00000000] ExecQueryFInstances():
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> coord=impala-ec2-centos74-m5-4xlarge-ondemand-018e.vpc.cloudera.com:27000
> #instances=5
> I1116 13:25:49.192133 367 coordinator.cc:532]
> 504b3a2511f3cd0e:e27bec6b00000000] started execution on 3 backends for
> query_id=504b3a2511f3cd0e:e27bec6b
> 00000000
> ...
> I1116 13:26:49.278456 32665 impala-server.cc:1320] UnregisterQuery():
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> I1116 13:26:49.278478 32665 coordinator.cc:672] ExecState: query
> id=504b3a2511f3cd0e:e27bec6b00000000 execution cancelled
> I1116 13:26:49.278493 32665 coordinator-backend-state.cc:921]
> query_id=504b3a2511f3cd0e:e27bec6b00000000 target backend=127.0.0.1:27000:
> Sending CancelQueryFInstances rpc
> ...
> {noformat}
> As far as when the instances finished - 13:25:55
> {noformat}
> tarmstrong@tarmstrong-box:~/tmplogs8/impala-asf-master-core-s3-DEBUG-673/logs/custom_cluster_tests/relevant$
> grep -h -R 504b3a2511f3cd0e.*'Instance' *
> I1116 13:25:49.189028 32071 control-service.cc:142]
> 504b3a2511f3cd0e:e27bec6b00000000] ExecQueryFInstances():
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> coord=impala-ec2-centos74-m5-4xlarge-ondemand-018e.vpc.cloudera.com:27000
> #instances=5
> I1116 13:25:55.125044 640 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000002] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000002 #in-flight=4 status=OK
> I1116 13:25:55.136926 642 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000004] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000004 #in-flight=3 status=OK
> I1116 13:25:55.139536 639 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000001] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000001 #in-flight=2 status=OK
> I1116 13:25:55.144346 641 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000003] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000003 #in-flight=1 status=OK
> I1116 13:26:49.278493 32665 coordinator-backend-state.cc:921]
> query_id=504b3a2511f3cd0e:e27bec6b00000000 target backend=127.0.0.1:27000:
> Sending CancelQueryFInstances rpc
> I1116 13:26:49.278884 32665 coordinator-backend-state.cc:921]
> query_id=504b3a2511f3cd0e:e27bec6b00000000 target backend=127.0.0.1:27002:
> Sending CancelQueryFInstances rpc
> I1116 13:26:49.278980 643 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000000] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000000 #in-flight=0 status=CANCELLED:
> Cancelled
> I1116 13:25:49.190251 32005 control-service.cc:142]
> 504b3a2511f3cd0e:e27bec6b00000000] ExecQueryFInstances():
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> coord=impala-ec2-centos74-m5-4xlarge-ondemand-018e.vpc.cloudera.com:27000
> #instances=4
> I1116 13:25:55.015192 645 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000006] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000006 #in-flight=3 status=OK
> I1116 13:25:55.015666 646 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000007] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000007 #in-flight=2 status=OK
> I1116 13:25:55.026870 644 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000005] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000005 #in-flight=1 status=OK
> I1116 13:25:55.034817 647 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000008] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000008 #in-flight=0 status=OK
> I1116 13:25:49.191587 31720 control-service.cc:142]
> 504b3a2511f3cd0e:e27bec6b00000000] ExecQueryFInstances():
> query_id=504b3a2511f3cd0e:e27bec6b00000000
> coord=impala-ec2-centos74-m5-4xlarge-ondemand-018e.vpc.cloudera.com:27000
> #instances=4
> I1116 13:25:55.641626 648 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b00000009] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b00000009 #in-flight=3 status=OK
> I1116 13:25:55.646078 649 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b0000000a] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b0000000a #in-flight=2 status=OK
> I1116 13:25:55.650281 652 query-state.cc:906]
> 504b3a2511f3cd0e:e27bec6b0000000b] Instance completed.
> instance_id=504b3a2511f3cd0e:e27bec6b0000000b #in-flight=1 status=OK
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org
For additional commands, e-mail: issues-all-h...@impala.apache.org
