[
https://issues.apache.org/jira/browse/IMPALA-6595?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Tim Armstrong resolved IMPALA-6595.
-----------------------------------
Resolution: Fixed
Fix Version/s: Impala 2.12.0
> Hit crash freeing buffer in NljBuilder::Close()
> -----------------------------------------------
>
> Key: IMPALA-6595
> URL: https://issues.apache.org/jira/browse/IMPALA-6595
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 2.8.0, Impala 2.9.0, Impala 2.10.0, Impala
> 2.11.0, Impala 3.0, Impala 2.12.0
> Reporter: Tim Armstrong
> Assignee: Tim Armstrong
> Priority: Blocker
> Labels: crash
> Fix For: Impala 3.0, Impala 2.12.0
>
>
> I think this is related to the ExchangeNode's use of buffers.
> Repro: increase the mem_limit for this test:
> {noformat}
> tarmstrong@tarmstrong-ubuntu:~/Impala$ git diff
> diff --git
> a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test
>
> b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test
> index a6b3cae..7e8e862 100644
> ---
> a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test
> +++
> b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test
> @@ -1,7 +1,7 @@
> ====
> ---- QUERY
> # IMPALA-2207: Analytic eval node feeding into build side of nested loop
> join.
> -set mem_limit=200m;
> +set mem_limit=220m;
> select straight_join * from (values(1 id), (2), (3)) v1,
> (select *, count(*) over() from tpch.lineitem where l_orderkey < 100000) v2
> order by id, l_orderkey, l_partkey, l_suppkey, l_linenumber
> {noformat}
> Loop the test:
> {noformat}
> while impala-py.test tests/query_test/test_join_queries.py -k
> 'test_single_node_nested_loop_joins_exhaustive'
> --workload_exploration_strategy=functional-query:exhaustive; do :; done
> {noformat}
> Boom:
> {noformat}
> (gdb) bt
> #0 0x00007fcffc535428 in __GI_raise (sig=sig@entry=6) at
> ../sysdeps/unix/sysv/linux/raise.c:54
> #1 0x00007fcffc53702a in __GI_abort () at abort.c:89
> #2 0x00007fcfff479069 in os::abort(bool) (dump_core=<optimized out>) at
> /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os/linux/vm/os_linux.cpp:1509
> #3 0x00007fcfff62c997 in VMError::report_and_die()
> (this=this@entry=0x7fcf8b201f50) at
> /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/share/vm/utilities/vmError.cpp:1060
> #4 0x00007fcfff48254f in JVM_handle_linux_signal(int, siginfo_t*, void*,
> int) (sig=sig@entry=11, info=info@entry=0x7fcf8b2021f0,
> ucVoid=ucVoid@entry=0x7fcf8b2020c0,
> abort_if_unrecognized=abort_if_unrecognized@entry=1) at
> /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os_cpu/linux_x86/vm/os_linux_x86.cpp:541
> #5 0x00007fcfff4761a8 in signalHandler(int, siginfo_t*, void*) (sig=11,
> info=0x7fcf8b2021f0, uc=0x7fcf8b2020c0) at
> /build/openjdk-8-W2Qe27/openjdk-8-8u151-b12/src/hotspot/src/os/linux/vm/os_linux.cpp:4346
> #6 0x00007fcffc8db390 in <signal handler called> () at
> /lib/x86_64-linux-gnu/libpthread.so.0
> #7 0x00007fcffc8d3d44 in __GI___pthread_mutex_lock (mutex=0xd8) at
> ../nptl/pthread_mutex_lock.c:67
> #8 0x0000000000d7f680 in
> impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*,
> impala::BufferPool::BufferHandle*) (m=0xd8) at
> toolchain/boost-1.57.0-p3/include/boost/thread/pthread/mutex.hpp:62
> #9 0x0000000000d7f680 in
> impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*,
> impala::BufferPool::BufferHandle*) (this=0xd8) at
> toolchain/boost-1.57.0-p3/include/boost/thread/pthread/mutex.hpp:116
> #10 0x0000000000d7f680 in
> impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*,
> impala::BufferPool::BufferHandle*) (m_=..., this=<synthetic pointer>)
> at toolchain/boost-1.57.0-p3/include/boost/thread/lock_guard.hpp:38
> #11 0x0000000000d7f680 in
> impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*,
> impala::BufferPool::BufferHandle*) (len=2097152, this=0x0) at
> be/src/runtime/bufferpool/buffer-pool-internal.h:262
> #12 0x0000000000d7f680 in
> impala::BufferPool::FreeBuffer(impala::BufferPool::ClientHandle*,
> impala::BufferPool::BufferHandle*) (this=<optimized out>, client=<optimized
> out>, handle=handle@entry=0xb971418)
> at be/src/runtime/bufferpool/buffer-pool.cc:254
> #13 0x0000000000b66fc2 in impala::RowBatch::FreeBuffers()
> (this=this@entry=0x1bfbfea0) at be/src/runtime/row-batch.cc:425
> #14 0x0000000000b67002 in impala::RowBatch::~RowBatch() (this=0x1bfbfea0,
> __in_chrg=<optimized out>) at be/src/runtime/row-batch.cc:220
> #15 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=<optimized out>, __ptr=0x1bfbfea0) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:76
> #16 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=0xa462628, __in_chrg=<optimized out>) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/unique_ptr.h:236
> #17 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (__pointer=0xa462628) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:93
> #18 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (__last=0xa462658, __first=0xa462628) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:103
> #19 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (__last=0xa462658, __first=0xa462000) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:126
> #20 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (__last=0xa462658, __first=0xa462000) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_construct.h:151
> #21 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=<optimized out>, __pos=0xa462000) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_vector.h:1438
> #22 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=0xf84ada8) at
> /home/tarmstrong/Impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_vector.h:1212
> #23 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=0xf84ad98) at be/src/exec/row-batch-cache.h:64
> #24 0x000000000107ecc0 in impala::NljBuilder::Close(impala::RuntimeState*)
> (this=0xf84ad20, state=0x111b8300) at
> be/src/exec/nested-loop-join-builder.cc:83
> #25 0x0000000000e38050 in
> impala::NestedLoopJoinNode::Close(impala::RuntimeState*) (this=0x111b5900,
> state=0x111b8300) at be/src/exec/nested-loop-join-node.cc:141
> #26 0x0000000000dc1041 in impala::ExecNode::Close(impala::RuntimeState*)
> (this=0x111b5400, state=0x111b8300) at be/src/exec/exec-node.cc:194
> #27 0x0000000000baa02a in impala::FragmentInstanceState::Close()
> (this=this@entry=0x9102000) at be/src/runtime/fragment-instance-state.cc:310
> #28 0x0000000000bad801 in impala::FragmentInstanceState::Exec()
> (this=this@entry=0x9102000) at be/src/runtime/fragment-instance-state.cc:95
> #29 0x0000000000b9c00a in
> impala::QueryState::ExecFInstance(impala::FragmentInstanceState*)
> (this=0x9129800, fis=0x9102000) at be/src/runtime/query-state.cc:388
> #30 0x0000000000d5be8f in impala::Thread::SuperviseThread(std::string const&,
> std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long>*) (this=0x7fcf8b202d50)
> at
> toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
> #31 0x0000000000d5be8f in impala::Thread::SuperviseThread(std::string const&,
> std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long>*) (name=..., category=..., functor=...,
> parent_thread_info=0x7fcf8c203ac0, thread_started=0x7fcf8c202fa0) at
> be/src/util/thread.cc:356
> #32 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void,
> void (*)(std::string const&, std::string const&, boost::function<void ()>,
> impala::ThreadDebugInfo const*, impala::Promise<long>*),
> boost::_bi::list5<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long>*> > > >::run() (f=@0x91023b8:
> 0xd5bb90 <impala::Thread::SuperviseThread(std::string const&, std::string
> const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
> impala::Promise<long>*)>, a=<synthetic pointer>, this=0x91023c0) at
> toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:525
> #33 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void,
> void (*)(std::string const&, std::string const&, boost::function<void ()>,
> impala::ThreadDebugInfo const*, impala::Promise<long>*),
> boost::_bi::list5<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x91023b8) at
> toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
> #34 0x0000000000d5c68a in boost::detail::thread_data<boost::_bi::bind_t<void,
> void (*)(std::string const&, std::string const&, boost::function<void ()>,
> impala::ThreadDebugInfo const*, impala::Promise<long>*),
> boost::_bi::list5<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
> boost::_bi::value<impala::ThreadDebugInfo*>,
> boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x9102200) at
> toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
> #35 0x00000000012d4e7a in thread_proxy ()
> #36 0x00007fcffc8d16ba in start_thread (arg=0x7fcf8b203700) at
> pthread_create.c:333
> #37 0x00007fcffc60741d in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
> {noformat}
> [~kwho] I think you fixed a similar bug elsewhere in a join, so I'll assign
> to you initially.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
