Tim Armstrong created IMPALA-5788:
-------------------------------------

             Summary: Spilling aggregation crashes when grouping by 
nondeterministic expression
                 Key: IMPALA-5788
                 URL: https://issues.apache.org/jira/browse/IMPALA-5788
             Project: IMPALA
          Issue Type: Bug
          Components: Backend
    Affects Versions: Impala 2.10.0
            Reporter: Tim Armstrong
            Priority: Blocker


This set of query options and query reliably crashes impala.
{code}
set buffer_pool_limit=5m;
set default_spillable_buffer_size=64k;
set num_nodes=1;
set buffer_pool_limit=5m; set default_spillable_buffer_size=64k; set 
num_nodes=1; select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment 
from tpch_parquet.lineitem group by 1,2,3,4,5, random() limit 5;
{code}

{code}
(gdb) bt
#0  0x00007fcb1d167428 in __GI_raise (sig=sig@entry=6) at 
../sysdeps/unix/sysv/linux/raise.c:54
#1  0x00007fcb1d16902a in __GI_abort () at abort.c:89
#2  0x00007fcb1f70cab9 in ?? () from 
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#3  0x00007fcb1f8c0667 in ?? () from 
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#4  0x00007fcb1f715f9f in JVM_handle_linux_signal () from 
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#5  0x00007fcb1f709bf8 in ?? () from 
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#6  <signal handler called>
#7  0x000000000189149c in boost::scoped_ptr<impala::HashTable>::get (this=0x18) 
at toolchain/boost-1.57.0-p3/include/boost/smart_ptr/scoped_ptr.hpp:105
#8  0x000000000189b0b2 in impala::PartitionedAggregationNode::GetHashTable 
(partition_idx=1, this=0x939e300) at 
be/src/exec/partitioned-aggregation-node.h:444
#9  impala::PartitionedAggregationNode::EvalAndHashPrefetchGroup<false> 
(this=0x939e300, batch=0x7fca79897e30, start_row_idx=0, 
prefetch_mode=impala::TPrefetchMode::HT_BUCKET, 
    ht_ctx=0x55c93480) at be/src/exec/partitioned-aggregation-node-ir.cc:86
#10 0x000000000189a115 in 
impala::PartitionedAggregationNode::ProcessBatch<false> (this=0x939e300, 
batch=0x7fca79897e30, prefetch_mode=impala::TPrefetchMode::HT_BUCKET, 
ht_ctx=0x55c93480)
    at be/src/exec/partitioned-aggregation-node-ir.cc:56
#11 0x0000000001892cf2 in 
impala::PartitionedAggregationNode::ProcessStream<false> (this=0x939e300, 
input_stream=0xd059600) at be/src/exec/partitioned-aggregation-node.cc:1353
#12 0x0000000001887fca in 
impala::PartitionedAggregationNode::BuildSpilledPartition (this=0x939e300, 
built_partition=0x7fca79898178) at 
be/src/exec/partitioned-aggregation-node.cc:1260
#13 0x000000000188767a in impala::PartitionedAggregationNode::NextPartition 
(this=0x939e300) at be/src/exec/partitioned-aggregation-node.cc:1221
#14 0x000000000188068d in 
impala::PartitionedAggregationNode::GetRowsFromPartition (this=0x939e300, 
state=0x95b4400, row_batch=0x980a9f0) at 
be/src/exec/partitioned-aggregation-node.cc:470
#15 0x000000000187fabc in impala::PartitionedAggregationNode::GetNextInternal 
(this=0x939e300, state=0x95b4400, row_batch=0x980a9f0, eos=0x7fca798988ff)
    at be/src/exec/partitioned-aggregation-node.cc:427
#16 0x000000000187ee21 in impala::PartitionedAggregationNode::GetNext 
(this=0x939e300, state=0x95b4400, row_batch=0x980a9f0, eos=0x7fca798988ff)
    at be/src/exec/partitioned-aggregation-node.cc:350
#17 0x0000000001496b04 in impala::FragmentInstanceState::ExecInternal 
(this=0xb97e580) at be/src/runtime/fragment-instance-state.cc:269
#18 0x000000000149447b in impala::FragmentInstanceState::Exec (this=0xb97e580) 
at be/src/runtime/fragment-instance-state.cc:89
#19 0x000000000145a98a in impala::QueryState::ExecFInstance (this=0x4d91f200, 
fis=0xb97e580) at be/src/runtime/query-state.cc:366
#20 0x00000000014594ba in impala::QueryState::<lambda()>::operator()(void) 
const (__closure=0x7fca79898d68) at be/src/runtime/query-state.cc:334
#21 0x000000000145b63b in 
boost::detail::function::void_function_obj_invoker0<impala::QueryState::StartFInstances()::<lambda()>,
 void>::invoke(boost::detail::function::function_buffer &) (
    function_obj_ptr=...) at 
toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
#22 0x00000000013cdfe8 in boost::function0<void>::operator() 
(this=0x7fca79898d60) at 
toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
#23 0x0000000001694aef in impala::Thread::SuperviseThread(std::string const&, 
std::string const&, boost::function<void ()>, impala::Promise<long>*) (
    name="exec-finstance (finst:984beb1b1c07ad43:394fe8fd00000000)", 
category="fragment-execution", functor=..., thread_started=0x7fca78597d80) at 
be/src/util/thread.cc:330
#24 0x000000000169d5f6 in boost::_bi::list4<boost::_bi::value<std::string>, 
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, 
boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string 
const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), 
boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, 
std::string const&, boost::function<void ()>, impala::Promise<long>*), 
boost::_bi::list0&, int) (this=0x57217bc0, 
    f=@0x57217bb8: 0x16947d0 <impala::Thread::SuperviseThread(std::string 
const&, std::string const&, boost::function<void ()>, impala::Promise<long>*)>, 
a=...)
    at toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
#25 0x000000000169d539 in boost::_bi::bind_t<void, void (*)(std::string const&, 
std::string const&, boost::function<void ()>, impala::Promise<long>*), 
boost::_bi::list4<boost::_bi::value<std::string>, 
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, 
boost::_bi::value<impala::Promise<long>*> > >::operator()() (this=0x57217bb8)
    at toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
#26 0x000000000169d4fc in boost::detail::thread_data<boost::_bi::bind_t<void, 
void (*)(std::string const&, std::string const&, boost::function<void ()>, 
impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, 
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, 
boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x57217a00)
    at toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
#27 0x0000000001b78faa in thread_proxy ()
#28 0x00007fcb1d5036ba in start_thread (arg=0x7fca79899700) at 
pthread_create.c:333
#29 0x00007fcb1d2393dd in clone () at 
../sysdeps/unix/sysv/linux/x86_64/clone.S:109
{code}

This is caused by the fix for IMPALA-2708. Before that we always repartitioned 
so all partitions were initialised. Now, however, only the partition that we 
expect the rows previously hashed to is populated in memory, so if the rows 
hash to something different we have a problem.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to