Tim Armstrong created IMPALA-5788:
-------------------------------------
Summary: Spilling aggregation crashes when grouping by
nondeterministic expression
Key: IMPALA-5788
URL: https://issues.apache.org/jira/browse/IMPALA-5788
Project: IMPALA
Issue Type: Bug
Components: Backend
Affects Versions: Impala 2.10.0
Reporter: Tim Armstrong
Priority: Blocker
This set of query options and query reliably crashes impala.
{code}
set buffer_pool_limit=5m;
set default_spillable_buffer_size=64k;
set num_nodes=1;
set buffer_pool_limit=5m; set default_spillable_buffer_size=64k; set
num_nodes=1; select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
from tpch_parquet.lineitem group by 1,2,3,4,5, random() limit 5;
{code}
{code}
(gdb) bt
#0 0x00007fcb1d167428 in __GI_raise (sig=sig@entry=6) at
../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007fcb1d16902a in __GI_abort () at abort.c:89
#2 0x00007fcb1f70cab9 in ?? () from
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#3 0x00007fcb1f8c0667 in ?? () from
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#4 0x00007fcb1f715f9f in JVM_handle_linux_signal () from
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#5 0x00007fcb1f709bf8 in ?? () from
/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
#6 <signal handler called>
#7 0x000000000189149c in boost::scoped_ptr<impala::HashTable>::get (this=0x18)
at toolchain/boost-1.57.0-p3/include/boost/smart_ptr/scoped_ptr.hpp:105
#8 0x000000000189b0b2 in impala::PartitionedAggregationNode::GetHashTable
(partition_idx=1, this=0x939e300) at
be/src/exec/partitioned-aggregation-node.h:444
#9 impala::PartitionedAggregationNode::EvalAndHashPrefetchGroup<false>
(this=0x939e300, batch=0x7fca79897e30, start_row_idx=0,
prefetch_mode=impala::TPrefetchMode::HT_BUCKET,
ht_ctx=0x55c93480) at be/src/exec/partitioned-aggregation-node-ir.cc:86
#10 0x000000000189a115 in
impala::PartitionedAggregationNode::ProcessBatch<false> (this=0x939e300,
batch=0x7fca79897e30, prefetch_mode=impala::TPrefetchMode::HT_BUCKET,
ht_ctx=0x55c93480)
at be/src/exec/partitioned-aggregation-node-ir.cc:56
#11 0x0000000001892cf2 in
impala::PartitionedAggregationNode::ProcessStream<false> (this=0x939e300,
input_stream=0xd059600) at be/src/exec/partitioned-aggregation-node.cc:1353
#12 0x0000000001887fca in
impala::PartitionedAggregationNode::BuildSpilledPartition (this=0x939e300,
built_partition=0x7fca79898178) at
be/src/exec/partitioned-aggregation-node.cc:1260
#13 0x000000000188767a in impala::PartitionedAggregationNode::NextPartition
(this=0x939e300) at be/src/exec/partitioned-aggregation-node.cc:1221
#14 0x000000000188068d in
impala::PartitionedAggregationNode::GetRowsFromPartition (this=0x939e300,
state=0x95b4400, row_batch=0x980a9f0) at
be/src/exec/partitioned-aggregation-node.cc:470
#15 0x000000000187fabc in impala::PartitionedAggregationNode::GetNextInternal
(this=0x939e300, state=0x95b4400, row_batch=0x980a9f0, eos=0x7fca798988ff)
at be/src/exec/partitioned-aggregation-node.cc:427
#16 0x000000000187ee21 in impala::PartitionedAggregationNode::GetNext
(this=0x939e300, state=0x95b4400, row_batch=0x980a9f0, eos=0x7fca798988ff)
at be/src/exec/partitioned-aggregation-node.cc:350
#17 0x0000000001496b04 in impala::FragmentInstanceState::ExecInternal
(this=0xb97e580) at be/src/runtime/fragment-instance-state.cc:269
#18 0x000000000149447b in impala::FragmentInstanceState::Exec (this=0xb97e580)
at be/src/runtime/fragment-instance-state.cc:89
#19 0x000000000145a98a in impala::QueryState::ExecFInstance (this=0x4d91f200,
fis=0xb97e580) at be/src/runtime/query-state.cc:366
#20 0x00000000014594ba in impala::QueryState::<lambda()>::operator()(void)
const (__closure=0x7fca79898d68) at be/src/runtime/query-state.cc:334
#21 0x000000000145b63b in
boost::detail::function::void_function_obj_invoker0<impala::QueryState::StartFInstances()::<lambda()>,
void>::invoke(boost::detail::function::function_buffer &) (
function_obj_ptr=...) at
toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
#22 0x00000000013cdfe8 in boost::function0<void>::operator()
(this=0x7fca79898d60) at
toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
#23 0x0000000001694aef in impala::Thread::SuperviseThread(std::string const&,
std::string const&, boost::function<void ()>, impala::Promise<long>*) (
name="exec-finstance (finst:984beb1b1c07ad43:394fe8fd00000000)",
category="fragment-execution", functor=..., thread_started=0x7fca78597d80) at
be/src/util/thread.cc:330
#24 0x000000000169d5f6 in boost::_bi::list4<boost::_bi::value<std::string>,
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string
const&, std::string const&, boost::function<void ()>, impala::Promise<long>*),
boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&,
std::string const&, boost::function<void ()>, impala::Promise<long>*),
boost::_bi::list0&, int) (this=0x57217bc0,
f=@0x57217bb8: 0x16947d0 <impala::Thread::SuperviseThread(std::string
const&, std::string const&, boost::function<void ()>, impala::Promise<long>*)>,
a=...)
at toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
#25 0x000000000169d539 in boost::_bi::bind_t<void, void (*)(std::string const&,
std::string const&, boost::function<void ()>, impala::Promise<long>*),
boost::_bi::list4<boost::_bi::value<std::string>,
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::Promise<long>*> > >::operator()() (this=0x57217bb8)
at toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
#26 0x000000000169d4fc in boost::detail::thread_data<boost::_bi::bind_t<void,
void (*)(std::string const&, std::string const&, boost::function<void ()>,
impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>,
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x57217a00)
at toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
#27 0x0000000001b78faa in thread_proxy ()
#28 0x00007fcb1d5036ba in start_thread (arg=0x7fca79899700) at
pthread_create.c:333
#29 0x00007fcb1d2393dd in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:109
{code}
This is caused by the fix for IMPALA-2708. Before that we always repartitioned
so all partitions were initialised. Now, however, only the partition that we
expect the rows previously hashed to is populated in memory, so if the rows
hash to something different we have a problem.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
