IMPALA-4564,IMPALA-4565: mt_dop fixes for old aggs and joins

Fix a test bug where we need to skip nested types tests for the old aggs
and joins.

Fix a product bug where *eos is not initialised by the MT scan node.
This causes incorrect results when the calling ExecNode does not
initialise the eos variable, e.g. the sort node and the old agg and join
nodes.

Testing:
Added a test that reproduces the incorrect results with the sort node
when run under ASAN

Tested the mt_dop tests locally with old aggs and joins to ensure they
pass.

Change-Id: I48c50c8aa0c23710eb099fba252bc3c0cb74b313
Reviewed-on: http://gerrit.cloudera.org:8080/5302
Reviewed-by: Alex Behm <alex.b...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b3740612
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b3740612
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b3740612

Branch: refs/heads/master
Commit: b3740612065d742f7ddb96154da64c590867759e
Parents: 56f4d0f
Author: Tim Armstrong <tarmstr...@cloudera.com>
Authored: Wed Nov 30 21:53:27 2016 -0800
Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org>
Committed: Fri Dec 2 01:46:55 2016 +0000

----------------------------------------------------------------------
 be/src/exec/hdfs-scan-node-mt.cc                |  1 +
 .../QueryTest/mt-dop-parquet-nested.test        | 34 ++++++++++++++++++++
 .../queries/QueryTest/mt-dop-parquet.test       | 33 -------------------
 .../queries/QueryTest/mt-dop.test               | 19 +++++++++++
 tests/query_test/test_mt_dop.py                 |  6 ++++
 5 files changed, 60 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3740612/be/src/exec/hdfs-scan-node-mt.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-mt.cc b/be/src/exec/hdfs-scan-node-mt.cc
index bde9f81..50936b8 100644
--- a/be/src/exec/hdfs-scan-node-mt.cc
+++ b/be/src/exec/hdfs-scan-node-mt.cc
@@ -67,6 +67,7 @@ Status HdfsScanNodeMt::GetNext(RuntimeState* state, RowBatch* 
row_batch, bool* e
   RETURN_IF_ERROR(ExecDebugAction(TExecNodePhase::GETNEXT, state));
   RETURN_IF_CANCELLED(state);
   RETURN_IF_ERROR(QueryMaintenance(state));
+  *eos = false;
 
   DCHECK(scan_range_ == NULL || scanner_ != NULL);
   if (scan_range_ == NULL || scanner_->eos()) {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3740612/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-nested.test
----------------------------------------------------------------------
diff --git 
a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-nested.test
 
b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-nested.test
new file mode 100644
index 0000000..9be983d
--- /dev/null
+++ 
b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-nested.test
@@ -0,0 +1,34 @@
+====
+---- QUERY
+# IMPALA-4554: Memory corruption of nested collection with MT_DOP > 0.
+select id, cnt
+from functional_parquet.complextypestbl t,
+  (select count(item) cnt from t.int_array) v
+order by id
+limit 10
+---- RESULTS
+1,3
+2,3
+3,0
+4,0
+5,0
+6,0
+7,0
+8,1
+---- TYPES
+bigint,bigint
+====
+---- QUERY
+# IMPALA-4458: Test proper resource cleanup for cancelled fragments.
+# This test is duplicated from nested-types-subplan.test
+select c_custkey, c_mktsegment, o_orderkey, o_orderdate
+from tpch_nested_parquet.customer c, c.c_orders o
+where c_custkey = 1
+limit 3
+---- RESULTS
+1,regex:.*,regex:.*,regex:.*
+1,regex:.*,regex:.*,regex:.*
+1,regex:.*,regex:.*,regex:.*
+---- TYPES
+bigint,string,bigint,string
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3740612/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet.test
----------------------------------------------------------------------
diff --git 
a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet.test 
b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet.test
index 0523f1d..39ec4b3 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet.test
@@ -5,36 +5,3 @@ select * from functional_parquet.bad_metadata_len
 ---- CATCH
 Invalid metadata size in file footer
 ====
----- QUERY
-# IMPALA-4554: Memory corruption of nested collection with MT_DOP > 0.
-select id, cnt
-from functional_parquet.complextypestbl t,
-  (select count(item) cnt from t.int_array) v
-order by id
-limit 10
----- RESULTS
-1,3
-2,3
-3,0
-4,0
-5,0
-6,0
-7,0
-8,1
----- TYPES
-bigint,bigint
-====
----- QUERY
-# IMPALA-4458: Test proper resource cleanup for cancelled fragments.
-# This test is duplicated from nested-types-subplan.test
-select c_custkey, c_mktsegment, o_orderkey, o_orderdate
-from tpch_nested_parquet.customer c, c.c_orders o
-where c_custkey = 1
-limit 3
----- RESULTS
-1,regex:.*,regex:.*,regex:.*
-1,regex:.*,regex:.*,regex:.*
-1,regex:.*,regex:.*,regex:.*
----- TYPES
-bigint,string,bigint,string
-====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3740612/testdata/workloads/functional-query/queries/QueryTest/mt-dop.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/mt-dop.test 
b/testdata/workloads/functional-query/queries/QueryTest/mt-dop.test
index ac453ca..a46693a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/mt-dop.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/mt-dop.test
@@ -7,3 +7,22 @@ select count(*) from alltypes
 ---- TYPES
 BIGINT
 ====
+---- QUERY
+# IMPALA-4565: incorrect results because mt scan node does not set eos
+# correctly and sort node only gets the first row batch.
+set batch_size=1;
+select id
+from alltypestiny
+order by id
+---- TYPES
+int
+---- RESULTS
+0
+1
+2
+3
+4
+5
+6
+7
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3740612/tests/query_test/test_mt_dop.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index d05d6b4..ff60b60 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -21,6 +21,7 @@ import pytest
 
 from copy import deepcopy
 from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfOldAggsJoins
 from tests.common.test_vector import TestDimension
 from tests.common.test_vector import TestVector
 
@@ -88,3 +89,8 @@ class TestMtDopParquet(ImpalaTestSuite):
   def test_parquet(self, vector):
     vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
     self.run_test_case('QueryTest/mt-dop-parquet', vector)
+
+  @SkipIfOldAggsJoins.nested_types
+  def test_parquet_nested(self, vector):
+    vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
+    self.run_test_case('QueryTest/mt-dop-parquet-nested', vector)

Reply via email to