Move tests related to the old join node. No tests were added/dropped or modified. They are consolidated into fewer .test files.
Change-Id: Idda4b34b5e6e9b5012b177a4c00077aa7fec394c Reviewed-on: http://gerrit.cloudera.org:8080/8153 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c1781b73 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c1781b73 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c1781b73 Branch: refs/heads/master Commit: c1781b73b35899ed9374befa2e04e14747224be7 Parents: 4d49099 Author: Alex Behm <[email protected]> Authored: Tue Sep 26 22:47:55 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Sep 28 18:36:17 2017 +0000 ---------------------------------------------------------------------- .../queries/QueryTest/exprs.test | 12 +++ .../queries/QueryTest/inline-view.test | 19 ++++ .../queries/QueryTest/joins-partitioned.test | 36 -------- .../queries/QueryTest/runtime_row_filters.test | 95 ++++++++++++++++++++ .../QueryTest/runtime_row_filters_phj.test | 94 ------------------- tests/query_test/test_join_queries.py | 3 - tests/query_test/test_runtime_filters.py | 3 - 7 files changed, 126 insertions(+), 136 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/exprs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test index 811a169..552dea2 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test @@ -876,6 +876,18 @@ order by int_col limit 20 int,string ==== ---- QUERY +# IMPALA-2495: In predicate containing a TupleIsNullPredicate +select count(id) from functional.alltypestiny t1 +left join (select coalesce(1, 10) as int_col + from functional.alltypessmall) t2 +on t1.id = t2.int_col +where t2.int_col in (t2.int_col, 10) +---- RESULTS +100 +---- TYPES +BIGINT +==== +---- QUERY select count(*) from alltypesagg where true in (bool_col, tinyint_col) ---- RESULTS http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/inline-view.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test index 7c7616d..dd0094b 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test +++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test @@ -499,3 +499,22 @@ true ---- TYPES boolean ==== +---- QUERY +# IMPALA-1987: Equi-join predicates of outer joins contain TupleIsNullPredicate exprs. +select t1.int_col, t2.int_col, t3.id +from alltypestiny t1 left outer join + (select coalesce(int_col, 384) as int_col from alltypestiny) t2 + on t1.int_col = t2.int_col +left outer join + (select 0 as id from alltypestiny) t3 + on t1.int_col = t3.id +order by 1 limit 5 +---- RESULTS +0,0,0 +0,0,0 +0,0,0 +0,0,0 +0,0,0 +---- TYPES +INT,INT,TINYINT +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test b/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test deleted file mode 100644 index d8acb9a..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test +++ /dev/null @@ -1,36 +0,0 @@ -==== ----- QUERY -# Regression test for IMPALA-1987. Equi-join predicates of outer joins contain -# TupleIsNullPredicate exprs. -# TODO: Move back to inline-views.test when this works with the non-partitioned hash -# join (IMPALA-2375). -select t1.int_col, t2.int_col, t3.id -from alltypestiny t1 left outer join - (select coalesce(int_col, 384) as int_col from alltypestiny) t2 - on t1.int_col = t2.int_col -left outer join - (select 0 as id from alltypestiny) t3 - on t1.int_col = t3.id -order by 1 limit 5 ----- RESULTS -0,0,0 -0,0,0 -0,0,0 -0,0,0 -0,0,0 ----- TYPES -INT,INT,TINYINT -==== ----- QUERY -# Regression test for IMPALA-2495: Crash: impala::InPredicate::SetLookupPrepare -# TODO: Move back to exprs.test when this works with the non-partitioned hash join -# (IMPALA-2375). -select count(id) from functional.alltypestiny t1 -left join (select coalesce(1, 10) as int_col -from functional.alltypessmall) t2 on t1.id = t2.int_col -where t2.int_col in (t2.int_col, 10); ----- RESULTS -100 ----- TYPES -BIGINT -==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test index 7cb5884..3af07af 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test +++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test @@ -322,10 +322,12 @@ from tpch_parquet.lineitem l1 join tpch_parquet.lineitem l2 ---- QUERY +#################################################### # Test case 15: filter with a predicate that has different decimal precision between # lhs expr and rhs expr. # IMPALA-5597: Runtime filter should be generated and assigned successfully when the # source expr and target expr have different decimal types. +#################################################### select count(*) from tpch_parquet.lineitem @@ -334,3 +336,96 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice; ---- RESULTS 1 ==== + + +---- QUERY +#################################################### +# Test case 16: PHJ nodes that spill should still produce filters. +# Run this for Parquet only to avoid variable memory +# consumption / spilling behaviour. +#################################################### + +SET BUFFER_POOL_LIMIT=40m; +SET RUNTIME_FILTER_MODE=GLOBAL; +SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_BLOOM_FILTER_SIZE=16M; +select STRAIGHT_JOIN count(a.l_comment) + from tpch_parquet.lineitem a join tpch_parquet.lineitem b + on a.l_comment = b.l_comment; +---- RESULTS +51495713 +---- TYPES +BIGINT +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: [1-9]\d* .* +row_regex: .*Rows processed: 16.38K.* +row_regex: .*Rows rejected: 0 .* +row_regex: .*1 of 1 Runtime Filter Published.* +==== + + +---- QUERY +#################################################### +# Test case 17: Filters are still effective inside subplans +# (in certain cases). The query has a HJ-scan pair inside a +# subplan (on the LHS). +#################################################### +SET RUNTIME_FILTER_MODE=GLOBAL; +SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +select straight_join count(1) + from alltypes a join complextypestbl b + # b.id + 10 confuses planner, so it doesn't think it + # can transitively push a.id < 20 to scan of a. + on a.id = b.id + 10 join b.int_array where b.id < 10 +---- RESULTS +10 +---- RUNTIME_PROFILE +row_regex: .*1 of 1 Runtime Filter Published.* +row_regex: .*Rows rejected: 2.43K \(2432\).* +==== + + +---- QUERY +#################################################### +# Test case 18: Filters will not be used if they exceed +# the configured memory limit on the coordinator. +# To test this, we need to construct a query where memory +# consumption on the coordinator exceeds MEM_LIMIT, but +# not on the backends (because otherwise they will disable +# the filters through another path). We set MEM_LIMIT to +# the minimum possible then set filter size to be roughly +# half that: since the coordinator must aggregate two of +# these filters (and indeed must create one as well), it +# will exceed the memory limit. This is checked for +# indirectly by confirming that the filter had no effect +# (when usually it would be selective). +#################################################### +SET RUNTIME_FILTER_MODE=GLOBAL; +SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_MIN_SIZE=128MB; +SET RUNTIME_FILTER_MAX_SIZE=500MB; +SET MEM_LIMIT=140MB; +select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b + on a.month = b.id and b.int_col = -3 +---- RESULTS +---- RUNTIME_PROFILE +row_regex: .*Filter 0 \(128.00 MB\).* +row_regex: .*Files processed: 8.* +row_regex: .*Files rejected: 0.* +==== +---- QUERY +# Confirm that with broadcast join, memory limit is not hit. +SET RUNTIME_FILTER_MODE=GLOBAL; +SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_MIN_SIZE=128MB; +SET RUNTIME_FILTER_MAX_SIZE=500MB; +# Allocate enough memory for the join + filter + scan +SET MEM_LIMIT=170MB; +select STRAIGHT_JOIN * from alltypes a join [BROADCAST] alltypes b + on a.month = b.id and b.int_col = -3 +---- RESULTS +---- RUNTIME_PROFILE +row_regex: .*Filter 0 \(128.00 MB\).* +row_regex: .*Files processed: 8.* +row_regex: .*Files rejected: 8.* +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test deleted file mode 100644 index 66391a5..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test +++ /dev/null @@ -1,94 +0,0 @@ -==== ----- QUERY -# Test cases that are only valid for PHJ joins are in this file. - -#################################################### -# Test case 1: PHJ nodes that spill should still produce filters. -# Run this for Parquet only to avoid variable memory -# consumption / spilling behaviour. -#################################################### - -SET BUFFER_POOL_LIMIT=40m; -SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; -SET RUNTIME_BLOOM_FILTER_SIZE=16M; -select STRAIGHT_JOIN count(a.l_comment) - from tpch_parquet.lineitem a join tpch_parquet.lineitem b - on a.l_comment = b.l_comment; ----- RESULTS -51495713 ----- TYPES -BIGINT ----- RUNTIME_PROFILE -row_regex: .*SpilledPartitions: [1-9]\d* .* -row_regex: .*Rows processed: 16.38K.* -row_regex: .*Rows rejected: 0 .* -row_regex: .*1 of 1 Runtime Filter Published.* -==== - - ----- QUERY -#################################################### -# Test case 2: Filters are still effective inside subplans -# (in certain cases). The query has a HJ-scan pair inside a -# subplan (on the LHS). -#################################################### -SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; -select straight_join count(1) - from alltypes a join complextypestbl b - # b.id + 10 confuses planner, so it doesn't think it - # can transitively push a.id < 20 to scan of a. - on a.id = b.id + 10 join b.int_array where b.id < 10 ----- RESULTS -10 ----- RUNTIME_PROFILE -row_regex: .*1 of 1 Runtime Filter Published.* -row_regex: .*Rows rejected: 2.43K \(2432\).* -==== - - ----- QUERY -#################################################### -# Test case 3: Filters will not be used if they exceed -# the configured memory limit on the coordinator. -# To test this, we need to construct a query where memory -# consumption on the coordinator exceeds MEM_LIMIT, but -# not on the backends (because otherwise they will disable -# the filters through another path). We set MEM_LIMIT to -# the minimum possible then set filter size to be roughly -# half that: since the coordinator must aggregate two of -# these filters (and indeed must create one as well), it -# will exceed the memory limit. This is checked for -# indirectly by confirming that the filter had no effect -# (when usually it would be selective). -#################################################### -SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; -SET RUNTIME_FILTER_MIN_SIZE=128MB; -SET RUNTIME_FILTER_MAX_SIZE=500MB; -SET MEM_LIMIT=140MB; -select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b - on a.month = b.id and b.int_col = -3 ----- RESULTS ----- RUNTIME_PROFILE -row_regex: .*Filter 0 \(128.00 MB\).* -row_regex: .*Files processed: 8.* -row_regex: .*Files rejected: 0.* -==== ----- QUERY -# Confirm that with broadcast join, memory limit is not hit. -SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; -SET RUNTIME_FILTER_MIN_SIZE=128MB; -SET RUNTIME_FILTER_MAX_SIZE=500MB; -# Allocate enough memory for the join + filter + scan -SET MEM_LIMIT=170MB; -select STRAIGHT_JOIN * from alltypes a join [BROADCAST] alltypes b - on a.month = b.id and b.int_col = -3 ----- RESULTS ----- RUNTIME_PROFILE -row_regex: .*Filter 0 \(128.00 MB\).* -row_regex: .*Files processed: 8.* -row_regex: .*Files rejected: 8.* -==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/tests/query_test/test_join_queries.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py index a8f2be0..30c659c 100644 --- a/tests/query_test/test_join_queries.py +++ b/tests/query_test/test_join_queries.py @@ -61,9 +61,6 @@ class TestJoinQueries(ImpalaTestSuite): new_vector.get_value('exec_option')['num_nodes'] = 1 self.run_test_case('QueryTest/single-node-joins-with-limits-exhaustive', new_vector) - def test_partitioned_joins(self, vector): - self.run_test_case('QueryTest/joins-partitioned', vector) - @SkipIfS3.hbase @SkipIfADLS.hbase @SkipIfIsilon.hbase http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/tests/query_test/test_runtime_filters.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_runtime_filters.py b/tests/query_test/test_runtime_filters.py index 7710f43..82b5792 100644 --- a/tests/query_test/test_runtime_filters.py +++ b/tests/query_test/test_runtime_filters.py @@ -62,6 +62,3 @@ class TestRuntimeRowFilters(ImpalaTestSuite): def test_row_filters(self, vector): self.run_test_case('QueryTest/runtime_row_filters', vector) - - def test_row_filters_phj_only(self, vector): - self.run_test_case('QueryTest/runtime_row_filters_phj', vector)
