Repository: incubator-impala Updated Branches: refs/heads/master e4f585240 -> 839c45777
IMPALA-6004: Fix test_row_filters failure on ASAN 'Test case 16' in test_row_filters has been failing occasionaly on ASAN as the runtime filters are not generated within the specified RUNTIME_FILTER_WAIT_TIME_MS. The fix is to increase RUNTIME_FILTER_WAIT_TIME_MS. This patch updates all of the tests in test_row_filters to use the same timeout, which is set to a higher value for ASAN builds. Change-Id: Ia098735594b36a72f02bf7edd051171689618051 Reviewed-on: http://gerrit.cloudera.org:8080/8358 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e7b1c321 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e7b1c321 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e7b1c321 Branch: refs/heads/master Commit: e7b1c321033c1201ee88c9695c44535bd02a2f4e Parents: e4f5852 Author: Thomas Tauber-Marshall <[email protected]> Authored: Mon Oct 23 11:33:41 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Oct 25 18:56:00 2017 +0000 ---------------------------------------------------------------------- .../queries/QueryTest/runtime_row_filters.test | 46 ++++++++++---------- tests/query_test/test_runtime_filters.py | 6 ++- 2 files changed, 28 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e7b1c321/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test index 3af07af..818a8d5 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test +++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test @@ -8,7 +8,7 @@ # With filtering, expect 618 / 3 = 206 rows. #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=OFF; select STRAIGHT_JOIN count(*) from alltypes p join [BROADCAST] alltypestiny b on p.id = b.id and b.month = 1 and b.string_col = "1" @@ -19,7 +19,7 @@ row_regex: .*RowsRead: 2.43K .* ==== ---- QUERY # Now turn on local filtering: we expect to see a reduction in scan volume. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes p join [BROADCAST] alltypestiny b on p.id = b.id and b.month = 1 and b.string_col = "1" @@ -39,7 +39,7 @@ row_regex: .*Rows rejected: 2.43K .* # Local mode. Filters won't be propagated to scan, so scans will read all rows. # Still give enough time for filters to show up (even if they won't) -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes p join [SHUFFLE] alltypestiny b on p.id = b.int_col and b.month = 1 and b.string_col = "1" @@ -50,7 +50,7 @@ row_regex: .*RowsRead: 2.43K .* ==== ---- QUERY # Shuffle join, global mode. Expect filters to be propagated. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes p join [SHUFFLE] alltypestiny b on p.id = b.int_col and b.month = 1 and b.string_col = "1" @@ -71,7 +71,7 @@ row_regex: .*Rows rejected: 2.43K .* # Local mode. Only the left-most scan will receive its filter, but since the scan of 'b' # will not, the lack of predicates means there is no filter effect. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypes b @@ -85,7 +85,7 @@ row_regex: .*Files rejected: 0 .* ---- QUERY # Global mode. Scan of 'b' will receive highly effective filter, and will propagate that # to left-most scan. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypes b @@ -106,7 +106,7 @@ row_regex: .*Files rejected: 8 .* # With filtering, expect 0 rows. #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=10000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypestiny b @@ -124,7 +124,7 @@ row_regex: .*Rows rejected: 2.43K .* #################################################### # Local mode. Coordinator should report 0 filter updates received. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypestiny b @@ -137,7 +137,7 @@ row_regex: .*Rows rejected: 2.43K .* ==== ---- QUERY # Global mode. Coordinator should report 0 filter updates received. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypestiny b @@ -156,7 +156,7 @@ row_regex: .*Rows rejected: 2.43K .* #################################################### # Local mode. Coordinator should report 0 filter updates received. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes a join [SHUFFLE] alltypestiny b @@ -168,7 +168,7 @@ row_regex: .*FiltersReceived: 0 .* ==== ---- QUERY # Global mode. Coordinator should report 1 filter updates per backend. -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a join [SHUFFLE] alltypestiny b @@ -184,7 +184,7 @@ row_regex: .*FiltersReceived: 3 .* # Test case 7: filters with target exprs bound by > 1 slotref #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=LOCAL; select STRAIGHT_JOIN count(*) from alltypes a join [BROADCAST] alltypestiny b @@ -201,7 +201,7 @@ row_regex: .*Rows rejected: 2.43K .* # Test case 8: filters do not pass through LOJ. #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a LEFT OUTER join alltypestiny b @@ -218,7 +218,7 @@ row_regex: .*RowsReturned: 2.43K .* # Test case 9: filters do pass through ROJ. #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a RIGHT OUTER join alltypestiny b @@ -235,7 +235,7 @@ row_regex: .*Rows rejected: 2.43K .* # Test case 10: filters do not pass through FOJ. #################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select STRAIGHT_JOIN count(*) from alltypes a FULL OUTER join alltypestiny b @@ -254,7 +254,7 @@ row_regex: .*RowsReturned: 2.43K .* #################################################### SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; with l as (select l_orderkey from tpch_parquet.lineitem UNION ALL select l_orderkey from tpch_parquet.lineitem UNION ALL select l_orderkey from tpch_parquet.lineitem UNION ALL @@ -276,7 +276,7 @@ row_regex: .*Filter 0 \(16.00 MB\).* # Test case 12: filter with both remote and local targets ################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select straight_join count(*) from alltypes a join [BROADCAST] alltypessmall c @@ -292,7 +292,7 @@ from alltypes a join [BROADCAST] alltypessmall c # Test case 13: filter with multiple remote targets ################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; select straight_join count(*) from alltypes a join [SHUFFLE] alltypessmall c @@ -310,7 +310,7 @@ from alltypes a join [SHUFFLE] alltypessmall c # mem_limit is calibrated so that the query fails if allocations are not freed. ################################################### -SET RUNTIME_FILTER_WAIT_TIME_MS=100000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MODE=GLOBAL; SET MEM_LIMIT=200MB; select straight_join count(*) @@ -347,7 +347,7 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice; SET BUFFER_POOL_LIMIT=40m; SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_BLOOM_FILTER_SIZE=16M; select STRAIGHT_JOIN count(a.l_comment) from tpch_parquet.lineitem a join tpch_parquet.lineitem b @@ -371,7 +371,7 @@ row_regex: .*1 of 1 Runtime Filter Published.* # subplan (on the LHS). #################################################### SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; select straight_join count(1) from alltypes a join complextypestbl b # b.id + 10 confuses planner, so it doesn't think it @@ -401,7 +401,7 @@ row_regex: .*Rows rejected: 2.43K \(2432\).* # (when usually it would be selective). #################################################### SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MIN_SIZE=128MB; SET RUNTIME_FILTER_MAX_SIZE=500MB; SET MEM_LIMIT=140MB; @@ -416,7 +416,7 @@ row_regex: .*Files rejected: 0.* ---- QUERY # Confirm that with broadcast join, memory limit is not hit. SET RUNTIME_FILTER_MODE=GLOBAL; -SET RUNTIME_FILTER_WAIT_TIME_MS=30000; +SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS; SET RUNTIME_FILTER_MIN_SIZE=128MB; SET RUNTIME_FILTER_MAX_SIZE=500MB; # Allocate enough memory for the join + filter + scan http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e7b1c321/tests/query_test/test_runtime_filters.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_runtime_filters.py b/tests/query_test/test_runtime_filters.py index f631ad1..313b058 100644 --- a/tests/query_test/test_runtime_filters.py +++ b/tests/query_test/test_runtime_filters.py @@ -20,9 +20,12 @@ import pytest import re import time +from tests.common.environ import specific_build_type_timeout from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal +WAIT_TIME_MS = specific_build_type_timeout(60000, slow_build_timeout=100000) + @SkipIfLocal.multiple_impalad class TestRuntimeFilters(ImpalaTestSuite): @classmethod @@ -71,4 +74,5 @@ class TestRuntimeRowFilters(ImpalaTestSuite): v.get_value('table_format').file_format in ['parquet']) def test_row_filters(self, vector): - self.run_test_case('QueryTest/runtime_row_filters', vector) + self.run_test_case('QueryTest/runtime_row_filters', vector, + test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS' : str(WAIT_TIME_MS)})
