IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test Add a targeted test that confirms that setting the query option will force spilling.
Testing: Ran test_spilling locally. Change-Id: Ida6b55b2dee0779b1739af5d75943518ec40d6ce Reviewed-on: http://gerrit.cloudera.org:8080/7809 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ebe8ddd4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ebe8ddd4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ebe8ddd4 Branch: refs/heads/release-2.10.0 Commit: ebe8ddd451b3f14d3f778339978a76bcd14b2589 Parents: 1faf89f Author: Tim Armstrong <[email protected]> Authored: Thu Aug 24 13:26:46 2017 -0700 Committer: Tim Armstrong <[email protected]> Committed: Wed Aug 30 14:54:49 2017 -0700 ---------------------------------------------------------------------- .../QueryTest/disable-unsafe-spills.test | 11 ---- .../QueryTest/spilling-query-options.test | 54 ++++++++++++++++++++ tests/common/impala_test_suite.py | 2 +- tests/query_test/test_spilling.py | 41 +++++++++++---- 4 files changed, 85 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test deleted file mode 100644 index a9b8f35..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test +++ /dev/null @@ -1,11 +0,0 @@ -==== ----- QUERY -# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should -# not spill to disk. -set disable_unsafe_spills=true; -set buffer_pool_limit=40m; -select distinct * -from tpch_avro.orders ----- CATCH -Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false -==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test new file mode 100644 index 0000000..5b9a641 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test @@ -0,0 +1,54 @@ +==== +---- QUERY +# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should +# not spill to disk. +set disable_unsafe_spills=true; +set buffer_pool_limit=40m; +select distinct * +from tpch_avro.orders +---- CATCH +Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on PREPARE. +set debug_action="-1:PREPARE:[email protected]"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on OPEN. +set debug_action="-1:OPEN:[email protected]"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on GETNEXT. +# This won't affect the merge aggregation, which accumulates its memory in Open(), but +# will affect the streaming aggregation. +set debug_action="-1:GETNEXT:[email protected]"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/common/impala_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index d5841b2..1b7d043 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -210,7 +210,7 @@ class ImpalaTestSuite(BaseTestSuite): if not query_option in self.default_query_options: continue default_val = self.default_query_options[query_option] - query_str = 'SET '+ query_option + '=' + default_val + ';' + query_str = 'SET '+ query_option + '="' + default_val + '"' try: impalad_client.execute(query_str) except Exception as e: http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/query_test/test_spilling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py index c818b65..0b36429 100644 --- a/tests/query_test/test_spilling.py +++ b/tests/query_test/test_spilling.py @@ -30,14 +30,14 @@ DEBUG_ACTION_DIMS = [None, @pytest.mark.xfail(pytest.config.option.testing_remote_cluster, reason='Queries may not spill on larger clusters') -class TestSpilling(ImpalaTestSuite): +class TestSpillingDebugActionDimensions(ImpalaTestSuite): @classmethod def get_workload(self): return 'functional-query' @classmethod def add_test_dimensions(cls): - super(TestSpilling, cls).add_test_dimensions() + super(TestSpillingDebugActionDimensions, cls).add_test_dimensions() cls.ImpalaTestMatrix.clear_constraints() cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch')) # Tests are calibrated so that they can execute and spill with this page size. @@ -60,19 +60,38 @@ class TestSpilling(ImpalaTestSuite): """Test spilling null-aware anti-joins""" self.run_test_case('QueryTest/spilling-naaj', vector) + def test_spilling_sorts_exhaustive(self, vector): + if self.exploration_strategy() != 'exhaustive': + pytest.skip("only run large sorts on exhaustive") + self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector) + + [email protected](pytest.config.option.testing_remote_cluster, + reason='Queries may not spill on larger clusters') +class TestSpillingNoDebugActionDimensions(ImpalaTestSuite): + """Spilling tests to which we don't want to apply the debug_action dimension.""" + @classmethod + def get_workload(self): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions() + cls.ImpalaTestMatrix.clear_constraints() + cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch')) + # Tests are calibrated so that they can execute and spill with this page size. + cls.ImpalaTestMatrix.add_dimension( + create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']})) + def test_spilling_naaj_no_deny_reservation(self, vector): """ Null-aware anti-join tests that depend on getting more than the minimum reservation and therefore will not reliably pass with the deny reservation debug action enabled. """ - if vector.get_value('exec_option')['debug_action'] is None: - self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector) + self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector) - def test_spilling_sorts_exhaustive(self, vector): - if self.exploration_strategy() != 'exhaustive': - pytest.skip("only run large sorts on exhaustive") - self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector) + def test_spilling_query_options(self, vector): + """Test that spilling-related query options work end-to-end. These tests rely on + setting debug_action to alternative values via query options.""" + self.run_test_case('QueryTest/spilling-query-options', vector) - def test_disable_unsafe_spills(self, vector): - """Test that the disable_unsafe_spills query options works end-to-end.""" - self.run_test_case('QueryTest/disable-unsafe-spills', vector)
