IMPALA-3669: test_scratch_disk fails on S3 Make the test deterministic by using max_block_mgr_memory instead of mem_limit, so that the non-deterministic scanner memory usage does not influence the spilling behaviour of the queries.
Testing: Ran the test locally to confirm that it succeeded. Also manually computed the memory requirement. The data size to be sorted is ~220MB, so with a 64MB block manager limit per node, at least one node must spill. Change-Id: I9525a029ac020bb5b8bea210a741c9f9c5ec3c75 Reviewed-on: http://gerrit.cloudera.org:8080/3318 Reviewed-by: Dan Hecht <[email protected]> Reviewed-by: Michael Brown <[email protected]> Reviewed-by: Silvius Rus <[email protected]> Tested-by: Tim Armstrong <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c1093ed8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c1093ed8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c1093ed8 Branch: refs/heads/master Commit: c1093ed8611e8cc2702f81575e6c7193e0580386 Parents: 4e7172f Author: Tim Armstrong <[email protected]> Authored: Mon Jun 6 14:35:03 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Thu Jun 9 17:31:00 2016 -0700 ---------------------------------------------------------------------- tests/custom_cluster/test_scratch_disk.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1093ed8/tests/custom_cluster/test_scratch_disk.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py index 611bb89..7a48f1a 100644 --- a/tests/custom_cluster/test_scratch_disk.py +++ b/tests/custom_cluster/test_scratch_disk.py @@ -38,9 +38,10 @@ class TestScratchDir(CustomClusterTestSuite): in_mem_query = """ select o_orderdate, o_custkey, o_comment from tpch.orders """ - # Memory limit that is low enough to get Impala to spill to disk when executing - # spill_query and high enough that we can execute in_mem_query without spilling. - mem_limit = "200m" + # Block manager memory limit that is low enough to force Impala to spill to disk when + # executing spill_query and high enough that we can execute in_mem_query without + # spilling. + max_block_mgr_memory = "64m" def count_nonempty_dirs(self, dirs): count = 0 @@ -99,7 +100,7 @@ class TestScratchDir(CustomClusterTestSuite): self.assert_impalad_log_contains("INFO", "Using scratch directory ", expected_count=1) exec_option = vector.get_value('exec_option') - exec_option['mem_limit'] = self.mem_limit + exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory impalad = self.cluster.get_any_impalad() client = impalad.service.create_beeswax_client() self.execute_query_expect_success(client, self.spill_query, exec_option) @@ -113,7 +114,7 @@ class TestScratchDir(CustomClusterTestSuite): self.assert_impalad_log_contains("WARNING", "Running without spill to disk: no scratch directories provided\.") exec_option = vector.get_value('exec_option') - exec_option['mem_limit'] = self.mem_limit + exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory impalad = self.cluster.get_any_impalad() client = impalad.service.create_beeswax_client() # Expect spill to disk to fail @@ -132,7 +133,7 @@ class TestScratchDir(CustomClusterTestSuite): self.assert_impalad_log_contains("WARNING", "Could not remove and recreate directory " + ".*: cannot use it for scratch\. Error was: .*", expected_count=5) exec_option = vector.get_value('exec_option') - exec_option['mem_limit'] = self.mem_limit + exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory impalad = self.cluster.get_any_impalad() client = impalad.service.create_beeswax_client() # Expect spill to disk to fail @@ -153,7 +154,7 @@ class TestScratchDir(CustomClusterTestSuite): + "Encountered exception while verifying existence of directory path", expected_count=5) exec_option = vector.get_value('exec_option') - exec_option['mem_limit'] = self.mem_limit + exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory impalad = self.cluster.get_any_impalad() client = impalad.service.create_beeswax_client() # Expect spill to disk to fail
