IMPALA-5780,IMPALA-5779: extra spilling tests * Test for disable_unsafe_spills * Test for buffer size > I/O size (--read_size)
Change-Id: I03de00394bb6bbcf381250f816e22a4b987f1135 Reviewed-on: http://gerrit.cloudera.org:8080/7787 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bb6b0ce2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bb6b0ce2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bb6b0ce2 Branch: refs/heads/master Commit: bb6b0ce249f561af04ea42752ca874cd93dd63c9 Parents: 88d00b4 Author: Tim Armstrong <[email protected]> Authored: Wed Aug 23 13:47:37 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Aug 24 06:27:05 2017 +0000 ---------------------------------------------------------------------- .../QueryTest/disable-unsafe-spills.test | 11 +++++++ .../QueryTest/spilling-sorts-exhaustive.test | 31 +++++++++++++++++++- tests/query_test/test_spilling.py | 4 +++ 3 files changed, 45 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test new file mode 100644 index 0000000..a9b8f35 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test @@ -0,0 +1,11 @@ +==== +---- QUERY +# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should +# not spill to disk. +set disable_unsafe_spills=true; +set buffer_pool_limit=40m; +select distinct * +from tpch_avro.orders +---- CATCH +Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test index d391884..415fe5d 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test @@ -192,4 +192,33 @@ STRING # Verify that the sort actually spilled row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\) - +==== +---- QUERY +# Test that we can spill with a buffer size larger than the default I/O size (8mb). +set default_spillable_buffer_size=16m; +set buffer_pool_limit=96m; +select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment +from tpch_parquet.lineitem +order by 1, 2, 3, 4, 5 +offset 6001200 +---- TYPES +BIGINT,BIGINT,BIGINT,INT,STRING +---- RESULTS +5999971,97922,7923,5,'place of the slyly quick pla' +5999971,132903,2904,4,'ckly above the boldly ' +5999971,144823,4824,2,'e blithely after the carefully pending ' +5999971,161882,6915,6,'luffy theodolites nag boldly bli' +5999972,102942,473,1,'s maintain carefully among the' +5999972,133109,8136,2,' the furiously express pearls. furi' +5999972,152761,2762,3,'sual accounts al' +5999973,176345,1380,1,'gular excuses. ' +5999974,10463,5466,2,'dolites wake' +5999974,25360,5361,1,'express dependencies. express, pendi' +5999975,6452,1453,2,'lar pinto beans aft' +5999975,7272,2273,1,'tructions. excu' +5999975,37131,2138,3,', quick deposits. ironic, unusual deposi' +6000000,32255,2256,1,'carefully ' +6000000,96127,6128,2,'ooze furiously about the pe' +---- RUNTIME_PROFILE +row_regex: .*SpilledRuns: .* \([1-9][0-9]*\) +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/tests/query_test/test_spilling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py index 437739c..c818b65 100644 --- a/tests/query_test/test_spilling.py +++ b/tests/query_test/test_spilling.py @@ -72,3 +72,7 @@ class TestSpilling(ImpalaTestSuite): if self.exploration_strategy() != 'exhaustive': pytest.skip("only run large sorts on exhaustive") self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector) + + def test_disable_unsafe_spills(self, vector): + """Test that the disable_unsafe_spills query options works end-to-end.""" + self.run_test_case('QueryTest/disable-unsafe-spills', vector)
