Repository: impala Updated Branches: refs/heads/2.x 258938ad9 -> 1e126dacc
IMPALA-6594: fix tests on local fs Skip most mem_usage_scaling tests, which are tuned for the memory requirements on 3 daemons. Also skip test_sort_reservation_usage, which similarly is tuned for 3 daemons. Fix the other test to wrap the HDFS path correctly. Testing: Ran the modified tests by hand against the minicluster to confirm they still ran as expected. Running full set of local tests. Change-Id: I76086ed695bf78e3e0f2745c1964dac8330d6c19 Reviewed-on: http://gerrit.cloudera.org:8080/9463 Reviewed-by: Dan Hecht <dhe...@cloudera.com> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/1e126dac Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/1e126dac Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/1e126dac Branch: refs/heads/2.x Commit: 1e126dacc0333c57d7f69ec89bffde2f5f018bdc Parents: 258938a Author: Tim Armstrong <tarmstr...@cloudera.com> Authored: Tue Feb 27 14:19:09 2018 -0800 Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org> Committed: Thu Mar 1 23:10:16 2018 +0000 ---------------------------------------------------------------------- tests/query_test/test_mem_usage_scaling.py | 6 ++++-- tests/query_test/test_sort.py | 2 ++ tests/query_test/test_udfs.py | 7 +++---- 3 files changed, 9 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/1e126dac/tests/query_test/test_mem_usage_scaling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py index f13e320..2ce02fa 100644 --- a/tests/query_test/test_mem_usage_scaling.py +++ b/tests/query_test/test_mem_usage_scaling.py @@ -32,6 +32,7 @@ MEM_LIMIT_TOO_LOW_FOR_RESERVATION = ("minimum memory reservation is greater than "available to the query for buffer reservations") MEM_LIMIT_ERROR_MSGS = [MEM_LIMIT_EXCEEDED_MSG, MEM_LIMIT_TOO_LOW_FOR_RESERVATION] +@SkipIfLocal.mem_usage_different class TestQueryMemLimitScaling(ImpalaTestSuite): """Test class to do functional validation of per query memory limits. """ QUERY = ["select * from lineitem where l_orderkey = -1", @@ -119,6 +120,7 @@ class TestLowMemoryLimits(ImpalaTestSuite): assert found_expected_error, str(e) +@SkipIfLocal.mem_usage_different class TestTpchMemLimitError(TestLowMemoryLimits): # The mem limits that will be used. MEM_IN_MB = [20, 140, 180, 220, 275, 450, 700] @@ -175,7 +177,6 @@ class TestTpchMemLimitError(TestLowMemoryLimits): def test_low_mem_limit_q9(self, vector): self.low_memory_limit_test(vector, 'tpch-q9', self.MIN_MEM_FOR_TPCH['Q9']) - @SkipIfLocal.mem_usage_different def test_low_mem_limit_q10(self, vector): self.low_memory_limit_test(vector, 'tpch-q10', self.MIN_MEM_FOR_TPCH['Q10']) @@ -209,7 +210,6 @@ class TestTpchMemLimitError(TestLowMemoryLimits): def test_low_mem_limit_q20(self, vector): self.low_memory_limit_test(vector, 'tpch-q20', self.MIN_MEM_FOR_TPCH['Q20']) - @SkipIfLocal.mem_usage_different def test_low_mem_limit_q21(self, vector): self.low_memory_limit_test(vector, 'tpch-q21', self.MIN_MEM_FOR_TPCH['Q21']) @@ -225,6 +225,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits): verifier = MetricVerifier(impalad.service) verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0) +@SkipIfLocal.mem_usage_different class TestTpchPrimitivesMemLimitError(TestLowMemoryLimits): """ Memory usage tests using targeted-perf queries to exercise specific operators. @@ -270,6 +271,7 @@ class TestTpchPrimitivesMemLimitError(TestLowMemoryLimits): self.run_primitive_query(vector, 'primitive_orderby_all') +@SkipIfLocal.mem_usage_different class TestTpcdsMemLimitError(TestLowMemoryLimits): # The mem limits that will be used. MEM_IN_MB = [20, 100, 116, 150] http://git-wip-us.apache.org/repos/asf/impala/blob/1e126dac/tests/query_test/test_sort.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py index 70629c4..ea0760f 100644 --- a/tests/query_test/test_sort.py +++ b/tests/query_test/test_sort.py @@ -18,6 +18,7 @@ from copy import copy from tests.common.impala_test_suite import ImpalaTestSuite +from tests.common.skip import SkipIfLocal def transpose_results(result, map_fn=lambda x: x): """Given a query result (list of strings, each string represents a row), return a list @@ -167,6 +168,7 @@ class TestQueryFullSort(ImpalaTestSuite): query, exec_option, table_format=table_format).data) assert(result[0] == sorted(result[0])) + @SkipIfLocal.mem_usage_different def test_sort_reservation_usage(self, vector): """Tests for sorter reservation usage.""" self.run_test_case('sort-reservation-usage', vector) http://git-wip-us.apache.org/repos/asf/impala/blob/1e126dac/tests/query_test/test_udfs.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py index 1be25e5..5252363 100644 --- a/tests/query_test/test_udfs.py +++ b/tests/query_test/test_udfs.py @@ -332,18 +332,17 @@ class TestUdfExecution(TestUdfBase): # 1. too many arguments # 2. IR UDF fd, dir_name = tempfile.mkstemp() + hdfs_path = get_fs_path("/test-warehouse/{0}_bad_udf.ll".format(unique_database)) try: with open(dir_name, "w") as f: f.write("Hello World") - check_call(["hadoop", "fs", "-put", "-f", f.name, "/test-warehouse/" + - unique_database + "_bad_udf.ll"]) + check_call(["hadoop", "fs", "-put", "-f", f.name, hdfs_path]) if vector.get_value('exec_option')['disable_codegen']: self.run_test_case('QueryTest/udf-errors', vector, use_db=unique_database) finally: if os.path.exists(f.name): os.remove(f.name) - call(["hadoop", "fs", "-rm", "-f", "/test-warehouse/" + - unique_database + "_bad_udf.ll"]) + call(["hadoop", "fs", "-rm", "-f", hdfs_path]) os.close(fd) # Run serially because this will blow the process limit, potentially causing other
