IMPALA-3491: Use unique_database fixture in test_hidden_files.py. Testing: Tested the changes locally by running them in a loop 10 times. Also did a private core/hdfs run.
Change-Id: I37e1528c02e598f3fb2d673b6559d55a34bf79b4 Reviewed-on: http://gerrit.cloudera.org:8080/3002 Reviewed-by: Michael Brown <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/12097a07 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/12097a07 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/12097a07 Branch: refs/heads/master Commit: 12097a07074efb1d83dfb43c4b851781345bb71a Parents: 96e18f9 Author: Alex Behm <[email protected]> Authored: Sat May 7 00:33:57 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Thu May 12 14:17:59 2016 -0700 ---------------------------------------------------------------------- .../queries/QueryTest/hidden-files.test | 4 +- tests/metadata/test_hidden_files.py | 47 +++++++++----------- 2 files changed, 22 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/12097a07/testdata/workloads/functional-query/queries/QueryTest/hidden-files.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/hidden-files.test b/testdata/workloads/functional-query/queries/QueryTest/hidden-files.test index 841f67d..aa67541 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/hidden-files.test +++ b/testdata/workloads/functional-query/queries/QueryTest/hidden-files.test @@ -1,6 +1,6 @@ ==== ---- QUERY -show partitions hidden_files_db.hf +show partitions test_hidden_files ---- LABELS YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION ---- RESULTS @@ -11,7 +11,7 @@ YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCRE STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING ==== ---- QUERY -select count(*) from hidden_files_db.hf +select count(*) from test_hidden_files ---- RESULTS 310 ---- TYPES http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/12097a07/tests/metadata/test_hidden_files.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_hidden_files.py b/tests/metadata/test_hidden_files.py index bdbf2f5..9126208 100644 --- a/tests/metadata/test_hidden_files.py +++ b/tests/metadata/test_hidden_files.py @@ -6,15 +6,15 @@ from tests.common.test_vector import * from tests.common.impala_test_suite import * from tests.util.filesystem_utils import WAREHOUSE, IS_S3 -TEST_DB = 'hidden_files_db' -TEST_TBL = 'hf' - class TestHiddenFiles(ImpalaTestSuite): """ Tests that files with special prefixes/suffixes are considered 'hidden' when loading table metadata and running queries. """ + # The .test file run in these tests relies this table name. + TBL_NAME = "test_hidden_files" + @classmethod def get_workload(self): return 'functional-query' @@ -28,28 +28,20 @@ class TestHiddenFiles(ImpalaTestSuite): if cls.exploration_strategy() != 'exhaustive' and not IS_S3: cls.TestMatrix.clear() - def setup_method(self, method): - self.cleanup_db(TEST_DB) - self.client.execute("create database %s location '%s/%s'" % (TEST_DB, WAREHOUSE, - TEST_DB)) + def __prepare_test_table(self, db_name, tbl_name): + """Creates a test table with two partitions, and copies files into the HDFS + directories of the two partitions. The goal is to have both an empty and non-empty + partition with hidden files.""" + self.client.execute( - "create table %s.%s like functional.alltypes" % (TEST_DB, TEST_TBL)) + "create table %s.%s like functional.alltypes" % (db_name, tbl_name)) self.client.execute( - "alter table %s.%s add partition (year=2010, month=1)" % (TEST_DB, TEST_TBL)) + "alter table %s.%s add partition (year=2010, month=1)" % (db_name, tbl_name)) self.client.execute( - "alter table %s.%s add partition (year=2010, month=2)" % (TEST_DB, TEST_TBL)) - - self.__populate_test_table() - - def teardown_method(self, method): - self.cleanup_db(TEST_DB) - - def __populate_test_table(self): - """Copy files into the HDFS directories of two partitions of the table. - The goal is to have both an empty and non-empty partition with hidden files.""" + "alter table %s.%s add partition (year=2010, month=2)" % (db_name, tbl_name)) ALLTYPES_LOC = "%s/alltypes" % WAREHOUSE - TEST_TBL_LOC = "%s/%s/%s" % (WAREHOUSE, TEST_DB, TEST_TBL) + TEST_TBL_LOC = "%s/%s.db/%s" % (WAREHOUSE, db_name, tbl_name) # Copy a visible file into one of the partitions. check_call(["hadoop", "fs", "-cp", "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC, @@ -81,15 +73,16 @@ class TestHiddenFiles(ImpalaTestSuite): "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC, "%s/year=2010/month=2/100201.txt.tmp" % TEST_TBL_LOC], shell=False) - @pytest.mark.execute_serially - def test_hidden_files_load(self, vector): + def test_hidden_files_load(self, vector, unique_database): """Tests that an incremental refresh ignores hidden files.""" - self.client.execute("invalidate metadata %s.%s" % (TEST_DB, TEST_TBL)) - self.run_test_case('QueryTest/hidden-files', vector) + self.__prepare_test_table(unique_database, self.TBL_NAME) + self.client.execute("invalidate metadata %s.%s" % (unique_database, self.TBL_NAME)) + self.run_test_case('QueryTest/hidden-files', vector, unique_database) # This test runs on one dimension. Therefore, running in it parallel is safe, given no # other method in this test class is run. - def test_hidden_files_refresh(self, vector): + def test_hidden_files_refresh(self, vector, unique_database): """Tests that an incremental refresh ignores hidden files.""" - self.client.execute("refresh %s.%s" % (TEST_DB, TEST_TBL)) - self.run_test_case('QueryTest/hidden-files', vector) + self.__prepare_test_table(unique_database, self.TBL_NAME) + self.client.execute("refresh %s.%s" % (unique_database, self.TBL_NAME)) + self.run_test_case('QueryTest/hidden-files', vector, unique_database)
