IMPALA-3491: Use unique_database fixture in test_metadata_query_statements.py.
Testing: Ran the test locally on exhaustive in a loop 10 times. Ran a private exhaustive build on hdfs. Change-Id: Ia0af1dc6534234508bd0fed03531f7fe8ff556aa Reviewed-on: http://gerrit.cloudera.org:8080/3103 Reviewed-by: Alex Behm <[email protected]> Tested-by: Alex Behm <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a9f7cf51 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a9f7cf51 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a9f7cf51 Branch: refs/heads/master Commit: a9f7cf51f4e422a048e871d97ef1b42d4a4cefe0 Parents: d23e550 Author: Alex Behm <[email protected]> Authored: Fri May 13 22:27:35 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue Jun 7 09:34:30 2016 -0700 ---------------------------------------------------------------------- .../queries/QueryTest/describe-db.test | 65 ++++++++ .../queries/QueryTest/describe-path.test | 137 ++++++++++++++++ .../queries/QueryTest/describe.test | 140 ---------------- .../queries/QueryTest/describedb.test | 65 -------- .../queries/QueryTest/show-stats.test | 5 - .../metadata/test_metadata_query_statements.py | 162 +++++++++---------- 6 files changed, 274 insertions(+), 300 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/testdata/workloads/functional-query/queries/QueryTest/describe-db.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test new file mode 100644 index 0000000..8f55325 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test @@ -0,0 +1,65 @@ +==== +---- QUERY +# Test printing of default database. +describe database default +---- RESULTS +'default','$NAMENODE/test-warehouse','Default Hive database' +---- TYPES +string, string, string +==== +---- QUERY +# Test printing of hive_test_desc_db database. +describe database hive_test_desc_db +---- RESULTS +'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment' +---- TYPES +string, string, string +==== +---- QUERY +# Test printing of hive_test_desc_db database with extended information. +describe database extended hive_test_desc_db +---- RESULTS +'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment' +'Parameter: ','','' +'','test','USER' +'Owner: ','','' +'','e','2.82' +'','pi','3.14' +---- TYPES +string, string, string +==== +---- QUERY +describe database extended impala_test_desc_db1 +---- RESULTS +'impala_test_desc_db1','$NAMENODE/test-warehouse/impala_test_desc_db1.db','' +---- TYPES +string, string, string +==== +---- QUERY +describe database extended impala_test_desc_db2 +---- RESULTS +'impala_test_desc_db2','$NAMENODE/test-warehouse/impala_test_desc_db2.db','test comment' +---- TYPES +string, string, string +==== +---- QUERY +describe database extended impala_test_desc_db3 +---- RESULTS +'impala_test_desc_db3','$NAMENODE/testdb','' +---- TYPES +string, string, string +==== +---- QUERY +describe database extended impala_test_desc_db4 +---- RESULTS +'impala_test_desc_db4','$NAMENODE/test2.db','test comment' +---- TYPES +string, string, string +==== +---- QUERY +describe database formatted impala_test_desc_db4 +---- RESULTS +'impala_test_desc_db4','$NAMENODE/test2.db','test comment' +---- TYPES +string, string, string +==== \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/testdata/workloads/functional-query/queries/QueryTest/describe-path.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-path.test b/testdata/workloads/functional-query/queries/QueryTest/describe-path.test new file mode 100644 index 0000000..2547267 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/describe-path.test @@ -0,0 +1,137 @@ +==== +---- QUERY +describe functional.alltypes +---- RESULTS +'id','int','Add a comment' +'bool_col','boolean','' +'tinyint_col','tinyint','' +'smallint_col','smallint','' +'int_col','int','' +'bigint_col','bigint','' +'float_col','float','' +'double_col','double','' +'date_string_col','string','' +'string_col','string','' +'timestamp_col','timestamp','' +'year','int','' +'month','int','' +---- TYPES +string, string, string +==== +---- QUERY +# Test printing of complex types. +describe functional.allcomplextypes +---- RESULTS +'id','int','' +'int_array_col','array<int>','' +'array_array_col','array<array<int>>','' +'map_array_col','array<map<string,int>>','' +'struct_array_col','array<struct<\n f1:bigint,\n f2:string\n>>','' +'int_map_col','map<string,int>','' +'array_map_col','map<string,array<int>>','' +'map_map_col','map<string,map<string,int>>','' +'struct_map_col','map<string,struct<\n f1:bigint,\n f2:string\n>>','' +'int_struct_col','struct<\n f1:int,\n f2:int\n>','' +'complex_struct_col','struct<\n f1:int,\n f2:array<int>,\n f3:map<string,int>\n>','' +'nested_struct_col','struct<\n f1:int,\n f2:struct<\n f11:bigint,\n f12:struct<\n f21:bigint\n >\n >\n>','' +'complex_nested_struct_col','struct<\n f1:int,\n f2:array<struct<\n f11:bigint,\n f12:map<string,struct<\n f21:bigint\n >>\n >>\n>','' +'year','int','' +'month','int','' +---- TYPES +string, string, string +==== +---- QUERY +describe functional_parquet.allcomplextypes.int_array_col +---- RESULTS +'item','int','' +'pos','bigint','' +---- TYPES +string,string,string +==== +---- QUERY +describe functional_parquet.allcomplextypes.map_array_col.item +---- RESULTS +'key','string','' +'value','int','' +---- TYPES +string,string,string +==== +---- QUERY +describe functional_parquet.allcomplextypes.complex_struct_col +---- RESULTS +'f1','int','' +'f2','array<int>','' +'f3','map<string,int>','' +---- TYPES +string,string,string +==== +---- QUERY +describe functional_parquet.allcomplextypes.complex_struct_col.f2 +---- RESULTS +'item','int','' +'pos','bigint','' +---- TYPES +string,string,string +==== +---- QUERY +describe functional_parquet.allcomplextypes.nested_struct_col +---- RESULTS +'f1','int','' +'f2','struct<\n f11:bigint,\n f12:struct<\n f21:bigint\n >\n>','' +---- TYPES +string,string,string +==== +---- QUERY +# Test describing structs within collections. +create table if not exists nested_structs ( + map_array_struct_col map<string, array<struct<f1:int, f2:string>>>, + struct_array_struct_col + struct<f1:int, f2:array<struct<f11:bigint, f12:string>>>, + map_array_map_struct_col + map<string, array<map<string, struct<f1:string, f2:int>>>>) +---- RESULTS +==== +---- QUERY +describe nested_structs +---- RESULTS +'map_array_struct_col','map<string,array<struct<\n f1:int,\n f2:string\n>>>','' +'struct_array_struct_col','struct<\n f1:int,\n f2:array<struct<\n f11:bigint,\n f12:string\n >>\n>','' +'map_array_map_struct_col','map<string,array<map<string,struct<\n f1:string,\n f2:int\n>>>>','' +---- TYPES +string,string,string +==== +---- QUERY +use functional +==== +---- QUERY +# Check that paths relative to current database work ok. +describe alltypes +---- RESULTS +'id','int','Add a comment' +'bool_col','boolean','' +'tinyint_col','tinyint','' +'smallint_col','smallint','' +'int_col','int','' +'bigint_col','bigint','' +'float_col','float','' +'double_col','double','' +'date_string_col','string','' +'string_col','string','' +'timestamp_col','timestamp','' +'year','int','' +'month','int','' +---- TYPES +string, string, string +==== +---- QUERY +use functional_parquet +==== +---- QUERY +# Check that paths relative to current database work ok. +describe allcomplextypes.int_array_col +---- RESULTS +'item','int','' +'pos','bigint','' +---- TYPES +string,string,string +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/testdata/workloads/functional-query/queries/QueryTest/describe.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe.test b/testdata/workloads/functional-query/queries/QueryTest/describe.test deleted file mode 100644 index 06ac191..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/describe.test +++ /dev/null @@ -1,140 +0,0 @@ -==== ----- QUERY -# Simple describe (note Hive does not support this) -describe functional.alltypes ----- RESULTS -'id','int','Add a comment' -'bool_col','boolean','' -'tinyint_col','tinyint','' -'smallint_col','smallint','' -'int_col','int','' -'bigint_col','bigint','' -'float_col','float','' -'double_col','double','' -'date_string_col','string','' -'string_col','string','' -'timestamp_col','timestamp','' -'year','int','' -'month','int','' ----- TYPES -string, string, string -==== ----- QUERY -USE functional -==== ----- QUERY -# Default database -describe alltypes ----- RESULTS -'id','int','Add a comment' -'bool_col','boolean','' -'tinyint_col','tinyint','' -'smallint_col','smallint','' -'int_col','int','' -'bigint_col','bigint','' -'float_col','float','' -'double_col','double','' -'date_string_col','string','' -'string_col','string','' -'timestamp_col','timestamp','' -'year','int','' -'month','int','' ----- TYPES -string, string, string -==== ----- QUERY -# Test printing of complex types. -describe functional.allcomplextypes ----- RESULTS -'id','int','' -'int_array_col','array<int>','' -'array_array_col','array<array<int>>','' -'map_array_col','array<map<string,int>>','' -'struct_array_col','array<struct<\n f1:bigint,\n f2:string\n>>','' -'int_map_col','map<string,int>','' -'array_map_col','map<string,array<int>>','' -'map_map_col','map<string,map<string,int>>','' -'struct_map_col','map<string,struct<\n f1:bigint,\n f2:string\n>>','' -'int_struct_col','struct<\n f1:int,\n f2:int\n>','' -'complex_struct_col','struct<\n f1:int,\n f2:array<int>,\n f3:map<string,int>\n>','' -'nested_struct_col','struct<\n f1:int,\n f2:struct<\n f11:bigint,\n f12:struct<\n f21:bigint\n >\n >\n>','' -'complex_nested_struct_col','struct<\n f1:int,\n f2:array<struct<\n f11:bigint,\n f12:map<string,struct<\n f21:bigint\n >>\n >>\n>','' -'year','int','' -'month','int','' ----- TYPES -string, string, string -==== ----- QUERY -describe functional_parquet.allcomplextypes.int_array_col ----- RESULTS -'item','int','' -'pos','bigint','' ----- TYPES -string,string,string -==== ----- QUERY -describe functional_parquet.allcomplextypes.map_array_col.item ----- RESULTS -'key','string','' -'value','int','' ----- TYPES -string,string,string -==== ----- QUERY -describe functional_parquet.allcomplextypes.complex_struct_col ----- RESULTS -'f1','int','' -'f2','array<int>','' -'f3','map<string,int>','' ----- TYPES -string,string,string -==== ----- QUERY -describe functional_parquet.allcomplextypes.complex_struct_col.f2 ----- RESULTS -'item','int','' -'pos','bigint','' ----- TYPES -string,string,string -==== ----- QUERY -use functional_parquet -==== ----- QUERY -describe functional_parquet.allcomplextypes.complex_struct_col.f2 ----- RESULTS -'item','int','' -'pos','bigint','' ----- TYPES -string,string,string -==== ----- QUERY -use functional_parquet -==== ----- QUERY -# Check that paths relative to current database work ok. -describe allcomplextypes.int_array_col ----- RESULTS -'item','int','' -'pos','bigint','' ----- TYPES -string,string,string -==== ----- QUERY -# Pretty-printing should work for fields too. -describe allcomplextypes.nested_struct_col ----- RESULTS -'f1','int','' -'f2','struct<\n f11:bigint,\n f12:struct<\n f21:bigint\n >\n>','' ----- TYPES -string,string,string -==== ----- QUERY -describe impala_test_desc_db1.complex_types_tbl ----- RESULTS -'map_array_struct_col','map<string,array<struct<\n f1:int,\n f2:string\n>>>','' -'struct_array_struct_col','struct<\n f1:int,\n f2:array<struct<\n f11:bigint,\n f12:string\n >>\n>','' -'map_array_map_struct_col','map<string,array<map<string,struct<\n f1:string,\n f2:int\n>>>>','' ----- TYPES -string,string,string -==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/testdata/workloads/functional-query/queries/QueryTest/describedb.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/describedb.test b/testdata/workloads/functional-query/queries/QueryTest/describedb.test deleted file mode 100644 index 8f55325..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/describedb.test +++ /dev/null @@ -1,65 +0,0 @@ -==== ----- QUERY -# Test printing of default database. -describe database default ----- RESULTS -'default','$NAMENODE/test-warehouse','Default Hive database' ----- TYPES -string, string, string -==== ----- QUERY -# Test printing of hive_test_desc_db database. -describe database hive_test_desc_db ----- RESULTS -'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment' ----- TYPES -string, string, string -==== ----- QUERY -# Test printing of hive_test_desc_db database with extended information. -describe database extended hive_test_desc_db ----- RESULTS -'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment' -'Parameter: ','','' -'','test','USER' -'Owner: ','','' -'','e','2.82' -'','pi','3.14' ----- TYPES -string, string, string -==== ----- QUERY -describe database extended impala_test_desc_db1 ----- RESULTS -'impala_test_desc_db1','$NAMENODE/test-warehouse/impala_test_desc_db1.db','' ----- TYPES -string, string, string -==== ----- QUERY -describe database extended impala_test_desc_db2 ----- RESULTS -'impala_test_desc_db2','$NAMENODE/test-warehouse/impala_test_desc_db2.db','test comment' ----- TYPES -string, string, string -==== ----- QUERY -describe database extended impala_test_desc_db3 ----- RESULTS -'impala_test_desc_db3','$NAMENODE/testdb','' ----- TYPES -string, string, string -==== ----- QUERY -describe database extended impala_test_desc_db4 ----- RESULTS -'impala_test_desc_db4','$NAMENODE/test2.db','test comment' ----- TYPES -string, string, string -==== ----- QUERY -describe database formatted impala_test_desc_db4 ----- RESULTS -'impala_test_desc_db4','$NAMENODE/test2.db','test comment' ----- TYPES -string, string, string -==== \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/testdata/workloads/functional-query/queries/QueryTest/show-stats.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/show-stats.test b/testdata/workloads/functional-query/queries/QueryTest/show-stats.test index 879e5f1..5dfc82c 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/show-stats.test +++ b/testdata/workloads/functional-query/queries/QueryTest/show-stats.test @@ -1,11 +1,6 @@ ==== ---- QUERY -use functional -==== ----- QUERY # Stats on a partitioned Hdfs table stored as text -drop stats alltypes; -compute stats alltypes; show table stats alltypes ---- LABELS YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a9f7cf51/tests/metadata/test_metadata_query_statements.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py index 6721014..f84fbd7 100644 --- a/tests/metadata/test_metadata_query_statements.py +++ b/tests/metadata/test_metadata_query_statements.py @@ -3,7 +3,6 @@ import logging import os import pytest -from subprocess import call from tests.beeswax.impala_beeswax import ImpalaBeeswaxException from tests.common.impala_test_suite import * from tests.common.skip import SkipIfIsilon, SkipIfS3, SkipIfLocal @@ -12,11 +11,6 @@ from tests.util.filesystem_utils import get_fs_path # TODO: For these tests to pass, all table metadata must be created exhaustively. # the tests should be modified to remove that requirement. - -# Execute all tests serially to avoid the following concurrency conflicts: -# 1. In setup/teardown when dropping/creating databases and tables (IMPALA-2537) -# 2. Avoid running "invalidate metadata" concurrently with other pytests (IMPALA-2687) [email protected]_serially class TestMetadataQueryStatements(ImpalaTestSuite): CREATE_DATA_SRC_STMT = ("CREATE DATA SOURCE %s LOCATION '" + @@ -35,7 +29,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite): super(TestMetadataQueryStatements, cls).add_test_dimensions() sync_ddl_opts = [0, 1] if cls.exploration_strategy() != 'exhaustive': - # Cut down on test runtime by only running with SYNC_DDL=1 + # Cut down on test runtime by only running with SYNC_DDL=0 sync_ddl_opts = [0] cls.TestMatrix.add_dimension(create_exec_option_dimension( @@ -45,83 +39,24 @@ class TestMetadataQueryStatements(ImpalaTestSuite): sync_ddl=sync_ddl_opts)) cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload())) - def __drop_data_sources(self): - for name in self.TEST_DATA_SRC_NAMES: - self.client.execute(self.DROP_DATA_SRC_STMT % (name,)) - - def __create_data_sources(self): - self.__drop_data_sources() - for name in self.TEST_DATA_SRC_NAMES: - self.client.execute(self.CREATE_DATA_SRC_STMT % (name,)) - - def setup_method(self, method): - self.cleanup_db('impala_test_desc_db1') - self.cleanup_db('impala_test_desc_db2') - self.cleanup_db('impala_test_desc_db3') - self.cleanup_db('impala_test_desc_db4') - self.cleanup_db('hive_test_desc_db') - self.cleanup_db('hive_test_db') - - self.client.execute("create database if not exists impala_test_desc_db1") - self.client.execute( - "create database if not exists impala_test_desc_db2 " - "comment \"test comment\"") - self.client.execute( - "create database if not exists impala_test_desc_db3 " - "location \"" + get_fs_path("/testdb") + "\"") - self.client.execute( - "create database if not exists impala_test_desc_db4 " - "comment \"test comment\" location \"" + get_fs_path("/test2.db") + "\"") - - self.client.execute( - "create table if not exists impala_test_desc_db1.complex_types_tbl (" - "map_array_struct_col map<string, array<struct<f1:int, f2:string>>>, " - "struct_array_struct_col " - "struct<f1:int, f2:array<struct<f11:bigint, f12:string>>>, " - "map_array_map_struct_col " - "map<string, array<map<string, struct<f1:string, f2:int>>>>)") - - def teardown_method(self, method): - self.cleanup_db('impala_test_desc_db1') - self.cleanup_db('impala_test_desc_db2') - self.cleanup_db('impala_test_desc_db3') - self.cleanup_db('impala_test_desc_db4') - self.cleanup_db('hive_test_desc_db') - self.cleanup_db('hive_test_db') + def test_use(self, vector): + self.run_test_case('QueryTest/use', vector) def test_show(self, vector): self.run_test_case('QueryTest/show', vector) - @SkipIfS3.hive - @SkipIfIsilon.hive - @SkipIfLocal.hive - def test_describe_db(self, vector): - call([ - "hive", "-e", "create database hive_test_desc_db comment 'test comment' " - "with dbproperties('pi' = '3.14', 'e' = '2.82')"]) - call(["hive", "-e", "alter database hive_test_desc_db set owner user test"]) - self.client.execute("invalidate metadata") - self.run_test_case('QueryTest/describedb', vector) - - def test_show_data_sources(self, vector): - try: - self.__create_data_sources() - self.run_test_case('QueryTest/show-data-sources', vector) - finally: - self.__drop_data_sources() - def test_show_stats(self, vector): - self.run_test_case('QueryTest/show-stats', vector) + self.run_test_case('QueryTest/show-stats', vector, "functional") - def test_describe_table(self, vector): - self.run_test_case('QueryTest/describe', vector) + def test_describe_path(self, vector, unique_database): + self.run_test_case('QueryTest/describe-path', vector, unique_database) # Missing Coverage: Describe formatted compatibility between Impala and Hive when the # data doesn't reside in hdfs. @SkipIfIsilon.hive @SkipIfS3.hive @SkipIfLocal.hive - def test_describe_formatted(self, vector): + def test_describe_formatted(self, vector, unique_database): # Describe a partitioned table. self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.alltypes") self.exec_and_compare_hive_and_impala_hs2( @@ -134,17 +69,16 @@ class TestMetadataQueryStatements(ImpalaTestSuite): # by Impala without any column definitions. # TODO: Instead of creating new tables here, change one of the existing # Avro tables to be created without any column definitions. - db_name = "hive_test_db" - self.client.execute("create database if not exists %s" % db_name) + self.client.execute("create database if not exists %s" % unique_database) self.client.execute(( "create table %s.%s with serdeproperties ('avro.schema.url'='%s') stored as avro" - % (db_name, "avro_alltypes_nopart", self.AVRO_SCHEMA_LOC))) + % (unique_database, "avro_alltypes_nopart", self.AVRO_SCHEMA_LOC))) self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_nopart") self.client.execute(( "create table %s.%s partitioned by (year int, month int) " "with serdeproperties ('avro.schema.url'='%s') stored as avro" - % (db_name, "avro_alltypes_part", self.AVRO_SCHEMA_LOC))) + % (unique_database, "avro_alltypes_part", self.AVRO_SCHEMA_LOC))) self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_part") try: @@ -154,23 +88,71 @@ class TestMetadataQueryStatements(ImpalaTestSuite): except AssertionError: pytest.xfail("Investigate minor difference in displaying null vs empty values") - def test_use_table(self, vector): - self.run_test_case('QueryTest/use', vector) + @pytest.mark.execute_serially # due to data src setup/teardown + def test_show_data_sources(self, vector): + try: + self.__create_data_sources() + self.run_test_case('QueryTest/show-data-sources', vector) + finally: + self.__drop_data_sources() + + def __drop_data_sources(self): + for name in self.TEST_DATA_SRC_NAMES: + self.client.execute(self.DROP_DATA_SRC_STMT % (name,)) + + def __create_data_sources(self): + self.__drop_data_sources() + for name in self.TEST_DATA_SRC_NAMES: + self.client.execute(self.CREATE_DATA_SRC_STMT % (name,)) + + @SkipIfS3.hive + @SkipIfIsilon.hive + @SkipIfLocal.hive + @pytest.mark.execute_serially # because of invalidate metadata + def test_describe_db(self, vector): + self.__test_describe_db_cleanup() + try: + self.client.execute("create database impala_test_desc_db1") + self.client.execute("create database impala_test_desc_db2 " + "comment 'test comment'") + self.client.execute("create database impala_test_desc_db3 " + "location '" + get_fs_path("/testdb") + "'") + self.client.execute("create database impala_test_desc_db4 comment 'test comment' " + "location \"" + get_fs_path("/test2.db") + "\"") + self.run_stmt_in_hive("create database hive_test_desc_db comment 'test comment' " + "with dbproperties('pi' = '3.14', 'e' = '2.82')") + self.run_stmt_in_hive("alter database hive_test_desc_db set owner user test") + self.client.execute("invalidate metadata") + self.run_test_case('QueryTest/describe-db', vector) + finally: + self.__test_describe_db_cleanup() + + def __test_describe_db_cleanup(self): + self.cleanup_db('hive_test_desc_db') + self.cleanup_db('impala_test_desc_db1') + self.cleanup_db('impala_test_desc_db2') + self.cleanup_db('impala_test_desc_db3') + self.cleanup_db('impala_test_desc_db4') # Missing Coverage: ddl by hive being visible to Impala for data not residing in hdfs. @SkipIfIsilon.hive @SkipIfS3.hive @SkipIfLocal.hive - def test_impala_sees_hive_created_tables_and_databases(self, vector): + @pytest.mark.execute_serially # because of invalidate metadata + def test_impala_sees_hive_tables_and_dbs(self, vector): self.client.set_configuration(vector.get_value('exec_option')) - db_name = 'hive_test_db' - tbl_name = 'testtbl' - call(["hive", "-e", "DROP DATABASE IF EXISTS %s CASCADE" % db_name]) - self.client.execute("invalidate metadata") + DB_NAME = 'hive_test_db' + TBL_NAME = 'testtbl' + self.cleanup_db(DB_NAME) + try: + self.__run_test_impala_sees_hive_tables_and_dbs(DB_NAME, TBL_NAME) + finally: + self.cleanup_db(DB_NAME) + def __run_test_impala_sees_hive_tables_and_dbs(self, db_name, tbl_name): assert db_name not in self.all_db_names() - call(["hive", "-e", "CREATE DATABASE %s" % db_name]) + self.run_stmt_in_hive("create database %s" % db_name) # Run 'invalidate metadata <table name>' when the parent database does not exist. try: @@ -183,7 +165,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite): assert db_name not in self.all_db_names() # Create a table external to Impala. - call(["hive", "-e", "CREATE TABLE %s.%s (i int)" % (db_name, tbl_name)]) + self.run_stmt_in_hive("create table %s.%s (i int)" % (db_name, tbl_name)) # Impala does not know about this database or table. assert db_name not in self.all_db_names() @@ -198,10 +180,10 @@ class TestMetadataQueryStatements(ImpalaTestSuite): assert len(result.data) == 1 self.client.execute("create table %s.%s (j int)" % (db_name, tbl_name + "_test")) - call(["hive", "-e", "drop table %s.%s" % (db_name, tbl_name + "_test")]) + self.run_stmt_in_hive("drop table %s.%s" % (db_name, tbl_name + "_test")) # Re-create the table in Hive. Use the same name, but different casing. - call(["hive", "-e", "CREATE TABLE %s.%s (i bigint)" % (db_name, tbl_name + "_TEST")]) + self.run_stmt_in_hive("create table %s.%s (i bigint)" % (db_name, tbl_name + "_TEST")) self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name + "_Test")) result = self.client.execute("show tables in %s" % db_name) assert tbl_name + "_test" in result.data @@ -244,7 +226,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite): assert tbl_name in result.data # Create another table - call(["hive", "-e", "CREATE TABLE %s.%s (i int)" % (db_name, tbl_name + '2')]) + self.run_stmt_in_hive("create table %s.%s (i int)" % (db_name, tbl_name + '2')) self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name + '2')) result = self.client.execute("show tables in %s" % db_name) assert tbl_name + '2' in result.data @@ -252,7 +234,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite): # Drop the table, and then verify invalidate metadata <table name> removes the # table from the catalog. - call(["hive", "-e", "DROP TABLE %s.%s " % (db_name, tbl_name)]) + self.run_stmt_in_hive("drop table %s.%s " % (db_name, tbl_name)) self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name)) result = self.client.execute("show tables in %s" % db_name) assert tbl_name + '2' in result.data @@ -272,7 +254,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite): assert tbl_name not in result.data # Drop the parent database (this will drop all tables). Then invalidate the table - call(["hive", "-e", "DROP DATABASE %s CASCADE" % db_name]) + self.run_stmt_in_hive("drop database %s CASCADE" % db_name) self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name + '2')) result = self.client.execute("show tables in %s" % db_name); assert len(result.data) == 0
