This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit e7376466c56c15101fc699428b6bf3a00de7332b Author: Riza Suminto <[email protected]> AuthorDate: Thu Sep 12 20:50:18 2024 -0700 IMPALA-13377: Excercise exec_option in test_recover_partitions.py Before this patch, test_recover_partitions.py declare exec_option dimension, but none of the test function exercise it. vector arg is never used anywhere. This patch fix the test by setting client configuration with given vector.get_value('exec_option'). Testing: - Run and pass test_recover_partitions.py in exhaustive mode. - Confirm in coordinator log file that there are equal number of queries run with either sync_ddl true or false (95 each). Change-Id: I4e938dd8667937c996854032a1e13184c62d7b48 Reviewed-on: http://gerrit.cloudera.org:8080/21796 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- tests/common/impala_test_suite.py | 23 ++++++++++++++++++++--- tests/metadata/test_recover_partitions.py | 22 +++++++++++++++------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index d822194c4..9f28cc29c 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -895,7 +895,9 @@ class ImpalaTestSuite(BaseTestSuite): @execute_wrapper def execute_query_expect_success(cls, impalad_client, query, query_options=None, user=None): - """Executes a query and asserts if the query fails""" + """Executes a query and asserts that the query succeded. + Remember to pass vector.get_value('exec_option') as 'query_options' argument + if the test has one.""" result = cls.__execute_query(impalad_client, query, query_options, user) assert result.success return result @@ -904,7 +906,9 @@ class ImpalaTestSuite(BaseTestSuite): @execute_wrapper def execute_query_expect_failure(cls, impalad_client, query, query_options=None, user=None): - """Executes a query and asserts if the query succeeds""" + """Executes a query and asserts that the query failed. + Remember to pass vector.get_value('exec_option') as 'query_options' argument + if the test has one.""" result = None try: result = cls.__execute_query(impalad_client, query, query_options, user) @@ -916,10 +920,16 @@ class ImpalaTestSuite(BaseTestSuite): @execute_wrapper def execute_query_unchecked(self, impalad_client, query, query_options=None, user=None): + """Execute a query against sepecific impalad without checking whether the query + succeded or failed. Remember to pass vector.get_value('exec_option') as + 'query_options' argument if the test has one.""" return self.__execute_query(impalad_client, query, query_options, user) @execute_wrapper def execute_query(self, query, query_options=None): + """Execute a query against the default impalad client without checking whether + the query succeded or failed. Remember to pass vector.get_value('exec_option') + as 'query_options' argument if the test has one.""" return self.__execute_query(self.client, query, query_options) def exec_and_time(self, query, query_options=None, impalad=0): @@ -961,6 +971,10 @@ class ImpalaTestSuite(BaseTestSuite): @execute_wrapper def execute_scalar(self, query, query_options=None): + """Executes a scalar query return the single row result. + Only validate that query return just one row. + Remember to pass vector.get_value('exec_option') as 'query_options' argument + if the test has one.""" result = self.__execute_query(self.client, query, query_options) assert len(result.data) <= 1, 'Multiple values returned from scalar' return result.data[0] if len(result.data) == 1 else None @@ -969,7 +983,10 @@ class ImpalaTestSuite(BaseTestSuite): @execute_wrapper def execute_scalar_expect_success(cls, impalad_client, query, query_options=None, user=None): - """Executes a query and asserts if the query fails""" + """Executes a scalar query return the single row result. + Validate that query execution is indeed successful and return just one row. + Remember to pass vector.get_value('exec_option') as 'query_options' argument + if the test has one.""" result = cls.__execute_query(impalad_client, query, query_options, user) assert result.success assert len(result.data) <= 1, 'Multiple values returned from scalar' diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py index 4f0f4b081..a23d5f084 100644 --- a/tests/metadata/test_recover_partitions.py +++ b/tests/metadata/test_recover_partitions.py @@ -24,12 +24,13 @@ import shutil from six.moves import urllib from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal, SkipIfFS, SkipIfCatalogV2 -from tests.common.test_dimensions import ALL_NODES_ONLY -from tests.common.test_dimensions import create_exec_option_dimension -from tests.util.filesystem_utils import WAREHOUSE, IS_S3 - +from tests.common.test_dimensions import ( + ALL_NODES_ONLY, + create_exec_option_dimension) +from tests.util.filesystem_utils import WAREHOUSE from tests.common.test_dimensions import create_uncompressed_text_dimension + # Validates ALTER TABLE RECOVER PARTITIONS statement class TestRecoverPartitions(ImpalaTestSuite): @@ -65,6 +66,7 @@ class TestRecoverPartitions(ImpalaTestSuite): """Test that RECOVER PARTITIONS correctly discovers new partitions added externally by the hdfs client. """ + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME) @@ -133,6 +135,7 @@ class TestRecoverPartitions(ImpalaTestSuite): def test_nondefault_location_partitions(self, vector, unique_database): """If the location of data files in one partition is changed, test that data files in the default location will not be loaded after partition recovery.""" + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME) @@ -170,7 +173,7 @@ class TestRecoverPartitions(ImpalaTestSuite): def test_recover_many_partitions(self, vector, unique_database): """Test that RECOVER PARTITIONS correctly discovers new partitions added externally by the hdfs client, recovered in batches""" - + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME DB_LOCATION = '%s/%s.db/' % (WAREHOUSE, unique_database) @@ -211,6 +214,7 @@ class TestRecoverPartitions(ImpalaTestSuite): """Test that RECOVER PARTITIONS does not recover equivalent partitions. Two partitions are considered equivalent if they correspond to distinct paths but can be converted to the same partition key values (e.g. "i=0005/p=p2" and "i=05/p=p2").""" + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME) @@ -261,6 +265,7 @@ class TestRecoverPartitions(ImpalaTestSuite): @SkipIfCatalogV2.impala_8489() def test_post_invalidate(self, vector, unique_database): """Test that RECOVER PARTITIONS works correctly after invalidate.""" + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME) @@ -297,6 +302,7 @@ class TestRecoverPartitions(ImpalaTestSuite): @SkipIfLocal.hdfs_client def test_support_all_types(self, vector, unique_database): """Test that RECOVER PARTITIONS works correctly on all supported data types.""" + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_recover_partitions" FQ_TBL_NAME = unique_database + "." + TBL_NAME TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME) @@ -344,6 +350,7 @@ class TestRecoverPartitions(ImpalaTestSuite): def test_encoded_partition(self, vector, unique_database): """IMPALA-6619: Test that RECOVER PARTITIONS does not create unnecessary partitions when dealing with URL encoded partition value.""" + self.client.set_configuration(vector.get_value('exec_option')) TBL_NAME = "test_encoded_partition" FQ_TBL_NAME = unique_database + "." + TBL_NAME @@ -369,6 +376,7 @@ class TestRecoverPartitions(ImpalaTestSuite): def test_unescaped_string_partition(self, vector, unique_database): """IMPALA-7784: Test that RECOVER PARTITIONS correctly parses unescaped string values""" + self.client.set_configuration(vector.get_value('exec_option')) tbl_name = "test_unescaped_string_partition" fq_tbl_name = unique_database + "." + tbl_name tbl_location = self.__get_fs_location(unique_database, tbl_name) @@ -403,7 +411,7 @@ class TestRecoverPartitions(ImpalaTestSuite): # Adds partition directories. num_partitions = 10 for i in range(1, num_partitions): - PART_DIR = "i=%d/s=part%d" % (i,i) + PART_DIR = "i=%d/s=part%d" % (i, i) self.filesystem_client.make_dir(TBL_LOCATION + PART_DIR) # Adds a duplicate directory name. @@ -429,7 +437,7 @@ class TestRecoverPartitions(ImpalaTestSuite): for S3, so enforcing a non-empty directory is less error prone across filesystems.""" partition_dir = os.path.join(root_path, new_dir) - self.filesystem_client.make_dir(partition_dir); + self.filesystem_client.make_dir(partition_dir) self.filesystem_client.create_file(os.path.join(partition_dir, new_file), value)
