This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit d3134bfb4ad50f8bb680e54be6c27e587ae02817 Author: Pranav Lodha <[email protected]> AuthorDate: Mon Jun 24 15:24:11 2024 -0700 IMPALA-13101: test_data_source_tables fails with Data source does not exist The test failed with a "Data source does not exist" due to name conflicts with pre-existing Data source objects. To resolve this, each datasource name is made unique for each concurrently running test dimension. The fix ensures that the test runs smoothly without encountering errors related to conflicting Data source names. Testing: To test this change, it needs to be built with -ubsan flag, post which a bash script is triggered to set some environment variables, followed by './bin/run-all-tests.sh' command to make sure all tests are run. Some important environment variables of the bash script includes: 1. EXPLORATION_STRATEGY set to exhaustive to ensure all possible scenarios are covered. 2. The specific test file to run is query_test/ test_ext_data_sources.py::TestExtDataSources ::test_data_source_tables and custom_cluster/ test_ext_data_sources.py, while frontend (FE), backend (BE), and cluster tests are disabled. End-to-end tests are enabled (EE_TEST=true), with iteration and failure limits also specified. Change-Id: I29822855da8136e013c8a62bb0489a181bf131ae Reviewed-on: http://gerrit.cloudera.org:8080/21815 Reviewed-by: Wenzhe Zhou <[email protected]> Reviewed-by: Riza Suminto <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../queries/QueryTest/data-source-tables.test | 12 ++++++------ tests/custom_cluster/test_ext_data_sources.py | 15 +++++++++------ tests/query_test/test_ext_data_sources.py | 5 +++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test index 8830a65a4..7957006ee 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test +++ b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test @@ -1,8 +1,8 @@ ==== ---- QUERY # Create DataSource -DROP DATA SOURCE IF EXISTS TestGenericDataSource; -CREATE DATA SOURCE TestGenericDataSource +DROP DATA SOURCE IF EXISTS $UNIQUE_DATASOURCE; +CREATE DATA SOURCE $UNIQUE_DATASOURCE LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/test-data-source.jar' CLASS 'org.apache.impala.extdatasource.AllTypesDataSource' API_VERSION 'V1'; @@ -11,11 +11,11 @@ API_VERSION 'V1'; ==== ---- QUERY # Show created DataSource -SHOW DATA SOURCES LIKE 'testgenericdatasource'; +SHOW DATA SOURCES LIKE '$UNIQUE_DATASOURCE'; ---- LABELS NAME,LOCATION,CLASS NAME,API VERSION ---- RESULTS -'testgenericdatasource',regex:'.*/test-warehouse/data-sources/test-data-source.jar','org.apache.impala.extdatasource.AllTypesDataSource','V1' +'$UNIQUE_DATASOURCE',regex:'.*/test-warehouse/data-sources/test-data-source.jar','org.apache.impala.extdatasource.AllTypesDataSource','V1' ---- TYPES STRING,STRING,STRING,STRING ==== @@ -39,7 +39,7 @@ CREATE TABLE alltypes_datasource ( dec_col4 DECIMAL(38,37), dec_col5 DECIMAL(10,5), date_col DATE) -PRODUCED BY DATA SOURCE TestGenericDataSource("TestInitString"); +PRODUCED BY DATA SOURCE $UNIQUE_DATASOURCE("TestInitString"); ---- RESULTS 'Table has been created.' ==== @@ -181,7 +181,7 @@ DROP TABLE alltypes_datasource; ==== ---- QUERY # Drop DataSource -DROP DATA SOURCE TestGenericDataSource; +DROP DATA SOURCE $UNIQUE_DATASOURCE; ---- RESULTS 'Data source has been dropped.' ==== diff --git a/tests/custom_cluster/test_ext_data_sources.py b/tests/custom_cluster/test_ext_data_sources.py index 67a94f100..b3dbde78d 100644 --- a/tests/custom_cluster/test_ext_data_sources.py +++ b/tests/custom_cluster/test_ext_data_sources.py @@ -45,9 +45,10 @@ class TestExtDataSources(CustomClusterTestSuite): @CustomClusterTestSuite.with_args( impalad_args="--use_local_catalog=true", catalogd_args="--catalog_topic_mode=minimal") - def test_data_source_tables(self, vector, unique_database): + def test_data_source_tables(self, vector, unique_database, unique_name): """Start Impala cluster in LocalCatalog Mode""" - self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database) + self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database, + test_file_vars={'$UNIQUE_DATASOURCE': unique_name}) @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( @@ -60,16 +61,18 @@ class TestExtDataSources(CustomClusterTestSuite): @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args='--data_source_batch_size=2048') - def test_data_source_big_batch_size(self, vector, unique_database): + def test_data_source_big_batch_size(self, vector, unique_database, unique_name): """Run test with batch size greater than default size 1024""" - self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database) + self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database, + test_file_vars={'$UNIQUE_DATASOURCE': unique_name}) @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args='--data_source_batch_size=512') - def test_data_source_small_batch_size(self, vector, unique_database): + def test_data_source_small_batch_size(self, vector, unique_database, unique_name): """Run test with batch size less than default size 1024""" - self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database) + self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database, + test_file_vars={'$UNIQUE_DATASOURCE': unique_name}) @SkipIfApacheHive.data_connector_not_supported @pytest.mark.execute_serially diff --git a/tests/query_test/test_ext_data_sources.py b/tests/query_test/test_ext_data_sources.py index 8d97e34f7..c799012ee 100644 --- a/tests/query_test/test_ext_data_sources.py +++ b/tests/query_test/test_ext_data_sources.py @@ -84,8 +84,9 @@ class TestExtDataSources(ImpalaTestSuite): break assert match, result.data - def test_data_source_tables(self, vector, unique_database): - self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database) + def test_data_source_tables(self, vector, unique_database, unique_name): + self.run_test_case('QueryTest/data-source-tables', vector, use_db=unique_database, + test_file_vars={'$UNIQUE_DATASOURCE': unique_name}) def test_jdbc_data_source(self, vector, unique_database): self.run_test_case('QueryTest/jdbc-data-source', vector, use_db=unique_database)
