This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 14fd2b1bd33a99c9c367771ef1b131eac90d9875 Author: Riza Suminto <[email protected]> AuthorDate: Thu Jan 8 21:44:19 2026 -0800 IMPALA-14669: Fix false-positive in test_ext_data_sources.py test_ext_data_sources.py has several negative tests that use pytest.raise + match such as test_invalid_postgres_jdbc_table, test_invalid_postgres_hive_jdbc_table, and test_invalid_mysql_hive_jdbc_table_properties. This pytest.raise + match does not work in pytest-2.9.2, and silently pass without actually matching the expected error message. On pytest-6.2.5, these tests fail for not finding the match in thrown exception. The tests are only throwing exception because the CREATE QUERY sent to Hive instead of Impala, and Hive exception message is not validated. This patch modify test_invalid_postgres_jdbc_table to run both CREATE QUERY and SELECT via Impala and validate error messages coming out of Impala. test_invalid_postgres_hive_jdbc_table and test_invalid_mysql_hive_jdbc_table_properties stays running against Hive, but the expected error messages are adjusted. None of them validate error message from Impala because Hive will fail the malformed CREATE QUERY. Testing: - Run and pass test_ext_data_sources.py exhaustively using both pytest-2.9.2 and pytest-6.2.5 Change-Id: I3f332a36bd0c4a796d3f474be666e4cc1dc46b45 Reviewed-on: http://gerrit.cloudera.org:8080/23843 Reviewed-by: Wenzhe Zhou <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../jdbc/conf/JdbcStorageConfigManager.java | 28 ++++----- tests/custom_cluster/test_ext_data_sources.py | 66 ++++++++++++++++++---- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java index 143353a81..d83814604 100644 --- a/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java +++ b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java @@ -101,22 +101,24 @@ public class JdbcStorageConfigManager { try { String dbTypeName = props.get(JdbcStorageConfig.DATABASE_TYPE.getPropertyName()); dbType = DatabaseType.valueOf(dbTypeName.toUpperCase()); - - if (dbType != DatabaseType.HIVE) { - String table = props.get(JdbcStorageConfig.TABLE.getPropertyName()); - String query = props.get(JdbcStorageConfig.QUERY.getPropertyName()); - if (Strings.isNullOrEmpty(table) && Strings.isNullOrEmpty(query)) { - throw new IllegalArgumentException( - "For JDBC tables, either 'table' or 'query' property must be set."); - } - if (!Strings.isNullOrEmpty(table) && !Strings.isNullOrEmpty(query)) { - throw new IllegalArgumentException("Only one of 'hive.sql.table' or" + - " 'hive.sql.query' should be set."); - } - } } catch (Exception e) { throw new IllegalArgumentException("Unknown database type.", e); } + + // For non-Hive databases, either 'table' or 'query' property must be set. + if (dbType != DatabaseType.HIVE) { + String table = props.get(JdbcStorageConfig.TABLE.getPropertyName()); + String query = props.get(JdbcStorageConfig.QUERY.getPropertyName()); + if (Strings.isNullOrEmpty(table) && Strings.isNullOrEmpty(query)) { + throw new IllegalArgumentException( + "For JDBC tables, either 'table' or 'query' property must be set."); + } + if (!Strings.isNullOrEmpty(table) && !Strings.isNullOrEmpty(query)) { + throw new IllegalArgumentException( + "Only one of 'table' or 'query' should be set."); + } + } + // Check the required parameters for (JdbcStorageConfig config : JdbcStorageConfig.values()) { if (config.isRequired() && !props.containsKey(config.getPropertyName())) { diff --git a/tests/custom_cluster/test_ext_data_sources.py b/tests/custom_cluster/test_ext_data_sources.py index 1f0c6d919..30279ceb8 100644 --- a/tests/custom_cluster/test_ext_data_sources.py +++ b/tests/custom_cluster/test_ext_data_sources.py @@ -392,8 +392,11 @@ class TestPostgresJdbcTables(CustomClusterTestSuite): "query"="SELECT * FROM country"); """.format(unique_database) - with pytest.raises(Exception, match="Only one of 'table' or 'query' should be set"): - self.run_stmt_in_hive(sql_both_set) + self.execute_query(sql_both_set) + ex = self.execute_query_expect_failure( + self.client, + "select count(*) from {0}.invalid_both_props".format(unique_database)) + assert "Only one of 'table' or 'query' should be set" in str(ex) sql_none_set = """ CREATE EXTERNAL TABLE {0}.invalid_no_props ( @@ -411,8 +414,12 @@ class TestPostgresJdbcTables(CustomClusterTestSuite): "dbcp.password"="password"); """.format(unique_database) - with pytest.raises(Exception, match="Either 'table' or 'query' must be set"): - self.run_stmt_in_hive(sql_none_set) + self.execute_query(sql_none_set) + ex = self.execute_query_expect_failure( + self.client, + "select count(*) from {0}.invalid_no_props".format(unique_database)) + assert "either 'table' or 'query' property must be set" in str(ex) + class TestHivePostgresJdbcTables(CustomClusterTestSuite): """Tests for hive jdbc postgres tables. """ @@ -605,8 +612,16 @@ class TestHivePostgresJdbcTables(CustomClusterTestSuite): ) """.format(unique_database) - with pytest.raises(Exception, match="Only one of 'hive.sql.table' or" - " 'hive.sql.query' should be set"): self.run_stmt_in_hive(sql_both_set) + try: + self.run_stmt_in_hive(sql_both_set) + # This test expect querying from Impala will fail. + # However, it is not possible to test this because the Hive CREATE TABLE + # query will fail when both "hive.sql.table" and "hive.sql.query" are set. + # ex = self.execute_query_expect_failure( + # "select count(*) from {0}.invalid_both_props".format(unique_database)) + # assert "Only one of 'hive.sql.table' or 'hive.sql.query' should be set" in str(ex) + except Exception as ex: + assert "Caught exception while initializing the SqlSerDe: null" in str(ex) # Neither hive.sql.table nor hive.sql.query is set sql_none_set = """ @@ -624,8 +639,18 @@ class TestHivePostgresJdbcTables(CustomClusterTestSuite): ) """.format(unique_database) - with pytest.raises(Exception, match="Either 'hive.sql.table' or" - " 'hive.sql.query' must be set"): self.run_stmt_in_hive(sql_none_set) + try: + self.run_stmt_in_hive(sql_none_set) + # This test expect querying from Impala will fail. + # However, it is not possible to test this because the Hive CREATE TABLE + # query will fail when none of "hive.sql.table" and "hive.sql.query" are set. + # ex = self.execute_query_expect_failure( + # "select count(*) from {0}.invalid_no_props".format(unique_database)) + # assert "Either 'hive.sql.table' or 'hive.sql.query' must be set" in str(ex) + except Exception as ex: + assert ("Caught exception while trying to get columns: " + "Both parameters are null") in str(ex) + class TestMySqlExtJdbcTables(CustomClusterTestSuite): """Impala query tests for external jdbc tables on MySQL server. @@ -863,8 +888,16 @@ class TestMySqlExtJdbcTables(CustomClusterTestSuite): ) """.format(unique_database) - with pytest.raises(Exception, match="Only one of 'hive.sql.table' or" - " 'hive.sql.query' should be set"): self.run_stmt_in_hive(sql_both_set) + try: + self.run_stmt_in_hive(sql_both_set) + # This test expect querying from Impala will fail. + # However, it is not possible to test this because the Hive CREATE TABLE + # query will fail when both "hive.sql.table" and "hive.sql.query" are set. + # ex = self.execute_query_expect_failure( + # "select count(*) from {0}.invalid_both_props_mysql".format(unique_database)) + # assert "Only one of 'hive.sql.table' or 'hive.sql.query' should be set" in str(ex) + except Exception as ex: + assert "Caught exception while initializing the SqlSerDe: null" in str(ex) # Neither hive.sql.table nor hive.sql.query is set sql_none_set = """ @@ -882,8 +915,17 @@ class TestMySqlExtJdbcTables(CustomClusterTestSuite): ) """.format(unique_database) - with pytest.raises(Exception, match="Either 'hive.sql.table' or" - " 'hive.sql.query' must be set"): self.run_stmt_in_hive(sql_none_set) + try: + self.run_stmt_in_hive(sql_none_set) + # This test expect querying from Impala will fail. + # However, it is not possible to test this because the Hive CREATE TABLE + # query will fail when none of "hive.sql.table" and "hive.sql.query" are set. + # ex = self.execute_query_expect_failure( + # "select count(*) from {0}.invalid_no_props".format(unique_database)) + # assert "Either 'hive.sql.table' or 'hive.sql.query' must be set" in str(ex) + except Exception as ex: + assert ("Cannot load JDBC driver class 'com.mysql.cj.jdbc.Driver'") in str(ex) + class TestImpalaExtJdbcTables(CustomClusterTestSuite): """Impala query tests for external jdbc tables in Impala cluster."""
