This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit b072f29552d7329c2559837666d0636020f729cc Author: Joe McDonnell <[email protected]> AuthorDate: Thu Oct 10 19:13:41 2024 -0700 IMPALA-13446: Bump CDP GBN to 58457853 to get Ranger improvements This updates the GBN to include the RANGER-4585 functionality to support multi-column policy creation. IMPALA-12554 will use this to create a single multi-column policy for a GRANT statement rather than many single-column policies. This fixes a few issues encountered during the upgrade: 1. This includes the fix for IMPALA-13433 to make test_sfs.py resilient to HMS versions that do not properly create the database directories. 2. This modified test_metadata_query_statements.py to use unique directories for the databases to avoid HMS bugs. 3. The version of Avro changed, which changed the version of Jackson and the package name of the JsonParseException. This adds code to tolerate both the old and new package name in the error message. 4. This includes the fix for IMPALA-13391 to exclude log4j-slf4j-impl from hadoop-cloud-storage. 5. This excludes an unnecessary org.cloudera.logredactor dependency. Testing: - Ran a core job Change-Id: I32727020a69a66c3af4f4096fe15bc81600e2215 Reviewed-on: http://gerrit.cloudera.org:8080/21921 Reviewed-by: Michael Smith <[email protected]> Reviewed-by: Fang-Yu Rao <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- bin/impala-config.sh | 22 +++++++++++----------- .../org/apache/impala/analysis/AnalyzeDDLTest.java | 6 +++--- .../org/apache/impala/common/FrontendTestBase.java | 7 +++++++ java/executor-deps/pom.xml | 4 ++++ java/shaded-deps/hive-exec/pom.xml | 4 ++++ java/test-hive-udfs/pom.xml | 4 ++++ .../queries/QueryTest/describe-db.test | 12 ++++++------ tests/metadata/test_metadata_query_statements.py | 9 ++++++--- tests/query_test/test_sfs.py | 5 +++++ 9 files changed, 50 insertions(+), 23 deletions(-) diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 363236b93..e36ad542a 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -228,19 +228,19 @@ fi : ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com} export IMPALA_TOOLCHAIN_HOST -export CDP_BUILD_NUMBER=49623641 +export CDP_BUILD_NUMBER=58457853 export CDP_MAVEN_REPOSITORY=\ "https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven"; -export CDP_AVRO_JAVA_VERSION=1.8.2.7.3.0.0-128 -export CDP_HADOOP_VERSION=3.1.1.7.3.0.0-128 -export CDP_HBASE_VERSION=2.4.17.7.3.0.0-128 -export CDP_HIVE_VERSION=3.1.3000.7.3.0.0-128 -export CDP_ICEBERG_VERSION=1.3.1.7.3.0.0-128 -export CDP_KNOX_VERSION=2.0.0.7.3.0.0-128 -export CDP_OZONE_VERSION=1.3.0.7.3.0.0-128 -export CDP_PARQUET_VERSION=1.12.3.7.3.0.0-128 -export CDP_RANGER_VERSION=2.4.0.7.3.0.0-128 -export CDP_TEZ_VERSION=0.9.1.7.3.0.0-128 +export CDP_AVRO_JAVA_VERSION=1.11.1.7.3.1.0-160 +export CDP_HADOOP_VERSION=3.1.1.7.3.1.0-160 +export CDP_HBASE_VERSION=2.4.17.7.3.1.0-160 +export CDP_HIVE_VERSION=3.1.3000.7.3.1.0-160 +export CDP_ICEBERG_VERSION=1.3.1.7.3.1.0-160 +export CDP_KNOX_VERSION=2.0.0.7.3.1.0-160 +export CDP_OZONE_VERSION=1.3.0.7.3.1.0-160 +export CDP_PARQUET_VERSION=1.12.3.7.3.1.0-160 +export CDP_RANGER_VERSION=2.4.0.7.3.1.0-160 +export CDP_TEZ_VERSION=0.9.1.7.3.1.0-160 # Ref: https://infra.apache.org/release-download-pages.html#closer : ${APACHE_MIRROR:="https://www.apache.org/dyn/closer.cgi"} diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java index d111f4c1c..6c96dfdd8 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java @@ -1337,7 +1337,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { "('avro.schema.literal'='{\"name\": \"my_record\", \"type\": \"record\", " + "\"fields\": {\"name\": \"string1\", \"type\": \"string\"}]}')", propertyType), "Error parsing Avro schema for table 'functional.alltypes': " + - "org.codehaus.jackson.JsonParseException: Unexpected close marker ']': " + + "com.fasterxml.jackson.core.JsonParseException: Unexpected close marker ']': "+ "expected '}'"); AnalysisError(String.format("alter table functional.alltypes set %s " + "('avro.schema.literal'='')", propertyType), @@ -1372,7 +1372,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { "\"fields\": {\"name\": \"string1\", \"type\": \"string\"}]}', " + "'avro.schema.url'='')", propertyType), "Error parsing Avro schema for table 'functional.alltypes': " + - "org.codehaus.jackson.JsonParseException: Unexpected close marker ']': " + + "com.fasterxml.jackson.core.JsonParseException: Unexpected close marker ']': "+ "expected '}'"); // Url is invalid but ignored because literal is provided. AnalyzesOk(String.format("alter table functional.alltypes set %s " + @@ -3249,7 +3249,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { "('avro.schema.literal'='{\"name\": \"my_record\", \"type\": \"record\", " + "\"fields\": {\"name\": \"string1\", \"type\": \"string\"}]}')", "Error parsing Avro schema for table 'default.foo_avro': " + - "org.codehaus.jackson.JsonParseException: Unexpected close marker ']': "+ + "com.fasterxml.jackson.core.JsonParseException: Unexpected close marker ']': "+ "expected '}'"); // Map/Array types in Avro schema. diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java index a51aed0d8..14e7371fb 100644 --- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java +++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java @@ -320,6 +320,13 @@ public class FrontendTestBase extends AbstractFrontendTest { errorString = errorString.replace("No FileSystem for scheme ", "No FileSystem for scheme: "); } + // Different versions of Avro use different versions of Jackson. That can result + // in a different package name for JsonParseException. Tolerate either version + // by replacing the old package (org.codehaus.jackson.JsonParseException) for + // the new package (com.fasterxml.jackson.core.JsonParseException). + errorString = errorString.replace("org.codehaus.jackson.JsonParseException", + "com.fasterxml.jackson.core.JsonParseException"); + Assert.assertTrue(msg, errorString.startsWith(expectedErrorString)); return; } diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml index 360d09605..62a31d54d 100644 --- a/java/executor-deps/pom.xml +++ b/java/executor-deps/pom.xml @@ -112,6 +112,10 @@ under the License. <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + </exclusion> </exclusions> </dependency> diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml index 16476dc07..18be87df6 100644 --- a/java/shaded-deps/hive-exec/pom.xml +++ b/java/shaded-deps/hive-exec/pom.xml @@ -52,6 +52,10 @@ the same dependencies <groupId>com.cloudera</groupId> <artifactId>*</artifactId> </exclusion> + <exclusion> + <groupId>org.cloudera.logredactor</groupId> + <artifactId>*</artifactId> + </exclusion> <exclusion> <groupId>org.bouncycastle</groupId> <artifactId>*</artifactId> diff --git a/java/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml index 0565d806e..5120c46d7 100644 --- a/java/test-hive-udfs/pom.xml +++ b/java/test-hive-udfs/pom.xml @@ -59,6 +59,10 @@ under the License. <groupId>com.cloudera</groupId> <artifactId>*</artifactId> </exclusion> + <exclusion> + <groupId>org.cloudera.logredactor</groupId> + <artifactId>*</artifactId> + </exclusion> <exclusion> <groupId>org.bouncycastle</groupId> <artifactId>*</artifactId> diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test index a43b3f81c..8dca36f98 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test +++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test @@ -30,7 +30,7 @@ describe database extended impala_test_desc_db3 ---- RESULTS '','$USER','USER' 'Owner: ','','' -'impala_test_desc_db3','$NAMENODE/testdb','' +'impala_test_desc_db3','$NAMENODE/testdb3','' ---- TYPES string, string, string ==== @@ -39,7 +39,7 @@ describe database extended impala_test_desc_db4 ---- RESULTS '','$USER','USER' 'Owner: ','','' -'impala_test_desc_db4','$NAMENODE/test2.db','test comment' +'impala_test_desc_db4','$NAMENODE/test4.db','test comment' ---- TYPES string, string, string ==== @@ -47,7 +47,7 @@ string, string, string describe database impala_test_desc_db5 ---- RESULTS 'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment' -'managedlocation:','$NAMENODE/test2.db','' +'managedlocation:','$NAMENODE/test5.db','' ---- TYPES string, string, string ==== @@ -57,7 +57,7 @@ describe database extended impala_test_desc_db5 '','$USER','USER' 'Owner: ','','' 'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment' -'managedlocation:','$NAMENODE/test2.db','' +'managedlocation:','$NAMENODE/test5.db','' ---- TYPES string, string, string ==== @@ -67,7 +67,7 @@ describe database extended impala_test_desc_db5 '','$USER','USER' 'Owner: ','','' 'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment' -'managedlocation:','$NAMENODE/test2.db','' +'managedlocation:','$NAMENODE/test5.db','' ---- TYPES string, string, string ==== @@ -77,7 +77,7 @@ describe database formatted impala_test_desc_db5 '','$USER','USER' 'Owner: ','','' 'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment' -'managedlocation:','$NAMENODE/test2.db','' +'managedlocation:','$NAMENODE/test5.db','' ---- TYPES string, string, string ==== diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py index aa3608a62..d94698551 100644 --- a/tests/metadata/test_metadata_query_statements.py +++ b/tests/metadata/test_metadata_query_statements.py @@ -167,15 +167,18 @@ class TestMetadataQueryStatements(ImpalaTestSuite): def test_describe_db(self, vector, cluster_properties): self.__test_describe_db_cleanup() try: + # Some versions of HMS will fail to create the database if the managed directory + # is already present. This is not a test for HMS, so this uses unique directory + # names to workaround the issue. self.client.execute("create database impala_test_desc_db1") self.client.execute("create database impala_test_desc_db2 " "comment 'test comment'") self.client.execute("create database impala_test_desc_db3 " - "location '" + get_fs_path("/testdb") + "'") + "location '" + get_fs_path("/testdb3") + "'") self.client.execute("create database impala_test_desc_db4 comment 'test comment' " - "location \"" + get_fs_path("/test2.db") + "\"") + "location \"" + get_fs_path("/test4.db") + "\"") self.client.execute("create database impala_test_desc_db5 comment 'test comment' " - "managedlocation \"" + get_fs_path("/test2.db") + "\"") + "managedlocation \"" + get_fs_path("/test5.db") + "\"") self.run_stmt_in_hive("create database hive_test_desc_db comment 'test comment' " "with dbproperties('pi' = '3.14', 'e' = '2.82')") self.run_stmt_in_hive("create database hive_test_desc_db2 comment 'test comment' " diff --git a/tests/query_test/test_sfs.py b/tests/query_test/test_sfs.py index 4a1ee1caf..cc4a602ac 100644 --- a/tests/query_test/test_sfs.py +++ b/tests/query_test/test_sfs.py @@ -48,6 +48,11 @@ class TestSFS(ImpalaTestSuite): hdfs_dir_for_managed_tables =\ "{0}/managed/{1}.db/".format(WAREHOUSE, unique_database) + # Some versions of HMS do not properly create the database directories. + # This isn't a test for HMS, so workaround the issue by creating the + # directories manually. + self.filesystem_client.make_dir(hdfs_dir_for_external_tables) + self.filesystem_client.make_dir(hdfs_dir_for_managed_tables) copy_files_to_hdfs_dir(files_for_external_tables, hdfs_dir_for_external_tables) copy_files_to_hdfs_dir(files_for_managed_tables, hdfs_dir_for_managed_tables)
