This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit c3ec9272c591624e8dfc130a85c98da318735f14 Author: Michael Smith <[email protected]> AuthorDate: Mon Nov 14 13:26:21 2022 -0800 IMPALA-11724: Use CDP Ozone in test environment Updates the test environment to default to the CDP build of Ozone, as the latest build of CDP Hive depends on pre-release features unavailable in Ozone 1.2.1. Apache Ozone 1.2 can still be used by setting USE_APACHE_OZONE=true. The latest CDP build also includes a version of Ozone based on ozone#master with a candidate version of 1.3.0. Both Apache and CDP therefore have builds of Ozone we can test with that use the new artifact names introduced in Ozone 1.2, so this patch cleans up setup that was only needed for Ozone versions prior to 1.2. Change-Id: I1177a1b820fe21adca9f8c1cc51ff73ee001d3f2 Reviewed-on: http://gerrit.cloudera.org:8080/19247 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Michael Smith <[email protected]> --- bin/bootstrap_toolchain.py | 3 +-- bin/impala-config.sh | 17 ++--------------- java/executor-deps/pom.xml | 4 ++-- java/pom.xml | 2 -- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index e6c4d0423..7e8371a4d 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -517,8 +517,7 @@ def get_hadoop_downloads(): if use_apache_ozone: ozone = ApacheComponent("ozone", component_path_tmpl="ozone/${version}") else: - ozone = CdpComponent("ozone", archive_basename_tmpl="hadoop-ozone-${version}", - unpack_directory_tmpl="ozone-${version}") + ozone = CdpComponent("ozone") use_apache_hive = os.environ["USE_APACHE_HIVE"] == "true" if use_apache_hive: diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 188f21879..a8eb649bd 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -349,28 +349,15 @@ if [[ "${IMPALA_HIVE_MAJOR_VERSION}" == "1" || return 1 fi -# Defaults to Apache because Ozone in newer CDP releases is currently unmaintained -export USE_APACHE_OZONE=${USE_APACHE_OZONE-true} +export USE_APACHE_OZONE=${USE_APACHE_OZONE-false} if $USE_APACHE_OZONE; then - export IMPALA_OZONE_DIST_TYPE="apache-ozone" export IMPALA_OZONE_VERSION=${APACHE_OZONE_VERSION} export IMPALA_OZONE_URL=${APACHE_OZONE_URL-} else - export IMPALA_OZONE_DIST_TYPE="ozone" export IMPALA_OZONE_VERSION=${CDP_OZONE_VERSION} export IMPALA_OZONE_URL=${CDP_OZONE_URL-} fi -# Ozone changed jar groupId and artifactId in Ozone 1.2 -export IMPALA_OZONE_MINOR_VERSION=$(echo "$IMPALA_OZONE_VERSION" | cut -d . -f 2) -if [[ ${IMPALA_OZONE_MINOR_VERSION} < 2 ]]; then - export IMPALA_OZONE_JAR_GROUP_ID="org.apache.hadoop" - export IMPALA_OZONE_JAR_ARTIFACT_ID="hadoop-ozone-filesystem-hadoop3" -else - export IMPALA_OZONE_JAR_GROUP_ID="org.apache.ozone" - export IMPALA_OZONE_JAR_ARTIFACT_ID="ozone-filesystem-hadoop3" -fi - # It is important to have a coherent view of the JAVA_HOME and JAVA executable. # The JAVA_HOME should be determined first, then the JAVA executable should be # derived from JAVA_HOME. bin/bootstrap_development.sh adds code to @@ -797,7 +784,7 @@ export HADOOP_LIB_DIR=${HADOOP_LIB_DIR_OVERRIDE:-"${HADOOP_HOME}/lib"} export HADOOP_CLASSPATH="${HADOOP_CLASSPATH-}" # Add Ozone Hadoop filesystem implementation when using Ozone if [ "${TARGET_FILESYSTEM}" = "ozone" ]; then - OZONE_JAR="${IMPALA_OZONE_JAR_ARTIFACT_ID}-${IMPALA_OZONE_VERSION}.jar" + OZONE_JAR="ozone-filesystem-hadoop3-${IMPALA_OZONE_VERSION}.jar" HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${OZONE_HOME}/share/ozone/lib/${OZONE_JAR}" fi # Add the path containing the hadoop-aws jar, which is required to access AWS from the diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml index 4f1ea5029..bf607480f 100644 --- a/java/executor-deps/pom.xml +++ b/java/executor-deps/pom.xml @@ -158,8 +158,8 @@ under the License. </dependency> <dependency> - <groupId>${ozone.groupId}</groupId> - <artifactId>${ozone.artifactId}</artifactId> + <groupId>org.apache.ozone</groupId> + <artifactId>ozone-filesystem-hadoop3</artifactId> <version>${ozone.version}</version> <!-- Remove all transitive dependencies from the Apache Ozone dependency. ozone-filesystem-hadoop3 is a shaded-jar, which already includes diff --git a/java/pom.xml b/java/pom.xml index 41ce4f0a7..20e26adf2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -46,8 +46,6 @@ under the License. <hbase.version>${env.IMPALA_HBASE_VERSION}</hbase.version> <avro.version>${env.IMPALA_AVRO_JAVA_VERSION}</avro.version> <orc.version>${env.IMPALA_ORC_JAVA_VERSION}</orc.version> - <ozone.groupId>${env.IMPALA_OZONE_JAR_GROUP_ID}</ozone.groupId> - <ozone.artifactId>${env.IMPALA_OZONE_JAR_ARTIFACT_ID}</ozone.artifactId> <ozone.version>${env.IMPALA_OZONE_VERSION}</ozone.version> <parquet.version>${env.IMPALA_PARQUET_VERSION}</parquet.version> <kite.version>${env.IMPALA_KITE_VERSION}</kite.version>
