This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch branch-3.4.2 in repository https://gitbox.apache.org/repos/asf/impala.git
commit 54faec0523d3049d6a84c9027d4cde5268a62d15 Author: Joe McDonnell <[email protected]> AuthorDate: Wed Apr 13 15:18:17 2022 -0700 IMPALA-10057: Fix log spew by using jars in the classpath Some tests saw log spew that causes the INFO log files to be filled with output like this: E0903 02:25:39.453887 12060 TransactionKeepalive.java:137] Unexpected exception thrown Java exception follows: java.lang.BootstrapMethodError: java.lang.NoClassDefFoundError: org/apache/impala/common/TransactionKeepalive$HeartbeatContext at org.apache.impala.common.TransactionKeepalive$DaemonThread.run(TransactionKeepalive.java:114) at java.lang.Thread.run(Thread.java:748) ... It turns out that the catalogd/impalad use a CLASSPATH in tests that refers to fe/target/classes. The maven command that runs frontend tests recompiles these classes and causes the files in fe/target/classes to be deleted and recreated. There are race conditions where this causes the symptoms above. This changes the CLASSPATH to use the frontend jars, which are not impacted by the machinations on fe/target/classes. To find the appropriate jar, set-classpath.sh needs to know the Impala version. This adds IMPALA_VERSION in bin/impala-config.sh to provide an easy to use environment variable. To make the versioning more uniform, this modifies bin/save-version.sh to use this environment variable. It also adds a check to make sure that the Java pom.xml files use the same version as the environment variable. It fails the build if the Java pom.xml files do not match. Testing: - Ran core jobs - Checked the log file sizes on jobs - Changed a Java pom.xml's version and verified that bin/validate-java-pom-versions.sh fails Merge conflicts: - Change version string "4.1.0" to "3.4.2". Change-Id: Id35544e446c5bf283c322d3fe2e7ad475cfa12eb Reviewed-on: http://gerrit.cloudera.org:8080/18415 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Michael Smith <[email protected]> Reviewed-by: Quanlong Huang <[email protected]> Reviewed-on: http://gerrit.cloudera.org:8080/18879 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Quanlong Huang <[email protected]> Reviewed-on: http://gerrit.cloudera.org:8080/21129 Reviewed-by: Zihao Ye <[email protected]> --- bin/impala-config.sh | 10 ++++++ bin/save-version.sh | 2 +- bin/set-classpath.sh | 8 +++-- bin/validate-java-pom-versions.sh | 68 +++++++++++++++++++++++++++++++++++++++ common/yarn-extras/pom.xml | 4 +-- ext-data-source/api/pom.xml | 4 +-- ext-data-source/pom.xml | 4 +-- ext-data-source/sample/pom.xml | 4 +-- ext-data-source/test/pom.xml | 6 ++-- fe/pom.xml | 4 +-- impala-parent/CMakeLists.txt | 4 +++ impala-parent/pom.xml | 6 ++-- query-event-hook-api/pom.xml | 4 +-- shaded-deps/pom.xml | 2 +- testdata/TableFlattener/pom.xml | 4 +-- testdata/pom.xml | 4 +-- tests/test-hive-udfs/pom.xml | 2 +- 17 files changed, 113 insertions(+), 27 deletions(-) diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 6233a2f34..4218fa6e7 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -63,6 +63,15 @@ fi # variable from a previously-sourced impala-config.sh overrides the new value. # ####################################################################################### +# The current Impala version that will be embedded in the Impala binary. This is +# also used to find the Impala frontend jar files, so the version must match +# the version in our Maven pom.xml files. This is validated via +# bin/validate-java-pom-version.sh during the build. +# WARNING: If changing this value, also run these commands: +# cd ${IMPALA_HOME}/java +# mvn versions:set -DnewVersion=YOUR_NEW_VERSION +export IMPALA_VERSION=3.4.2-RELEASE + # The unique build id of the toolchain to use if bootstrapping. This is generated by the # native-toolchain build when publishing its build artifacts. This should be changed when # moving to a different build of the toolchain, e.g. when a version is bumped or a @@ -793,6 +802,7 @@ CLASSPATH="$LZO_JAR_PATH:$CLASSPATH" # A marker in the environment to prove that we really did source this file export IMPALA_CONFIG_SOURCED=1 +echo "IMPALA_VERSION = $IMPALA_VERSION" echo "IMPALA_HOME = $IMPALA_HOME" echo "HADOOP_HOME = $HADOOP_HOME" echo "HADOOP_CONF_DIR = $HADOOP_CONF_DIR" diff --git a/bin/save-version.sh b/bin/save-version.sh index b56dca1e3..375702463 100755 --- a/bin/save-version.sh +++ b/bin/save-version.sh @@ -21,7 +21,7 @@ # Note: for internal (aka pre-release) versions, the version should have # "-INTERNAL" appended. Parts of the code will look for this to distinguish # between released and internal versions. -VERSION=3.4.1-RELEASE +VERSION=${IMPALA_VERSION} GIT_HASH=eb1ed66fa435a722fa8c6a7c58ff53edc10c572e if [ -z $GIT_HASH ] then diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh index d5a168230..b311c4b15 100644 --- a/bin/set-classpath.sh +++ b/bin/set-classpath.sh @@ -28,11 +28,15 @@ if [ "$0" = "$BASH_SOURCE" ]; then exit 1 fi +# IMPALA-10057: The fe/target/classes and fe/target/test-classes +# directory can get cleaned out / rebuilt as part of running the +# frontend tests. This uses jars to avoid issues related to that +# inconsistency. CLASSPATH=\ "$IMPALA_HOME"/fe/src/test/resources:\ -"$IMPALA_HOME"/fe/target/classes:\ +"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}.jar:\ "$IMPALA_HOME"/fe/target/dependency:\ -"$IMPALA_HOME"/fe/target/test-classes: +"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar: FE_CP_FILE="$IMPALA_HOME/fe/target/build-classpath.txt" diff --git a/bin/validate-java-pom-versions.sh b/bin/validate-java-pom-versions.sh new file mode 100755 index 000000000..a996eb666 --- /dev/null +++ b/bin/validate-java-pom-versions.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This script verifies that the Java pom.xml files use the +# IMPALA_VERSION. This can fail if the IMPALA_VERSION changes +# without the version in the pom.xml changing. To update the +# pom.xml files, run: +# cd $IMPALA_HOME/java +# mvn versions:set -DnewVersion=NEW_MAVEN_VERSION + +set -euo pipefail + +MAVEN_VERSION_STRING="<version>${IMPALA_VERSION}</version>" + +# Detect if IMPALA_HOME is a git repository +pushd ${IMPALA_HOME} > /dev/null 2>&1 +IS_GIT_CHECKOUT=false +if git ls-files --error-unmatch > /dev/null 2>&1 ; then + IS_GIT_CHECKOUT=true +fi; +popd > /dev/null 2>&1 + +RETVAL=0 +NO_MATCH_FILES=() +for pom_file in $(find ${IMPALA_HOME} -name pom.xml); do + # If this is a git checkout, then only do the check for pom.xml + # files known to git. If this is not a git checkout, then it should + # be building from a tarball, and there should not be extra + # pom.xml files. + if ${IS_GIT_CHECKOUT} && + ! git ls-files --error-unmatch ${pom_file} > /dev/null 2>&1 ; then + # This pom.xml file is not known to git. + continue; + fi + if ! grep $MAVEN_VERSION_STRING "${pom_file}" > /dev/null; then + NO_MATCH_FILES+=(${pom_file}) + RETVAL=1 + fi +done + +if [[ $RETVAL != 0 ]]; then + echo "Check for Java pom.xml versions FAILED" + echo "Expected ${MAVEN_VERSION_STRING}" + echo "Not found in:" + for pom_file in ${NO_MATCH_FILES[@]}; do + echo " ${pom_file}" + done + echo "The pom.xml files can be updated automatically via:" + echo 'cd ${IMPALA_HOME}/java; mvn versions:set -DnewVersion=YOUR_NEW_VERSION' +fi + +exit $RETVAL diff --git a/common/yarn-extras/pom.xml b/common/yarn-extras/pom.xml index 48a609a8b..0f6dbc34d 100644 --- a/common/yarn-extras/pom.xml +++ b/common/yarn-extras/pom.xml @@ -22,13 +22,13 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>yarn-extras</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <name>YARN Extras</name> <description>Extra Hadoop classes from YARN needed by Impala</description> <packaging>jar</packaging> diff --git a/ext-data-source/api/pom.xml b/ext-data-source/api/pom.xml index 82e6a8838..121d48981 100644 --- a/ext-data-source/api/pom.xml +++ b/ext-data-source/api/pom.xml @@ -23,11 +23,11 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source</artifactId> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> </parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source-api</artifactId> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <name>Apache Impala External Data Source API</name> <description>External Data Source API interface definition</description> <packaging>jar</packaging> diff --git a/ext-data-source/pom.xml b/ext-data-source/pom.xml index b4b004508..ddf7ea52d 100644 --- a/ext-data-source/pom.xml +++ b/ext-data-source/pom.xml @@ -22,14 +22,14 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source</artifactId> <name>Apache Impala External Data Source</name> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>pom</packaging> <modules> diff --git a/ext-data-source/sample/pom.xml b/ext-data-source/sample/pom.xml index c0df755dd..3f8a82fd0 100644 --- a/ext-data-source/sample/pom.xml +++ b/ext-data-source/sample/pom.xml @@ -23,11 +23,11 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source</artifactId> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> </parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source-sample</artifactId> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <name>Apache Impala External Data Source Sample</name> <description>External Data Source Sample</description> <packaging>jar</packaging> diff --git a/ext-data-source/test/pom.xml b/ext-data-source/test/pom.xml index c96c86a2a..94050a79c 100644 --- a/ext-data-source/test/pom.xml +++ b/ext-data-source/test/pom.xml @@ -23,18 +23,18 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source</artifactId> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> </parent> <groupId>org.apache.impala</groupId> <artifactId>impala-data-source-test</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <name>Apache Impala External Data Source Test Library</name> <description>Test External Data Source</description> <packaging>jar</packaging> <url>.</url> <properties> - <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version> + <impala.extdatasrc.api.version>${project.version}</impala.extdatasrc.api.version> </properties> <dependencies> diff --git a/fe/pom.xml b/fe/pom.xml index 84e30bfa6..949975140 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -23,13 +23,13 @@ under the License. <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>impala-frontend</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>jar</packaging> <name>Apache Impala Query Engine Frontend</name> diff --git a/impala-parent/CMakeLists.txt b/impala-parent/CMakeLists.txt index fdd6e9818..127c412c9 100644 --- a/impala-parent/CMakeLists.txt +++ b/impala-parent/CMakeLists.txt @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +add_custom_target(validate_java_pom_versions ALL + COMMAND $ENV{IMPALA_HOME}/bin/validate-java-pom-versions.sh +) + add_custom_target(impala-parent ALL COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests ) diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml index 400755212..5f368f598 100644 --- a/impala-parent/pom.xml +++ b/impala-parent/pom.xml @@ -21,7 +21,7 @@ under the License. <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>pom</packaging> <name>Apache Impala Parent POM</name> @@ -45,8 +45,8 @@ under the License. <kite.version>${env.IMPALA_KITE_VERSION}</kite.version> <knox.version>${env.IMPALA_KNOX_VERSION}</knox.version> <thrift.version>0.9.3-1</thrift.version> - <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version> - <impala.query.event.hook.api.version>1.0-SNAPSHOT</impala.query.event.hook.api.version> + <impala.extdatasrc.api.version>${project.version}</impala.extdatasrc.api.version> + <impala.query.event.hook.api.version>${project.version}</impala.query.event.hook.api.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <kudu.version>${env.IMPALA_KUDU_JAVA_VERSION}</kudu.version> <commons-io.version>2.6</commons-io.version> diff --git a/query-event-hook-api/pom.xml b/query-event-hook-api/pom.xml index 8466e0b11..e958c7e30 100644 --- a/query-event-hook-api/pom.xml +++ b/query-event-hook-api/pom.xml @@ -22,14 +22,14 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>query-event-hook-api</artifactId> <name>Apache Impala Query Event Hook API</name> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>jar</packaging> <url>.</url> diff --git a/shaded-deps/pom.xml b/shaded-deps/pom.xml index eefd73b94..bd55fce68 100644 --- a/shaded-deps/pom.xml +++ b/shaded-deps/pom.xml @@ -27,7 +27,7 @@ the same dependencies <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/testdata/TableFlattener/pom.xml b/testdata/TableFlattener/pom.xml index cff546568..8d6e52104 100644 --- a/testdata/TableFlattener/pom.xml +++ b/testdata/TableFlattener/pom.xml @@ -22,14 +22,14 @@ <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>nested-table-flattener</artifactId> <name>Impala Nested Table Flattener</name> - <version>1.0-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>jar</packaging> <build> diff --git a/testdata/pom.xml b/testdata/pom.xml index 74ce872c7..96ab6259b 100644 --- a/testdata/pom.xml +++ b/testdata/pom.xml @@ -23,14 +23,14 @@ under the License. <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.impala</groupId> <artifactId>impala-testdata</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <packaging>jar</packaging> <name>Build some test data</name> diff --git a/tests/test-hive-udfs/pom.xml b/tests/test-hive-udfs/pom.xml index e51d598d4..bab67c4c7 100644 --- a/tests/test-hive-udfs/pom.xml +++ b/tests/test-hive-udfs/pom.xml @@ -22,7 +22,7 @@ under the License. <parent> <groupId>org.apache.impala</groupId> <artifactId>impala-parent</artifactId> - <version>0.1-SNAPSHOT</version> + <version>3.4.2-RELEASE</version> <relativePath>../../impala-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion>
