This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.15.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit bbebda457cf42ed5b7fb4397b9fd2c642f1feadc Author: Y Ethan Guo <[email protected]> AuthorDate: Mon May 27 02:10:02 2024 -0700 [HUDI-7707] Enable bundle validation on Java 8 and 11 (#11313) --- .github/workflows/bot.yml | 8 +++++--- .github/workflows/release_candidate_validation.yml | 6 +++--- packaging/bundle-validation/ci_run.sh | 9 +++++---- packaging/bundle-validation/validate.sh | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index fd5835afb14..c649b502529 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -499,21 +499,23 @@ jobs: - name: IT - Bundle Validation - OpenJDK 8 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} SCALA_PROFILE: ${{ matrix.scalaProfile }} if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 run: | HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8 + ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 - name: IT - Bundle Validation - OpenJDK 11 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} SCALA_PROFILE: ${{ matrix.scalaProfile }} if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 run: | HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11 + ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 - name: IT - Bundle Validation - OpenJDK 17 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} @@ -523,7 +525,7 @@ jobs: if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17 run: | HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17 + ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 integration-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml index 02a598888ea..d9872cc7906 100644 --- a/.github/workflows/release_candidate_validation.yml +++ b/.github/workflows/release_candidate_validation.yml @@ -81,7 +81,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} run: | - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8 $STAGING_REPO_NUM + ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 $STAGING_REPO_NUM - name: IT - Bundle Validation - OpenJDK 11 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} @@ -89,7 +89,7 @@ jobs: SPARK_RUNTIME: ${{ matrix.sparkRuntime }} if: ${{ startsWith(env.SPARK_PROFILE, 'spark3') }} # Only Spark 3.x supports Java 11 as of now run: | - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11 $STAGING_REPO_NUM + ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 $STAGING_REPO_NUM - name: IT - Bundle Validation - OpenJDK 17 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} @@ -97,4 +97,4 @@ jobs: SPARK_RUNTIME: ${{ matrix.sparkRuntime }} if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3 supports Java 17 as of now run: | - ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17 $STAGING_REPO_NUM + ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM diff --git a/packaging/bundle-validation/ci_run.sh b/packaging/bundle-validation/ci_run.sh index 669278b9f61..58ef6f37588 100755 --- a/packaging/bundle-validation/ci_run.sh +++ b/packaging/bundle-validation/ci_run.sh @@ -27,9 +27,10 @@ # This is to run by GitHub Actions CI tasks from the project root directory # and it contains the CI environment-specific variables. -HUDI_VERSION=$1 -JAVA_RUNTIME_VERSION=$2 -STAGING_REPO_NUM=$3 +CONTAINER_NAME=$1 +HUDI_VERSION=$2 +JAVA_RUNTIME_VERSION=$3 +STAGING_REPO_NUM=$4 echo "HUDI_VERSION: $HUDI_VERSION JAVA_RUNTIME_VERSION: $JAVA_RUNTIME_VERSION" echo "SPARK_RUNTIME: $SPARK_RUNTIME SPARK_PROFILE (optional): $SPARK_PROFILE" echo "SCALA_PROFILE: $SCALA_PROFILE" @@ -237,7 +238,7 @@ docker build \ . # run validation script in docker -docker run --name hudi_docker \ +docker run --name $CONTAINER_NAME \ -v ${GITHUB_WORKSPACE}:/opt/bundle-validation/docker-test \ -v $TMP_JARS_DIR:/opt/bundle-validation/jars \ -v $TMP_DATA_DIR:/opt/bundle-validation/data \ diff --git a/packaging/bundle-validation/validate.sh b/packaging/bundle-validation/validate.sh index de319e7d9dd..d81f3771f0b 100755 --- a/packaging/bundle-validation/validate.sh +++ b/packaging/bundle-validation/validate.sh @@ -299,7 +299,7 @@ if [ "$?" -ne 0 ]; then fi echo "::warning::validate.sh done validating utilities slim bundle" -if [[ ${JAVA_RUNTIME_VERSION} == 'openjdk8' && ${SCALA_PROFILE} != 'scala-2.13' ]]; then +if [[ ${JAVA_RUNTIME_VERSION} == 'openjdk8' && ${SCALA_PROFILE} != 'scala-2.13' && ! "${FLINK_HOME}" == *"1.18"* ]]; then echo "::warning::validate.sh validating flink bundle" test_flink_bundle if [ "$?" -ne 0 ]; then
