This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/branch-0.x by this push:
new 6913999fcd5 [HUDI-7707] Enable bundle validation on Java 8 and 11
(#11313)
6913999fcd5 is described below
commit 6913999fcd5df5adf1c68261b93ab8322f2db431
Author: Y Ethan Guo <[email protected]>
AuthorDate: Mon May 27 02:10:02 2024 -0700
[HUDI-7707] Enable bundle validation on Java 8 and 11 (#11313)
---
.github/workflows/bot.yml | 8 +++++---
.github/workflows/release_candidate_validation.yml | 6 +++---
packaging/bundle-validation/ci_run.sh | 9 +++++----
packaging/bundle-validation/validate.sh | 2 +-
4 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index fd5835afb14..c649b502529 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -499,21 +499,23 @@ jobs:
- name: IT - Bundle Validation - OpenJDK 8
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
SCALA_PROFILE: ${{ matrix.scalaProfile }}
if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on
Spark 3
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q
-DforceStdout)
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java8
$HUDI_VERSION openjdk8
- name: IT - Bundle Validation - OpenJDK 11
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
SCALA_PROFILE: ${{ matrix.scalaProfile }}
if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on
Spark 3
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q
-DforceStdout)
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java11
$HUDI_VERSION openjdk11
- name: IT - Bundle Validation - OpenJDK 17
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
@@ -523,7 +525,7 @@ jobs:
if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above
support Java 17
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q
-DforceStdout)
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java17
$HUDI_VERSION openjdk17
integration-tests:
runs-on: ubuntu-latest
diff --git a/.github/workflows/release_candidate_validation.yml
b/.github/workflows/release_candidate_validation.yml
index 02a598888ea..d9872cc7906 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -81,7 +81,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
run: |
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8
$STAGING_REPO_NUM
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java8
$HUDI_VERSION openjdk8 $STAGING_REPO_NUM
- name: IT - Bundle Validation - OpenJDK 11
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
@@ -89,7 +89,7 @@ jobs:
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
if: ${{ startsWith(env.SPARK_PROFILE, 'spark3') }} # Only Spark 3.x
supports Java 11 as of now
run: |
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11
$STAGING_REPO_NUM
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java11
$HUDI_VERSION openjdk11 $STAGING_REPO_NUM
- name: IT - Bundle Validation - OpenJDK 17
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
@@ -97,4 +97,4 @@ jobs:
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3
supports Java 17 as of now
run: |
- ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17
$STAGING_REPO_NUM
+ ./packaging/bundle-validation/ci_run.sh hudi_docker_java17
$HUDI_VERSION openjdk17 $STAGING_REPO_NUM
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 669278b9f61..58ef6f37588 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -27,9 +27,10 @@
# This is to run by GitHub Actions CI tasks from the project root directory
# and it contains the CI environment-specific variables.
-HUDI_VERSION=$1
-JAVA_RUNTIME_VERSION=$2
-STAGING_REPO_NUM=$3
+CONTAINER_NAME=$1
+HUDI_VERSION=$2
+JAVA_RUNTIME_VERSION=$3
+STAGING_REPO_NUM=$4
echo "HUDI_VERSION: $HUDI_VERSION JAVA_RUNTIME_VERSION: $JAVA_RUNTIME_VERSION"
echo "SPARK_RUNTIME: $SPARK_RUNTIME SPARK_PROFILE (optional): $SPARK_PROFILE"
echo "SCALA_PROFILE: $SCALA_PROFILE"
@@ -237,7 +238,7 @@ docker build \
.
# run validation script in docker
-docker run --name hudi_docker \
+docker run --name $CONTAINER_NAME \
-v ${GITHUB_WORKSPACE}:/opt/bundle-validation/docker-test \
-v $TMP_JARS_DIR:/opt/bundle-validation/jars \
-v $TMP_DATA_DIR:/opt/bundle-validation/data \
diff --git a/packaging/bundle-validation/validate.sh
b/packaging/bundle-validation/validate.sh
index de319e7d9dd..d81f3771f0b 100755
--- a/packaging/bundle-validation/validate.sh
+++ b/packaging/bundle-validation/validate.sh
@@ -299,7 +299,7 @@ if [ "$?" -ne 0 ]; then
fi
echo "::warning::validate.sh done validating utilities slim bundle"
-if [[ ${JAVA_RUNTIME_VERSION} == 'openjdk8' && ${SCALA_PROFILE} !=
'scala-2.13' ]]; then
+if [[ ${JAVA_RUNTIME_VERSION} == 'openjdk8' && ${SCALA_PROFILE} !=
'scala-2.13' && ! "${FLINK_HOME}" == *"1.18"* ]]; then
echo "::warning::validate.sh validating flink bundle"
test_flink_bundle
if [ "$?" -ne 0 ]; then
