This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.15.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 27e45ace224a177ba36d20e09bb5f7c0aabda98a Author: Y Ethan Guo <[email protected]> AuthorDate: Mon May 27 02:12:21 2024 -0700 [HUDI-7802] Fix bundle validation scripts (#11332) --- .github/workflows/release_candidate_validation.yml | 52 ++++++++++++---------- packaging/bundle-validation/ci_run.sh | 9 +++- scripts/release/validate_staged_bundles.sh | 38 +++++++++++----- 3 files changed, 64 insertions(+), 35 deletions(-) diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml index d9872cc7906..451e3b6c8fb 100644 --- a/.github/workflows/release_candidate_validation.yml +++ b/.github/workflows/release_candidate_validation.yml @@ -5,17 +5,6 @@ on: branches: - 'release-*' pull_request: - paths-ignore: - - '**.bmp' - - '**.gif' - - '**.jpg' - - '**.jpeg' - - '**.md' - - '**.pdf' - - '**.png' - - '**.svg' - - '**.yaml' - - '.gitignore' branches: - 'release-*' @@ -36,34 +25,48 @@ jobs: strategy: matrix: include: - - flinkProfile: 'flink1.18' + - scalaProfile: 'scala-2.13' + flinkProfile: 'flink1.18' + sparkProfile: 'spark3.5' + sparkRuntime: 'spark3.5.0' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.18' sparkProfile: 'spark3' sparkRuntime: 'spark3.5.0' - - flinkProfile: 'flink1.18' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.18' sparkProfile: 'spark3.5' sparkRuntime: 'spark3.5.0' - - flinkProfile: 'flink1.18' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.18' sparkProfile: 'spark3.4' sparkRuntime: 'spark3.4.0' - - flinkProfile: 'flink1.17' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.17' sparkProfile: 'spark3.3' sparkRuntime: 'spark3.3.2' - - flinkProfile: 'flink1.16' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.16' sparkProfile: 'spark3.3' sparkRuntime: 'spark3.3.1' - - flinkProfile: 'flink1.15' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.15' sparkProfile: 'spark3.2' sparkRuntime: 'spark3.2.3' - - flinkProfile: 'flink1.14' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.14' sparkProfile: 'spark3.1' sparkRuntime: 'spark3.1.3' - - flinkProfile: 'flink1.14' + - scalaProfile: 'scala-2.12' + flinkProfile: 'flink1.14' sparkProfile: 'spark3.0' sparkRuntime: 'spark3.0.2' - - flinkProfile: 'flink1.14' + - scalaProfile: 'scala-2.11' + flinkProfile: 'flink1.14' sparkProfile: 'spark' sparkRuntime: 'spark2.4.8' - - flinkProfile: 'flink1.14' + - scalaProfile: 'scala-2.11' + flinkProfile: 'flink1.11' sparkProfile: 'spark2.4' sparkRuntime: 'spark2.4.8' steps: @@ -80,6 +83,7 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} + SCALA_PROFILE: ${{ matrix.scalaProfile }} run: | ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 $STAGING_REPO_NUM - name: IT - Bundle Validation - OpenJDK 11 @@ -87,7 +91,8 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} - if: ${{ startsWith(env.SPARK_PROFILE, 'spark3') }} # Only Spark 3.x supports Java 11 as of now + SCALA_PROFILE: ${{ matrix.scalaProfile }} + if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 run: | ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 $STAGING_REPO_NUM - name: IT - Bundle Validation - OpenJDK 17 @@ -95,6 +100,7 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_RUNTIME: ${{ matrix.sparkRuntime }} - if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3 supports Java 17 as of now + SCALA_PROFILE: ${{ matrix.scalaProfile }} + if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17 run: | ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM diff --git a/packaging/bundle-validation/ci_run.sh b/packaging/bundle-validation/ci_run.sh index 58ef6f37588..6a388ea215d 100755 --- a/packaging/bundle-validation/ci_run.sh +++ b/packaging/bundle-validation/ci_run.sh @@ -132,7 +132,8 @@ fi TMP_JARS_DIR=/tmp/jars/$(date +%s) mkdir -p $TMP_JARS_DIR -if [[ "$HUDI_VERSION" == *"SNAPSHOT" ]]; then +if [[ -z "$STAGING_REPO_NUM" ]]; then + echo 'Adding built bundle jars for validation' if [[ "$SCALA_PROFILE" != 'scala-2.13' ]]; then # For Scala 2.13, Flink is not support, so skipping the Flink bundle validation cp ${GITHUB_WORKSPACE}/packaging/hudi-flink-bundle/target/hudi-*-$HUDI_VERSION.jar $TMP_JARS_DIR/ @@ -159,6 +160,10 @@ else HUDI_SPARK_BUNDLE_NAME=hudi-spark2.4-bundle_2.11 HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.11 HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.11 + elif [[ ${SPARK_PROFILE} == 'spark3.0' ]]; then + HUDI_SPARK_BUNDLE_NAME=hudi-spark3.0-bundle_2.12 + HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12 + HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12 elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then HUDI_SPARK_BUNDLE_NAME=hudi-spark3.1-bundle_2.12 HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12 @@ -179,7 +184,7 @@ else HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.12 HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12 HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12 - elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.12' ]]; then + elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.13' ]]; then HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.13 HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.13 HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.13 diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh index 1fc7b9f6e1c..843f590900a 100755 --- a/scripts/release/validate_staged_bundles.sh +++ b/scripts/release/validate_staged_bundles.sh @@ -36,26 +36,44 @@ declare -a bundles=("hudi-aws-bundle" "hudi-cli-bundle_2.11" "hudi-cli-bundle_2. "hudi-flink1.15-bundle" "hudi-flink1.16-bundle" "hudi-flink1.17-bundle" "hudi-flink1.18-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle" "hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle" "hudi-presto-bundle" "hudi-spark-bundle_2.11" "hudi-spark-bundle_2.12" "hudi-spark2.4-bundle_2.11" "hudi-spark2.4-bundle_2.12" "hudi-spark3-bundle_2.12" "hudi-spark3.0-bundle_2.12" "hudi-spark3.1-bundle_2.12" -"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12" "hudi-timeline-server-bundle" -"hudi-trino-bundle" "hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12" "hudi-utilities-slim-bundle_2.11" -"hudi-utilities-slim-bundle_2.12") +"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12" +"hudi-spark3.5-bundle_2.13" "hudi-timeline-server-bundle" "hudi-trino-bundle" +"hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13" +"hudi-utilities-slim-bundle_2.11" "hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13") + +curl_with_url() { + local url="$1" + if curl -s -o /dev/null --head --fail "$url"; then + echo "Artifact exists: $url" + else + echo "Artifact missing: $url" + exit 1 + fi +} + +export -f curl_with_url NOW=$(date +%s) TMP_DIR_FOR_BUNDLES=/tmp/${NOW} mkdir "$TMP_DIR_FOR_BUNDLES" +ALL_URLS="" + for bundle in "${bundles[@]}" do for extension in "${extensions[@]}" do url=${STAGING_REPO}/$bundle/${VERSION}/$bundle-${VERSION}$extension - if curl --output "$TMP_DIR_FOR_BUNDLES/$bundle-${VERSION}$extension" --head --fail "$url"; then - echo "Artifact exists: $url" - else - echo "Artifact missing: $url" - exit 1 - fi + ALL_URLS+="$url\n" done done -echo "All artifacts exist. Validation succeeds." +echo "-- All bundles to check:" +echo -e "$ALL_URLS" + +if echo -e "$ALL_URLS" | xargs -n 1 -P 16 -I {} bash -c 'curl_with_url "{}"'; then + echo "All artifacts exist. Validation succeeds." +else + echo "Some artifact(s) missing." + exit 1 +fi
