This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 0c7f5c2191d [HUDI-7802] Fix bundle validation scripts (#11314)
0c7f5c2191d is described below
commit 0c7f5c2191df8c5f1a5f3e36ebae80329ea3e131
Author: Y Ethan Guo <[email protected]>
AuthorDate: Mon May 27 09:11:58 2024 -0700
[HUDI-7802] Fix bundle validation scripts (#11314)
---
.github/workflows/release_candidate_validation.yml | 52 ++++++++++++----------
packaging/bundle-validation/ci_run.sh | 9 +++-
scripts/release/validate_staged_bundles.sh | 38 +++++++++++-----
3 files changed, 64 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/release_candidate_validation.yml
b/.github/workflows/release_candidate_validation.yml
index d9872cc7906..451e3b6c8fb 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -5,17 +5,6 @@ on:
branches:
- 'release-*'
pull_request:
- paths-ignore:
- - '**.bmp'
- - '**.gif'
- - '**.jpg'
- - '**.jpeg'
- - '**.md'
- - '**.pdf'
- - '**.png'
- - '**.svg'
- - '**.yaml'
- - '.gitignore'
branches:
- 'release-*'
@@ -36,34 +25,48 @@ jobs:
strategy:
matrix:
include:
- - flinkProfile: 'flink1.18'
+ - scalaProfile: 'scala-2.13'
+ flinkProfile: 'flink1.18'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.0'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.18'
sparkProfile: 'spark3'
sparkRuntime: 'spark3.5.0'
- - flinkProfile: 'flink1.18'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.18'
sparkProfile: 'spark3.5'
sparkRuntime: 'spark3.5.0'
- - flinkProfile: 'flink1.18'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.18'
sparkProfile: 'spark3.4'
sparkRuntime: 'spark3.4.0'
- - flinkProfile: 'flink1.17'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.17'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.2'
- - flinkProfile: 'flink1.16'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.16'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.1'
- - flinkProfile: 'flink1.15'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.15'
sparkProfile: 'spark3.2'
sparkRuntime: 'spark3.2.3'
- - flinkProfile: 'flink1.14'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.14'
sparkProfile: 'spark3.1'
sparkRuntime: 'spark3.1.3'
- - flinkProfile: 'flink1.14'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.14'
sparkProfile: 'spark3.0'
sparkRuntime: 'spark3.0.2'
- - flinkProfile: 'flink1.14'
+ - scalaProfile: 'scala-2.11'
+ flinkProfile: 'flink1.14'
sparkProfile: 'spark'
sparkRuntime: 'spark2.4.8'
- - flinkProfile: 'flink1.14'
+ - scalaProfile: 'scala-2.11'
+ flinkProfile: 'flink1.11'
sparkProfile: 'spark2.4'
sparkRuntime: 'spark2.4.8'
steps:
@@ -80,6 +83,7 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
run: |
./packaging/bundle-validation/ci_run.sh hudi_docker_java8
$HUDI_VERSION openjdk8 $STAGING_REPO_NUM
- name: IT - Bundle Validation - OpenJDK 11
@@ -87,7 +91,8 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
- if: ${{ startsWith(env.SPARK_PROFILE, 'spark3') }} # Only Spark 3.x
supports Java 11 as of now
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on
Spark 3
run: |
./packaging/bundle-validation/ci_run.sh hudi_docker_java11
$HUDI_VERSION openjdk11 $STAGING_REPO_NUM
- name: IT - Bundle Validation - OpenJDK 17
@@ -95,6 +100,7 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
- if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3
supports Java 17 as of now
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above
support Java 17
run: |
./packaging/bundle-validation/ci_run.sh hudi_docker_java17
$HUDI_VERSION openjdk17 $STAGING_REPO_NUM
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 58ef6f37588..6a388ea215d 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -132,7 +132,8 @@ fi
TMP_JARS_DIR=/tmp/jars/$(date +%s)
mkdir -p $TMP_JARS_DIR
-if [[ "$HUDI_VERSION" == *"SNAPSHOT" ]]; then
+if [[ -z "$STAGING_REPO_NUM" ]]; then
+ echo 'Adding built bundle jars for validation'
if [[ "$SCALA_PROFILE" != 'scala-2.13' ]]; then
# For Scala 2.13, Flink is not support, so skipping the Flink bundle
validation
cp
${GITHUB_WORKSPACE}/packaging/hudi-flink-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
@@ -159,6 +160,10 @@ else
HUDI_SPARK_BUNDLE_NAME=hudi-spark2.4-bundle_2.11
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.11
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.11
+ elif [[ ${SPARK_PROFILE} == 'spark3.0' ]]; then
+ HUDI_SPARK_BUNDLE_NAME=hudi-spark3.0-bundle_2.12
+ HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
+ HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.1-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
@@ -179,7 +184,7 @@ else
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
- elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
+ elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.13
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.13
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.13
diff --git a/scripts/release/validate_staged_bundles.sh
b/scripts/release/validate_staged_bundles.sh
index 1fc7b9f6e1c..843f590900a 100755
--- a/scripts/release/validate_staged_bundles.sh
+++ b/scripts/release/validate_staged_bundles.sh
@@ -36,26 +36,44 @@ declare -a bundles=("hudi-aws-bundle"
"hudi-cli-bundle_2.11" "hudi-cli-bundle_2.
"hudi-flink1.15-bundle" "hudi-flink1.16-bundle" "hudi-flink1.17-bundle"
"hudi-flink1.18-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle"
"hudi-hive-sync-bundle" "hudi-integ-test-bundle"
"hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle"
"hudi-presto-bundle" "hudi-spark-bundle_2.11" "hudi-spark-bundle_2.12"
"hudi-spark2.4-bundle_2.11" "hudi-spark2.4-bundle_2.12"
"hudi-spark3-bundle_2.12" "hudi-spark3.0-bundle_2.12"
"hudi-spark3.1-bundle_2.12"
-"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12"
"hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12"
"hudi-timeline-server-bundle"
-"hudi-trino-bundle" "hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12"
"hudi-utilities-slim-bundle_2.11"
-"hudi-utilities-slim-bundle_2.12")
+"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12"
"hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12"
+"hudi-spark3.5-bundle_2.13" "hudi-timeline-server-bundle" "hudi-trino-bundle"
+"hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12"
"hudi-utilities-bundle_2.13"
+"hudi-utilities-slim-bundle_2.11" "hudi-utilities-slim-bundle_2.12"
"hudi-utilities-slim-bundle_2.13")
+
+curl_with_url() {
+ local url="$1"
+ if curl -s -o /dev/null --head --fail "$url"; then
+ echo "Artifact exists: $url"
+ else
+ echo "Artifact missing: $url"
+ exit 1
+ fi
+}
+
+export -f curl_with_url
NOW=$(date +%s)
TMP_DIR_FOR_BUNDLES=/tmp/${NOW}
mkdir "$TMP_DIR_FOR_BUNDLES"
+ALL_URLS=""
+
for bundle in "${bundles[@]}"
do
for extension in "${extensions[@]}"
do
url=${STAGING_REPO}/$bundle/${VERSION}/$bundle-${VERSION}$extension
- if curl --output "$TMP_DIR_FOR_BUNDLES/$bundle-${VERSION}$extension"
--head --fail "$url"; then
- echo "Artifact exists: $url"
- else
- echo "Artifact missing: $url"
- exit 1
- fi
+ ALL_URLS+="$url\n"
done
done
-echo "All artifacts exist. Validation succeeds."
+echo "-- All bundles to check:"
+echo -e "$ALL_URLS"
+
+if echo -e "$ALL_URLS" | xargs -n 1 -P 16 -I {} bash -c 'curl_with_url "{}"';
then
+ echo "All artifacts exist. Validation succeeds."
+else
+ echo "Some artifact(s) missing."
+ exit 1
+fi
