This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 67b39465f405 chore(ci): run bundle validation on Spark 3.5.9 instead
of 3.5.1 (#20428)
67b39465f405 is described below
commit 67b39465f405280bf785f95818388d8347884704
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Oct 23 16:48:31 2026 -0700
chore(ci): run bundle validation on Spark 3.5.9 instead of 3.5.1 (#20428)
---
.github/workflows/java_ci_engines.yml | 10 ++++----
.github/workflows/maven_artifact_validation.yml | 10 ++++----
.github/workflows/release_candidate_validation.yml | 10 ++++----
...h => build_flink1190hive313spark359scala213.sh} | 6 ++---
...h => build_flink1200hive313spark359scala213.sh} | 6 ++---
...park351.sh => build_flink200hive313spark359.sh} | 6 ++---
...park351.sh => build_flink211hive313spark359.sh} | 6 ++---
...park351.sh => build_flink221hive313spark359.sh} | 6 ++---
packaging/bundle-validation/ci_run.sh | 30 ++++++++--------------
9 files changed, 41 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/java_ci_engines.yml
b/.github/workflows/java_ci_engines.yml
index ce71426c8f75..753c9ef9754d 100644
--- a/.github/workflows/java_ci_engines.yml
+++ b/.github/workflows/java_ci_engines.yml
@@ -238,7 +238,7 @@ jobs:
flinkAvroVersion: '1.11.4'
flinkParquetVersion: '1.13.1'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink1.20'
flinkAvroVersion: '1.11.4'
@@ -250,14 +250,14 @@ jobs:
flinkAvroVersion: '1.11.4'
flinkParquetVersion: '1.15.2'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
# [CI-TRIM] to revisit for CI improvement
# - scalaProfile: 'scala-2.13'
# flinkProfile: 'flink1.19'
# flinkAvroVersion: '1.11.4'
# flinkParquetVersion: '1.13.1'
# sparkProfile: 'spark3.5'
-# sparkRuntime: 'spark3.5.1'
+# sparkRuntime: 'spark3.5.9'
# - scalaProfile: 'scala-2.12'
# flinkProfile: 'flink1.18'
# flinkAvroVersion: '1.11.4'
@@ -381,13 +381,13 @@ jobs:
# flinkAvroVersion: '1.11.4'
# flinkParquetVersion: '1.14.4'
# sparkProfile: 'spark3.5'
-# sparkRuntime: 'spark3.5.1'
+# sparkRuntime: 'spark3.5.9'
# - scalaProfile: 'scala-2.12'
# flinkProfile: 'flink2.1'
# flinkAvroVersion: '1.11.4'
# flinkParquetVersion: '1.15.2'
# sparkProfile: 'spark3.5'
-# sparkRuntime: 'spark3.5.1'
+# sparkRuntime: 'spark3.5.9'
# steps:
# - uses: actions/checkout@v5
diff --git a/.github/workflows/maven_artifact_validation.yml
b/.github/workflows/maven_artifact_validation.yml
index fb2b007ce038..f3d09fef423c 100644
--- a/.github/workflows/maven_artifact_validation.yml
+++ b/.github/workflows/maven_artifact_validation.yml
@@ -26,23 +26,23 @@ jobs:
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.2'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.1'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.0'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.20'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.19'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink1.18'
sparkProfile: 'spark3.4'
diff --git a/.github/workflows/release_candidate_validation.yml
b/.github/workflows/release_candidate_validation.yml
index 4880cbed343d..edf29b955bd5 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -26,11 +26,11 @@ jobs:
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.20'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.19'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink1.18'
sparkProfile: 'spark3.4'
@@ -116,15 +116,15 @@ jobs:
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.0'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.1'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.2'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.1'
+ sparkRuntime: 'spark3.5.9'
steps:
- uses: actions/checkout@v5
- name: Set up JDK 11
diff --git
a/packaging/bundle-validation/base/build_flink1190hive313spark351scala213.sh
b/packaging/bundle-validation/base/build_flink1190hive313spark359scala213.sh
similarity index 84%
rename from
packaging/bundle-validation/base/build_flink1190hive313spark351scala213.sh
rename to
packaging/bundle-validation/base/build_flink1190hive313spark359scala213.sh
index adb26b8fc0b7..be4eef563869 100755
--- a/packaging/bundle-validation/base/build_flink1190hive313spark351scala213.sh
+++ b/packaging/bundle-validation/base/build_flink1190hive313spark359scala213.sh
@@ -20,9 +20,9 @@
docker build \
--build-arg HIVE_VERSION=3.1.3 \
--build-arg FLINK_VERSION=1.19.0 \
- --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
--build-arg SPARK_HADOOP_VERSION=3 \
--build-arg HADOOP_VERSION=3.3.5 \
--build-arg SCALA_VERSION=2.13 \
- -t hudi-ci-bundle-validation-base:flink1190hive313spark351scala213 .
-docker image tag
hudi-ci-bundle-validation-base:flink1190hive313spark351scala213
apachehudi/hudi-ci-bundle-validation-base:flink1190hive313spark351scala213
+ -t hudi-ci-bundle-validation-base:flink1190hive313spark359scala213 .
+docker image tag
hudi-ci-bundle-validation-base:flink1190hive313spark359scala213
apachehudi/hudi-ci-bundle-validation-base:flink1190hive313spark359scala213
diff --git
a/packaging/bundle-validation/base/build_flink1200hive313spark351scala213.sh
b/packaging/bundle-validation/base/build_flink1200hive313spark359scala213.sh
similarity index 84%
rename from
packaging/bundle-validation/base/build_flink1200hive313spark351scala213.sh
rename to
packaging/bundle-validation/base/build_flink1200hive313spark359scala213.sh
index a47bb230ecb0..a349e9a5ec66 100755
--- a/packaging/bundle-validation/base/build_flink1200hive313spark351scala213.sh
+++ b/packaging/bundle-validation/base/build_flink1200hive313spark359scala213.sh
@@ -20,9 +20,9 @@
docker build \
--build-arg HIVE_VERSION=3.1.3 \
--build-arg FLINK_VERSION=1.20.0 \
- --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
--build-arg SPARK_HADOOP_VERSION=3 \
--build-arg HADOOP_VERSION=3.3.5 \
--build-arg SCALA_VERSION=2.13 \
- -t hudi-ci-bundle-validation-base:flink1200hive313spark351scala213 .
-docker image tag
hudi-ci-bundle-validation-base:flink1200hive313spark351scala213
apachehudi/hudi-ci-bundle-validation-base:flink1200hive313spark351scala213
+ -t hudi-ci-bundle-validation-base:flink1200hive313spark359scala213 .
+docker image tag
hudi-ci-bundle-validation-base:flink1200hive313spark359scala213
apachehudi/hudi-ci-bundle-validation-base:flink1200hive313spark359scala213
diff --git a/packaging/bundle-validation/base/build_flink200hive313spark351.sh
b/packaging/bundle-validation/base/build_flink200hive313spark359.sh
similarity index 85%
rename from packaging/bundle-validation/base/build_flink200hive313spark351.sh
rename to packaging/bundle-validation/base/build_flink200hive313spark359.sh
index e52b98b5fded..a98e439d1d55 100644
--- a/packaging/bundle-validation/base/build_flink200hive313spark351.sh
+++ b/packaging/bundle-validation/base/build_flink200hive313spark359.sh
@@ -20,8 +20,8 @@
docker build \
--build-arg HIVE_VERSION=3.1.3 \
--build-arg FLINK_VERSION=2.0.0 \
- --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
--build-arg SPARK_HADOOP_VERSION=3 \
--build-arg HADOOP_VERSION=3.3.5 \
- -t hudi-ci-bundle-validation-base:flink200hive313spark351 .
-docker image tag hudi-ci-bundle-validation-base:flink200hive313spark351
apachehudi/hudi-ci-bundle-validation-base:flink200hive313spark351
+ -t hudi-ci-bundle-validation-base:flink200hive313spark359 .
+docker image tag hudi-ci-bundle-validation-base:flink200hive313spark359
apachehudi/hudi-ci-bundle-validation-base:flink200hive313spark359
diff --git a/packaging/bundle-validation/base/build_flink211hive313spark351.sh
b/packaging/bundle-validation/base/build_flink211hive313spark359.sh
similarity index 85%
rename from packaging/bundle-validation/base/build_flink211hive313spark351.sh
rename to packaging/bundle-validation/base/build_flink211hive313spark359.sh
index 7cb2fd388e83..9df185ac531e 100644
--- a/packaging/bundle-validation/base/build_flink211hive313spark351.sh
+++ b/packaging/bundle-validation/base/build_flink211hive313spark359.sh
@@ -20,8 +20,8 @@
docker build \
--build-arg HIVE_VERSION=3.1.3 \
--build-arg FLINK_VERSION=2.1.1 \
- --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
--build-arg SPARK_HADOOP_VERSION=3 \
--build-arg HADOOP_VERSION=3.3.5 \
- -t hudi-ci-bundle-validation-base:flink211hive313spark351 .
-docker image tag hudi-ci-bundle-validation-base:flink211hive313spark351
apachehudi/hudi-ci-bundle-validation-base:flink211hive313spark351
+ -t hudi-ci-bundle-validation-base:flink211hive313spark359 .
+docker image tag hudi-ci-bundle-validation-base:flink211hive313spark359
apachehudi/hudi-ci-bundle-validation-base:flink211hive313spark359
diff --git a/packaging/bundle-validation/base/build_flink221hive313spark351.sh
b/packaging/bundle-validation/base/build_flink221hive313spark359.sh
similarity index 85%
rename from packaging/bundle-validation/base/build_flink221hive313spark351.sh
rename to packaging/bundle-validation/base/build_flink221hive313spark359.sh
index c611c84ea6c7..3b127e00f29f 100755
--- a/packaging/bundle-validation/base/build_flink221hive313spark351.sh
+++ b/packaging/bundle-validation/base/build_flink221hive313spark359.sh
@@ -20,8 +20,8 @@
docker build \
--build-arg HIVE_VERSION=3.1.3 \
--build-arg FLINK_VERSION=2.2.1 \
- --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
--build-arg SPARK_HADOOP_VERSION=3 \
--build-arg HADOOP_VERSION=3.3.5 \
- -t hudi-ci-bundle-validation-base:flink221hive313spark351 .
-docker image tag hudi-ci-bundle-validation-base:flink221hive313spark351
apachehudi/hudi-ci-bundle-validation-base:flink221hive313spark351
+ -t hudi-ci-bundle-validation-base:flink221hive313spark359 .
+docker image tag hudi-ci-bundle-validation-base:flink221hive313spark359
apachehudi/hudi-ci-bundle-validation-base:flink221hive313spark359
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 757219d4361f..eefbc864d6e4 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -95,50 +95,42 @@ elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' &&
${SCALA_PROFILE} == 'scala-2.13' ]];
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1200hive313spark350scala213
elif [[ ${SPARK_RUNTIME} == 'spark3.5.9' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
- # Apache DataFusion Comet 1.1.0 is built against Spark 3.5.9 and needs at
least 3.5.2 at runtime.
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.20.1
SPARK_VERSION=3.5.9
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1200hive313spark359
-elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
- HADOOP_VERSION=3.3.5
- HIVE_VERSION=3.1.3
- DERBY_VERSION=10.14.1.0
- SPARK_VERSION=3.5.1
- SPARK_HADOOP_VERSION=3
- CONFLUENT_VERSION=5.5.12
- KAFKA_CONNECT_HDFS_VERSION=10.1.13
- if [[ ${FLINK_PROFILE} == 'flink2.0' ]]; then
- IMAGE_TAG=flink200hive313spark351
+ if [[ ${FLINK_PROFILE} == 'flink1.20' ]]; then
+ IMAGE_TAG=flink1200hive313spark359
+ FLINK_VERSION=1.20.1
+ elif [[ ${FLINK_PROFILE} == 'flink2.0' ]]; then
+ IMAGE_TAG=flink200hive313spark359
FLINK_VERSION=2.0.0
elif [[ ${FLINK_PROFILE} == 'flink2.1' ]]; then
- IMAGE_TAG=flink211hive313spark351
+ IMAGE_TAG=flink211hive313spark359
FLINK_VERSION=2.1.1
elif [[ ${FLINK_PROFILE} == 'flink2.2' ]]; then
- IMAGE_TAG=flink221hive313spark351
+ IMAGE_TAG=flink221hive313spark359
FLINK_VERSION=2.2.1
else
echo "Unsupported Flink profile ${FLINK_PROFILE}"
exit 1
fi
-elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.9' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- SPARK_VERSION=3.5.1
+ SPARK_VERSION=3.5.9
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
if [[ ${FLINK_PROFILE} == 'flink1.19' ]]; then
- IMAGE_TAG=flink1190hive313spark351scala213
+ IMAGE_TAG=flink1190hive313spark359scala213
FLINK_VERSION=1.19.0
else
- IMAGE_TAG=flink1200hive313spark351scala213
+ IMAGE_TAG=flink1200hive313spark359scala213
FLINK_VERSION=1.20.1
fi
elif [[ ${SPARK_RUNTIME} == 'spark4.0.0' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
