This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d45970cdd88 [HUDI-8002] Added back Flink 1.15 and 1.14 bundle
validation (#11653)
d45970cdd88 is described below
commit d45970cdd880a691fa18056c4edfaa9ef56f7d07
Author: Vova Kolmakov <[email protected]>
AuthorDate: Thu Jul 25 06:52:26 2024 +0700
[HUDI-8002] Added back Flink 1.15 and 1.14 bundle validation (#11653)
Co-authored-by: Vova Kolmakov <[email protected]>
---
.github/workflows/bot.yml | 16 +++----
packaging/bundle-validation/ci_run.sh | 78 +++++++++++++----------------------
2 files changed, 37 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 545603fa522..39610cc68aa 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -494,21 +494,21 @@ jobs:
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.18'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.0'
+ sparkRuntime: 'spark3.5.1'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.18'
+ flinkProfile: 'flink1.17'
sparkProfile: 'spark3.5'
- sparkRuntime: 'spark3.5.0'
+ sparkRuntime: 'spark3.5.1'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.18'
+ flinkProfile: 'flink1.16'
sparkProfile: 'spark3.4'
- sparkRuntime: 'spark3.4.0'
+ sparkRuntime: 'spark3.4.3'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.17'
+ flinkProfile: 'flink1.15'
sparkProfile: 'spark3.3'
- sparkRuntime: 'spark3.3.2'
+ sparkRuntime: 'spark3.3.4'
- scalaProfile: 'scala-2.12'
- flinkProfile: 'flink1.16'
+ flinkProfile: 'flink1.14'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.1'
steps:
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 6a388ea215d..1eb5c34e5f7 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -36,96 +36,76 @@ echo "SPARK_RUNTIME: $SPARK_RUNTIME SPARK_PROFILE
(optional): $SPARK_PROFILE"
echo "SCALA_PROFILE: $SCALA_PROFILE"
# choose versions based on build profiles
-if [[ ${SPARK_RUNTIME} == 'spark2.4.8' ]]; then
- HADOOP_VERSION=2.7.7
- HIVE_VERSION=2.3.9
- DERBY_VERSION=10.10.2.0
- FLINK_VERSION=1.14.6
- SPARK_VERSION=2.4.8
- SPARK_HADOOP_VERSION=2.7
- CONFLUENT_VERSION=5.5.12
- KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1146hive239spark248
-elif [[ ${SPARK_RUNTIME} == 'spark3.0.2' ]]; then
- HADOOP_VERSION=2.7.7
- HIVE_VERSION=3.1.3
- DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.14.6
- SPARK_VERSION=3.0.2
- SPARK_HADOOP_VERSION=2.7
- CONFLUENT_VERSION=5.5.12
- KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1146hive313spark302
-elif [[ ${SPARK_RUNTIME} == 'spark3.1.3' ]]; then
+if [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
FLINK_VERSION=1.14.6
- SPARK_VERSION=3.1.3
- SPARK_HADOOP_VERSION=2.7
+ SPARK_VERSION=3.3.1
+ SPARK_HADOOP_VERSION=2
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1146hive313spark313
-elif [[ ${SPARK_RUNTIME} == 'spark3.2.3' ]]; then
+ IMAGE_TAG=flink1146hive313spark331
+elif [[ ${SPARK_RUNTIME} == 'spark3.3.4' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
FLINK_VERSION=1.15.3
- SPARK_VERSION=3.2.3
- SPARK_HADOOP_VERSION=2.7
+ SPARK_VERSION=3.3.4
+ SPARK_HADOOP_VERSION=2
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1153hive313spark323
-elif [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
- HADOOP_VERSION=2.7.7
+ IMAGE_TAG=flink1153hive313spark334
+elif [[ ${SPARK_RUNTIME} == 'spark3.4.3' ]]; then
+ HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
FLINK_VERSION=1.16.2
- SPARK_VERSION=3.3.1
- SPARK_HADOOP_VERSION=2
+ SPARK_VERSION=3.4.3
+ SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1162hive313spark331
-elif [[ ${SPARK_RUNTIME} == 'spark3.3.2' ]]; then
- HADOOP_VERSION=2.7.7
+ IMAGE_TAG=flink1162hive313spark343
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
+ HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.17.0
- SPARK_VERSION=3.3.2
- SPARK_HADOOP_VERSION=2
+ FLINK_VERSION=1.18.0
+ SPARK_VERSION=3.5.0
+ SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1170hive313spark332
-elif [[ ${SPARK_RUNTIME} == 'spark3.4.0' ]]; then
+ IMAGE_TAG=flink1180hive313spark350
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
FLINK_VERSION=1.18.0
- SPARK_VERSION=3.4.0
+ SPARK_VERSION=3.5.0
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1180hive313spark340
-elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
+ IMAGE_TAG=flink1180hive313spark350scala213
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.18.0
- SPARK_VERSION=3.5.0
+ FLINK_VERSION=1.17.0
+ SPARK_VERSION=3.5.1
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1180hive313spark350
-elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
+ IMAGE_TAG=flink1170hive313spark351
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
FLINK_VERSION=1.18.0
- SPARK_VERSION=3.5.0
+ SPARK_VERSION=3.5.1
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1180hive313spark350scala213
+ IMAGE_TAG=flink1180hive313spark351scala213
fi
# Copy bundle jars to temp dir for mounting
