This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 90c40d48b06 [HUDI-6204] Add bundle validation on Spark 3.3.2 (#8692)
90c40d48b06 is described below
commit 90c40d48b06de732e54b5926c5f6bd39992fbddc
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri May 12 18:23:49 2023 -0600
[HUDI-6204] Add bundle validation on Spark 3.3.2 (#8692)
This commit adds the bundle validation on Spark 3.3.2 in Github Java CI to
ensure compatibility after we fixed the compatibility issue in #8082.
---
.github/workflows/bot.yml | 10 ++++++++--
packaging/bundle-validation/ci_run.sh | 18 ++++++++++++++----
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 384c39eebad..59eada9cc64 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -146,14 +146,19 @@ jobs:
include:
- flinkProfile: 'flink1.17'
sparkProfile: 'spark3.3'
+ sparkRuntime: 'spark3.3.2'
- flinkProfile: 'flink1.16'
sparkProfile: 'spark3.3'
+ sparkRuntime: 'spark3.3.2'
- flinkProfile: 'flink1.15'
sparkProfile: 'spark3.3'
+ sparkRuntime: 'spark3.3.1'
- flinkProfile: 'flink1.14'
sparkProfile: 'spark3.2'
+ sparkRuntime: 'spark3.2.3'
- flinkProfile: 'flink1.13'
sparkProfile: 'spark3.1'
+ sparkRuntime: 'spark3.1.3'
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
@@ -175,7 +180,7 @@ jobs:
- name: IT - Bundle Validation - OpenJDK 8
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
- SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
SCALA_PROFILE: 'scala-2.12'
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q
-DforceStdout)
@@ -183,7 +188,7 @@ jobs:
- name: IT - Bundle Validation - OpenJDK 11
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
- SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
SCALA_PROFILE: 'scala-2.12'
run: |
HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q
-DforceStdout)
@@ -192,6 +197,7 @@ jobs:
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
SCALA_PROFILE: 'scala-2.12'
if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3
supports Java 17 as of now
run: |
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 70faebe563d..bef2ab3282c 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -31,7 +31,7 @@ HUDI_VERSION=$1
JAVA_RUNTIME_VERSION=$2
# choose versions based on build profiles
-if [[ ${SPARK_PROFILE} == 'spark2.4' ]]; then
+if [[ ${SPARK_RUNTIME} == 'spark2.4' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=2.3.9
DERBY_VERSION=10.10.2.0
@@ -41,7 +41,7 @@ if [[ ${SPARK_PROFILE} == 'spark2.4' ]]; then
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1136hive239spark248
-elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then
+elif [[ ${SPARK_RUNTIME} == 'spark3.1.3' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
@@ -51,7 +51,7 @@ elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1136hive313spark313
-elif [[ ${SPARK_PROFILE} == 'spark3.2' ]]; then
+elif [[ ${SPARK_RUNTIME} == 'spark3.2.3' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
@@ -61,7 +61,7 @@ elif [[ ${SPARK_PROFILE} == 'spark3.2' ]]; then
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1146hive313spark323
-elif [[ ${SPARK_PROFILE} == 'spark3.3' ]]; then
+elif [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
@@ -71,6 +71,16 @@ elif [[ ${SPARK_PROFILE} == 'spark3.3' ]]; then
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1153hive313spark331
+elif [[ ${SPARK_RUNTIME} == 'spark3.3.2' ]]; then
+ HADOOP_VERSION=2.7.7
+ HIVE_VERSION=3.1.3
+ DERBY_VERSION=10.14.1.0
+ FLINK_VERSION=1.15.3
+ SPARK_VERSION=3.3.2
+ SPARK_HADOOP_VERSION=2
+ CONFLUENT_VERSION=5.5.12
+ KAFKA_CONNECT_HDFS_VERSION=10.1.13
+ IMAGE_TAG=flink1153hive313spark332
fi
# Copy bundle jars to temp dir for mounting
