This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new f88a73f09e7 [HUDI-7123] Improve CI scripts (#10136)
f88a73f09e7 is described below
commit f88a73f09e753ffe0a8029b490a8a430b05b8eaf
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Wed Nov 22 10:48:48 2023 -0800
[HUDI-7123] Improve CI scripts (#10136)
Improves the CI scripts in the following aspects:
- Removes `hudi-common` tests from `test-spark` job in GH CI as they are
already covered by Azure CI
- Removes unnecesary bundle validation jobs and adds new bundle validation
images (`flink1153hive313spark323`, `flink1162hive313spark331`)
- Updates `validate-release-candidate-bundles` jobs
- Moves functional tests of `hudi-spark-datasource/hudi-spark` from job 4
(3 hours) to job 2 (1 hour) in Azure CI to rebalance the finish time.
---
.github/workflows/bot.yml | 30 +++++++++++++---------
azure-pipelines-20230430.yml | 6 +++--
.../base/build_flink1153hive313spark323.sh | 26 +++++++++++++++++++
.../base/build_flink1162hive313spark331.sh | 26 +++++++++++++++++++
packaging/bundle-validation/ci_run.sh | 20 ++++++++++-----
5 files changed, 88 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index cff377ed13f..67c7ac16eaa 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -98,7 +98,7 @@ jobs:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
run:
- mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Quickstart Test
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -112,7 +112,7 @@ jobs:
SPARK_MODULES: ${{ matrix.sparkModules }}
if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
- name: FT - Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -299,19 +299,13 @@ jobs:
- flinkProfile: 'flink1.18'
sparkProfile: 'spark3.4'
sparkRuntime: 'spark3.4.0'
- - flinkProfile: 'flink1.18'
- sparkProfile: 'spark3.3'
- sparkRuntime: 'spark3.3.2'
- flinkProfile: 'flink1.17'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.2'
- flinkProfile: 'flink1.16'
- sparkProfile: 'spark3.3'
- sparkRuntime: 'spark3.3.2'
- - flinkProfile: 'flink1.15'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.1'
- - flinkProfile: 'flink1.14'
+ - flinkProfile: 'flink1.15'
sparkProfile: 'spark3.2'
sparkRuntime: 'spark3.2.3'
- flinkProfile: 'flink1.14'
@@ -380,18 +374,30 @@ jobs:
strategy:
matrix:
include:
- - flinkProfile: 'flink1.16'
+ - flinkProfile: 'flink1.18'
sparkProfile: 'spark3'
+ sparkRuntime: 'spark3.5.0'
+ - flinkProfile: 'flink1.18'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.0'
+ - flinkProfile: 'flink1.18'
+ sparkProfile: 'spark3.4'
+ sparkRuntime: 'spark3.4.0'
+ - flinkProfile: 'flink1.17'
+ sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.2'
- - flinkProfile: 'flink1.15'
+ - flinkProfile: 'flink1.16'
sparkProfile: 'spark3.3'
sparkRuntime: 'spark3.3.1'
- - flinkProfile: 'flink1.14'
+ - flinkProfile: 'flink1.15'
sparkProfile: 'spark3.2'
sparkRuntime: 'spark3.2.3'
- flinkProfile: 'flink1.14'
sparkProfile: 'spark3.1'
sparkRuntime: 'spark3.1.3'
+ - flinkProfile: 'flink1.14'
+ sparkProfile: 'spark3.0'
+ sparkRuntime: 'spark3.0.2'
- flinkProfile: 'flink1.14'
sparkProfile: 'spark'
sparkRuntime: 'spark2.4.8'
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 21c6d932ef9..c2a5f9d5a44 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -41,6 +41,7 @@ parameters:
type: object
default:
- 'hudi-client/hudi-spark-client'
+ - 'hudi-spark-datasource/hudi-spark'
- name: job3UTModules
type: object
default:
@@ -92,6 +93,7 @@ parameters:
- '!hudi-flink-datasource/hudi-flink1.16.x'
- '!hudi-flink-datasource/hudi-flink1.17.x'
- '!hudi-flink-datasource/hudi-flink1.18.x'
+ - '!hudi-spark-datasource/hudi-spark'
variables:
BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
@@ -141,7 +143,7 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_2
- displayName: FT client/spark-client
+ displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark
timeoutInMinutes: '150'
steps:
- task: Maven@4
@@ -153,7 +155,7 @@ stages:
publishJUnitResults: false
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: FT client/spark-client
+ displayName: FT client/spark-client &
hudi-spark-datasource/hudi-spark
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
diff --git a/packaging/bundle-validation/base/build_flink1153hive313spark323.sh
b/packaging/bundle-validation/base/build_flink1153hive313spark323.sh
new file mode 100755
index 00000000000..4f20a42d095
--- /dev/null
+++ b/packaging/bundle-validation/base/build_flink1153hive313spark323.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+docker build \
+ --build-arg HIVE_VERSION=3.1.3 \
+ --build-arg FLINK_VERSION=1.15.3 \
+ --build-arg SPARK_VERSION=3.2.3 \
+ --build-arg SPARK_HADOOP_VERSION=2.7 \
+ -t hudi-ci-bundle-validation-base:flink1153hive313spark323 .
+docker image tag hudi-ci-bundle-validation-base:flink1153hive313spark323
apachehudi/hudi-ci-bundle-validation-base:flink1153hive313spark323
diff --git a/packaging/bundle-validation/base/build_flink1162hive313spark331.sh
b/packaging/bundle-validation/base/build_flink1162hive313spark331.sh
new file mode 100755
index 00000000000..03e01729ef6
--- /dev/null
+++ b/packaging/bundle-validation/base/build_flink1162hive313spark331.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+docker build \
+ --build-arg HIVE_VERSION=3.1.3 \
+ --build-arg FLINK_VERSION=1.16.2 \
+ --build-arg SPARK_VERSION=3.3.1 \
+ --build-arg SPARK_HADOOP_VERSION=2 \
+ -t hudi-ci-bundle-validation-base:flink1162hive313spark331 .
+docker image tag hudi-ci-bundle-validation-base:flink1162hive313spark331
apachehudi/hudi-ci-bundle-validation-base:flink1162hive313spark331
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 6b80ab7078d..987d093a531 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -68,22 +68,22 @@ elif [[ ${SPARK_RUNTIME} == 'spark3.2.3' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.14.6
+ FLINK_VERSION=1.15.3
SPARK_VERSION=3.2.3
SPARK_HADOOP_VERSION=2.7
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1146hive313spark323
+ IMAGE_TAG=flink1153hive313spark323
elif [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.15.3
+ FLINK_VERSION=1.16.2
SPARK_VERSION=3.3.1
SPARK_HADOOP_VERSION=2
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1153hive313spark331
+ IMAGE_TAG=flink1162hive313spark331
elif [[ ${SPARK_RUNTIME} == 'spark3.3.2' ]]; then
HADOOP_VERSION=2.7.7
HIVE_VERSION=3.1.3
@@ -98,12 +98,12 @@ elif [[ ${SPARK_RUNTIME} == 'spark3.4.0' ]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
DERBY_VERSION=10.14.1.0
- FLINK_VERSION=1.17.0
+ FLINK_VERSION=1.18.0
SPARK_VERSION=3.4.0
SPARK_HADOOP_VERSION=3
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
- IMAGE_TAG=flink1170hive313spark340
+ IMAGE_TAG=flink1180hive313spark340
elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' ]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
@@ -156,6 +156,14 @@ else
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.3-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
+ elif [[ ${SPARK_PROFILE} == 'spark3.4' ]]; then
+ HUDI_SPARK_BUNDLE_NAME=hudi-spark3.4-bundle_2.12
+ HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
+ HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
+ elif [[ ${SPARK_PROFILE} == 'spark3.5' ]]; then
+ HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.12
+ HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
+ HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
elif [[ ${SPARK_PROFILE} == 'spark3' ]]; then
HUDI_SPARK_BUNDLE_NAME=hudi-spark3-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
