This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new eeccdf9bb0f [HUDI-7465] Split tests in CI further to reduce total CI
elapsed time (#10795)
eeccdf9bb0f is described below
commit eeccdf9bb0f2885c37e0b480c330400fd2f80a1b
Author: Y Ethan Guo <[email protected]>
AuthorDate: Sat Mar 2 13:59:58 2024 -0800
[HUDI-7465] Split tests in CI further to reduce total CI elapsed time
(#10795)
---
.github/workflows/bot.yml | 139 +++++++++++++++++++++++++++++++++++++++----
azure-pipelines-20230430.yml | 58 ++++++++++++++----
2 files changed, 176 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 0bfd9541bcc..3007c752534 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -53,7 +53,7 @@ jobs:
- name: RAT check
run: ./scripts/release/validate_source_rat.sh
- test-spark:
+ test-spark-java-tests:
runs-on: ubuntu-latest
strategy:
matrix:
@@ -107,22 +107,87 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
run:
mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
hudi-examples/hudi-examples-spark $MVN_ARGS
- - name: UT - Common & Spark
+ - name: Java UT - Common & Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
run:
- mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
- - name: FT - Spark
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Java FT - Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
run:
- mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+
+ test-spark-scala-tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.11"
+ sparkProfile: "spark2.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark2"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.0"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.0.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.1"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.1.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.2"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.2.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.3"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.3.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.4.x"
+
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.5.x"
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ - name: Scala UT - Common & Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
+ run:
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-Dtest=skipJavaTests -DfailIfNoTests=false -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Scala FT - Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
+ run:
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-Dtest=skipJavaTests -DfailIfNoTests=false -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
test-hudi-hadoop-mr-and-hudi-java-client:
runs-on: ubuntu-latest
@@ -161,7 +226,7 @@ jobs:
run:
./mvnw test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS
- test-spark-java17:
+ test-spark-java17-java-tests:
runs-on: ubuntu-latest
strategy:
matrix:
@@ -203,16 +268,16 @@ jobs:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
run:
- mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-pl hudi-examples/hudi-examples-spark $MVN_ARGS
- - name: UT - Common & Spark
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl
hudi-examples/hudi-examples-spark $MVN_ARGS
+ - name: Java UT - Common & Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
run:
- mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
- - name: FT - Spark
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Java FT - Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
@@ -221,6 +286,60 @@ jobs:
run:
mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ test-spark-java17-scala-tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.3"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.3.x"
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.4"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.4.x"
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ sparkModules: "hudi-spark-datasource/hudi-spark3.5.x"
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'adopt'
+ architecture: x64
+ cache: maven
+ - name: Scala UT - Common & Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
+ run:
+ mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-Dtest=skipJavaTests -DfailIfNoTests=false -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+ - name: Scala FT - Spark
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ SPARK_MODULES: ${{ matrix.sparkModules }}
+ if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2
as it's covered by Azure CI
+ run:
+ mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+
test-flink:
runs-on: ubuntu-latest
strategy:
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 4d7ef3578b5..a511c2aed5a 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -52,6 +52,15 @@ parameters:
- 'hudi-spark-datasource/hudi-spark3-common'
- 'hudi-spark-datasource/hudi-spark-common'
- name: job4UTModules
+ type: object
+ default:
+ - 'hudi-spark-datasource'
+ - 'hudi-spark-datasource/hudi-spark'
+ - 'hudi-spark-datasource/hudi-spark3.2.x'
+ - 'hudi-spark-datasource/hudi-spark3.2plus-common'
+ - 'hudi-spark-datasource/hudi-spark3-common'
+ - 'hudi-spark-datasource/hudi-spark-common'
+ - name: job5UTModules
type: object
default:
- '!hudi-hadoop-mr'
@@ -76,7 +85,7 @@ parameters:
- '!hudi-spark-datasource/hudi-spark3.2plus-common'
- '!hudi-spark-datasource/hudi-spark3-common'
- '!hudi-spark-datasource/hudi-spark-common'
- - name: job4FTModules
+ - name: job5FTModules
type: object
default:
- '!hudi-client/hudi-spark-client'
@@ -98,13 +107,14 @@ parameters:
variables:
BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp
-B -V -Pwarn-log
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
- MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES)
$(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5'
+ MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests
$(BUILD_PROFILES) $(PLUGIN_OPTS)
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-Dmaven.wagon.http.retryHandler.count=5'
MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
JOB3_MODULES: ${{ join(',',parameters.job3UTModules) }}
- JOB4_UT_MODULES: ${{ join(',',parameters.job4UTModules) }}
- JOB4_FT_MODULES: ${{ join(',',parameters.job4FTModules) }}
+ JOB4_MODULES: ${{ join(',',parameters.job4UTModules) }}
+ JOB5_UT_MODULES: ${{ join(',',parameters.job5UTModules) }}
+ JOB5_FT_MODULES: ${{ join(',',parameters.job5FTModules) }}
stages:
- stage: test
@@ -157,7 +167,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB2_MODULES) -am
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
@@ -175,7 +185,7 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_3
- displayName: UT spark-datasource
+ displayName: Java UT spark-datasource
timeoutInMinutes: '240'
steps:
- task: Maven@4
@@ -183,16 +193,16 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB3_MODULES) -am
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: UT spark-datasource
+ displayName: Java UT spark-datasource
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
+ options: $(MVN_OPTS_TEST) -DwildcardSuites=skipScalaTests
-DfailIfNoTests=false -Punit-tests -pl $(JOB3_MODULES)
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
@@ -201,6 +211,32 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_4
+ displayName: Scala UT spark-datasource
+ timeoutInMinutes: '240'
+ steps:
+ - task: Maven@4
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'clean install'
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB4_MODULES) -am
+ publishJUnitResults: true
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
+ jdkVersionOption: '1.8'
+ - task: Maven@4
+ displayName: Scala UT spark-datasource
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: $(MVN_OPTS_TEST) -Dtest=skipJavaTests
-DfailIfNoTests=false -Punit-tests -pl $(JOB4_MODULES)
+ publishJUnitResults: true
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx4g'
+ - script: |
+ grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
+ displayName: Top 100 long-running testcases
+ - job: UT_FT_5
displayName: UT FT other modules
timeoutInMinutes: '240'
steps:
@@ -226,6 +262,6 @@ stages:
arguments: >
-i
docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
/bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
-Phudi-platform-service -Pthrift-gen-source
- && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl
$(JOB4_UT_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl
$(JOB4_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl
$(JOB5_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl
$(JOB5_UT_MODULES)
&& grep \"testcase\" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort
-nr | head -n 100"
