This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e9f7a0c06ef [MINOR] Rebalance Azure CI jobs (2025-02-13) (#12840)
e9f7a0c06ef is described below
commit e9f7a0c06ef58d1f1e8e82097a1013d0eef75327
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Feb 13 22:21:41 2025 -0800
[MINOR] Rebalance Azure CI jobs (2025-02-13) (#12840)
---
azure-pipelines-20230430.yml | 84 +++++++++++++++++++++++++++++---------------
1 file changed, 56 insertions(+), 28 deletions(-)
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 4f0f51e6eae..4cf9a383168 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -60,7 +60,7 @@ parameters:
- 'hudi-hadoop-common'
- 'hudi-client/hudi-spark-client'
- 'hudi-spark-datasource/hudi-spark'
- - name: job34UTModules
+ - name: job345UTModules
type: object
default:
- 'hudi-spark-datasource'
@@ -68,7 +68,7 @@ parameters:
- 'hudi-spark-datasource/hudi-spark3.5.x'
- 'hudi-spark-datasource/hudi-spark3-common'
- 'hudi-spark-datasource/hudi-spark-common'
- - name: job6UTModules
+ - name: job7UTModules
type: object
default:
- '!hudi-hadoop-mr'
@@ -97,7 +97,7 @@ parameters:
- '!hudi-spark-datasource/hudi-spark3-common'
- '!hudi-spark-datasource/hudi-spark-common'
- '!hudi-utilities'
- - name: job6FTModules
+ - name: job7FTModules
type: object
default:
- '!hudi-client/hudi-spark-client'
@@ -120,7 +120,7 @@ parameters:
- '!hudi-flink-datasource/hudi-flink1.20.x'
- '!hudi-spark-datasource/hudi-spark'
- '!hudi-utilities'
- - name: job4HudiSparkDmlOthersWildcardSuites
+ - name: job5HudiSparkDdlOthersWildcardSuites
type: object
default:
- 'org.apache.hudi'
@@ -130,7 +130,7 @@ parameters:
- 'org.apache.spark.sql.hudi.analysis'
- 'org.apache.spark.sql.hudi.command'
- 'org.apache.spark.sql.hudi.common'
- - 'org.apache.spark.sql.hudi.dml'
+ - 'org.apache.spark.sql.hudi.ddl'
variables:
BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink1.18'
@@ -142,12 +142,15 @@ variables:
JOB1_UT_MODULES: ${{ join(',',parameters.job1UTModules) }}
JOB1_FT_MODULES: ${{ join(',',parameters.job1FTModules) }}
JOB2_FT_MODULES: ${{ join(',',parameters.job2FTModules) }}
- JOB34_MODULES: ${{ join(',',parameters.job34UTModules) }}
- JOB3_SPARK_DDL_WILDCARD_SUITES: 'org.apache.spark.sql.hudi.ddl'
- JOB6_SPARK_PROCEDURE_WILDCARD_SUITES: 'org.apache.spark.sql.hudi.procedure'
- JOB4_SPARK_DML_OTHERS_WILDCARD_SUITES: ${{
join(',',parameters.job4HudiSparkDmlOthersWildcardSuites) }}
- JOB6_UT_MODULES: ${{ join(',',parameters.job6UTModules) }}
- JOB6_FT_MODULES: ${{ join(',',parameters.job6FTModules) }}
+ JOB345_MODULES: ${{ join(',',parameters.job345UTModules) }}
+ JOB5_JAVA_TEST_FILTER: '**/org/apache/hudi/functional/**/*'
+ JOB5_MVN_ARG_TEST: "-Dtest=\"$(JOB5_JAVA_TEST_FILTER)\""
+ JOB3_MVN_ARG_TEST: "-Dtest=\"!$(JOB5_JAVA_TEST_FILTER)\""
+ JOB4_SPARK_DML_WILDCARD_SUITES: 'org.apache.spark.sql.hudi.dml'
+ JOB7_SPARK_PROCEDURE_WILDCARD_SUITES: 'org.apache.spark.sql.hudi.procedure'
+ JOB5_SPARK_DDL_OTHERS_WILDCARD_SUITES: ${{
join(',',parameters.job5HudiSparkDdlOthersWildcardSuites) }}
+ JOB7_UT_MODULES: ${{ join(',',parameters.job7UTModules) }}
+ JOB7_FT_MODULES: ${{ join(',',parameters.job7FTModules) }}
stages:
- stage: test
@@ -215,7 +218,7 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_3
- displayName: UT spark-datasource Java Tests & DDL
+ displayName: UT spark-datasource Java Test 1
timeoutInMinutes: '120'
steps:
- task: Maven@4
@@ -223,24 +226,40 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
- options: $(MVN_OPTS_INSTALL) -pl $(JOB34_MODULES) -am
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB345_MODULES) -am
publishJUnitResults: false
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: Java UT spark-datasource
+ displayName: Java UT spark-datasource 1
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests $(JAVA_MVN_TEST_FILTER)
-pl $(JOB34_MODULES)
- publishJUnitResults: false
+ options: $(MVN_OPTS_TEST) -Punit-tests $(JAVA_MVN_TEST_FILTER)
$(JOB3_MVN_ARG_TEST) -pl $(JOB345_MODULES)
+ publishJUnitResults: true
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g'
+ - script: |
+ grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
+ displayName: Top 100 long-running testcases
+ - job: UT_FT_4
+ displayName: UT spark-datasource DML
+ timeoutInMinutes: '120'
+ steps:
+ - task: Maven@4
+ displayName: maven install
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'clean install'
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB345_MODULES) -am
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
- task: Maven@4
- displayName: Scala UT spark-datasource DDL
+ displayName: Scala UT spark-datasource DML
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests $(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB3_SPARK_DDL_WILDCARD_SUITES)" -pl $(JOB34_MODULES)
+ options: $(MVN_OPTS_TEST) -Punit-tests $(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB4_SPARK_DML_WILDCARD_SUITES)" -pl $(JOB345_MODULES)
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
@@ -248,8 +267,8 @@ stages:
- script: |
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- - job: UT_FT_4
- displayName: UT spark-datasource DML & others
+ - job: UT_FT_5
+ displayName: UT spark-datasource Java Test 2 & DDL & Others
timeoutInMinutes: '120'
steps:
- task: Maven@4
@@ -257,15 +276,24 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
- options: $(MVN_OPTS_INSTALL) -pl $(JOB34_MODULES) -am
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB345_MODULES) -am
publishJUnitResults: false
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: Scala UT spark-datasource DML & others
+ displayName: Java UT spark-datasource
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests $(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB4_SPARK_DML_OTHERS_WILDCARD_SUITES)" -pl $(JOB34_MODULES)
+ options: $(MVN_OPTS_TEST) -Punit-tests $(JAVA_MVN_TEST_FILTER)
$(JOB5_MVN_ARG_TEST) -pl $(JOB345_MODULES)
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx4g'
+ - task: Maven@4
+ displayName: Scala UT spark-datasource DDL & Others
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: $(MVN_OPTS_TEST) -Punit-tests $(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB5_SPARK_DDL_OTHERS_WILDCARD_SUITES)" -pl
$(JOB345_MODULES)
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
@@ -273,7 +301,7 @@ stages:
- script: |
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- - job: UT_FT_5
+ - job: UT_FT_6
displayName: UT FT Hudi Utilities
timeoutInMinutes: '90'
steps:
@@ -312,7 +340,7 @@ stages:
- script: |
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- - job: UT_FT_6
+ - job: UT_FT_7
displayName: UT FT other modules
timeoutInMinutes: '90'
steps:
@@ -339,9 +367,9 @@ stages:
-v $(Build.SourcesDirectory):/hudi
-i
docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
/bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
-Phudi-platform-service -Pthrift-gen-source
- && mvn test $(MVN_OPTS_TEST) -Punit-tests
$(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB6_SPARK_PROCEDURE_WILDCARD_SUITES)" -pl $(JOB34_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Punit-tests
-DfailIfNoTests=false -pl $(JOB6_UT_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-DfailIfNoTests=false -pl $(JOB6_FT_MODULES)"
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests
$(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB7_SPARK_PROCEDURE_WILDCARD_SUITES)" -pl $(JOB345_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests
-DfailIfNoTests=false -pl $(JOB7_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-DfailIfNoTests=false -pl $(JOB7_FT_MODULES)"
- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
