This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 387ece77516 [MINOR] Rebalance Azure CI jobs (2025-01-24) (#12705)
387ece77516 is described below
commit 387ece77516e5833a96b78b26f5bb2c7ccd2a030
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Jan 24 17:16:51 2025 -0800
[MINOR] Rebalance Azure CI jobs (2025-01-24) (#12705)
---
azure-pipelines-20230430.yml | 62 +++++++++++++++++++++++---------------------
1 file changed, 32 insertions(+), 30 deletions(-)
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 9a699127873..4f0f51e6eae 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -26,15 +26,12 @@ pool:
vmImage: 'ubuntu-22.04'
parameters:
- - name: job1Modules
+ - name: job1UTModules
type: object
default:
- 'hudi-common'
- 'hudi-hadoop-common'
- 'hudi-client/hudi-spark-client'
- - name: job2UTModules
- type: object
- default:
- 'hudi-flink-datasource'
- 'hudi-flink-datasource/hudi-flink'
- 'hudi-flink-datasource/hudi-flink1.14.x'
@@ -44,11 +41,9 @@ parameters:
- 'hudi-flink-datasource/hudi-flink1.18.x'
- 'hudi-flink-datasource/hudi-flink1.19.x'
- 'hudi-flink-datasource/hudi-flink1.20.x'
- - name: job2FTModules
+ - name: job1FTModules
type: object
default:
- - 'hudi-common'
- - 'hudi-hadoop-common'
- 'hudi-flink-datasource'
- 'hudi-flink-datasource/hudi-flink'
- 'hudi-flink-datasource/hudi-flink1.14.x'
@@ -58,6 +53,11 @@ parameters:
- 'hudi-flink-datasource/hudi-flink1.18.x'
- 'hudi-flink-datasource/hudi-flink1.19.x'
- 'hudi-flink-datasource/hudi-flink1.20.x'
+ - name: job2FTModules
+ type: object
+ default:
+ - 'hudi-common'
+ - 'hudi-hadoop-common'
- 'hudi-client/hudi-spark-client'
- 'hudi-spark-datasource/hudi-spark'
- name: job34UTModules
@@ -96,6 +96,7 @@ parameters:
- '!hudi-spark-datasource/hudi-spark3.5.x'
- '!hudi-spark-datasource/hudi-spark3-common'
- '!hudi-spark-datasource/hudi-spark-common'
+ - '!hudi-utilities'
- name: job6FTModules
type: object
default:
@@ -118,6 +119,7 @@ parameters:
- '!hudi-flink-datasource/hudi-flink1.19.x'
- '!hudi-flink-datasource/hudi-flink1.20.x'
- '!hudi-spark-datasource/hudi-spark'
+ - '!hudi-utilities'
- name: job4HudiSparkDmlOthersWildcardSuites
type: object
default:
@@ -137,8 +139,8 @@ variables:
MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
JAVA_MVN_TEST_FILTER: '-DwildcardSuites=skipScalaTests -DfailIfNoTests=false'
SCALA_MVN_TEST_FILTER: '-Dtest=skipJavaTests -DfailIfNoTests=false'
- JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
- JOB2_UT_MODULES: ${{ join(',',parameters.job2UTModules) }}
+ JOB1_UT_MODULES: ${{ join(',',parameters.job1UTModules) }}
+ JOB1_FT_MODULES: ${{ join(',',parameters.job1FTModules) }}
JOB2_FT_MODULES: ${{ join(',',parameters.job2FTModules) }}
JOB34_MODULES: ${{ join(',',parameters.job34UTModules) }}
JOB3_SPARK_DDL_WILDCARD_SUITES: 'org.apache.spark.sql.hudi.ddl'
@@ -154,7 +156,7 @@ stages:
value: 1
jobs:
- job: UT_FT_1
- displayName: UT common & client/spark-client
+ displayName: UT common & client/spark-client & UT FT flink
timeoutInMinutes: '90'
steps:
- task: Maven@4
@@ -162,15 +164,24 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
- options: $(MVN_OPTS_INSTALL) -pl $(JOB1_MODULES) -am
+ options: $(MVN_OPTS_INSTALL) -pl $(JOB1_UT_MODULES) -am
publishJUnitResults: false
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: UT common & client/spark-client
+ displayName: UT common & client/spark-client & flink
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES)
+ options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_UT_MODULES)
+ publishJUnitResults: false
+ jdkVersionOption: '1.8'
+ mavenOptions: '-Xmx4g'
+ - task: Maven@4
+ displayName: FT flink
+ inputs:
+ mavenPomFile: 'pom.xml'
+ goals: 'test'
+ options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl
$(JOB1_FT_MODULES)
publishJUnitResults: true
testResultsFiles: '**/surefire-reports/TEST-*.xml'
jdkVersionOption: '1.8'
@@ -179,7 +190,7 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_2
- displayName: UT flink & FT common & flink & spark-client & hudi-spark
+ displayName: FT common & spark-client & hudi-spark
timeoutInMinutes: '120'
steps:
- task: Maven@4
@@ -191,16 +202,7 @@ stages:
publishJUnitResults: false
jdkVersionOption: '1.8'
- task: Maven@4
- displayName: UT flink
- inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB2_UT_MODULES)
- publishJUnitResults: false
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - task: Maven@4
- displayName: FT common & flink & client/spark-client &
hudi-spark-datasource/hudi-spark
+ displayName: FT common & client/spark-client &
hudi-spark-datasource/hudi-spark
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
@@ -272,7 +274,7 @@ stages:
grep "testcase" */target/surefire-reports/*.xml
*/*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
| head -n 100
displayName: Top 100 long-running testcases
- job: UT_FT_5
- displayName: UT FT Hudi Streamer
+ displayName: UT FT Hudi Utilities
timeoutInMinutes: '90'
steps:
- task: Docker@2
@@ -289,7 +291,7 @@ stages:
Dockerfile: '**/Dockerfile'
ImageName: $(Build.BuildId)
- task: Docker@2
- displayName: "UT FT other modules"
+ displayName: "UT FT Hudi Utilities"
inputs:
containerRegistry: 'apachehudi-docker-hub'
repository: 'apachehudi/hudi-ci-bundle-validation-base'
@@ -298,8 +300,8 @@ stages:
-v $(Build.SourcesDirectory):/hudi
-i
docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
/bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
-Phudi-platform-service -Pthrift-gen-source -pl hudi-utilities -am
- && mvn test $(MVN_OPTS_TEST) -Punit-tests
-Dtest="Test*DeltaStreamer*" -DfailIfNoTests=false -pl hudi-utilities
- && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-Dtest="Test*DeltaStreamer*" -DfailIfNoTests=false -pl hudi-utilities"
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests
-DfailIfNoTests=false -pl hudi-utilities
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-DfailIfNoTests=false -pl hudi-utilities"
- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
@@ -338,8 +340,8 @@ stages:
-i
docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
/bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
-Phudi-platform-service -Pthrift-gen-source
&& mvn test $(MVN_OPTS_TEST) -Punit-tests
$(SCALA_MVN_TEST_FILTER)
-DwildcardSuites="$(JOB6_SPARK_PROCEDURE_WILDCARD_SUITES)" -pl $(JOB34_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Punit-tests
-Dtest="!Test*DeltaStreamer*" -DfailIfNoTests=false -pl $(JOB6_UT_MODULES)
- && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-Dtest="!Test*DeltaStreamer*" -DfailIfNoTests=false -pl $(JOB6_FT_MODULES)"
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests
-DfailIfNoTests=false -pl $(JOB6_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-DfailIfNoTests=false -pl $(JOB6_FT_MODULES)"
- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
