This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 68659a1c32f6e40d42b14d81a61d9b4e7f80bf27 Author: Y Ethan Guo <[email protected]> AuthorDate: Sat Mar 2 13:59:58 2024 -0800 [HUDI-7465] Split tests in CI further to reduce total CI elapsed time (#10795) --- .github/workflows/bot.yml | 139 +++++++++++++++++++++++++++++++++++++++---- azure-pipelines-20230430.yml | 58 ++++++++++++++---- 2 files changed, 176 insertions(+), 21 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 0bfd9541bcc..3007c752534 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -53,7 +53,7 @@ jobs: - name: RAT check run: ./scripts/release/validate_source_rat.sh - test-spark: + test-spark-java-tests: runs-on: ubuntu-latest strategy: matrix: @@ -107,22 +107,87 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} run: mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS - - name: UT - Common & Spark + - name: Java UT - Common & Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI run: - mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS - - name: FT - Spark + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Java FT - Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI run: - mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + + test-spark-scala-tests: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - scalaProfile: "scala-2.11" + sparkProfile: "spark2.4" + sparkModules: "hudi-spark-datasource/hudi-spark2" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.0" + sparkModules: "hudi-spark-datasource/hudi-spark3.0.x" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.1" + sparkModules: "hudi-spark-datasource/hudi-spark3.1.x" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.2" + sparkModules: "hudi-spark-datasource/hudi-spark3.2.x" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.3" + sparkModules: "hudi-spark-datasource/hudi-spark3.3.x" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.4" + sparkModules: "hudi-spark-datasource/hudi-spark3.4.x" + + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.5" + sparkModules: "hudi-spark-datasource/hudi-spark3.5.x" + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + architecture: x64 + cache: maven + - name: Build Project + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + run: + mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES" + - name: Scala UT - Common & Spark + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_MODULES: ${{ matrix.sparkModules }} + if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI + run: + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Scala FT - Spark + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_MODULES: ${{ matrix.sparkModules }} + if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI + run: + mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS test-hudi-hadoop-mr-and-hudi-java-client: runs-on: ubuntu-latest @@ -161,7 +226,7 @@ jobs: run: ./mvnw test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS - test-spark-java17: + test-spark-java17-java-tests: runs-on: ubuntu-latest strategy: matrix: @@ -203,16 +268,16 @@ jobs: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} run: - mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS - - name: UT - Common & Spark + mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl hudi-examples/hudi-examples-spark $MVN_ARGS + - name: Java UT - Common & Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI run: - mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS - - name: FT - Spark + mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Java FT - Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} @@ -221,6 +286,60 @@ jobs: run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + test-spark-java17-scala-tests: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.3" + sparkModules: "hudi-spark-datasource/hudi-spark3.3.x" + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.4" + sparkModules: "hudi-spark-datasource/hudi-spark3.4.x" + - scalaProfile: "scala-2.12" + sparkProfile: "spark3.5" + sparkModules: "hudi-spark-datasource/hudi-spark3.5.x" + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + architecture: x64 + cache: maven + - name: Build Project + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + run: + mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'adopt' + architecture: x64 + cache: maven + - name: Scala UT - Common & Spark + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_MODULES: ${{ matrix.sparkModules }} + if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI + run: + mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Scala FT - Spark + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_MODULES: ${{ matrix.sparkModules }} + if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI + run: + mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests -DfailIfNoTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + test-flink: runs-on: ubuntu-latest strategy: diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 4d7ef3578b5..a511c2aed5a 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -52,6 +52,15 @@ parameters: - 'hudi-spark-datasource/hudi-spark3-common' - 'hudi-spark-datasource/hudi-spark-common' - name: job4UTModules + type: object + default: + - 'hudi-spark-datasource' + - 'hudi-spark-datasource/hudi-spark' + - 'hudi-spark-datasource/hudi-spark3.2.x' + - 'hudi-spark-datasource/hudi-spark3.2plus-common' + - 'hudi-spark-datasource/hudi-spark3-common' + - 'hudi-spark-datasource/hudi-spark-common' + - name: job5UTModules type: object default: - '!hudi-hadoop-mr' @@ -76,7 +85,7 @@ parameters: - '!hudi-spark-datasource/hudi-spark3.2plus-common' - '!hudi-spark-datasource/hudi-spark3-common' - '!hudi-spark-datasource/hudi-spark-common' - - name: job4FTModules + - name: job5FTModules type: object default: - '!hudi-client/hudi-spark-client' @@ -98,13 +107,14 @@ parameters: variables: BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18' PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' - MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' + MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' JOB1_MODULES: ${{ join(',',parameters.job1Modules) }} JOB2_MODULES: ${{ join(',',parameters.job2Modules) }} JOB3_MODULES: ${{ join(',',parameters.job3UTModules) }} - JOB4_UT_MODULES: ${{ join(',',parameters.job4UTModules) }} - JOB4_FT_MODULES: ${{ join(',',parameters.job4FTModules) }} + JOB4_MODULES: ${{ join(',',parameters.job4UTModules) }} + JOB5_UT_MODULES: ${{ join(',',parameters.job5UTModules) }} + JOB5_FT_MODULES: ${{ join(',',parameters.job5FTModules) }} stages: - stage: test @@ -157,7 +167,7 @@ stages: inputs: mavenPomFile: 'pom.xml' goals: 'clean install' - options: $(MVN_OPTS_INSTALL) + options: $(MVN_OPTS_INSTALL) -pl $(JOB2_MODULES) -am publishJUnitResults: true testResultsFiles: '**/surefire-reports/TEST-*.xml' jdkVersionOption: '1.8' @@ -175,7 +185,7 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_3 - displayName: UT spark-datasource + displayName: Java UT spark-datasource timeoutInMinutes: '240' steps: - task: Maven@4 @@ -183,16 +193,16 @@ stages: inputs: mavenPomFile: 'pom.xml' goals: 'clean install' - options: $(MVN_OPTS_INSTALL) + options: $(MVN_OPTS_INSTALL) -pl $(JOB3_MODULES) -am publishJUnitResults: true testResultsFiles: '**/surefire-reports/TEST-*.xml' jdkVersionOption: '1.8' - task: Maven@4 - displayName: UT spark-datasource + displayName: Java UT spark-datasource inputs: mavenPomFile: 'pom.xml' goals: 'test' - options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) + options: $(MVN_OPTS_TEST) -DwildcardSuites=skipScalaTests -DfailIfNoTests=false -Punit-tests -pl $(JOB3_MODULES) publishJUnitResults: true testResultsFiles: '**/surefire-reports/TEST-*.xml' jdkVersionOption: '1.8' @@ -201,6 +211,32 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_4 + displayName: Scala UT spark-datasource + timeoutInMinutes: '240' + steps: + - task: Maven@4 + displayName: maven install + inputs: + mavenPomFile: 'pom.xml' + goals: 'clean install' + options: $(MVN_OPTS_INSTALL) -pl $(JOB4_MODULES) -am + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + - task: Maven@4 + displayName: Scala UT spark-datasource + inputs: + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Dtest=skipJavaTests -DfailIfNoTests=false -Punit-tests -pl $(JOB4_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 + displayName: Top 100 long-running testcases + - job: UT_FT_5 displayName: UT FT other modules timeoutInMinutes: '240' steps: @@ -226,6 +262,6 @@ stages: arguments: > -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source - && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) - && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES) + && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB5_UT_MODULES) + && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB5_UT_MODULES) && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
