This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch fix/hudi-cli-tests-azure-ci-16448 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b646dd04baa0907306b5483cf35a382ec6f21da1 Author: voon <[email protected]> AuthorDate: Wed Sep 2 00:35:41 2026 +0800 ci: fold the hudi-cli tests into the spark-client job hudi-cli ran as its own job at 18m10, of which 7m14 was a build whose dependency closure is a superset of hudi-spark-client's. Running the module inside test-spark-client-and-hadoop-common shares that build, so the only added cost is the 10m28 the tests themselves take. Measured on this PR's last run against 12 recent master runs: the host job goes from 27 to about 41 min against a 70 min critical path, so the wall clock is unchanged and the run sheds about 4 runner-minutes. Its timeout goes 60 -> 75 to keep the same relative headroom. Coverage rides under the existing spark-client-hadoop-common flag, so the .codecov.yml flag note reverts. hudi-cli stays excluded from the catch-all UT_MODULES/FT_MODULES lists so it is not run twice. --- .codecov.yml | 2 +- .github/workflows/bot.yml | 88 +++++++++++++---------------------------------- 2 files changed, 25 insertions(+), 65 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index c5ef8e0ea23d..21791ebbd48d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -84,7 +84,7 @@ comment: # path filter on a given PR, so a partial run does not zero out that flag's coverage. # Flags uploaded by .github/workflows/bot.yml: spark-client-hadoop-common, utilities, # common-and-other-modules, spark-java-tests, spark-scala-tests, hadoop-mr-java-client, -# hudi-cli, integration-tests. +# integration-tests. flag_management: default_rules: carryforward: true diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 32225808e290..81831a101292 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -114,10 +114,16 @@ jobs: - name: RAT check run: ./scripts/release/validate_source_rat.sh + # hudi-cli rides along with hudi-spark-client here rather than joining the catch-all + # module lists in test-common-and-other-modules: its functional suite starts a + # SparkSession per test class, and this job has the headroom for it while that one does + # not. hudi-cli's dependency closure is a superset of hudi-spark-client's, so it shares + # this job's build instead of paying for a second one. It stays listed in the catch-all + # job's UT_MODULES/FT_MODULES exclusions so the tests are not run twice. test-spark-client-and-hadoop-common: runs-on: ubuntu-latest needs: changes - timeout-minutes: 60 + timeout-minutes: 75 strategy: matrix: include: @@ -143,7 +149,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-client/hudi-spark-client + mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-client/hudi-spark-client,hudi-cli - name: UT - hudi-hadoop-common if: needs.changes.outputs.relevant == 'true' env: @@ -168,6 +174,22 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} run: mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -fae -pl hudi-client/hudi-spark-client $MVN_ARGS -Djacoco.skip=false + - name: UT - hudi-cli + if: needs.changes.outputs.relevant == 'true' + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + run: + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -fae -pl hudi-cli $MVN_ARGS -Djacoco.skip=false + - name: FT - hudi-cli + if: needs.changes.outputs.relevant == 'true' + env: + SCALA_PROFILE: ${{ matrix.scalaProfile }} + SPARK_PROFILE: ${{ matrix.sparkProfile }} + FLINK_PROFILE: ${{ matrix.flinkProfile }} + run: + mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -fae -pl hudi-cli $MVN_ARGS -Djacoco.skip=false - name: Generate merged coverage report if: always() && needs.changes.outputs.relevant == 'true' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -716,68 +738,6 @@ jobs: flags: hadoop-mr-java-client token: ${{ secrets.CODECOV_TOKEN }} - # hudi-cli runs in its own job rather than joining the catch-all module lists in - # test-common-and-other-modules: its functional suite starts a SparkSession per test - # class, so folding it in would extend that job's critical path. It stays listed in - # that job's UT_MODULES/FT_MODULES exclusions so the tests are not run twice. - test-hudi-cli: - runs-on: ubuntu-latest - needs: changes - timeout-minutes: 60 - strategy: - matrix: - include: - - scalaProfile: "scala-2.12" - sparkProfile: "spark3.5" - flinkProfile: "flink2.2" - - steps: - - if: needs.changes.outputs.relevant == 'true' - uses: actions/checkout@v5 - - name: Set up JDK 11 - if: needs.changes.outputs.relevant == 'true' - uses: actions/setup-java@v5 - with: - java-version: '11' - distribution: 'temurin' - architecture: x64 - cache: maven - - name: Build Project - if: needs.changes.outputs.relevant == 'true' - env: - SCALA_PROFILE: ${{ matrix.scalaProfile }} - SPARK_PROFILE: ${{ matrix.sparkProfile }} - FLINK_PROFILE: ${{ matrix.flinkProfile }} - run: - mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-cli - - name: UT - hudi-cli - if: needs.changes.outputs.relevant == 'true' - env: - SCALA_PROFILE: ${{ matrix.scalaProfile }} - SPARK_PROFILE: ${{ matrix.sparkProfile }} - FLINK_PROFILE: ${{ matrix.flinkProfile }} - run: - mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-cli $MVN_ARGS -Djacoco.skip=false - - name: FT - hudi-cli - if: needs.changes.outputs.relevant == 'true' - env: - SCALA_PROFILE: ${{ matrix.scalaProfile }} - SPARK_PROFILE: ${{ matrix.sparkProfile }} - FLINK_PROFILE: ${{ matrix.flinkProfile }} - run: - mvn test -Pfunctional-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-cli $MVN_ARGS -Djacoco.skip=false - - name: Generate merged coverage report - if: always() && needs.changes.outputs.relevant == 'true' - run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE - - name: Upload coverage to Codecov - if: always() && needs.changes.outputs.relevant == 'true' - uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 - with: - files: ./jacoco-report.xml - disable_search: true - flags: hudi-cli - token: ${{ secrets.CODECOV_TOKEN }} - test-spark-java17-java-tests-part1: runs-on: ubuntu-latest needs: changes
