This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-19909-1d2e87269cf3981f91043e1891a7492e71e34ed1
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit e9e589796533ba08902ddee3dd039f0c3acdeba7
Author: voonhous <[email protected]>
AuthorDate: Mon Nov 16 11:14:07 2026 +0000

    chore: trim CI builds to one full chain per module and lane (#19909)
    
    * chore(ci): one full build per module and lane
    
    Part of #19524. Workflow files only.
    
    - flink-2's failsafe filter -Dit.test=!ITTestHoodieDataSource
      dropped failsafe's IT* include and ran 170 unit-test classes the
      docker job already runs. New filter: ITTest*,!ITTestHoodieDataSource.
    - The docker job built the whole reactor, 7.8 minutes, of which the
      15 bundles and the examples are 6. The build, UT and FT reactors
      now skip the bundles, the examples and hudi-cli, each of which is
      built, tested or validated in the job that owns it.
    - hudi-examples-spark is built only in the jobs that run its
      quickstart.
    
    Measured over three runs: 577 to a mean of 548 runner-minutes,
    flink-2 39 to 23 minutes, the docker job 37 to about 30.
    
    * chore(ci): match flink-2's IT filter to the failsafe include
    
    ITTest* is narrower than the integration-tests profile's
    **/IT*.java include, so a future hudi-flink IT not named ITTest*
    would run in no job. Use IT*,!ITTestHoodieDataSource. Today both
    select the same 19 classes.
    
    * chore(ci): check BUNDLE_MODULES against the reactor's bundles
    
    A bundle missing from BUNDLE_MODULES stays in the docker job's
    reactor and fails to resolve the modules the trimmed build no longer
    installs, as hudi-spark-native-bundle would have. The comment now
    says every new packaging/ module must be listed, and a node test in
    validate-pr-and-source compares the list with the packaging/ modules
    of the root pom's default <modules>.
---
 .github/ci/scripts/test-fork-ci-status.cjs | 15 +++++++++
 .github/workflows/bot.yml                  | 52 +++++++++++++++++++-----------
 2 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/.github/ci/scripts/test-fork-ci-status.cjs 
b/.github/ci/scripts/test-fork-ci-status.cjs
index bbfd556dd12a..cc2c6d6ecf8e 100644
--- a/.github/ci/scripts/test-fork-ci-status.cjs
+++ b/.github/ci/scripts/test-fork-ci-status.cjs
@@ -452,6 +452,21 @@ test('the matrix-expanded test jobs stay within the ASF 
cap of 20 jobs at once',
   assert.equal(Number(stated?.[1]), total);
 });
 
+// test-common-and-other-modules builds and tests the default reactor minus 
its bundles. A bundle
+// missing from BUNDLE_MODULES stays in the reactor and fails to resolve the 
modules the trimmed
+// build no longer installs.
+test('BUNDLE_MODULES lists every bundle of the default reactor', () => {
+  const pom = readFileSync(path.join(__dirname, '../../../pom.xml'), 'utf8');
+  // The first <modules> block is the default reactor; profiles declare theirs 
later in the file.
+  const modules = 
[.../<modules>([\s\S]*?)<\/modules>/.exec(pom)[1].matchAll(/<module>([^<]+)<\/module>/g)]
+    .map((match) => match[1]);
+  const bundles = modules.filter((module) => module.startsWith('packaging/'));
+  assert.ok(bundles.length > 0);
+  const listed = 
ci.workflow.jobs['test-common-and-other-modules'].env.BUNDLE_MODULES.split(',')
+    .map((module) => module.replace(/^!/, ''));
+  assert.deepEqual([...listed].sort(), [...bundles].sort());
+});
+
 // [case, context, whether the test jobs run, whether the aggregate jobs run, 
the `changes` job]
 for (const [name, github, tests, aggregate, needs] of [
   ['fork PR to master', forkPr, false, true],
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index f9d919797179..4372831bd867 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -76,7 +76,10 @@ env:
   SCALA_TEST_OTHERS1_FILTER: 
-DwildcardSuites=org.apache.spark.sql.hudi.procedure,org.apache.spark.sql.hudi.ddl
   SCALA_TEST_OTHERS2_FILTER: 
-DwildcardSuites=org.apache.hudi,org.apache.spark.hudi,org.apache.spark.sql.avro,org.apache.spark.sql.execution,org.apache.spark.sql.hive,org.apache.spark.sql.hudi.analysis,org.apache.spark.sql.hudi.blob,org.apache.spark.sql.hudi.catalog,org.apache.spark.sql.hudi.command,org.apache.spark.sql.hudi.common,org.apache.spark.sql.hudi.feature
   FLINK_IT_FILTER1: -Dit.test=ITTestHoodieDataSource
-  FLINK_IT_FILTER2: -Dit.test=!ITTestHoodieDataSource
+  # An include is required next to the exclusion: a bare exclusion replaces 
failsafe's IT*
+  # include pattern with every test class, which ran hudi-flink's unit tests a 
second time here.
+  # The include repeats the integration-tests profile's **/IT*.java, so no IT 
class is left out.
+  FLINK_IT_FILTER2: -Dit.test=IT*,!ITTestHoodieDataSource
 
 # Three conditions route the jobs. GitHub expressions cannot be named, so the 
first test job
 # spells them out in full under the YAML anchor `route`, every other test job 
reuses it through
@@ -340,9 +343,22 @@ jobs:
 
     env:
       UT_MODULES: >-
-        
!hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities
+        
!hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities
       FT_MODULES: >-
-        
!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities
+        
!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities
+      # Every bundle of the default reactor. None has tests, validate-bundles 
shades and exercises
+      # every one, and nothing this job tests depends on them, so the build 
and the two test steps
+      # below all leave them out (a bundle left in a test reactor would try to 
resolve the modules
+      # the trimmed build no longer installs). A new packaging/ module in the 
root pom's <modules>
+      # must be added here; .github/ci/scripts/test-fork-ci-status.cjs checks 
the two match.
+      BUNDLE_MODULES: >-
+        
!packaging/hudi-hadoop-mr-bundle,!packaging/hudi-datahub-sync-bundle,!packaging/hudi-hive-sync-bundle,!packaging/hudi-aws-bundle,!packaging/hudi-gcp-bundle,!packaging/hudi-azure-bundle,!packaging/hudi-spark-bundle,!packaging/hudi-spark-native-bundle,!packaging/hudi-presto-bundle,!packaging/hudi-utilities-slim-bundle,!packaging/hudi-timeline-server-bundle,!packaging/hudi-flink-bundle,!packaging/hudi-kafka-connect-bundle,!packaging/hudi-cli-bundle,!packaging/hudi-utilities-bundle
+      # Also skipped by the build: the metaserver bundle, which only exists 
under
+      # -Phudi-platform-service and so cannot be listed for the test steps; 
the examples, which
+      # this job never tests; and hudi-cli, which depends on the utilities 
bundle and is built
+      # and tested in test-spark-client-and-hadoop-common.
+      BUILD_SKIP_MODULES: >-
+        
!packaging/hudi-metaserver-server-bundle,!hudi-cli,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-spark,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s
 
     steps:
       - uses: actions/checkout@v5
@@ -367,7 +383,7 @@ jobs:
             -v /var/run/docker.sock:/var/run/docker.sock \
             -w /hudi \
             hudi-ci \
-            /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service 
-Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m 
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS"
+            /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service 
-Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m 
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -pl 
$BUILD_SKIP_MODULES,$BUNDLE_MODULES"
       - name: UT - common and other modules
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -380,7 +396,7 @@ jobs:
             -v /var/run/docker.sock:/var/run/docker.sock \
             -w /hudi \
             hudi-ci \
-            /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $UT_MODULES 
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+            /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl 
$UT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS 
-Djacoco.skip=false"
       - name: FT - common and other modules
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -393,7 +409,7 @@ jobs:
             -v /var/run/docker.sock:/var/run/docker.sock \
             -w /hudi \
             hudi-ci \
-            /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES 
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+            /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests 
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl 
$FT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS 
-Djacoco.skip=false"
       - name: Generate merged coverage report
         if: always() && github.event_name != 'merge_group'
         run: ./scripts/jacoco/generate_merged_coverage_report.sh 
$GITHUB_WORKSPACE
@@ -437,7 +453,7 @@ jobs:
 #          SPARK_PROFILE: ${{ matrix.sparkProfile }}
 #          SPARK_MODULES: ${{ matrix.sparkModules }}
 #        run:
-#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
 #      - name: Java UT 1 - Common & Spark
 #        env:
 #          SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -556,7 +572,7 @@ jobs:
 #          SPARK_PROFILE: ${{ matrix.sparkProfile }}
 #          SPARK_MODULES: ${{ matrix.sparkModules }}
 #        run:
-#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
 #      - name: Java FTB - Spark
 #        env:
 #          SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -616,7 +632,7 @@ jobs:
 #          SPARK_PROFILE: ${{ matrix.sparkProfile }}
 #          SPARK_MODULES: ${{ matrix.sparkModules }}
 #        run:
-#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
 #      - name: Scala UT - Common & Spark
 #        env:
 #          SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -676,7 +692,7 @@ jobs:
 #          SPARK_PROFILE: ${{ matrix.sparkProfile }}
 #          SPARK_MODULES: ${{ matrix.sparkModules }}
 #        run:
-#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+#          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
 #      - name: Scala UT - Common & Spark
 #        env:
 #          SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -803,7 +819,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Java UT - functional package
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -921,7 +937,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Java UT 2 - Common & Spark
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -982,7 +998,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Java FTA - Spark
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1036,7 +1052,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Java FTB - Spark
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1090,7 +1106,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Scala UT - Common & Spark (dml.others)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1151,7 +1167,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Scala UT - Common & Spark (dml.insert, dml.schema)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1248,7 +1264,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Scala UT - Common & Spark (procedure, ddl)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1309,7 +1325,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+          mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
       - name: Scala UT - Common & Spark (remaining packages)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}

Reply via email to