This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-19919-47958f5309ffb46cb6ffda3b9e9d7fc512c004ad
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit a40cbd09dc4931e70c7ebe001e0ccefa52a676fc
Author: voonhous <[email protected]>
AuthorDate: Tue Nov 17 08:11:56 2026 +0000

    chore(ci): build and verify only the IT modules in integration-tests 
(#19919)
    
    * chore(ci): build and verify only the IT modules in integration-tests
    
    Part of #19524.
    
    IT_MODULES names the four modules with integration-test classes
    outside hudi-flink: hudi-cli, hudi-integ-test, hudi-aws and
    hudi-azure. The build passes them with -am, which Maven resolves to
    30 modules; the IT step verifies just the four instead of the whole
    reactor minus hudi-flink. Every module no longer built here is
    built, and where a bundle shaded and run, in the job that owns it.
    
    Measured: the job from 37 to 28 minutes, same 25 test classes.
    
    * chore(ci): check IT_MODULES against the IT classes
    
    The integration-tests job lists its IT modules by hand. A node test
    now walks the source tree for IT*.java test classes and fails when
    the modules holding them, hudi-flink aside, differ from the list.
---
 .github/ci/scripts/test-fork-ci-status.cjs | 28 ++++++++++++++++++++++++++++
 .github/workflows/bot.yml                  | 11 +++++++++--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/.github/ci/scripts/test-fork-ci-status.cjs 
b/.github/ci/scripts/test-fork-ci-status.cjs
index 79aa181d1500..55d27fbc91c7 100644
--- a/.github/ci/scripts/test-fork-ci-status.cjs
+++ b/.github/ci/scripts/test-fork-ci-status.cjs
@@ -466,6 +466,34 @@ test('BUNDLE_MODULES lists every bundle of the default 
reactor', () => {
   assert.deepEqual([...listed].sort(), [...bundles].sort());
 });
 
+test('IT_MODULES lists every module with integration-test classes but 
hudi-flink', () => {
+  const root = path.join(__dirname, '../../..');
+  // The integration-tests profile of the root pom runs **/IT*.java; 
hudi-flink's ITs run in the
+  // flink jobs.
+  const modules = new Set();
+  const walk = (dir) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (entry.name.startsWith('.') || entry.name === 'target' || entry.name 
=== 'node_modules') {
+        continue;
+      }
+      const file = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(file);
+      } else if (/^IT.*\.java$/.test(entry.name)) {
+        const relative = path.relative(root, file).split(path.sep).join('/');
+        const module = /^(.+?)\/src\/test\//.exec(relative);
+        if (module) {
+          modules.add(module[1]);
+        }
+      }
+    }
+  };
+  walk(root);
+  assert.ok(modules.delete('hudi-flink-datasource/hudi-flink'));
+  const listed = 
ci.workflow.jobs['integration-tests'].env.IT_MODULES.split(',');
+  assert.deepEqual([...listed].sort(), [...modules].sort());
+});
+
 // [case, context, whether the test jobs run, whether the aggregate job runs, 
the `changes` job]
 for (const [name, github, tests, aggregate, needs] of [
   ['fork PR to master', forkPr, false, true],
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index d8b4206192ae..f9fbd1fafdfd 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1875,6 +1875,13 @@ jobs:
           - sparkProfile: 'spark3.5'
             flinkProfile: 'flink2.2'
             sparkArchive: 'spark-3.5.9/spark-3.5.9-bin-hadoop3.tgz'
+    env:
+      # The only modules with integration-test classes besides hudi-flink, 
whose ITs run in the
+      # flink jobs. The build below builds these and what they depend on (-am) 
and the IT step
+      # verifies just these, instead of building and re-packaging all 66 
modules. A module that
+      # adds an IT class must be listed here, or its ITs stop running in CI;
+      # .github/ci/scripts/test-fork-ci-status.cjs checks the list against the 
IT classes.
+      IT_MODULES: 
hudi-cli,hudi-integ-test,hudi-aws,hudi-azure,packaging/hudi-gcp-bundle
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK 11
@@ -1901,7 +1908,7 @@ jobs:
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
           SCALA_PROFILE: '-Dscala-2.12 -Dscala.binary.version=2.12'
         run:
-          mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" 
-D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS 
-Ddocker.compose.skip=true
+          mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" 
-D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS 
-Ddocker.compose.skip=true -pl "$IT_MODULES" -am
       - name: 'UT integ-test'
         env:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
@@ -1943,7 +1950,7 @@ jobs:
           SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE)
           export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*}
           rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE
-          mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl 
!hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
+          mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl 
"$IT_MODULES" $MVN_ARGS -Djacoco.skip=false
       - name: Test timing summary
         if: always()
         run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job 
}}"

Reply via email to