This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e9e589796533 chore: trim CI builds to one full chain per module and
lane (#19909)
e9e589796533 is described below
commit e9e589796533ba08902ddee3dd039f0c3acdeba7
Author: voonhous <[email protected]>
AuthorDate: Mon Nov 16 11:14:07 2026 +0000
chore: trim CI builds to one full chain per module and lane (#19909)
* chore(ci): one full build per module and lane
Part of #19524. Workflow files only.
- flink-2's failsafe filter -Dit.test=!ITTestHoodieDataSource
dropped failsafe's IT* include and ran 170 unit-test classes the
docker job already runs. New filter: ITTest*,!ITTestHoodieDataSource.
- The docker job built the whole reactor, 7.8 minutes, of which the
15 bundles and the examples are 6. The build, UT and FT reactors
now skip the bundles, the examples and hudi-cli, each of which is
built, tested or validated in the job that owns it.
- hudi-examples-spark is built only in the jobs that run its
quickstart.
Measured over three runs: 577 to a mean of 548 runner-minutes,
flink-2 39 to 23 minutes, the docker job 37 to about 30.
* chore(ci): match flink-2's IT filter to the failsafe include
ITTest* is narrower than the integration-tests profile's
**/IT*.java include, so a future hudi-flink IT not named ITTest*
would run in no job. Use IT*,!ITTestHoodieDataSource. Today both
select the same 19 classes.
* chore(ci): check BUNDLE_MODULES against the reactor's bundles
A bundle missing from BUNDLE_MODULES stays in the docker job's
reactor and fails to resolve the modules the trimmed build no longer
installs, as hudi-spark-native-bundle would have. The comment now
says every new packaging/ module must be listed, and a node test in
validate-pr-and-source compares the list with the packaging/ modules
of the root pom's default <modules>.
---
.github/ci/scripts/test-fork-ci-status.cjs | 15 +++++++++
.github/workflows/bot.yml | 52 +++++++++++++++++++-----------
2 files changed, 49 insertions(+), 18 deletions(-)
diff --git a/.github/ci/scripts/test-fork-ci-status.cjs
b/.github/ci/scripts/test-fork-ci-status.cjs
index bbfd556dd12a..cc2c6d6ecf8e 100644
--- a/.github/ci/scripts/test-fork-ci-status.cjs
+++ b/.github/ci/scripts/test-fork-ci-status.cjs
@@ -452,6 +452,21 @@ test('the matrix-expanded test jobs stay within the ASF
cap of 20 jobs at once',
assert.equal(Number(stated?.[1]), total);
});
+// test-common-and-other-modules builds and tests the default reactor minus
its bundles. A bundle
+// missing from BUNDLE_MODULES stays in the reactor and fails to resolve the
modules the trimmed
+// build no longer installs.
+test('BUNDLE_MODULES lists every bundle of the default reactor', () => {
+ const pom = readFileSync(path.join(__dirname, '../../../pom.xml'), 'utf8');
+ // The first <modules> block is the default reactor; profiles declare theirs
later in the file.
+ const modules =
[.../<modules>([\s\S]*?)<\/modules>/.exec(pom)[1].matchAll(/<module>([^<]+)<\/module>/g)]
+ .map((match) => match[1]);
+ const bundles = modules.filter((module) => module.startsWith('packaging/'));
+ assert.ok(bundles.length > 0);
+ const listed =
ci.workflow.jobs['test-common-and-other-modules'].env.BUNDLE_MODULES.split(',')
+ .map((module) => module.replace(/^!/, ''));
+ assert.deepEqual([...listed].sort(), [...bundles].sort());
+});
+
// [case, context, whether the test jobs run, whether the aggregate jobs run,
the `changes` job]
for (const [name, github, tests, aggregate, needs] of [
['fork PR to master', forkPr, false, true],
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index f9d919797179..4372831bd867 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -76,7 +76,10 @@ env:
SCALA_TEST_OTHERS1_FILTER:
-DwildcardSuites=org.apache.spark.sql.hudi.procedure,org.apache.spark.sql.hudi.ddl
SCALA_TEST_OTHERS2_FILTER:
-DwildcardSuites=org.apache.hudi,org.apache.spark.hudi,org.apache.spark.sql.avro,org.apache.spark.sql.execution,org.apache.spark.sql.hive,org.apache.spark.sql.hudi.analysis,org.apache.spark.sql.hudi.blob,org.apache.spark.sql.hudi.catalog,org.apache.spark.sql.hudi.command,org.apache.spark.sql.hudi.common,org.apache.spark.sql.hudi.feature
FLINK_IT_FILTER1: -Dit.test=ITTestHoodieDataSource
- FLINK_IT_FILTER2: -Dit.test=!ITTestHoodieDataSource
+ # An include is required next to the exclusion: a bare exclusion replaces
failsafe's IT*
+ # include pattern with every test class, which ran hudi-flink's unit tests a
second time here.
+ # The include repeats the integration-tests profile's **/IT*.java, so no IT
class is left out.
+ FLINK_IT_FILTER2: -Dit.test=IT*,!ITTestHoodieDataSource
# Three conditions route the jobs. GitHub expressions cannot be named, so the
first test job
# spells them out in full under the YAML anchor `route`, every other test job
reuses it through
@@ -340,9 +343,22 @@ jobs:
env:
UT_MODULES: >-
-
!hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities
+
!hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities
FT_MODULES: >-
-
!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities
+
!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities
+ # Every bundle of the default reactor. None has tests, validate-bundles
shades and exercises
+ # every one, and nothing this job tests depends on them, so the build
and the two test steps
+ # below all leave them out (a bundle left in a test reactor would try to
resolve the modules
+ # the trimmed build no longer installs). A new packaging/ module in the
root pom's <modules>
+ # must be added here; .github/ci/scripts/test-fork-ci-status.cjs checks
the two match.
+ BUNDLE_MODULES: >-
+
!packaging/hudi-hadoop-mr-bundle,!packaging/hudi-datahub-sync-bundle,!packaging/hudi-hive-sync-bundle,!packaging/hudi-aws-bundle,!packaging/hudi-gcp-bundle,!packaging/hudi-azure-bundle,!packaging/hudi-spark-bundle,!packaging/hudi-spark-native-bundle,!packaging/hudi-presto-bundle,!packaging/hudi-utilities-slim-bundle,!packaging/hudi-timeline-server-bundle,!packaging/hudi-flink-bundle,!packaging/hudi-kafka-connect-bundle,!packaging/hudi-cli-bundle,!packaging/hudi-utilities-bundle
+ # Also skipped by the build: the metaserver bundle, which only exists
under
+ # -Phudi-platform-service and so cannot be listed for the test steps;
the examples, which
+ # this job never tests; and hudi-cli, which depends on the utilities
bundle and is built
+ # and tested in test-spark-client-and-hadoop-common.
+ BUILD_SKIP_MODULES: >-
+
!packaging/hudi-metaserver-server-bundle,!hudi-cli,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-spark,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s
steps:
- uses: actions/checkout@v5
@@ -367,7 +383,7 @@ jobs:
-v /var/run/docker.sock:/var/run/docker.sock \
-w /hudi \
hudi-ci \
- /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service
-Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS"
+ /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service
-Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -pl
$BUILD_SKIP_MODULES,$BUNDLE_MODULES"
- name: UT - common and other modules
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -380,7 +396,7 @@ jobs:
-v /var/run/docker.sock:/var/run/docker.sock \
-w /hudi \
hudi-ci \
- /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $UT_MODULES
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl
$UT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS
-Djacoco.skip=false"
- name: FT - common and other modules
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -393,7 +409,7 @@ jobs:
-v /var/run/docker.sock:/var/run/docker.sock \
-w /hudi \
hudi-ci \
- /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl
$FT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS
-Djacoco.skip=false"
- name: Generate merged coverage report
if: always() && github.event_name != 'merge_group'
run: ./scripts/jacoco/generate_merged_coverage_report.sh
$GITHUB_WORKSPACE
@@ -437,7 +453,7 @@ jobs:
# SPARK_PROFILE: ${{ matrix.sparkProfile }}
# SPARK_MODULES: ${{ matrix.sparkModules }}
# run:
-# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
# - name: Java UT 1 - Common & Spark
# env:
# SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -556,7 +572,7 @@ jobs:
# SPARK_PROFILE: ${{ matrix.sparkProfile }}
# SPARK_MODULES: ${{ matrix.sparkModules }}
# run:
-# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
# - name: Java FTB - Spark
# env:
# SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -616,7 +632,7 @@ jobs:
# SPARK_PROFILE: ${{ matrix.sparkProfile }}
# SPARK_MODULES: ${{ matrix.sparkModules }}
# run:
-# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
# - name: Scala UT - Common & Spark
# env:
# SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -676,7 +692,7 @@ jobs:
# SPARK_PROFILE: ${{ matrix.sparkProfile }}
# SPARK_MODULES: ${{ matrix.sparkModules }}
# run:
-# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES"
# - name: Scala UT - Common & Spark
# env:
# SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -803,7 +819,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Java UT - functional package
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -921,7 +937,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Java UT 2 - Common & Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -982,7 +998,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Java FTA - Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1036,7 +1052,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Java FTB - Spark
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1090,7 +1106,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Scala UT - Common & Spark (dml.others)
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1151,7 +1167,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Scala UT - Common & Spark (dml.insert, dml.schema)
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1248,7 +1264,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Scala UT - Common & Spark (procedure, ddl)
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -1309,7 +1325,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SPARK_MODULES: ${{ matrix.sparkModules }}
run:
- mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
+ mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES"
- name: Scala UT - Common & Spark (remaining packages)
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}