This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 7767309775 [Improve][CI] Deduplicate dedicated all-connectors shards
(#11291)
7767309775 is described below
commit 7767309775be948fbe15c5904e97c58a92ca1f6d
Author: Daniel <[email protected]>
AuthorDate: Sun Aug 16 22:41:14 2026 +0800
[Improve][CI] Deduplicate dedicated all-connectors shards (#11291)
Co-authored-by: DanielLeens <[email protected]>
Co-authored-by: davidzollo <[email protected]>
---
.github/workflows/backend.yml | 7 ---
.../test_update_modules_check.py | 64 ++++++++++++++++++++++
tools/update_modules_check/update_modules_check.py | 25 +++++++--
3 files changed, 84 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 15a0ae5161..dcae986970 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -982,13 +982,6 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 1`
- # Keep the longest Iceberg and HBase suites in a dedicated shard so
this hosted runner
- # does not lose heartbeat before the rest of the part-2 modules
finish.
- # Handle both first-position (no leading comma) and
mid/last-position (leading comma).
- run_it_modules=${run_it_modules//:connector-iceberg-e2e,/}
- run_it_modules=${run_it_modules//,:connector-iceberg-e2e/}
- run_it_modules=${run_it_modules//:connector-hbase-e2e,/}
- run_it_modules=${run_it_modules//,:connector-hbase-e2e/}
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
diff --git a/tools/update_modules_check/test_update_modules_check.py
b/tools/update_modules_check/test_update_modules_check.py
new file mode 100644
index 0000000000..843795784a
--- /dev/null
+++ b/tools/update_modules_check/test_update_modules_check.py
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from update_modules_check import build_sub_it_modules
+
+
+class UpdateModulesCheckTest(unittest.TestCase):
+ """
+ Guard the all-connectors shard contract used by backend CI.
+ """
+
+ def test_dedicated_heavy_suites_do_not_stay_in_regular_shards(self):
+ """
+ Iceberg and HBase should run only in the dedicated shard.
+ """
+ modules = ",".join(
+ [
+ "",
+ "connector-assert-e2e",
+ "connector-jdbc-e2e",
+ "connector-redis-e2e",
+ "connector-cdc-sqlserver-e2e",
+ "connector-kafka-e2e",
+ "connector-iceberg-e2e",
+ "connector-hbase-e2e",
+ "connector-http-e2e",
+ "connector-rocketmq-e2e",
+ "connector-kudu-e2e",
+ "connector-amazonsqs-e2e",
+ "connector-doris-e2e",
+ "connector-paimon-e2e",
+ "connector-cdc-oracle-e2e",
+ "connector-file-local-e2e",
+ "connector-file-sftp-e2e",
+ "connector-sensorsdata-e2e",
+ ]
+ )
+
+ shard_outputs = [build_sub_it_modules(modules, 7, shard) for shard in
range(7)]
+ combined_outputs = ",".join(shard_outputs)
+
+ self.assertIn("connector-assert-e2e", combined_outputs)
+ for shard_modules in shard_outputs:
+ self.assertNotIn("connector-iceberg-e2e", shard_modules)
+ self.assertNotIn("connector-hbase-e2e", shard_modules)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 6e020202a8..0238bb1f5c 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -141,7 +141,13 @@ def get_deleted_modules(files):
print(output_module)
-def get_sub_it_modules(modules, total_num, current_num):
+def build_sub_it_modules(modules, total_num, current_num):
+ """
+ Build one all-connectors shard while excluding suites with dedicated jobs.
+
+ Heavy suites that already have their own workflow shard must stay out of
the
+ round-robin shards, otherwise CI runs them twice and wastes runner time.
+ """
modules_arr = list(dict.fromkeys(modules.split(",")))
modules_arr.remove("connector-jdbc-e2e")
modules_arr.remove("connector-kafka-e2e")
@@ -155,8 +161,14 @@ def get_sub_it_modules(modules, total_num, current_num):
modules_arr.remove("connector-file-sftp-e2e")
modules_arr.remove("connector-redis-e2e")
modules_arr.remove("connector-sensorsdata-e2e")
- modules_arr.remove("connector-elasticsearch-e2e")
- modules_arr.remove("connector-cdc-mysql-e2e")
+ if "connector-elasticsearch-e2e" in modules_arr:
+ modules_arr.remove("connector-elasticsearch-e2e")
+ if "connector-cdc-mysql-e2e" in modules_arr:
+ modules_arr.remove("connector-cdc-mysql-e2e")
+ if "connector-iceberg-e2e" in modules_arr:
+ modules_arr.remove("connector-iceberg-e2e")
+ if "connector-hbase-e2e" in modules_arr:
+ modules_arr.remove("connector-hbase-e2e")
if "connector-seatunnel-e2e-base" in modules_arr:
modules_arr.remove("connector-seatunnel-e2e-base")
if "connector-console-seatunnel-e2e" in modules_arr:
@@ -168,8 +180,11 @@ def get_sub_it_modules(modules, total_num, current_num):
if len(module) > 0 and i % int(total_num) == int(current_num):
output = output + ",:" + module
- output = output[1:len(output)]
- print(output)
+ return output[1:len(output)]
+
+
+def get_sub_it_modules(modules, total_num, current_num):
+ print(build_sub_it_modules(modules, total_num, current_num))
def get_sub_update_it_modules(modules, total_num, current_num):
