This is an automated email from the ASF dual-hosted git repository.
zhangshenghang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new ab25ea0ead [Improve][CI] Harden dedicated all-connectors shard guards
(#11828)
ab25ea0ead is described below
commit ab25ea0eadeb22f23d5a7388a98ad52b25ae7345
Author: David Zollo <[email protected]>
AuthorDate: Sun Aug 23 21:12:53 2026 +0800
[Improve][CI] Harden dedicated all-connectors shard guards (#11828)
---
.../test_update_modules_check.py | 123 +++++++++++++++++----
tools/update_modules_check/update_modules_check.py | 92 ++++++++++-----
2 files changed, 162 insertions(+), 53 deletions(-)
diff --git a/tools/update_modules_check/test_update_modules_check.py
b/tools/update_modules_check/test_update_modules_check.py
index 843795784a..59c4f2e13f 100644
--- a/tools/update_modules_check/test_update_modules_check.py
+++ b/tools/update_modules_check/test_update_modules_check.py
@@ -15,8 +15,15 @@
# limitations under the License.
import unittest
+from collections import Counter
+from pathlib import Path
+import re
-from update_modules_check import build_sub_it_modules
+from update_modules_check import (
+ ALL_CONNECTORS_DEDICATED_SHARD_MODULES,
+ ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES,
+ build_sub_it_modules,
+)
class UpdateModulesCheckTest(unittest.TestCase):
@@ -24,40 +31,108 @@ class UpdateModulesCheckTest(unittest.TestCase):
Guard the all-connectors shard contract used by backend CI.
"""
- def test_dedicated_heavy_suites_do_not_stay_in_regular_shards(self):
+ @staticmethod
+ def parse_modules(modules):
+ return [module.lstrip(":") for module in modules.split(",") if module]
+
+ def test_regular_shards_keep_only_remaining_modules_once(self):
+ """
+ Regular all-connectors shards should keep each surviving module once.
+ """
+ expected_modules = {
+ "connector-assert-e2e",
+ "connector-cdc-sqlserver-e2e",
+ "connector-http-e2e",
+ }
+ modules = ",".join(
+ ["", *sorted(expected_modules),
*ALL_CONNECTORS_DEDICATED_SHARD_MODULES]
+ )
+
+ shard_outputs = [build_sub_it_modules(modules, 7, shard) for shard in
range(7)]
+ shard_modules = [self.parse_modules(output) for output in
shard_outputs]
+ combined_counter = Counter(
+ module for output_modules in shard_modules for module in
output_modules
+ )
+
+ self.assertEqual(expected_modules, set(combined_counter))
+ self.assertEqual(Counter(expected_modules), combined_counter)
+ self.assertTrue(
+
set(ALL_CONNECTORS_DEDICATED_SHARD_MODULES).isdisjoint(set(combined_counter))
+ )
+ for output_modules in shard_modules:
+ self.assertNotIn("connector-iceberg-e2e", output_modules)
+ self.assertNotIn("connector-hbase-e2e", output_modules)
+
+ def test_regular_shards_fail_fast_when_dedicated_modules_disappear(self):
"""
- Iceberg and HBase should run only in the dedicated shard.
+ The all-connectors source list should fail loudly if a dedicated
module drifts.
"""
modules = ",".join(
[
"",
"connector-assert-e2e",
- "connector-jdbc-e2e",
- "connector-redis-e2e",
- "connector-cdc-sqlserver-e2e",
- "connector-kafka-e2e",
- "connector-iceberg-e2e",
- "connector-hbase-e2e",
- "connector-http-e2e",
- "connector-rocketmq-e2e",
- "connector-kudu-e2e",
- "connector-amazonsqs-e2e",
- "connector-doris-e2e",
- "connector-paimon-e2e",
- "connector-cdc-oracle-e2e",
- "connector-file-local-e2e",
- "connector-file-sftp-e2e",
- "connector-sensorsdata-e2e",
+ *[
+ module
+ for module in
ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES
+ if module != "connector-elasticsearch-e2e"
+ ],
]
)
+ with self.assertRaisesRegex(ValueError, "connector-elasticsearch-e2e"):
+ build_sub_it_modules(modules, 7, 0)
+
+ def
test_regular_shards_allow_optional_dedicated_modules_to_be_absent(self):
+ """
+ Dedicated suites outside connector-v2 input should stay optional.
+ """
+ modules = ",".join(
+ ["", "connector-assert-e2e",
*ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES]
+ )
+
shard_outputs = [build_sub_it_modules(modules, 7, shard) for shard in
range(7)]
- combined_outputs = ",".join(shard_outputs)
+ combined_modules = {
+ module
+ for output in shard_outputs
+ for module in self.parse_modules(output)
+ }
+
+ self.assertEqual({"connector-assert-e2e"}, combined_modules)
- self.assertIn("connector-assert-e2e", combined_outputs)
- for shard_modules in shard_outputs:
- self.assertNotIn("connector-iceberg-e2e", shard_modules)
- self.assertNotIn("connector-hbase-e2e", shard_modules)
+ def test_workflow_keeps_dedicated_jobs_for_excluded_modules(self):
+ """
+ Workflow job lists should continue covering modules excluded from
regular shards.
+ """
+ workflow = (
+ Path(__file__).resolve().parents[2] / ".github" / "workflows" /
"backend.yml"
+ ).read_text(encoding="utf-8")
+ workflow_modules = set()
+ for modules in re.findall(
+ r"-pl\s+(:[A-Za-z0-9._-]+(?:,:[A-Za-z0-9._-]+)*)", workflow
+ ):
+ workflow_modules.update(
+ module.lstrip(":") for module in modules.split(",") if module
+ )
+
+ expected_workflow_modules = set(ALL_CONNECTORS_DEDICATED_SHARD_MODULES)
+ expected_workflow_modules.remove("connector-jdbc-e2e")
+ expected_workflow_modules.update(
+ {
+ "connector-jdbc-e2e-part-1",
+ "connector-jdbc-e2e-part-2",
+ "connector-jdbc-e2e-part-3",
+ "connector-jdbc-e2e-part-4",
+ "connector-jdbc-e2e-part-5",
+ "connector-jdbc-e2e-part-6",
+ "connector-jdbc-e2e-part-7",
+ "connector-jdbc-e2e-ddl",
+ }
+ )
+
+ self.assertFalse(
+ expected_workflow_modules - workflow_modules,
+ f"Missing dedicated workflow modules:
{sorted(expected_workflow_modules - workflow_modules)}",
+ )
if __name__ == "__main__":
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 0238bb1f5c..844bd6c48a 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -17,6 +17,38 @@
import json
import sys
+ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES = (
+ "connector-jdbc-e2e",
+ "connector-kafka-e2e",
+ "connector-rocketmq-e2e",
+ "connector-kudu-e2e",
+ "connector-amazonsqs-e2e",
+ "connector-doris-e2e",
+ "connector-paimon-e2e",
+ "connector-cdc-oracle-e2e",
+ "connector-file-local-e2e",
+ "connector-file-sftp-e2e",
+ "connector-redis-e2e",
+ "connector-sensorsdata-e2e",
+ "connector-elasticsearch-e2e",
+ "connector-cdc-mysql-e2e",
+ "connector-iceberg-e2e",
+ "connector-hbase-e2e",
+)
+
+# These suites have dedicated jobs in backend.yml, but they are not listed by
+# seatunnel-connector-v2-e2e's project.modules input.
+ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES = (
+ "connector-seatunnel-e2e-base",
+ "connector-console-seatunnel-e2e",
+ "seatunnel-edge-agent-e2e",
+)
+
+ALL_CONNECTORS_DEDICATED_SHARD_MODULES = (
+ ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES
+ + ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES
+)
+
def get_cv2_modules(files):
get_modules(files, 1, "connector-", "seatunnel-connectors-v2")
@@ -141,6 +173,28 @@ def get_deleted_modules(files):
print(output_module)
+def filter_dedicated_shard_modules(modules_arr, dedicated_modules,
fail_on_missing):
+ """
+ Remove modules that already run in dedicated workflow jobs.
+
+ The all-connectors path stays strict because a missing dedicated module
+ means the exclusion list drifted away from backend.yml.
+ """
+ module_set = set(modules_arr)
+ if fail_on_missing:
+ missing_modules = [
+ module for module in dedicated_modules if module not in module_set
+ ]
+ if missing_modules:
+ raise ValueError(
+ "Missing dedicated shard modules from all-connectors input: "
+ + ",".join(missing_modules)
+ )
+
+ dedicated_modules_set = set(dedicated_modules)
+ return [module for module in modules_arr if module not in
dedicated_modules_set]
+
+
def build_sub_it_modules(modules, total_num, current_num):
"""
Build one all-connectors shard while excluding suites with dedicated jobs.
@@ -149,38 +203,18 @@ def build_sub_it_modules(modules, total_num, current_num):
round-robin shards, otherwise CI runs them twice and wastes runner time.
"""
modules_arr = list(dict.fromkeys(modules.split(",")))
- modules_arr.remove("connector-jdbc-e2e")
- modules_arr.remove("connector-kafka-e2e")
- modules_arr.remove("connector-rocketmq-e2e")
- modules_arr.remove("connector-kudu-e2e")
- modules_arr.remove("connector-amazonsqs-e2e")
- modules_arr.remove("connector-doris-e2e")
- modules_arr.remove("connector-paimon-e2e")
- modules_arr.remove("connector-cdc-oracle-e2e")
- modules_arr.remove("connector-file-local-e2e")
- modules_arr.remove("connector-file-sftp-e2e")
- modules_arr.remove("connector-redis-e2e")
- modules_arr.remove("connector-sensorsdata-e2e")
- if "connector-elasticsearch-e2e" in modules_arr:
- modules_arr.remove("connector-elasticsearch-e2e")
- if "connector-cdc-mysql-e2e" in modules_arr:
- modules_arr.remove("connector-cdc-mysql-e2e")
- if "connector-iceberg-e2e" in modules_arr:
- modules_arr.remove("connector-iceberg-e2e")
- if "connector-hbase-e2e" in modules_arr:
- modules_arr.remove("connector-hbase-e2e")
- if "connector-seatunnel-e2e-base" in modules_arr:
- modules_arr.remove("connector-seatunnel-e2e-base")
- if "connector-console-seatunnel-e2e" in modules_arr:
- modules_arr.remove("connector-console-seatunnel-e2e")
- if "seatunnel-edge-agent-e2e" in modules_arr:
- modules_arr.remove("seatunnel-edge-agent-e2e")
- output = ""
+ modules_arr = filter_dedicated_shard_modules(
+ modules_arr, ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES, True
+ )
+ modules_arr = filter_dedicated_shard_modules(
+ modules_arr, ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES, False
+ )
+ output = []
for i, module in enumerate(modules_arr):
if len(module) > 0 and i % int(total_num) == int(current_num):
- output = output + ",:" + module
+ output.append(":" + module)
- return output[1:len(output)]
+ return ",".join(output)
def get_sub_it_modules(modules, total_num, current_num):
