This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 8d1b1c9ac8 [Improve][CI] Scope backend matrix for fork changes (#11290)
8d1b1c9ac8 is described below
commit 8d1b1c9ac853152178733a60ee9ae997567c3100
Author: Daniel <[email protected]>
AuthorDate: Mon Jul 27 22:45:17 2026 +0800
[Improve][CI] Scope backend matrix for fork changes (#11290)
---
.github/workflows/backend.yml | 32 ++++++-
docs/en/developer/contribution-path.md | 15 ++++
docs/zh/developer/contribution-path.md | 15 ++++
tools/update_modules_check/ci_scope.py | 134 ++++++++++++++++++++++++++++
tools/update_modules_check/test_ci_scope.py | 128 ++++++++++++++++++++++++++
5 files changed, 321 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 6ec74cd558..e70dc458c7 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -58,6 +58,8 @@ jobs:
run: ./mvnw -B -T 1 clean test -D"license.skipAddThirdParty"=true -pl
seatunnel-ci-tools -am --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx512m
+ - name: Check CI scope rules
+ run: python3 -m unittest discover -s tools/update_modules_check -p
'test_*.py'
- name: Check for .class files in git
run: |
echo "Checking for .class files tracked by git..."
@@ -209,10 +211,25 @@ jobs:
echo "edge-agent=$true_or_false" >> $GITHUB_OUTPUT
echo "edge-agent_files=$file_list" >> $GITHUB_OUTPUT
+ dist_files=`python tools/update_modules_check/check_file_updates.py
ua $workspace apache/dev origin/$current_branch "seatunnel-dist/**"
"bin/install-plugin.sh"`
+ true_or_false=${dist_files%%$'\n'*}
+ file_list=${dist_files#*$'\n'}
+ echo "dist=$true_or_false" >> $GITHUB_OUTPUT
+ echo "dist_files=$file_list" >> $GITHUB_OUTPUT
+
api_files=`python tools/update_modules_check/check_file_updates.py
ua $workspace apache/dev origin/$current_branch "seatunnel-api/**"
"seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**"
"seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**"
"seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**"
"seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**"
"pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"`
true_or_false=${api_files%%$'\n'*}
file_list=${api_files#*$'\n'}
- if [[ $repository_owner == 'apache' ]];then
+ # Apache dev, main, master, and numeric *-release pushes force full
API coverage.
+ # On fork pushes, workflow files, tools/update_modules_check,
seatunnel-dist, and
+ # bin/install-plugin.sh do not trigger the full connector matrix by
themselves.
+ ci_scope_ref="${GITHUB_BASE_REF:-$GITHUB_REF}"
+ if ! true_or_false=$(python tools/update_modules_check/ci_scope.py
"$repository_owner" "$ci_scope_ref" --api-changed "$true_or_false"
--api-files-json "$file_list"); then
+ echo "::warning::CI scope helper failed; forcing full API coverage"
+ true_or_false='true'
+ fi
+ if [[ $true_or_false != 'true' && $true_or_false != 'false' ]]; then
+ echo "::warning::CI scope helper returned an invalid value;
forcing full API coverage"
true_or_false='true'
fi
echo "api=$true_or_false" >> $GITHUB_OUTPUT
@@ -266,9 +283,13 @@ jobs:
- name: Make unit test modules
id: ut-modules
timeout-minutes: 60
- if: ${{ steps.filter.outputs.api == 'false' &&
(steps.engine-modules.outputs.modules != '' ||
steps.cv2-modules.outputs.modules != '') }}
+ if: ${{ steps.filter.outputs.api == 'false' &&
(steps.engine-modules.outputs.modules != '' ||
steps.cv2-modules.outputs.modules != '' || steps.filter.outputs.dist == 'true')
}}
run: |
- modules='${{ steps.engine-modules.outputs.modules }}${{
steps.cv2-modules.outputs.modules }}'
+ dist_modules=''
+ if [[ '${{ steps.filter.outputs.dist }}' == 'true' ]]; then
+ dist_modules=',seatunnel-dist'
+ fi
+ modules='${{ steps.engine-modules.outputs.modules }}${{
steps.cv2-modules.outputs.modules }}'$dist_modules
modules=${modules: 1}
pl_modules=`python
tools/update_modules_check/update_modules_check.py replace "$modules"`
# remove deleted modules
@@ -283,6 +304,11 @@ jobs:
engine_modules='${{ steps.engine-modules.outputs.modules }}'
connector_modules='${{ steps.cv2-modules.outputs.modules }}'
+ if [[ "zz${engine_modules}${connector_modules}" == "zz" &&
"zz"$dist_modules != "zz" ]];then
+ echo $pl_modules
+ echo "modules=$pl_modules" >> $GITHUB_OUTPUT
+ exit 0
+ fi
if [[ "zz"$connector_modules == "zz" && "zz"$engine_modules != "zz"
]];then
# Engine changes already route the full downstream engine test set
through engine=true.
# Keep the fixed smoke modules as the output without resolving the
Maven dependency tree.
diff --git a/docs/en/developer/contribution-path.md
b/docs/en/developer/contribution-path.md
index 23595e7195..e675fbca3f 100644
--- a/docs/en/developer/contribution-path.md
+++ b/docs/en/developer/contribution-path.md
@@ -118,6 +118,21 @@ A contribution is much easier to review when it includes:
For code contributions, avoid mixing unrelated cleanup with the real fix.
+## Understand Backend CI Coverage
+
+The required Backend `Build` on a pull request mirrors the `push` workflow run
in the PR head repository, which is usually the contributor's fork. It is not a
separate full-matrix run in the base repository.
+
+The workflow selects jobs as follows:
+
+- pushes in the Apache repository to `dev`, `main`, `master`, and numeric
release branches such as `2.3.13-release` force the full API-triggered backend
matrix
+- broad API-impacting changes, including API, core, common, format, transform,
translation, and root build changes, trigger the full API matrix
+- connector-only changes use changed-module detection to select
integration-test shards; when the unit-test job is selected, it still verifies
all modules
+- engine changes keep their existing engine and connector integration-test
paths
+- fork pushes that only change `.github/workflows/**`,
`tools/update_modules_check/**`, `seatunnel-dist/**`, or
`bin/install-plugin.sh` do not trigger every connector integration-test shard
by themselves; the full unit-test job can still run when a changed module
requires it
+- if the CI scope helper fails or returns an invalid result, the workflow
falls back to full API coverage
+
+Check the jobs listed under the Backend `Build` before assuming that a green
result includes every connector integration-test shard.
+
## Good First Contribution Shapes
These contribution shapes tend to land faster:
diff --git a/docs/zh/developer/contribution-path.md
b/docs/zh/developer/contribution-path.md
index aa10f775db..342233637c 100644
--- a/docs/zh/developer/contribution-path.md
+++ b/docs/zh/developer/contribution-path.md
@@ -118,6 +118,21 @@ title: 贡献路径
对代码贡献来说，不要把无关清理和真实修复混在一起。
+## 理解 Backend CI 的覆盖范围
+
+Pull Request 上要求通过的 Backend `Build`，映射自 PR head 仓库（通常是贡献者的 fork）中的 `push`
workflow；它不是 base 仓库另外执行的一次完整矩阵。
+
+workflow 按下面的规则选择 jobs：
+
+- 推送到 Apache 仓库的 `dev`、`main`、`master` 和数字版本发布分支（例如 `2.3.13-release`）时，强制运行由
API 变更触发的完整后端矩阵
+- 修改 API、core、common、format、transform、translation 和根构建文件等具有广泛 API 影响的范围时，运行完整
API 矩阵
+- 仅修改 connector 时，通过 changed-module 判断选择集成测试分片；如果命中 unit-test job，该 job
仍会验证所有模块
+- 修改 engine 时，沿用现有的 engine 和 connector 集成测试路径
+- fork 分支仅修改
`.github/workflows/**`、`tools/update_modules_check/**`、`seatunnel-dist/**` 或
`bin/install-plugin.sh` 时，这些文件本身不会触发所有 connector 集成测试分片；如果变更模块要求运行单元测试，完整
unit-test job 仍可能执行
+- 如果 CI 范围判断工具执行失败或返回非法结果，workflow 会回退到完整 API 覆盖
+
+判断一次绿色 Backend `Build` 是否包含所有 connector 集成测试分片前，请先查看该 Build 实际列出的 jobs。
+
## 哪类首个贡献更容易落地
这些类型通常更容易合入：
diff --git a/tools/update_modules_check/ci_scope.py
b/tools/update_modules_check/ci_scope.py
new file mode 100644
index 0000000000..8f15f6d988
--- /dev/null
+++ b/tools/update_modules_check/ci_scope.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for deciding when the backend workflow must keep full API
coverage."""
+
+import argparse
+import json
+import re
+from typing import Iterable
+
+
+PROTECTED_BRANCH_NAMES = {"dev", "main", "master"}
+RELEASE_BRANCH_PATTERN = re.compile(r"\d+\.\d+(?:\.\d+)?-release")
+LIGHTWEIGHT_API_FILE_PREFIXES = (
+ ".github/workflows/",
+ "seatunnel-dist/",
+ "tools/update_modules_check/",
+)
+LIGHTWEIGHT_API_FILES = {"bin/install-plugin.sh"}
+
+
+def should_force_full_api_check(repository_owner: str, github_ref: str) ->
bool:
+ """Return whether the backend workflow should bypass incremental CI
scoping."""
+
+ if repository_owner != "apache":
+ return False
+
+ ref_prefix = "refs/heads/"
+ if github_ref.startswith(ref_prefix):
+ ref_name = github_ref[len(ref_prefix) :]
+ elif github_ref.startswith("refs/"):
+ return False
+ else:
+ # GITHUB_BASE_REF is a bare branch name on pull_request events.
+ ref_name = github_ref
+
+ return (
+ ref_name in PROTECTED_BRANCH_NAMES
+ or RELEASE_BRANCH_PATTERN.fullmatch(ref_name) is not None
+ )
+
+
+def is_lightweight_api_file(path: str) -> bool:
+ """Return true when a broad API glob matched a file that has a cheaper
check path."""
+
+ return path in LIGHTWEIGHT_API_FILES or any(
+ path.startswith(prefix) for prefix in LIGHTWEIGHT_API_FILE_PREFIXES
+ )
+
+
+def should_run_full_api_check(
+ repository_owner: str,
+ github_ref: str,
+ api_changed: bool,
+ api_changed_files: Iterable[str],
+) -> bool:
+ """Return whether the backend workflow should run the full API-triggered
matrix."""
+
+ if should_force_full_api_check(repository_owner, github_ref):
+ return True
+
+ if not api_changed:
+ return False
+
+ return any(not is_lightweight_api_file(path) for path in api_changed_files)
+
+
+def parse_bool(value: str) -> bool:
+ """Parse the shell-friendly booleans emitted by check_file_updates.py."""
+
+ if value == "true":
+ return True
+ if value == "false":
+ return False
+ raise ValueError(f"invalid boolean: {value}")
+
+
+def parse_changed_files(value: str) -> list[str]:
+ """Parse a JSON list of changed file paths."""
+
+ parsed = json.loads(value)
+ if not isinstance(parsed, list) or not all(isinstance(item, str) for item
in parsed):
+ raise ValueError("api files must be a JSON array of strings")
+ return parsed
+
+
+def main() -> None:
+ """Print a shell-friendly boolean for the workflow caller."""
+
+ parser = argparse.ArgumentParser(
+ description="Decide whether the backend workflow should force full API
coverage."
+ )
+ parser.add_argument("repository_owner", help="Value of
GITHUB_REPOSITORY_OWNER")
+ parser.add_argument("github_ref", help="Value of GITHUB_BASE_REF or
GITHUB_REF")
+ parser.add_argument(
+ "--api-changed",
+ choices=("true", "false"),
+ help="Whether API-scoped globs matched changed files",
+ )
+ parser.add_argument(
+ "--api-files-json",
+ default="[]",
+ help="JSON array emitted by check_file_updates.py for API-scoped
globs",
+ )
+ args = parser.parse_args()
+
+ if args.api_changed is None:
+ result = should_force_full_api_check(args.repository_owner,
args.github_ref)
+ else:
+ result = should_run_full_api_check(
+ args.repository_owner,
+ args.github_ref,
+ parse_bool(args.api_changed),
+ parse_changed_files(args.api_files_json),
+ )
+
+ print("true" if result else "false")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/update_modules_check/test_ci_scope.py
b/tools/update_modules_check/test_ci_scope.py
new file mode 100644
index 0000000000..886cd0ebea
--- /dev/null
+++ b/tools/update_modules_check/test_ci_scope.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Regression tests for backend workflow CI scoping decisions."""
+
+import unittest
+
+from ci_scope import should_force_full_api_check, should_run_full_api_check
+
+
+class CiScopeTest(unittest.TestCase):
+ """Verify that heavyweight CI only runs when a change really needs it."""
+
+ def test_force_full_api_check_on_dev_branch(self) -> None:
+ self.assertTrue(should_force_full_api_check("apache",
"refs/heads/dev"))
+
+ def test_force_full_api_check_on_main_and_master_branches(self) -> None:
+ for branch_name in ("main", "master"):
+ with self.subTest(branch_name=branch_name):
+ self.assertTrue(
+ should_force_full_api_check("apache",
f"refs/heads/{branch_name}")
+ )
+
+ def test_force_full_api_check_on_release_branch(self) -> None:
+ self.assertTrue(should_force_full_api_check("apache",
"refs/heads/2.3.13-release"))
+
+ def test_force_full_api_check_on_two_component_release_branch(self) ->
None:
+ self.assertTrue(should_force_full_api_check("apache",
"refs/heads/2.4-release"))
+
+ def test_force_full_api_check_on_pull_request_base_branch(self) -> None:
+ self.assertTrue(should_force_full_api_check("apache", "dev"))
+
+ def test_skip_force_full_api_check_for_fork_owner(self) -> None:
+ self.assertFalse(should_force_full_api_check("DanielLeens",
"refs/heads/dev"))
+
+ def test_skip_force_full_api_check_on_pull_request_merge_ref(self) -> None:
+ self.assertFalse(should_force_full_api_check("apache",
"refs/pull/123/merge"))
+
+ def test_skip_force_full_api_check_on_non_version_release_branch(self) ->
None:
+ self.assertFalse(
+ should_force_full_api_check("apache", "refs/heads/hotfix-release")
+ )
+
+ def test_skip_force_full_api_check_on_release_tag(self) -> None:
+ self.assertFalse(
+ should_force_full_api_check("apache", "refs/tags/2.3.13-release")
+ )
+
+ def test_skip_full_api_check_for_ci_only_fork_change(self) -> None:
+ self.assertFalse(
+ should_run_full_api_check(
+ "DanielLeens",
+ "refs/heads/ci-scope",
+ True,
+ [
+ ".github/workflows/backend.yml",
+ "tools/update_modules_check/ci_scope.py",
+ ],
+ )
+ )
+
+ def test_skip_full_api_check_for_dist_only_fork_change(self) -> None:
+ self.assertFalse(
+ should_run_full_api_check(
+ "DanielLeens",
+ "refs/heads/install-plugin",
+ True,
+ [
+ "bin/install-plugin.sh",
+ "seatunnel-dist/src/test/java/"
+
"org/apache/seatunnel/installer/InstallPluginScriptTest.java",
+ ],
+ )
+ )
+
+ def test_run_full_api_check_for_source_api_change(self) -> None:
+ self.assertTrue(
+ should_run_full_api_check(
+ "DanielLeens",
+ "refs/heads/api-change",
+ True,
+ [
+ "seatunnel-api/src/main/java/"
+ "org/apache/seatunnel/api/table/type/SeaTunnelRow.java"
+ ],
+ )
+ )
+
+ def test_run_full_api_check_for_mixed_lightweight_and_source_change(self)
-> None:
+ self.assertTrue(
+ should_run_full_api_check(
+ "DanielLeens",
+ "refs/heads/mixed-change",
+ True,
+ [
+ ".github/workflows/backend.yml",
+ "seatunnel-core/seatunnel-starter/src/main/java/"
+ "org/apache/seatunnel/core/starter/SeaTunnel.java",
+ ],
+ )
+ )
+
+ def test_protected_branch_keeps_full_api_for_lightweight_change(self) ->
None:
+ self.assertTrue(
+ should_run_full_api_check(
+ "apache",
+ "refs/heads/dev",
+ True,
+ [".github/workflows/backend.yml"],
+ )
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
