This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12475-fae69186f180edde7efe02e9bf6e8eb2fcd13640 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit c364063d7543a0601aa0729a29f3f934dd6d23ac Author: Goutam Adwant <[email protected]> AuthorDate: Fri Oct 9 09:26:07 2026 +0000 [Improve][CI] Skip updated-modules IT shards that have no modules (#12475) --- .github/workflows/backend.yml | 31 ++++++--- .../test_update_modules_check.py | 81 ++++++++++++++++++++++ tools/update_modules_check/update_modules_check.py | 26 ++++++- 3 files changed, 128 insertions(+), 10 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 43f68c3656..ad900a484f 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -217,6 +217,7 @@ jobs: helm: ${{ steps.filter.outputs.helm }} ut-modules: ${{ steps.ut-modules.outputs.modules || '[]' }} it-modules: ${{ steps.it-modules.outputs.modules || '[]' }} + updated-it-shards: ${{ steps.updated-it-shards.outputs.shards || '[]' }} steps: - uses: actions/checkout@v4 with: @@ -484,6 +485,20 @@ jobs: modules_json=$(python tools/update_modules_check/update_modules_check.py json "$build_modules") echo "modules=$modules_json" >> "$GITHUB_OUTPUT" + - name: Make updated integration test shards + id: updated-it-shards + if: ${{ steps.it-modules.outputs.modules != '' && steps.it-modules.outputs.modules != '[]' }} + env: + IT_MODULES: ${{ steps.it-modules.outputs.modules }} + run: | + # Only start the updated-modules-integration-test-part-N jobs that have modules to test. + if ! shards=$(python tools/update_modules_check/update_modules_check.py sub_update_it_shards "$IT_MODULES" 8); then + echo "::warning::Could not compute updated integration test shards; running all shards" + shards='["part-1","part-2","part-3","part-4","part-5","part-6","part-7","part-8"]' + fi + echo "$shards" + echo "shards=$shards" >> "$GITHUB_OUTPUT" + dependency-license: if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' name: Dependency licenses @@ -678,7 +693,7 @@ jobs: updated-modules-integration-test-part-1: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-1') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -716,7 +731,7 @@ jobs: updated-modules-integration-test-part-2: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-2') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -754,7 +769,7 @@ jobs: updated-modules-integration-test-part-3: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-3') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -792,7 +807,7 @@ jobs: updated-modules-integration-test-part-4: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-4') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -829,7 +844,7 @@ jobs: id: ${{ matrix.java }}-${{ matrix.os }} updated-modules-integration-test-part-5: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-5') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -866,7 +881,7 @@ jobs: id: ${{ matrix.java }}-${{ matrix.os }} updated-modules-integration-test-part-6: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-6') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -903,7 +918,7 @@ jobs: id: ${{ matrix.java }}-${{ matrix.os }} updated-modules-integration-test-part-7: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-7') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -941,7 +956,7 @@ jobs: updated-modules-integration-test-part-8: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '[]' && contains(fromJSON(needs.changes.outputs.updated-it-shards), 'part-8') runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/tools/update_modules_check/test_update_modules_check.py b/tools/update_modules_check/test_update_modules_check.py index f6b277e47c..38d3fa1280 100644 --- a/tools/update_modules_check/test_update_modules_check.py +++ b/tools/update_modules_check/test_update_modules_check.py @@ -31,6 +31,8 @@ from update_modules_check import ( STANDALONE_MODULE_PATHS, build_standalone_modules, build_sub_it_modules, + build_sub_update_it_modules, + build_sub_update_it_shards, get_sub_it_modules, get_sub_update_it_modules, modules_to_json, @@ -263,6 +265,85 @@ class ConnectorItShardingTest(unittest.TestCase): workflow, ) + def test_updated_shards_list_only_shards_with_modules(self) -> None: + # A single changed connector used to start all 8 shard jobs on 2 JDKs. + self.assertEqual( + ["part-1"], + build_sub_update_it_shards( + modules_to_json(":connector-cassandra-e2e,:connector-cassandra-e2e"), 8 + ), + ) + self.assertEqual( + ["part-1", "part-2", "part-3"], + build_sub_update_it_shards( + modules_to_json(":connector-a-e2e,:connector-b-e2e,:connector-c-e2e"), + 8, + ), + ) + # Modules owned by dedicated jobs leave every shared shard empty. + self.assertEqual( + [], + build_sub_update_it_shards( + modules_to_json( + ":connector-kafka-e2e,:seatunnel-engine-k8s-e2e," + ":connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e" + ), + 8, + ), + ) + self.assertEqual([], build_sub_update_it_shards("[]", 8)) + many_modules = [f"connector-m{i}-e2e" for i in range(11)] + self.assertEqual( + [f"part-{i}" for i in range(1, 9)], + build_sub_update_it_shards(json.dumps(many_modules), 8), + ) + + def test_updated_shards_agree_with_per_shard_modules(self) -> None: + modules = modules_to_json( + ":connector-a-e2e,:connector-kafka-e2e,:connector-b-e2e," + ":connector-jdbc-e2e,:seatunnel-engine-k8s-e2e,:connector-c-e2e" + ) + shards = build_sub_update_it_shards(modules, 8) + for current_num in range(8): + with self.subTest(current_num=current_num): + self.assertEqual( + f"part-{current_num + 1}" in shards, + bool(build_sub_update_it_modules(modules, 8, current_num)), + ) + + def test_updated_shards_reject_non_positive_shard_count(self) -> None: + with self.assertRaisesRegex( + ValueError, "total shard count must be positive, got 0" + ): + build_sub_update_it_shards("[]", 0) + + def test_updated_module_jobs_are_gated_on_their_shard(self) -> None: + workflow = self.workflow_text() + jobs = dict( + re.findall( + r"^ (updated-modules-integration-test-part-\d+):\n(.*?)(?=^ \S|\Z)", + workflow, + re.MULTILINE | re.DOTALL, + ) + ) + self.assertEqual(8, len(jobs)) + self.assertTrue( + 'sub_update_it_shards "$IT_MODULES" 8' in workflow, + "changes job must compute the non-empty updated-modules shards", + ) + for part in range(1, 9): + job = jobs[f"updated-modules-integration-test-part-{part}"] + with self.subTest(part=part): + if_line = re.search(r"^ if: (.*)$", job, re.MULTILINE).group(1) + self.assertIn( + "contains(fromJSON(needs.changes.outputs.updated-it-shards), " + f"'part-{part}')", + if_line, + ) + self.assertIn( + f'sub_update_it_module "$IT_MODULES" 8 {part - 1}', job + ) + def test_full_shard_rejects_non_positive_shard_count(self) -> None: with self.assertRaisesRegex( ValueError, "total shard count must be positive, got 0" diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py index aa662e14c3..13164a07f4 100644 --- a/tools/update_modules_check/update_modules_check.py +++ b/tools/update_modules_check/update_modules_check.py @@ -291,7 +291,7 @@ def get_sub_it_modules(modules, total_num, current_num): print(build_sub_it_modules(modules, total_num, current_num)) -def get_sub_update_it_modules(modules, total_num, current_num): +def build_sub_update_it_modules(modules, total_num, current_num): final_modules = list() module_names = json.loads(modules) module_list = _filter_shared_it_modules( @@ -306,7 +306,27 @@ def get_sub_update_it_modules(modules, total_num, current_num): for i, module in enumerate(module_list): if len(module) > 0 and i % int(total_num) == int(current_num): final_modules.append(":" + module) - print(",".join(final_modules)) + return final_modules + + +def get_sub_update_it_modules(modules, total_num, current_num): + print(",".join(build_sub_update_it_modules(modules, total_num, current_num))) + + +def build_sub_update_it_shards(modules, total_num): + """Return the 1-based updated-modules shards that have at least one module.""" + total_num = int(total_num) + if total_num <= 0: + raise ValueError(f"total shard count must be positive, got {total_num}") + return [ + f"part-{current_num + 1}" + for current_num in range(total_num) + if build_sub_update_it_modules(modules, total_num, current_num) + ] + + +def get_sub_update_it_shards(modules, total_num): + print(json.dumps(build_sub_update_it_shards(modules, total_num))) def main(argv): @@ -340,6 +360,8 @@ def main(argv): get_sub_it_modules(argv[2], argv[3], argv[4]) elif argv[1] == "sub_update_it_module": get_sub_update_it_modules(argv[2], argv[3], argv[4]) + elif argv[1] == "sub_update_it_shards": + get_sub_update_it_shards(argv[2], argv[3]) if __name__ == "__main__":
