This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-3-test by this push:
new 15ec9f0e0b0 [v3-3-test] Catch lang-SDK Go example module drift before
it reaches main (#70568) (#70624)
15ec9f0e0b0 is described below
commit 15ec9f0e0b0c4fd6f01a7aa0f594567fb197fb9b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jul 28 18:22:59 2026 +0200
[v3-3-test] Catch lang-SDK Go example module drift before it reaches main
(#70568) (#70624)
* Catch lang-SDK Go example module drift before it reaches main
kubernetes-tests/lang_sdk/go_example is a separate Go module that resolves
the SDK through a `replace` onto ../../../go-sdk, so it carries its own copy
of the SDK's indirect requirements. Nothing re-tidies it when a dependency
moves inside /go-sdk, and Dependabot bumps exactly one module per PR.
The blast radius is what makes this worth guarding. Go refuses to build an
inconsistent module, so once such a bump merges, "Kubernetes tests / K8S
Lang-SDK" fails at the Build Go bundle step on every pull request until
someone notices and tidies the module by hand — not just on the PR that
caused it.
Dependabot security updates do not consult .github/dependabot.yml, so no
per-directory configuration prevents this, and a second Dependabot PR for
the
example module would merge at a different time and leave main red in
between.
The drift has to fail the bump PR itself.
The check is `go mod tidy -diff` in the example module: it asks exactly the
question the failing CI step asks, never writes to the working tree, and
exits non-zero when the module is untidy.
* Let prek provide the Go toolchain for the tidy check
Static checks run on a runner whose preinstalled toolchains are deleted to
free disk space before prek starts, so the check could never find `go` there
and failed on every run. Asking prek for the toolchain is how the Go SDK's
own tidy hook already gets one, and it pins the same version everywhere.
(cherry picked from commit bce20ff30bbe9b96d0de5ee6456f2015611f4e53)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.pre-commit-config.yaml | 17 +++
scripts/ci/prek/check_go_example_mod_tidy.py | 126 +++++++++++++++++++++
.../ci/prek/test_check_go_example_mod_tidy.py | 99 ++++++++++++++++
3 files changed, 242 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 49b66ca2079..86510db86ed 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -284,6 +284,23 @@ repos:
^\.pre-commit-config\.yaml$
pass_filenames: false
require_serial: true
+ - id: check-go-example-mod-tidy
+ name: Check lang-SDK Go example module is tidy against the Go SDK
+ entry: ./scripts/ci/prek/check_go_example_mod_tidy.py
+ # golang so prek provisions the toolchain the check needs: static
checks run on a
+ # runner whose preinstalled toolchains are wiped to free disk space,
and the SDK's
+ # own `go mod tidy` hook gets its Go the same way.
+ language: golang
+ # The example module keeps its own copy of the SDK's indirect
requirements
+ # (it resolves the SDK through a `replace`), so a dependency moving in
+ # go-sdk/go.mod leaves it stale. Watching both modules' manifests is
enough:
+ # any new requirement in the SDK necessarily lands in go-sdk/go.mod
first.
+ files: >
+ (?x)
+ ^go-sdk/go\.(mod|sum)$|
+ ^kubernetes-tests/lang_sdk/go_example/go\.(mod|sum)$
+ pass_filenames: false
+ require_serial: true
- id: check-partition-mapper-defaults-in-sync
name: Check partition-mapper core/SDK sync (FanOutMapper table +
SegmentWindow/FixedKeyMapper)
entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py
diff --git a/scripts/ci/prek/check_go_example_mod_tidy.py
b/scripts/ci/prek/check_go_example_mod_tidy.py
new file mode 100755
index 00000000000..613e900106b
--- /dev/null
+++ b/scripts/ci/prek/check_go_example_mod_tidy.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Keep the lang-SDK Go example module tidy against the Go SDK.
+
+``kubernetes-tests/lang_sdk/go_example`` is a **separate** Go module that
+resolves the SDK from the in-repo sources::
+
+ replace github.com/apache/airflow/go-sdk => ../../../go-sdk
+
+Because of that ``replace`` it carries its own copy of the SDK's indirect
+requirements. Nothing re-tidies it when a dependency moves inside
+``/go-sdk`` — and Dependabot bumps exactly one module per PR. The example
+module is then left pinning the old versions, Go refuses to build an
+inconsistent module, and ``Kubernetes tests / K8S Lang-SDK`` fails at the
+"Build Go bundle" step::
+
+ go: updates to go.mod needed; to update it:
+ go mod tidy
+
+The damage is not limited to the bump PR: once it merges, that job is red on
+*every* pull request until someone notices and tidies the example module by
+hand. This happened with #70226 (``google.golang.org/grpc`` 1.79.3 -> 1.82.1
+in ``/go-sdk`` only) and was cleaned up after the fact by #70561.
+
+Note that Dependabot **security** updates do not consult
+``.github/dependabot.yml`` at all, so no amount of per-directory config
+prevents this — and a second Dependabot PR for the example module would merge
+at a different time, leaving ``main`` red in between. The drift has to fail
+the bump PR itself, which is what this check does.
+
+The check is ``go mod tidy -diff`` in the example module: it is the exact
+question the failing CI step asks, it never writes to the working tree, and it
+exits non-zero when the module is untidy.
+
+Run from the repo root:
+
+ uv run --project scripts python
scripts/ci/prek/check_go_example_mod_tidy.py
+
+Exits 0 if the example module is tidy, 1 otherwise.
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+EXAMPLE_MODULE = pathlib.Path("kubernetes-tests/lang_sdk/go_example")
+GO_SDK_MODULE = pathlib.Path("go-sdk")
+
+
+def run_tidy_diff(module_dir: pathlib.Path, go_binary: str = "go") ->
tuple[int, str]:
+ """Ask Go whether ``module_dir`` is tidy. Returns ``(returncode,
combined_output)``."""
+ completed = subprocess.run(
+ [go_binary, "mod", "tidy", "-diff"],
+ cwd=module_dir,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ return completed.returncode, (completed.stdout + completed.stderr).strip()
+
+
+def format_report(returncode: int, output: str) -> tuple[int, str]:
+ """Turn a ``go mod tidy -diff`` result into ``(exit_code, report)``."""
+ if returncode == 0:
+ return 0, f"OK: {EXAMPLE_MODULE} is tidy against {GO_SDK_MODULE}."
+ lines = [
+ f"ERROR: {EXAMPLE_MODULE} is not tidy.",
+ "",
+ f"It is a separate Go module that resolves the SDK via a `replace`
onto {GO_SDK_MODULE},",
+ "so it keeps its own copy of the SDK's indirect requirements. A
dependency moved in",
+ f"{GO_SDK_MODULE} without this module being re-tidied, which breaks
the",
+ "'Kubernetes tests / K8S Lang-SDK' bundle build on every pull request
once merged.",
+ "",
+ "Fix it in this PR by running:",
+ "",
+ f" (cd {EXAMPLE_MODULE} && go mod tidy)",
+ "",
+ "and committing the resulting go.mod / go.sum changes.",
+ "",
+ "`go mod tidy -diff` reported:",
+ "",
+ output or "(no output)",
+ ]
+ return 1, "\n".join(lines)
+
+
+def main() -> int:
+ module_dir = REPO_ROOT / EXAMPLE_MODULE
+ if not (module_dir / "go.mod").is_file():
+ print(f"ERROR: {EXAMPLE_MODULE}/go.mod not found — has the example
module moved?")
+ return 1
+ if shutil.which("go") is None:
+ if os.environ.get("CI"):
+ print("ERROR: `go` is not on PATH but this is a CI run — the
toolchain is required here.")
+ return 1
+ print(f"SKIPPED: `go` is not on PATH, cannot verify that
{EXAMPLE_MODULE} is tidy.")
+ return 0
+ returncode, output = run_tidy_diff(module_dir)
+ exit_code, report = format_report(returncode, output)
+ print(report)
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
b/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
new file mode 100644
index 00000000000..286d09efecc
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+from unittest import mock
+
+import check_go_example_mod_tidy as checker
+import pytest
+
+# Trimmed to the shape that matters: the drift #70226 introduced and #70561
cleaned up.
+GRPC_DRIFT_DIFF = """\
+diff current/go.mod tidy/go.mod
+--- current/go.mod
++++ tidy/go.mod
+@@ -37,9 +37,9 @@
+- google.golang.org/grpc v1.79.3 // indirect
++ google.golang.org/grpc v1.82.1 // indirect
+"""
+
+
+def test_tidy_module_passes():
+ exit_code, report = checker.format_report(0, "")
+
+ assert exit_code == 0
+ assert "is tidy" in report
+
+
+def test_untidy_module_fails_with_the_fix_command_and_the_diff():
+ exit_code, report = checker.format_report(1, GRPC_DRIFT_DIFF)
+
+ assert exit_code == 1
+ assert "is not tidy" in report
+ assert "(cd kubernetes-tests/lang_sdk/go_example && go mod tidy)" in report
+ # The reason the contributor cares: this is what turns K8S Lang-SDK red
for everyone.
+ assert "K8S Lang-SDK" in report
+ assert "google.golang.org/grpc v1.82.1" in report
+
+
+def test_untidy_module_without_diff_output_still_reports():
+ exit_code, report = checker.format_report(1, "")
+
+ assert exit_code == 1
+ assert "(no output)" in report
+
+
[email protected]("check_go_example_mod_tidy.subprocess.run", autospec=True)
+def test_run_tidy_diff_never_writes_to_the_working_tree(mock_run, tmp_path):
+ mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0,
stdout="", stderr="")
+
+ checker.run_tidy_diff(tmp_path)
+
+ args = mock_run.call_args.args[0]
+ assert args == ["go", "mod", "tidy", "-diff"]
+ assert mock_run.call_args.kwargs["cwd"] == tmp_path
+
+
[email protected]("check_go_example_mod_tidy.subprocess.run", autospec=True)
+def test_run_tidy_diff_combines_stdout_and_stderr(mock_run, tmp_path):
+ mock_run.return_value = subprocess.CompletedProcess(
+ args=[], returncode=1, stdout="diff current/go.mod tidy/go.mod\n",
stderr="go: downloading\n"
+ )
+
+ returncode, output = checker.run_tidy_diff(tmp_path)
+
+ assert returncode == 1
+ assert "diff current/go.mod tidy/go.mod" in output
+ assert "go: downloading" in output
+
+
[email protected](
+ ("ci_env", "expected_exit", "expected_text"),
+ [
+ pytest.param({"CI": "true"}, 1, "this is a CI run",
id="ci-fails-loudly"),
+ pytest.param({}, 0, "SKIPPED", id="local-skips"),
+ ],
+)
[email protected]("check_go_example_mod_tidy.shutil.which", return_value=None)
+def test_missing_go_toolchain(mock_which, ci_env, expected_exit,
expected_text, monkeypatch, capsys):
+ monkeypatch.delenv("CI", raising=False)
+ for key, value in ci_env.items():
+ monkeypatch.setenv(key, value)
+
+ assert checker.main() == expected_exit
+ assert expected_text in capsys.readouterr().out
