This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-3-test by this push:
new a84d2def196 [v3-3-test] Add prek hook to keep the Go toolchain version
in sync (#69338) (#69498)
a84d2def196 is described below
commit a84d2def1967e075f655d36d840cdb2a17c72b1d
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jul 7 00:28:46 2026 +0200
[v3-3-test] Add prek hook to keep the Go toolchain version in sync (#69338)
(#69498)
The Go toolchain version is pinned in eight places (go-sdk/go.mod, the
go_example module, the setup-go step in both CI workflow copies, the
golang alpine builder image used by the e2e conftest and by breeze, and
the default_language_version.golang in the top-level and go-sdk prek
configs). They have no cross-file include, so a dependency bump that
raises the go directive in go-sdk/go.mod leaves the rest behind and the
drift only surfaces as an opaque "go.mod requires go >= <new>" failure
deep in CI — as happened in PR #69214.
Add a prek hook that treats go-sdk/go.mod as the single source of truth
and fails when any other site disagrees at major.minor granularity,
pointing directly at the drifting file.
(cherry picked from commit 0a48feeb1d58f2cb608e9b2258f83c2df1e8b06a)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.pre-commit-config.yaml | 15 ++
scripts/ci/prek/check_go_version_in_sync.py | 197 +++++++++++++++++++++
.../tests/ci/prek/test_check_go_version_in_sync.py | 130 ++++++++++++++
3 files changed, 342 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 95bb77f184c..3998ee80bb5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -269,6 +269,21 @@ repos:
^Dockerfile\.ci$
pass_filenames: false
require_serial: true
+ - id: check-go-version-in-sync
+ name: Check Go toolchain version is consistent across build files
+ entry: ./scripts/ci/prek/check_go_version_in_sync.py
+ language: python
+ files: >
+ (?x)
+ ^go-sdk/go\.mod$|
+ ^kubernetes-tests/lang_sdk/go_example/go\.mod$|
+ ^\.github/workflows/ci-(arm|amd)\.yml$|
+ ^airflow-e2e-tests/tests/airflow_e2e_tests/constants\.py$|
+ ^dev/breeze/src/airflow_breeze/commands/kubernetes_commands\.py$|
+ ^go-sdk/\.pre-commit-config\.yaml$|
+ ^\.pre-commit-config\.yaml$
+ pass_filenames: false
+ require_serial: true
- id: check-partition-mapper-defaults-in-sync
name: Check partition-mapper core/SDK sync (FanOutMapper table +
SegmentWindow/FixedKeyMapper)
entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py
diff --git a/scripts/ci/prek/check_go_version_in_sync.py
b/scripts/ci/prek/check_go_version_in_sync.py
new file mode 100755
index 00000000000..250bb2246b2
--- /dev/null
+++ b/scripts/ci/prek/check_go_version_in_sync.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Keep the Go toolchain version in sync across every file that pins it.
+
+``golang.org/x/net`` (and other ``golang.org/x`` modules) periodically raise
+their minimum Go version. When that happens a dependency bump changes the
+``go`` directive in ``go-sdk/go.mod`` but leaves the *other* places that pin
+the toolchain untouched — and CI then fails deep inside static checks and the
+Go SDK e2e bundle build with ``go.mod requires go >= <new>`` (this is exactly
+what happened in PR #69214). Those pins have no cross-file include, so nothing
+catches the drift until CI does.
+
+The single source of truth is the ``go`` directive in ``go-sdk/go.mod``. Every
+other site below must agree with it at ``major.minor`` granularity (the minor
+version is what selects the toolchain and the builder image; the go.mod patch
+suffix is not meaningful for those):
+
+- ``go-sdk/go.mod`` -> ``go <maj.min[.patch]>``
(SOURCE OF TRUTH)
+- ``kubernetes-tests/lang_sdk/go_example/go.mod`` -> ``go <maj.min[.patch]>``
(resolves the SDK via ``replace``)
+- ``.github/workflows/ci-amd.yml`` -> Setup Go ``go-version:
<maj.min>``
+- ``.github/workflows/ci-arm.yml`` -> Setup Go ``go-version:
<maj.min>``
+- ``airflow-e2e-tests/tests/airflow_e2e_tests/constants.py`` ->
``GO_BUILDER_IMAGE`` default ``golang:<maj.min>-alpine``
+- ``dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py`` ->
``LANG_SDK_GO_BUILDER_IMAGE`` default ``golang:<maj.min>-alpine``
+- ``go-sdk/.pre-commit-config.yaml`` ->
``default_language_version.golang`` (the toolchain prek uses for the SDK's
``language: golang`` hooks — this is the pin that broke static checks in
#69214, not the setup-go one)
+- ``.pre-commit-config.yaml`` ->
``default_language_version.golang`` (top-level default for any ``language:
golang`` hook)
+
+When they disagree, bump the drifting sites to the source-of-truth minor
+version. If you add a new place that pins the Go version, register it here too.
+
+Run from the repo root:
+
+ uv run --project scripts python scripts/ci/prek/check_go_version_in_sync.py
+
+Exits 0 if every site agrees with the source of truth, 1 otherwise.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import re
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+
+# The ``go`` directive: ``go 1.25`` or ``go 1.25.0``.
+GO_MOD_DIRECTIVE = re.compile(r"^go\s+(\d+\.\d+(?:\.\d+)?)\s*$", re.MULTILINE)
+# Setup Go step: ``go-version: 1.25`` (optionally quoted).
+WORKFLOW_GO_VERSION =
re.compile(r"^\s*go-version:\s*[\"']?(\d+\.\d+(?:\.\d+)?)[\"']?\s*$",
re.MULTILINE)
+# Builder image default: ``golang:1.25-alpine``.
+GOLANG_BUILDER_IMAGE = re.compile(r"golang:(\d+\.\d+(?:\.\d+)?)-alpine")
+# prek ``default_language_version`` entry: `` golang: 1.25.0``.
+PREK_GOLANG_VERSION = re.compile(r"^\s*golang:\s*(\d+\.\d+(?:\.\d+)?)\s*$",
re.MULTILINE)
+
+
[email protected]
+class VersionSite:
+ """A single location that pins the Go toolchain version."""
+
+ label: str
+ path: pathlib.Path
+ pattern: re.Pattern[str]
+ is_source_of_truth: bool = False
+
+ def extract(self) -> str | None:
+ """Return the captured version string, or ``None`` if the pattern is
missing."""
+ if not self.path.exists():
+ return None
+ if m := self.pattern.search(self.path.read_text()):
+ return m.group(1)
+ return None
+
+
+def major_minor(version: str) -> str:
+ """Reduce ``1.25`` / ``1.25.0`` to its ``major.minor`` (``1.25``)."""
+ parts = version.split(".")
+ return ".".join(parts[:2])
+
+
+def build_sites(repo_root: pathlib.Path) -> list[VersionSite]:
+ """Build the list of Go-version pin sites, resolved against
``repo_root``."""
+ workflows = repo_root / ".github" / "workflows"
+ return [
+ VersionSite(
+ label="go-sdk/go.mod (go directive)",
+ path=repo_root / "go-sdk" / "go.mod",
+ pattern=GO_MOD_DIRECTIVE,
+ is_source_of_truth=True,
+ ),
+ VersionSite(
+ label="kubernetes-tests/lang_sdk/go_example/go.mod (go
directive)",
+ path=repo_root / "kubernetes-tests" / "lang_sdk" / "go_example" /
"go.mod",
+ pattern=GO_MOD_DIRECTIVE,
+ ),
+ VersionSite(
+ label=".github/workflows/ci-amd.yml (Setup Go go-version)",
+ path=workflows / "ci-amd.yml",
+ pattern=WORKFLOW_GO_VERSION,
+ ),
+ VersionSite(
+ label=".github/workflows/ci-arm.yml (Setup Go go-version)",
+ path=workflows / "ci-arm.yml",
+ pattern=WORKFLOW_GO_VERSION,
+ ),
+ VersionSite(
+ label="airflow-e2e-tests/.../constants.py (GO_BUILDER_IMAGE)",
+ path=repo_root / "airflow-e2e-tests" / "tests" /
"airflow_e2e_tests" / "constants.py",
+ pattern=GOLANG_BUILDER_IMAGE,
+ ),
+ VersionSite(
+ label="dev/breeze/.../kubernetes_commands.py
(LANG_SDK_GO_BUILDER_IMAGE)",
+ path=repo_root
+ / "dev"
+ / "breeze"
+ / "src"
+ / "airflow_breeze"
+ / "commands"
+ / "kubernetes_commands.py",
+ pattern=GOLANG_BUILDER_IMAGE,
+ ),
+ VersionSite(
+ label="go-sdk/.pre-commit-config.yaml
(default_language_version.golang)",
+ path=repo_root / "go-sdk" / ".pre-commit-config.yaml",
+ pattern=PREK_GOLANG_VERSION,
+ ),
+ VersionSite(
+ label=".pre-commit-config.yaml (default_language_version.golang)",
+ path=repo_root / ".pre-commit-config.yaml",
+ pattern=PREK_GOLANG_VERSION,
+ ),
+ ]
+
+
+def check_sync(sites: list[VersionSite], repo_root: pathlib.Path) ->
tuple[int, str]:
+ """Compare every site against the source of truth. Returns ``(exit_code,
report)``."""
+ results = [(site, site.extract()) for site in sites]
+
+ if missing := [site for site, version in results if version is None]:
+ lines = [f"ERROR: Go version pin not found in
{site.path.relative_to(repo_root)}" for site in missing]
+ lines += [f" (expected pattern {site.pattern.pattern!r})" for
site in missing]
+ return 1, "\n".join(lines)
+
+ source = next(site for site, _ in results if site.is_source_of_truth)
+ expected = major_minor(source.extract()) # type: ignore[arg-type]
+
+ drifted = [(site, version) for site, version in results if
major_minor(version) != expected] # type: ignore[arg-type]
+ if not drifted:
+ return 0, f"OK: Go version is consistently {expected} across all
{len(sites)} pin sites."
+
+ col = max(len(site.label) for site, _ in results)
+ lines = [
+ f"ERROR: Go version drifted from the source of truth ({source.label} =
{expected}).",
+ "",
+ f" {'PIN SITE':<{col}} VERSION",
+ ]
+ for site, version in results:
+ marker = (
+ " <- source of truth"
+ if site.is_source_of_truth
+ else (
+ " <- DRIFT" if major_minor(version) != expected else "" #
type: ignore[arg-type]
+ )
+ )
+ lines.append(f" {site.label:<{col}} {version}{marker}")
+ lines += [
+ "",
+ f"Bump the drifting site(s) to {expected} to match {source.label},",
+ "or, if you intentionally changed the source of truth, update the
other pins to match.",
+ "After changing a go.mod, run its `go mod tidy` so go.sum stays
consistent.",
+ ]
+ return 1, "\n".join(lines)
+
+
+def main() -> int:
+ exit_code, report = check_sync(build_sites(REPO_ROOT), REPO_ROOT)
+ print(report)
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_check_go_version_in_sync.py
b/scripts/tests/ci/prek/test_check_go_version_in_sync.py
new file mode 100644
index 00000000000..8092f17e034
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_go_version_in_sync.py
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from check_go_version_in_sync import build_sites, check_sync, major_minor
+
+# Default (all-consistent) version at every pin site. Individual tests override
+# one entry to simulate drift. go.mod carries the patch suffix; the workflow
and
+# image sites carry only major.minor — the check must treat these as equal.
+DEFAULT_VERSIONS = {
+ "go_sdk_mod": "1.25.0",
+ "go_example_mod": "1.25.0",
+ "ci_amd": "1.25",
+ "ci_arm": "1.25",
+ "constants": "1.25",
+ "kubernetes_commands": "1.25",
+ "go_sdk_prek": "1.25.0",
+ "root_prek": "1.25.0",
+}
+
+
+def _write_tree(root: Path, versions: dict[str, str]) -> None:
+ """Materialise a minimal repo tree with a Go version pin at each of the
eight sites."""
+
+ def write(rel: str, content: str) -> None:
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content)
+
+ write("go-sdk/go.mod", f"module github.com/apache/airflow/go-sdk\n\ngo
{versions['go_sdk_mod']}\n")
+ write(
+ "kubernetes-tests/lang_sdk/go_example/go.mod",
+ f"module
github.com/apache/airflow/kubernetes-tests/lang_sdk/go_example\n\n"
+ f"go {versions['go_example_mod']}\n",
+ )
+ for site, rel in (("ci_amd", "ci-amd.yml"), ("ci_arm", "ci-arm.yml")):
+ write(
+ f".github/workflows/{rel}",
+ f" - name: Setup Go\n with:\n go-version:
{versions[site]}\n",
+ )
+ write(
+ "airflow-e2e-tests/tests/airflow_e2e_tests/constants.py",
+ f'GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE",
"golang:{versions["constants"]}-alpine")\n',
+ )
+ write(
+ "dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py",
+ f'LANG_SDK_GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", '
+ f'"golang:{versions["kubernetes_commands"]}-alpine")\n',
+ )
+ for site, rel in (
+ ("go_sdk_prek", "go-sdk/.pre-commit-config.yaml"),
+ ("root_prek", ".pre-commit-config.yaml"),
+ ):
+ write(
+ rel,
+ f"default_language_version:\n python: python3\n node: 22.19.0\n
golang: {versions[site]}\n",
+ )
+
+
[email protected](
+ ("version", "expected"),
+ [("1.25", "1.25"), ("1.25.0", "1.25"), ("1.24.6", "1.24")],
+)
+def test_major_minor(version: str, expected: str):
+ assert major_minor(version) == expected
+
+
+def test_all_sites_consistent(tmp_path: Path):
+ _write_tree(tmp_path, DEFAULT_VERSIONS)
+ exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
+ assert exit_code == 0
+ assert "consistently 1.25" in report
+
+
+def test_go_mod_patch_suffix_matches_major_minor_pins(tmp_path: Path):
+ """go.mod may pin 1.25.0 while the workflow/image sites pin 1.25 — that is
in sync."""
+ _write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.25.0",
"ci_amd": "1.25"})
+ exit_code, _ = check_sync(build_sites(tmp_path), tmp_path)
+ assert exit_code == 0
+
+
[email protected](
+ "drifted_site",
+ ["go_example_mod", "ci_amd", "ci_arm", "constants", "kubernetes_commands",
"go_sdk_prek", "root_prek"],
+)
+def test_drift_in_any_derived_site_is_flagged(tmp_path: Path, drifted_site:
str):
+ """A single derived site left on the old minor version must fail the
check."""
+ _write_tree(
+ tmp_path, {**DEFAULT_VERSIONS, drifted_site: "1.24" if "mod" not in
drifted_site else "1.24.6"}
+ )
+ exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
+ assert exit_code == 1
+ assert "drifted from the source of truth" in report
+ assert "<- DRIFT" in report
+
+
+def test_source_of_truth_bump_flags_all_stale_sites(tmp_path: Path):
+ """Bumping only go-sdk/go.mod (the source of truth) flags every other site
— the #69214 scenario."""
+ _write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.26.0"})
+ exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
+ assert exit_code == 1
+ # All seven non-source sites are still on the old version -> all flagged.
+ assert report.count("<- DRIFT") == 7
+ assert "<- source of truth" in report
+
+
+def test_missing_pin_is_reported(tmp_path: Path):
+ _write_tree(tmp_path, DEFAULT_VERSIONS)
+ (tmp_path / "go-sdk" / "go.mod").write_text("module
github.com/apache/airflow/go-sdk\n")
+ exit_code, report = check_sync(build_sites(tmp_path), tmp_path)
+ assert exit_code == 1
+ assert "Go version pin not found" in report
+ assert "go-sdk/go.mod" in report
