This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4dc4ebff93f Automate keeping documented "tested with" versions in sync
with the test matrix (#69124)
4dc4ebff93f is described below
commit 4dc4ebff93f7a1d60d1c5aa482007e6befca4233
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Jul 20 00:24:04 2026 +0200
Automate keeping documented "tested with" versions in sync with the test
matrix (#69124)
The Python, database and Kubernetes versions Airflow is tested with live in
global_constants.py, but are also listed by hand in the installation
prerequisites doc and the README. Those lists drifted: PostgreSQL 18 (and
MySQL 8.4) were added to the test matrix and the README, but the
prerequisites
doc was forgotten, so it still advertised PostgreSQL up to 17.
Add a prek hook that regenerates the tested-versions block in
prerequisites.rst
and the "Main version (dev)" column of the README from global_constants.py,
so
this drift becomes an auto-fixable check failure instead of a silent doc
bug.
Also correct the stale PostgreSQL and MySQL versions in prerequisites.rst.
Generated-by: Claude Code (Opus 4.8) following the guidelines
---
.pre-commit-config.yaml | 6 +
airflow-core/docs/installation/prerequisites.rst | 4 +
scripts/ci/prek/common_prek_utils.py | 36 +++-
scripts/ci/prek/update_tested_versions.py | 209 +++++++++++++++++++++
scripts/tests/ci/prek/test_common_prek_utils.py | 40 ++++
.../tests/ci/prek/test_update_tested_versions.py | 123 ++++++++++++
6 files changed, 417 insertions(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c28e71d1ed7..d4a3ffd372e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -707,6 +707,12 @@ repos:
^scripts/ci/prek/extract_permissions\.py$|
^airflow-core/docs/security/api_permissions_ref\.rst$
pass_filenames: false
+ - id: update-tested-versions
+ name: Update tested Python/DB/Kubernetes versions in docs
+ entry: ./scripts/ci/prek/update_tested_versions.py
+ language: python
+ files:
^airflow-core/docs/installation/prerequisites\.rst$|^README\.md$|^dev/breeze/src/airflow_breeze/global_constants\.py$|^scripts/ci/prek/update_tested_versions\.py$|^scripts/ci/prek/common_prek_utils\.py$
+ pass_filenames: false
- id: check-revision-heads-map
name: Check that the REVISION_HEADS_MAP is up-to-date
language: python
diff --git a/airflow-core/docs/installation/prerequisites.rst
b/airflow-core/docs/installation/prerequisites.rst
index 190dd397555..b92ae330e67 100644
--- a/airflow-core/docs/installation/prerequisites.rst
+++ b/airflow-core/docs/installation/prerequisites.rst
@@ -20,6 +20,8 @@ Prerequisites
Airflow® is tested with:
+ .. Beginning of the auto-generated tested versions
+
* Python: 3.10, 3.11, 3.12, 3.13, 3.14
* Databases:
@@ -30,6 +32,8 @@ Airflow® is tested with:
* Kubernetes: 1.30, 1.31, 1.32, 1.33, 1.34, 1.35
+ .. End of the auto-generated tested versions
+
While we recommend a minimum of 4GB of memory for Airflow, the actual
requirements heavily depend on your chosen deployment.
.. warning::
diff --git a/scripts/ci/prek/common_prek_utils.py
b/scripts/ci/prek/common_prek_utils.py
index 2f960aa3e32..5d50133906b 100644
--- a/scripts/ci/prek/common_prek_utils.py
+++ b/scripts/ci/prek/common_prek_utils.py
@@ -133,13 +133,21 @@ GLOBAL_CONSTANTS_PATH = (
def _read_global_constants_assignment(name: str) -> Any:
- """Read a top-level assignment from global_constants.py."""
+ """Read a top-level assignment from global_constants.py.
+
+ Handles both plain assignments (``NAME = ...``) and annotated assignments
+ (``NAME: type = ...``). The value must be a literal so it can be safely
+ evaluated with ``ast.literal_eval``.
+ """
tree = ast.parse(GLOBAL_CONSTANTS_PATH.read_text())
for node in tree.body:
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == name:
return ast.literal_eval(node.value)
+ elif isinstance(node, ast.AnnAssign):
+ if isinstance(node.target, ast.Name) and node.target.id == name
and node.value is not None:
+ return ast.literal_eval(node.value)
raise RuntimeError(f"{name} not found in global_constants.py")
@@ -152,6 +160,32 @@ def read_allowed_kubernetes_versions() -> list[str]:
return [v.lstrip("v") for v in versions]
+def read_allowed_python_major_minor_versions() -> list[str]:
+ """Parse ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS from global_constants.py
(single source of truth)."""
+ return
list(_read_global_constants_assignment("ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS"))
+
+
+def read_current_postgres_versions() -> list[str]:
+ """Parse CURRENT_POSTGRES_VERSIONS from global_constants.py (single source
of truth)."""
+ return list(_read_global_constants_assignment("CURRENT_POSTGRES_VERSIONS"))
+
+
+def read_current_mysql_versions() -> list[str]:
+ """The MySQL release versions Airflow currently tests with.
+
+ Mirrors how ``CURRENT_MYSQL_VERSIONS`` is built in global_constants.py: the
+ "old" releases plus the LTS releases, plus an innovation release when one
is
+ configured. Returns the numeric versions only (e.g. ``["8.0", "8.4"]``);
the
+ docs add the textual "Innovation" annotation on top of these.
+ """
+ versions: list[str] =
list(_read_global_constants_assignment("MYSQL_OLD_RELEASES"))
+ versions += list(_read_global_constants_assignment("MYSQL_LTS_RELEASES"))
+ innovation = _read_global_constants_assignment("MYSQL_INNOVATION_RELEASE")
+ if innovation:
+ versions.append(innovation)
+ return versions
+
+
def read_default_python_major_minor_version_for_images() -> str:
"""Parse DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES from
global_constants.py."""
value =
_read_global_constants_assignment("DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES")
diff --git a/scripts/ci/prek/update_tested_versions.py
b/scripts/ci/prek/update_tested_versions.py
new file mode 100755
index 00000000000..4b5936de276
--- /dev/null
+++ b/scripts/ci/prek/update_tested_versions.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = []
+# ///
+"""Keep the documented "tested with" versions in sync with global_constants.py.
+
+The list of Python, database and Kubernetes versions Airflow is tested with
lives
+in a single source of truth
(``dev/breeze/src/airflow_breeze/global_constants.py``)
+but is also rendered for humans in two hand-maintained places:
+
+* ``airflow-core/docs/installation/prerequisites.rst`` — the "Prerequisites"
bullet list.
+* ``README.md`` — the "Main version (dev)" column of the Requirements table.
+
+These used to drift out of sync silently (e.g. PostgreSQL 18 was added to the
test
+matrix and the README but the prerequisites doc was forgotten). This hook
regenerates
+both from the constants so the drift becomes a (auto-fixable) prek failure
instead of
+a documentation bug nobody notices.
+
+Only versions that have a clean single source in global_constants.py are
generated:
+Python, PostgreSQL, MySQL (numeric releases) and Kubernetes. SQLite has no
constant
+and the MySQL "Innovation" annotation is editorial, so both are defined here.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from common_prek_utils import (
+ AIRFLOW_ROOT_PATH,
+ read_allowed_kubernetes_versions,
+ read_allowed_python_major_minor_versions,
+ read_current_mysql_versions,
+ read_current_postgres_versions,
+)
+
+PREREQUISITES_RST = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" /
"installation" / "prerequisites.rst"
+README_MD = AIRFLOW_ROOT_PATH / "README.md"
+
+PREREQUISITES_START = " .. Beginning of the auto-generated tested versions\n"
+PREREQUISITES_END = " .. End of the auto-generated tested versions\n"
+
+README_DEV_HEADER = "Main version (dev)"
+
+# Editorial values without a single source of truth in global_constants.py.
+SQLITE_VERSION = "3.15.0+"
+MYSQL_INNOVATION_ANNOTATION = "Innovation"
+MYSQL_INNOVATION_RST_LINK = (
+ "`Innovation <https://dev.mysql.com/blog-archive/";
+ "introducing-mysql-innovation-and-long-term-support-lts-versions>`_"
+)
+
+
+def kubernetes_major_minor_versions() -> list[str]:
+ """Kubernetes versions reduced to ``major.minor`` (e.g. ``1.30``) for
display."""
+ return [".".join(version.split(".")[:2]) for version in
read_allowed_kubernetes_versions()]
+
+
+def comma(values: list[str]) -> str:
+ return ", ".join(values)
+
+
+def render_prerequisites_block() -> str:
+ """The bullet list rendered between the prerequisites.rst markers."""
+ python_versions = comma(read_allowed_python_major_minor_versions())
+ postgres_versions = comma(read_current_postgres_versions())
+ mysql_versions = comma([*read_current_mysql_versions(),
MYSQL_INNOVATION_RST_LINK])
+ kubernetes_versions = comma(kubernetes_major_minor_versions())
+ return (
+ "\n"
+ f"* Python: {python_versions}\n"
+ "\n"
+ "* Databases:\n"
+ "\n"
+ f" * PostgreSQL: {postgres_versions}\n"
+ f" * MySQL: {mysql_versions}\n"
+ f" * SQLite: {SQLITE_VERSION}\n"
+ "\n"
+ f"* Kubernetes: {kubernetes_versions}\n"
+ "\n"
+ )
+
+
+def dev_version_values() -> dict[str, str]:
+ """Generated value for each README row label whose dev column we keep in
sync."""
+ return {
+ "Python": comma(read_allowed_python_major_minor_versions()),
+ "Kubernetes": comma(kubernetes_major_minor_versions()),
+ "PostgreSQL": comma(read_current_postgres_versions()),
+ "MySQL": comma([*read_current_mysql_versions(),
MYSQL_INNOVATION_ANNOTATION]),
+ }
+
+
+def replace_text_between(text: str, start: str, end: str, replacement: str) ->
str:
+ if start not in text or end not in text:
+ raise RuntimeError(f"Could not find markers {start!r} / {end!r} in the
file")
+ leading = text.split(start)[0]
+ trailing = text.split(end)[1]
+ return leading + start + replacement + end + trailing
+
+
+def update_prerequisites() -> bool:
+ """Regenerate the tested-versions block in prerequisites.rst. Return True
if changed."""
+ original = PREREQUISITES_RST.read_text()
+ updated = replace_text_between(
+ original, PREREQUISITES_START, PREREQUISITES_END,
render_prerequisites_block()
+ )
+ if updated != original:
+ PREREQUISITES_RST.write_text(updated)
+ return True
+ return False
+
+
+def update_readme() -> bool:
+ """Sync the "Main version (dev)" column of the README Requirements table.
+
+ Only the dev cell of the matched rows is rewritten; the label and the
historical
+ "Stable version" columns are preserved byte-for-byte. The dev column is
widened (and
+ only the dev column) if a generated value no longer fits, keeping the
table aligned
+ without touching the stable columns.
+ """
+ lines = README_MD.read_text().splitlines(keepends=True)
+ header_index = next(
+ (i for i, line in enumerate(lines) if README_DEV_HEADER in line and
line.lstrip().startswith("|")),
+ None,
+ )
+ if header_index is None:
+ raise RuntimeError(
+ f"Could not find the Requirements table header
({README_DEV_HEADER!r}) in README.md"
+ )
+
+ # The table spans contiguous lines starting with '|', header first then
separator.
+ table_end = header_index
+ while table_end < len(lines) and lines[table_end].lstrip().startswith("|"):
+ table_end += 1
+ table_rows = list(range(header_index, table_end))
+ separator_index = header_index + 1
+
+ generated = dev_version_values()
+
+ def cells(line: str) -> list[str]:
+ # Drop the empty fragments before the first and after the last pipe.
+ return line.rstrip("\n").split("|")[1:-1]
+
+ # Determine the dev column width: max of the current width and every value
we render.
+ current_width = len(cells(lines[header_index])[1]) - 2
+ needed = [current_width]
+ for idx in table_rows:
+ if idx == separator_index:
+ continue
+ row = cells(lines[idx])
+ label = row[0].strip()
+ value = generated.get(label, row[1].strip())
+ needed.append(len(value))
+ dev_width = max(needed)
+
+ changed = False
+ for idx in table_rows:
+ row = cells(lines[idx])
+ if idx == separator_index:
+ new_dev = "-" * (dev_width + 2)
+ else:
+ label = row[0].strip()
+ value = generated.get(label, row[1].strip())
+ new_dev = f" {value.ljust(dev_width)} "
+ if new_dev == row[1]:
+ continue
+ row[1] = new_dev
+ lines[idx] = "|" + "|".join(row) + "|\n"
+ changed = True
+
+ if changed:
+ README_MD.write_text("".join(lines))
+ return changed
+
+
+def main() -> int:
+ changed_files: list[Path] = []
+ if update_prerequisites():
+ changed_files.append(PREREQUISITES_RST)
+ if update_readme():
+ changed_files.append(README_MD)
+ if changed_files:
+ for path in changed_files:
+ print(f"Updated tested versions in
{path.relative_to(AIRFLOW_ROOT_PATH)}")
+ print("Files were re-generated from global_constants.py - please
re-stage them.")
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_common_prek_utils.py
b/scripts/tests/ci/prek/test_common_prek_utils.py
index f28961f9006..e23baabc542 100644
--- a/scripts/tests/ci/prek/test_common_prek_utils.py
+++ b/scripts/tests/ci/prek/test_common_prek_utils.py
@@ -33,6 +33,9 @@ from ci.prek.common_prek_utils import (
pre_process_mypy_files,
read_airflow_version,
read_allowed_kubernetes_versions,
+ read_allowed_python_major_minor_versions,
+ read_current_mysql_versions,
+ read_current_postgres_versions,
read_default_python_major_minor_version_for_images,
resolve_github_token,
retrieve_gh_token,
@@ -386,6 +389,43 @@ class TestReadDefaultPythonMajorMinorVersionForImages:
assert parts[1].isdigit()
+class TestReadAllowedPythonMajorMinorVersions:
+ def test_returns_list_of_versions(self):
+ versions = read_allowed_python_major_minor_versions()
+ assert isinstance(versions, list)
+ assert len(versions) > 0
+
+ def test_versions_look_like_major_minor(self):
+ for version in read_allowed_python_major_minor_versions():
+ parts = version.split(".")
+ assert len(parts) == 2, f"Version {version!r} should be
major.minor"
+ assert parts[0].isdigit()
+ assert parts[1].isdigit()
+
+
+class TestReadCurrentPostgresVersions:
+ def test_returns_list_of_versions(self):
+ versions = read_current_postgres_versions()
+ assert isinstance(versions, list)
+ assert len(versions) > 0
+
+ def test_versions_are_numeric_major_versions(self):
+ for version in read_current_postgres_versions():
+ assert version.isdigit(), f"Postgres version {version!r} should be
a numeric major version"
+
+
+class TestReadCurrentMysqlVersions:
+ def test_returns_list_of_versions(self):
+ versions = read_current_mysql_versions()
+ assert isinstance(versions, list)
+ assert len(versions) > 0
+
+ def test_reads_annotated_assignment(self):
+ # MYSQL_LTS_RELEASES is an annotated assignment (``NAME: list[str] =
[...]``);
+ # the reader must handle it, so its value must show up here.
+ assert "8.4" in read_current_mysql_versions()
+
+
class TestConsoleDiff:
def test_dump_added_lines(self):
diff = ConsoleDiff()
diff --git a/scripts/tests/ci/prek/test_update_tested_versions.py
b/scripts/tests/ci/prek/test_update_tested_versions.py
new file mode 100644
index 00000000000..08afc23ae72
--- /dev/null
+++ b/scripts/tests/ci/prek/test_update_tested_versions.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import textwrap
+
+from ci.prek import update_tested_versions as hook
+from ci.prek.common_prek_utils import (
+ read_allowed_python_major_minor_versions,
+ read_current_postgres_versions,
+)
+
+
+def _dev_cell(line: str) -> str:
+ """The (stripped) "Main version (dev)" cell of a markdown table row."""
+ return line.split("|")[1:-1][1].strip()
+
+
+def _stable_cells(line: str) -> list[str]:
+ """The stable-column cells (everything after the dev column)."""
+ return [c.strip() for c in line.split("|")[1:-1][2:]]
+
+
+class TestRenderPrerequisitesBlock:
+ def test_block_reflects_constants(self):
+ block = hook.render_prerequisites_block()
+ assert f"* Python: {',
'.join(read_allowed_python_major_minor_versions())}\n" in block
+ assert f" * PostgreSQL: {',
'.join(read_current_postgres_versions())}\n" in block
+ # SQLite has no central constant - it is rendered from the hook's own
value.
+ assert f" * SQLite: {hook.SQLITE_VERSION}\n" in block
+ # MySQL always carries the editorial "Innovation" annotation.
+ assert "Innovation" in block
+
+ def test_block_is_wrapped_by_markers_content_only(self):
+ # The rendered block must not itself contain the markers (those wrap
it).
+ block = hook.render_prerequisites_block()
+ assert hook.PREREQUISITES_START not in block
+ assert hook.PREREQUISITES_END not in block
+
+
+class TestUpdateReadme:
+ TABLE = textwrap.dedent(
+ """\
+ ## Requirements
+
+ | | Main version (dev) | Stable version (3.2.0) | Stable
version (2.11.2) |
+
|------------|--------------------|------------------------|-------------------------|
+ | Python | WRONG | 3.10, 3.11 | 3.10
|
+ | Platform | AMD64/ARM64 | AMD64/ARM64 |
AMD64/ARM64 |
+ | PostgreSQL | WRONG | 14, 15 | 12, 13
|
+ | SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+
|
+
+ text after the table
+ """
+ )
+
+ def test_syncs_dev_column_and_preserves_stable_columns(self, tmp_path,
monkeypatch):
+ readme = tmp_path / "README.md"
+ readme.write_text(self.TABLE)
+ monkeypatch.setattr(hook, "README_MD", readme)
+
+ assert hook.update_readme() is True
+
+ lines = readme.read_text().splitlines()
+ rows = {line.split("|")[1].strip(): line for line in lines if
line.startswith("|")}
+
+ expected_python = ", ".join(read_allowed_python_major_minor_versions())
+ expected_postgres = ", ".join(read_current_postgres_versions())
+
+ # Dev column is now synced from the constants.
+ assert _dev_cell(rows["Python"]) == expected_python
+ assert _dev_cell(rows["PostgreSQL"]) == expected_postgres
+
+ # Stable columns are untouched.
+ assert _stable_cells(rows["Python"]) == ["3.10, 3.11", "3.10"]
+ assert _stable_cells(rows["PostgreSQL"]) == ["14, 15", "12, 13"]
+
+ # A non-target row keeps its dev value verbatim.
+ assert _dev_cell(rows["Platform"]) == "AMD64/ARM64"
+
+ # Text outside the table is preserved.
+ assert "text after the table" in readme.read_text()
+
+ def test_noop_when_already_synced(self, tmp_path, monkeypatch):
+ readme = tmp_path / "README.md"
+ readme.write_text(self.TABLE)
+ monkeypatch.setattr(hook, "README_MD", readme)
+
+ # First pass syncs and reports a change; a second pass is a no-op.
+ assert hook.update_readme() is True
+ synced = readme.read_text()
+ assert hook.update_readme() is False
+ assert readme.read_text() == synced
+
+
+class TestRepoDocsInSync:
+ """Regression guard: the committed docs must already match the
constants."""
+
+ def test_prerequisites_in_sync(self):
+ assert hook.update_prerequisites() is False, (
+ "prerequisites.rst is out of sync with global_constants.py - "
+ "run 'prek run update-tested-versions --all-files'"
+ )
+
+ def test_readme_in_sync(self):
+ assert hook.update_readme() is False, (
+ "README.md tested-versions table is out of sync with
global_constants.py - "
+ "run 'prek run update-tested-versions --all-files'"
+ )
