This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8f96fbd6f67 Auto-sync provider README.rst Requirements with
pyproject.toml (#67669)
8f96fbd6f67 is described below
commit 8f96fbd6f67f54dbb1dbfb109680bd68b6cd0306
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri May 29 20:15:04 2026 +0200
Auto-sync provider README.rst Requirements with pyproject.toml (#67669)
Provider README.rst files are generated from pyproject.toml at
distribution-build time. Between releases, dependency bumps in
pyproject.toml do not propagate to the committed README, leading to
silent drift that has to be fixed manually (#67554).
Add a prek hook that re-renders the Requirements table from the current
pyproject.toml whenever the latter changes, catching drift in pre-commit
and CI instead of waiting for a release build. Only the table block is
touched — surrounding prose, the cross-provider table, and the rest of
the README are left alone.
Also brings 7 currently-stale provider READMEs into sync as the hook
output (apache/beam, apache/spark, fab, ibm/mq, microsoft/azure, mysql,
tableau) — these are the same class of drift #67554 fixed manually for
celery, cncf/kubernetes, edge3, git, and openlineage.
---
.pre-commit-config.yaml | 11 +++
providers/apache/beam/README.rst | 6 +-
providers/apache/spark/README.rst | 2 +
providers/fab/README.rst | 2 +-
providers/ibm/mq/README.rst | 15 +--
providers/microsoft/azure/README.rst | 8 +-
providers/mysql/README.rst | 1 +
providers/tableau/README.rst | 2 +-
scripts/ci/prek/sync_provider_readme.py | 158 ++++++++++++++++++++++++++++++++
9 files changed, 190 insertions(+), 15 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ad15f0b15d6..c084a48f152 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -938,6 +938,17 @@ repos:
language: python
files: ^README\.md$
pass_filenames: false
+ - id: sync-provider-readme
+ name: Sync provider README.rst Requirements table with pyproject.toml
+ description: |
+ Provider README.rst files are regenerated from pyproject.toml at
distribution-build
+ time. Between releases, dependency bumps in pyproject.toml do not
propagate to the
+ committed README, leading to drift. This hook re-renders the
Requirements table from
+ the current pyproject.toml whenever the latter changes.
+ entry: ./scripts/ci/prek/sync_provider_readme.py
+ language: python
+ files: ^providers/.*/pyproject\.toml$
+ pass_filenames: true
- id: lint-markdown
name: Run markdownlint
description: Checks the style of Markdown files.
diff --git a/providers/apache/beam/README.rst b/providers/apache/beam/README.rst
index 9b8c9c54256..e152d0223ed 100644
--- a/providers/apache/beam/README.rst
+++ b/providers/apache/beam/README.rst
@@ -56,10 +56,12 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``apache-beam`` ``>=2.69.0``
-``pyarrow`` ``>=16.1.0``
+``pyarrow`` ``>=16.1.0; python_version <
"3.14"``
+``pyarrow`` ``>=22.0.0; python_version >=
"3.14"``
``numpy`` ``>=1.22.4; python_version <
"3.11"``
``numpy`` ``>=1.23.2; python_version <
"3.12" and python_version >= "3.11"``
-``numpy`` ``>=1.26.0; python_version >=
"3.12"``
+``numpy`` ``>=1.26.0; python_version >=
"3.12" and python_version < "3.14"``
+``numpy`` ``>=2.4.3; python_version >=
"3.14"``
==========================================
==================================================================
Cross provider package dependencies
diff --git a/providers/apache/spark/README.rst
b/providers/apache/spark/README.rst
index 1fdc944ef96..8ffc567a161 100644
--- a/providers/apache/spark/README.rst
+++ b/providers/apache/spark/README.rst
@@ -57,6 +57,8 @@ PIP package Version required
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``pyspark-client`` ``>=4.0.0``
``grpcio-status`` ``>=1.67.0``
+``requests`` ``>=2.32.0``
+``tenacity`` ``>=8.3.0``
========================================== ==================
Cross provider package dependencies
diff --git a/providers/fab/README.rst b/providers/fab/README.rst
index d4b350483a5..0856d28853b 100644
--- a/providers/fab/README.rst
+++ b/providers/fab/README.rst
@@ -72,7 +72,7 @@ PIP package Version required
``wtforms`` ``>=3.0``
``cachetools`` ``>=6.0``
``marshmallow`` ``>=3``
-``flask_limiter`` ``>3,!=3.13``
+``flask_limiter`` ``>3``
==========================================
=====================================
Cross provider package dependencies
diff --git a/providers/ibm/mq/README.rst b/providers/ibm/mq/README.rst
index f08c8d1bc0d..59efe30eefa 100644
--- a/providers/ibm/mq/README.rst
+++ b/providers/ibm/mq/README.rst
@@ -63,13 +63,14 @@ The package supports the following python versions:
3.10,3.11,3.12,3.13
Requirements
------------
-=============================================
=====================================
-PIP package Version required
-=============================================
=====================================
-``apache-airflow`` ``>=2.11.0``
-``apache-airflow-providers-common-messaging`` ``>=2.0.0``
-``importlib-resources`` ``>=1.3``
-=============================================
=====================================
+==========================================
======================================
+PIP package Version required
+==========================================
======================================
+``apache-airflow`` ``>=2.11.0``
+``apache-airflow-providers-common-compat`` ``>=1.12.0``
+``asgiref`` ``>=2.3.0; python_version <
"3.14"``
+``asgiref`` ``>=3.11.1; python_version >=
"3.14"``
+==========================================
======================================
========================================================================================================================
====================
diff --git a/providers/microsoft/azure/README.rst
b/providers/microsoft/azure/README.rst
index b22d4fa519e..fa58eff6bbc 100644
--- a/providers/microsoft/azure/README.rst
+++ b/providers/microsoft/azure/README.rst
@@ -50,9 +50,9 @@ The package supports the following python versions:
3.10,3.11,3.12,3.13,3.14
Requirements
------------
-========================================== ===========================
+========================================== ===================
PIP package Version required
-========================================== ===========================
+========================================== ===================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.13.0``
``adlfs`` ``>=2023.10.0``
@@ -71,7 +71,7 @@ PIP package Version required
``azure-synapse-spark`` ``>=0.2.0``
``azure-synapse-artifacts`` ``>=0.17.0``
``azure-storage-file-datalake`` ``>=12.9.1``
-``azure-kusto-data`` ``>=4.1.0,!=4.6.0,!=5.0.0``
+``azure-kusto-data`` ``>=4.1.0,!=5.0.0``
``azure-mgmt-datafactory`` ``>=2.0.0``
``azure-mgmt-containerregistry`` ``>=8.0.0``
``azure-mgmt-compute`` ``>=33.0.0``
@@ -84,7 +84,7 @@ PIP package Version required
``microsoft-kiota-abstractions`` ``>=1.9.4,<2.0.0``
``microsoft-kiota-authentication-azure`` ``>=1.9.4,<2.0.0``
``msal-extensions`` ``>=1.3.0``
-========================================== ===========================
+========================================== ===================
Cross provider package dependencies
-----------------------------------
diff --git a/providers/mysql/README.rst b/providers/mysql/README.rst
index ef562d7777e..87f9aa43327 100644
--- a/providers/mysql/README.rst
+++ b/providers/mysql/README.rst
@@ -60,6 +60,7 @@ PIP package Version required
``mysql-connector-python`` ``>=9.1.0; python_version <
"3.12"``
``mysql-connector-python`` ``>=9.1.0,!=9.7.0; python_version
>= "3.12"``
``aiomysql`` ``>=0.2.0``
+``pymysql`` ``>=1.0.3,<1.2``
==========================================
=============================================
Cross provider package dependencies
diff --git a/providers/tableau/README.rst b/providers/tableau/README.rst
index 73100612e48..1c1f55adbca 100644
--- a/providers/tableau/README.rst
+++ b/providers/tableau/README.rst
@@ -55,7 +55,7 @@ PIP package Version required
========================================== ==================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.10.1``
-``tableauserverclient`` ``>=0.27,!=0.39``
+``tableauserverclient`` ``>=0.27``
========================================== ==================
Cross provider package dependencies
diff --git a/scripts/ci/prek/sync_provider_readme.py
b/scripts/ci/prek/sync_provider_readme.py
new file mode 100755
index 00000000000..efe72981518
--- /dev/null
+++ b/scripts/ci/prek/sync_provider_readme.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+# "packaging>=25",
+# "tabulate>=0.9.0",
+# "tomli>=2.0.1; python_version < '3.11'",
+# ]
+# ///
+"""
+Sync the ``Requirements`` table in each provider ``README.rst`` with the
+``[project].dependencies`` block of its sibling ``pyproject.toml``.
+
+Provider README files are generated at distribution-build time by
+``dev/breeze/src/airflow_breeze/utils/packages.py``'s ``_prepare_readme_file``.
+Between releases they are committed in-repo as the rendered output. A
+dependency bump in ``pyproject.toml`` does not propagate to the committed
+README until the next release build, leading to drift like the one fixed
+manually in https://github.com/apache/airflow/pull/67554.
+
+This hook re-renders the ``Requirements`` table directly from the current
+``pyproject.toml`` whenever the latter changes, using the same column shape
+``convert_pip_requirements_to_table`` produces. Only the table block is
+touched — surrounding prose, the cross-provider table, and the rest of the
+README are left alone.
+
+Run as a prek hook with the changed ``providers/*/pyproject.toml`` paths as
+positional arguments.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+try:
+ import tomllib
+except ModuleNotFoundError: # py < 3.11 fallback (CI runs ≥3.10)
+ import tomli as tomllib # type: ignore[no-redef]
+
+from packaging.requirements import Requirement
+from tabulate import tabulate
+
+HEADERS = ["PIP package", "Version required"]
+
+# Matches the "Requirements" section + its following RST grid table:
+#
+# Requirements
+# ------------
+#
+# ============ ==============
+# PIP package Version required
+# ============ ==============
+# ``foo`` ``>=1.0``
+# ============ ==============
+#
+# The pattern captures the heading + leading blank line as group(1) so the
+# substitution can re-emit it verbatim, then matches the full table block —
+# three ``===`` separator lines bracketing the header row and the data rows.
+# The MULTILINE+DOTALL flags let the body span multiple lines while keeping
+# the boundary anchors tight.
+TABLE_PATTERN = re.compile(
+ r"(?ms)"
+ r"(^Requirements\s*\n-+\s*\n\s*\n)" # group(1): heading + blank
+ r"=+ +=+[^\S\n]*\n" # opening ===
+ r".+?\n" # header row
+ r"=+ +=+[^\S\n]*\n" # === after header
+ r".+?\n" # data rows (non-greedy)
+ r"=+ +=+[^\S\n]*\n" # closing ===
+)
+
+
+def render_requirements_table(dependencies: list[str]) -> str:
+ """Render the same RST table shape that breeze's release-builder
produces."""
+ rows: list[tuple[str, str]] = []
+ for dep in dependencies:
+ req = Requirement(dep)
+ package = req.name
+ if req.extras:
+ package += f"[{','.join(sorted(req.extras))}]"
+ version_required = ""
+ if req.specifier:
+ version_required = ",".join(
+ str(spec) for spec in sorted(req.specifier, key=lambda spec:
spec.version)
+ )
+ if req.marker:
+ version_required += f"; {req.marker}"
+ version_required = version_required.strip()
+ formatted_version = f"``{version_required}``" if version_required else
""
+ rows.append((f"``{package}``", formatted_version))
+ return tabulate(rows, headers=HEADERS, tablefmt="rst")
+
+
+def sync_one(pyproject_path: Path) -> bool:
+ """Sync the README.rst next to ``pyproject_path``. Returns True on
change."""
+ readme_path = pyproject_path.parent / "README.rst"
+ if not readme_path.exists():
+ return False
+ try:
+ data = tomllib.loads(pyproject_path.read_text())
+ except tomllib.TOMLDecodeError as exc:
+ print(f"[sync-provider-readme] could not parse {pyproject_path}:
{exc}", file=sys.stderr)
+ return False
+ dependencies = data.get("project", {}).get("dependencies") or []
+ if not dependencies:
+ return False
+ new_table = render_requirements_table(dependencies)
+ original = readme_path.read_text()
+ if not TABLE_PATTERN.search(original):
+ # README has no Requirements section in the expected shape — leave it
+ # alone rather than guess where to inject one. Real-world providers
+ # that have dependencies all carry this section; flag for visibility.
+ print(
+ f"[sync-provider-readme] {readme_path} has no Requirements table —
skipping",
+ file=sys.stderr,
+ )
+ return False
+ updated = TABLE_PATTERN.sub(rf"\1{new_table}\n", original)
+ if updated == original:
+ return False
+ readme_path.write_text(updated)
+ print(f"[sync-provider-readme] updated {readme_path}")
+ return True
+
+
+def main(argv: list[str]) -> int:
+ changed = False
+ for arg in argv:
+ path = Path(arg)
+ if path.name != "pyproject.toml" or "providers/" not in
path.as_posix():
+ continue
+ if sync_one(path):
+ changed = True
+ # prek detects modifications via git diff after the hook runs, so a zero
+ # exit code combined with file changes is the standard "files were
+ # modified by this hook" failure signal.
+ return 0 if not changed else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
