This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch v3-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-1-test by this push:
new 43e8bbbf75a [v3-1-test] Improve pyproject.toml synchronization hook
with fallback logic (#61597) (#61599)
43e8bbbf75a is described below
commit 43e8bbbf75a2ab00e8280e80d61c39a2b67327ad
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Feb 7 17:46:22 2026 +0100
[v3-1-test] Improve pyproject.toml synchronization hook with fallback logic
(#61597) (#61599)
Add fallback mechanism for provider minimum version detection when
metadata is unavailable. The hook now attempts to read local provider
pyproject.toml files as a fallback before giving up.
Changes:
- Add get_local_provider_version() to read from local pyproject.toml
- Extract fallback logic into _fallback_provider_version() helper
- Add extra_information parameter to insert_documentation() for clearer
update messages showing what was synchronized
This ensures the hook works correctly even for providers without
published metadata, improving reliability during development.
(cherry picked from commit e7ecdbda7cf5b4faced200568e23be3bf13cffd5)
Co-authored-by: Jarek Potiuk <[email protected]>
---
scripts/ci/prek/common_prek_utils.py | 9 ++-
scripts/ci/prek/update_airflow_pyproject_toml.py | 77 +++++++++++++++++++++---
2 files changed, 75 insertions(+), 11 deletions(-)
diff --git a/scripts/ci/prek/common_prek_utils.py
b/scripts/ci/prek/common_prek_utils.py
index 26730129814..2f0af7f9306 100644
--- a/scripts/ci/prek/common_prek_utils.py
+++ b/scripts/ci/prek/common_prek_utils.py
@@ -129,7 +129,12 @@ def pre_process_files(files: list[str]) -> list[str]:
def insert_documentation(
- file_path: Path, content: list[str], header: str, footer: str,
add_comment: bool = False
+ file_path: Path,
+ content: list[str],
+ header: str,
+ footer: str,
+ add_comment: bool = False,
+ extra_information: str | None = None,
) -> bool:
found = False
old_content = file_path.read_text()
@@ -155,7 +160,7 @@ def insert_documentation(
sys.exit(1)
if new_content != old_content:
file_path.write_text(new_content)
- console.print(f"Updated {file_path}")
+ console.print(f"Updated {file_path} with {extra_information or
'generated documentation'}")
return True
return False
diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py
b/scripts/ci/prek/update_airflow_pyproject_toml.py
index 6c31a6e5ff4..b55642baa58 100755
--- a/scripts/ci/prek/update_airflow_pyproject_toml.py
+++ b/scripts/ci/prek/update_airflow_pyproject_toml.py
@@ -105,7 +105,49 @@ all_providers_metadata =
json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
all_providers_dependencies =
json.loads(PROVIDER_DEPENDENCIES_FILE_PATH.read_text())
+def _read_toml(path: Path) -> dict[str, Any]:
+ try:
+ import tomllib
+ except ImportError:
+ import tomli as tomllib
+ return tomllib.loads(path.read_text())
+
+
+def get_local_provider_version(provider_id: str) -> Version | None:
+ provider_pyproject = PROVIDERS_DIR / provider_path(provider_id) /
"pyproject.toml"
+ if not provider_pyproject.exists():
+ return None
+ try:
+ provider_toml = _read_toml(provider_pyproject)
+ except Exception:
+ return None
+ version_str = provider_toml.get("project", {}).get("version")
+ if not version_str:
+ return None
+ return parse_version(version_str)
+
+
+def _fallback_provider_version(
+ provider_id: str, min_version_override: Version | None
+) -> tuple[Version | None, str]:
+ if min_version_override:
+ console.print(
+ f"[yellow]Provider id {provider_id} min version fallback:[/] "
+ f"MIN_VERSION_OVERRIDE -> {min_version_override}"
+ )
+ return min_version_override, f" # Set from MIN_VERSION_OVERRIDE in
{Path(__file__).name}"
+ local_version = get_local_provider_version(provider_id)
+ if local_version:
+ console.print(
+ f"[yellow]Provider id {provider_id} min version fallback:[/] "
+ f"local provider pyproject.toml -> {local_version}"
+ )
+ return local_version, " # Set from local provider pyproject.toml"
+ return None, ""
+
+
def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
+ console.print(f"[bright_blue]Finding min version for provider id:[/]
{provider_id}")
metadata = all_providers_metadata.get(provider_id)
# We should periodically update the starting date to avoid pip install
resolution issues
# TODO: when min Python version is 3.11 change back the code to
fromisoformat
@@ -116,10 +158,11 @@ def find_min_provider_version(provider_id: str) ->
tuple[Version | None, str]:
last_version_newer_than_cutoff: Version | None = None
date_released: datetime | None = None
min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
+ override_comment = ""
if not metadata:
- if not min_version_override:
- return None, ""
- last_version_newer_than_cutoff = min_version_override
+ last_version_newer_than_cutoff, override_comment =
_fallback_provider_version(
+ provider_id, min_version_override
+ )
else:
versions: list[Version] = sorted([parse_version(version) for version
in metadata], reverse=True)
for version in versions:
@@ -130,11 +173,19 @@ def find_min_provider_version(provider_id: str) ->
tuple[Version | None, str]:
if date_released < cut_off_date:
break
last_version_newer_than_cutoff = version
- console.print(
- f"[bright_blue]Provider id {provider_id} min version found:[/] "
- f"{last_version_newer_than_cutoff} (date {date_released}"
- )
- override_comment = ""
+ if not last_version_newer_than_cutoff:
+ last_version_newer_than_cutoff, override_comment =
_fallback_provider_version(
+ provider_id, min_version_override
+ )
+ if date_released:
+ console.print(
+ f"[bright_blue]Provider id {provider_id} min version found:[/] "
+ f"{last_version_newer_than_cutoff} (date {date_released})"
+ )
+ else:
+ console.print(
+ f"[yellow]Provider id {provider_id} has no released versions newer
than cutoff date {cut_off_date}![/]"
+ )
if last_version_newer_than_cutoff:
if min_version_override and min_version_override >
last_version_newer_than_cutoff:
console.print(
@@ -211,13 +262,17 @@ if __name__ == "__main__":
all_optional_dependencies,
START_OPTIONAL_DEPENDENCIES,
END_OPTIONAL_DEPENDENCIES,
+ False,
+ "optional dependencies",
)
all_mypy_paths = []
for provider_id in all_providers:
provider_mypy_path =
f"$MYPY_CONFIG_FILE_DIR/providers/{provider_path(provider_id)}"
all_mypy_paths.append(f' "{provider_mypy_path}/src",\n')
all_mypy_paths.append(f' "{provider_mypy_path}/tests",\n')
- insert_documentation(AIRFLOW_PYPROJECT_TOML_FILE, all_mypy_paths,
START_MYPY_PATHS, END_MYPY_PATHS)
+ insert_documentation(
+ AIRFLOW_PYPROJECT_TOML_FILE, all_mypy_paths, START_MYPY_PATHS,
END_MYPY_PATHS, False, "mypy paths"
+ )
all_workspace_items = []
for provider_id in all_providers:
all_workspace_items.append(f"{provider_distribution_name(provider_id)}
= {{ workspace = true }}\n")
@@ -226,6 +281,8 @@ if __name__ == "__main__":
all_workspace_items,
START_WORKSPACE_ITEMS,
END_WORKSPACE_ITEMS,
+ False,
+ "workspace items",
)
all_workspace_members = []
for provider_id in all_providers:
@@ -235,4 +292,6 @@ if __name__ == "__main__":
all_workspace_members,
START_PROVIDER_WORKSPACE_MEMBERS,
END_PROVIDER_WORKSPACE_MEMBERS,
+ False,
+ "provider workspace members",
)
