This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cf4c7690e04 Require a lower bound on every dependency in
pyproject.toml (#71378)
cf4c7690e04 is described below
commit cf4c7690e04ef59871a6fcb94d0752dfdf54db36
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Aug 11 23:32:51 2026 +0800
Require a lower bound on every dependency in pyproject.toml (#71378)
An unbounded requirement lets the resolver answer with any version that has
ever been published, so what our constraints pin - and what a user ends up
installing - depends on how the resolution went rather than on what the code
needs. Four such requirements were found and fixed by hand while pinning
providers in constraints (#71324); they sat in four different dependency
tables, which is why they went unnoticed for so long.
Distributions that are members of the uv workspace are exempt - they resolve
from the checkout, so a version range would say nothing - as are direct URL
requirements, where the URL already names the exact artifact.
---
.pre-commit-config.yaml | 7 +
clients/python/pyproject.toml | 2 +-
.../13_airflow_dependencies_and_extras.rst | 15 +++
providers/common/ai/docs/index.rst | 10 +-
providers/common/ai/pyproject.toml | 10 +-
scripts/ci/prek/check_dependency_lower_bounds.py | 146 +++++++++++++++++++++
.../ci/prek/test_check_dependency_lower_bounds.py | 143 ++++++++++++++++++++
uv.lock | 20 +--
8 files changed, 332 insertions(+), 21 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2ff236b9131..4c24bcc2b1d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1081,6 +1081,13 @@ repos:
pass_filenames: false
require_serial: true
additional_dependencies: ['packaging>=25', 'pyyaml', 'tomli>=2.0.1',
'rich>=13.6.0']
+ - id: check-dependency-lower-bounds
+ name: Check that dependencies in pyproject.toml have lower bounds
+ language: python
+ entry: ./scripts/ci/prek/check_dependency_lower_bounds.py
+ files: (^|/)pyproject\.toml$
+ require_serial: true
+ additional_dependencies: ['packaging>=25', 'tomli>=2.0.1',
'rich>=13.6.0']
- id: update-reproducible-source-date-epoch
name: Update Source Date Epoch for reproducible builds
language: python
diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml
index 2e246c7778e..0614c5f4529 100644
--- a/clients/python/pyproject.toml
+++ b/clients/python/pyproject.toml
@@ -59,7 +59,7 @@ classifiers = [
dependencies = [
"pydantic >= 2.11.0",
- "python-dateutil",
+ "python-dateutil>=2.7.0",
"urllib3>=2.1.0,!=2.6.0",
]
diff --git a/contributing-docs/13_airflow_dependencies_and_extras.rst
b/contributing-docs/13_airflow_dependencies_and_extras.rst
index 7c59ce25d17..1a64d5e2ca9 100644
--- a/contributing-docs/13_airflow_dependencies_and_extras.rst
+++ b/contributing-docs/13_airflow_dependencies_and_extras.rst
@@ -239,6 +239,21 @@ rules to remember:
stopped working (like in case of ``amazon``, ``fab``). You are free to
modify those versions to higher
versions if you need to, and ``prek`` will remove those comments
automatically.
+* Every dependency we resolve from PyPI must have a lower bound. Without one
the resolver is free to answer
+ with any version that has ever been published, so what our constraints pin -
and what a user ends up
+ installing - depends on how the resolution went rather than on what the code
needs. The
+ ``check-dependency-lower-bounds`` prek hook enforces this across
``project.dependencies``,
+ ``project.optional-dependencies``, ``dependency-groups`` and
``build-system.requires`` of every
+ ``pyproject.toml``. Use the oldest version you are willing to test against:
+
+ .. code-block:: python
+
+ "pyspark>=4.0.0",
+
+ Two kinds of requirement are exempt: distributions that are members of our
``uv`` workspace (they are
+ resolved from the checkout, so a version range would say nothing) and direct
URL requirements (the URL
+ already names the exact artifact).
+
Our CI system will do all the tests for you anyway - including running some
lower-bind checks on dependencies.
For example it will take each provider in a turn and will try to resolve
lowest-possible dependencies defined
for that provider and see if the tests are still passing, so we should be
relatively protected against putting
diff --git a/providers/common/ai/docs/index.rst
b/providers/common/ai/docs/index.rst
index 41a01074f44..4b0f82608b3 100644
--- a/providers/common/ai/docs/index.rst
+++ b/providers/common/ai/docs/index.rst
@@ -241,11 +241,11 @@ Install them when installing from PyPI. For example:
==============
=======================================================================================================================================
Extra Dependencies
==============
=======================================================================================================================================
-``anthropic`` ``pydantic-ai-slim[anthropic]``
-``bedrock`` ``pydantic-ai-slim[bedrock]``
-``google`` ``pydantic-ai-slim[google]``
-``openai`` ``pydantic-ai-slim[openai]``
-``mcp`` ``pydantic-ai-slim[mcp]``
+``anthropic`` ``pydantic-ai-slim[anthropic]>=2.0.0``
+``bedrock`` ``pydantic-ai-slim[bedrock]>=2.0.0``
+``google`` ``pydantic-ai-slim[google]>=2.0.0``
+``openai`` ``pydantic-ai-slim[openai]>=2.0.0``
+``mcp`` ``pydantic-ai-slim[mcp]>=2.0.0``
``code-mode`` ``pydantic-ai-harness[codemode]>=0.3.0``
``shields`` ``pydantic-ai-shields>=0.3.4``
``skills`` ``apache-airflow-providers-git>=0.4.0``,
``pydantic-ai-skills>=1.2.0``
diff --git a/providers/common/ai/pyproject.toml
b/providers/common/ai/pyproject.toml
index 07a65f18dd4..aa806092833 100644
--- a/providers/common/ai/pyproject.toml
+++ b/providers/common/ai/pyproject.toml
@@ -76,11 +76,11 @@ dependencies = [
# The optional dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
[project.optional-dependencies]
-"anthropic" = ["pydantic-ai-slim[anthropic]"]
-"bedrock" = ["pydantic-ai-slim[bedrock]"]
-"google" = ["pydantic-ai-slim[google]"]
-"openai" = ["pydantic-ai-slim[openai]"]
-"mcp" = ["pydantic-ai-slim[mcp]"]
+"anthropic" = ["pydantic-ai-slim[anthropic]>=2.0.0"]
+"bedrock" = ["pydantic-ai-slim[bedrock]>=2.0.0"]
+"google" = ["pydantic-ai-slim[google]>=2.0.0"]
+"openai" = ["pydantic-ai-slim[openai]>=2.0.0"]
+"mcp" = ["pydantic-ai-slim[mcp]>=2.0.0"]
# Code mode: collapse tool calls into a single `run_code` tool that the model
# drives by writing Python, executed in the Monty sandbox (pydantic-monty).
# Enables AgentOperator(code_mode=True). Monty is pre-1.0; pinned here as an
diff --git a/scripts/ci/prek/check_dependency_lower_bounds.py
b/scripts/ci/prek/check_dependency_lower_bounds.py
new file mode 100755
index 00000000000..4ec5863d278
--- /dev/null
+++ b/scripts/ci/prek/check_dependency_lower_bounds.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = [
+# "packaging>=25",
+# "rich>=13.6.0",
+# "tomli>=2.0.1",
+# ]
+# ///
+"""
+Validate that every external dependency declared in a ``pyproject.toml`` has a
lower bound.
+
+An unbounded requirement lets the resolver answer with any version that
happens to be on
+PyPI, so what a constraints file pins - and what a user ends up installing -
depends on how
+the resolution went rather than on what the code needs. Naming the oldest
supported version
+makes that answer deterministic and documents the floor the code is tested
against.
+
+Checked locations: ``project.dependencies``, ``project.optional-dependencies``,
+``dependency-groups`` and ``build-system.requires``.
+
+Requirements resolved from the uv workspace rather than from PyPI are exempt -
their source
+is the checkout, so a version range would say nothing. Direct URL requirements
are exempt
+too, since the URL already names the exact artifact.
+"""
+
+from __future__ import annotations
+
+import sys
+from functools import cache
+from pathlib import Path
+
+from common_prek_utils import AIRFLOW_ROOT_PATH, console
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+from rich.markup import escape
+
+try:
+ import tomllib
+except ImportError:
+ import tomli as tomllib # type: ignore[no-redef]
+
+# Operators that place a floor under the resolved version; anything else
(``!=``, ``<``,
+# ``<=``) leaves the resolver free to reach back to the oldest release ever
published.
+LOWER_BOUND_OPERATORS = {">=", ">", "==", "===", "~="}
+
+
+def _load_toml(path: Path) -> dict:
+ return tomllib.loads(path.read_text())
+
+
+@cache
+def get_workspace_distribution_names() -> frozenset[str]:
+ """Return the canonical names of the distributions uv resolves from this
checkout."""
+ root_pyproject = AIRFLOW_ROOT_PATH / "pyproject.toml"
+ root_data = _load_toml(root_pyproject)
+ members = root_data.get("tool", {}).get("uv", {}).get("workspace",
{}).get("members", [])
+ names = set()
+ for member in members:
+ for member_path in
sorted(AIRFLOW_ROOT_PATH.glob(f"{member}/pyproject.toml")):
+ if name := _load_toml(member_path).get("project", {}).get("name"):
+ names.add(canonicalize_name(name))
+ return frozenset(names)
+
+
+def extract_requirements(data: dict) -> list[tuple[str, str]]:
+ """Return ``(section, requirement)`` pairs for every dependency table we
guard."""
+ requirements: list[tuple[str, str]] = []
+ project = data.get("project") or {}
+ for dependency in project.get("dependencies") or []:
+ requirements.append(("project.dependencies", dependency))
+ for extra, dependencies in (project.get("optional-dependencies") or
{}).items():
+ for dependency in dependencies:
+ requirements.append((f'project.optional-dependencies."{extra}"',
dependency))
+ for group, dependencies in (data.get("dependency-groups") or {}).items():
+ for dependency in dependencies:
+ # A group may also pull in another group via ``{include-group =
"..."}``.
+ if isinstance(dependency, str):
+ requirements.append((f'dependency-groups."{group}"',
dependency))
+ for dependency in (data.get("build-system") or {}).get("requires") or []:
+ requirements.append(("build-system.requires", dependency))
+ return requirements
+
+
+def check_requirement(section: str, dependency: str, workspace_names:
frozenset[str]) -> str | None:
+ """Return an error message when ``dependency`` needs a lower bound,
otherwise ``None``."""
+ try:
+ requirement = Requirement(dependency)
+ except InvalidRequirement as error:
+ return f"[{section}] {dependency!r} is not a valid requirement:
{error}"
+ if (
+ requirement.url
+ or canonicalize_name(requirement.name) in workspace_names
+ or any(specifier.operator in LOWER_BOUND_OPERATORS for specifier in
requirement.specifier)
+ ):
+ return None
+ return f"[{section}] {dependency!r} has no lower bound - add one, for
example {requirement.name}>=X.Y.Z"
+
+
+def check_pyproject_file(path: Path, workspace_names: frozenset[str]) ->
list[str]:
+ return [
+ error
+ for section, dependency in extract_requirements(_load_toml(path))
+ if (error := check_requirement(section, dependency, workspace_names))
+ ]
+
+
+def main() -> int:
+ workspace_names = get_workspace_distribution_names()
+ failed = False
+ for file in sys.argv[1:]:
+ path = Path(file)
+ if errors := check_pyproject_file(path, workspace_names):
+ failed = True
+ console.print(f"\n[red]Missing lower bounds in {file}:[/]\n")
+ for error in errors:
+ console.print(f" {escape(error)}")
+ if failed:
+ console.print(
+ "\n[bright_yellow]Every dependency resolved from PyPI needs a
lower bound.[/]\n"
+ "Without one the resolver may pick any published version, so
constraints pin\n"
+ "whatever the resolution happened to produce rather than the
oldest version the\n"
+ "code supports. Use the oldest version you are willing to test
against.\n"
+ )
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py
b/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py
new file mode 100644
index 00000000000..4733e3b3dbc
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py
@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import textwrap
+
+import pytest
+from check_dependency_lower_bounds import check_pyproject_file,
check_requirement, extract_requirements
+
+WORKSPACE_NAMES = frozenset({"apache-airflow-core",
"apache-airflow-devel-common"})
+
+
+class TestCheckRequirement:
+ @pytest.mark.parametrize(
+ "dependency",
+ [
+ pytest.param("pyspark>=4.0.0", id="greater-or-equal"),
+ pytest.param("pyspark>4.0.0", id="greater"),
+ pytest.param("hatchling==1.31.0", id="pinned"),
+ pytest.param("hatchling===1.31.0", id="arbitrary-equality"),
+ pytest.param("hatchling~=1.31", id="compatible-release"),
+ pytest.param("urllib3>=2.1.0,!=2.6.0",
id="lower-bound-with-exclusion"),
+ pytest.param("pydantic-ai-slim[mcp]>=2.0.0", id="extras"),
+ pytest.param('fastavro>=1.10.0; python_version < "3.14"',
id="marker"),
+ ],
+ )
+ def test_no_error_when_lower_bound_present(self, dependency):
+ assert check_requirement("project.dependencies", dependency,
WORKSPACE_NAMES) is None
+
+ @pytest.mark.parametrize(
+ "dependency",
+ [
+ pytest.param("pyspark", id="bare"),
+ pytest.param("pydantic-ai-slim[mcp]", id="extras"),
+ pytest.param("pyspark<5.0.0", id="upper-bound-only"),
+ pytest.param("pyspark!=4.1.0", id="exclusion-only"),
+ pytest.param('krb5; python_version < "3.14"', id="marker-only"),
+ ],
+ )
+ def test_error_when_lower_bound_missing(self, dependency):
+ error = check_requirement("project.dependencies", dependency,
WORKSPACE_NAMES)
+ assert error is not None
+ assert "has no lower bound" in error
+ assert "[project.dependencies]" in error
+
+ @pytest.mark.parametrize(
+ "dependency",
+ [
+ pytest.param("apache-airflow-core", id="plain"),
+ pytest.param("apache_airflow_core", id="non-canonical-name"),
+ pytest.param("apache-airflow-devel-common[mypy]", id="extras"),
+ ],
+ )
+ def test_no_error_for_workspace_distribution(self, dependency):
+ assert check_requirement("dependency-groups.dev", dependency,
WORKSPACE_NAMES) is None
+
+ def test_no_error_for_direct_url_requirement(self):
+ dependency = (
+
"sphinx-airflow-theme@https://airflow.apache.org/sphinx-airflow-theme/";
+ "sphinx_airflow_theme-0.3.13-py3-none-any.whl"
+ )
+ assert check_requirement("project.optional-dependencies.docs",
dependency, WORKSPACE_NAMES) is None
+
+ def test_error_for_invalid_requirement(self):
+ error = check_requirement("project.dependencies", "not a
requirement!", WORKSPACE_NAMES)
+ assert error is not None
+ assert "is not a valid requirement" in error
+
+
+class TestExtractRequirements:
+ def test_extracts_every_guarded_table(self):
+ data = {
+ "build-system": {"requires": ["hatchling"]},
+ "project": {
+ "dependencies": ["pyspark"],
+ "optional-dependencies": {"kerberos": ["krb5"]},
+ },
+ "dependency-groups": {"dev": ["pytest", {"include-group":
"docs"}]},
+ }
+ assert extract_requirements(data) == [
+ ("project.dependencies", "pyspark"),
+ ('project.optional-dependencies."kerberos"', "krb5"),
+ ('dependency-groups."dev"', "pytest"),
+ ("build-system.requires", "hatchling"),
+ ]
+
+ def test_no_requirements_when_tables_absent(self):
+ assert extract_requirements({"tool": {"uv": {"required-version":
">=0.9.0"}}}) == []
+
+
+class TestCheckPyprojectFile:
+ def _write(self, tmp_path, content):
+ path = tmp_path / "pyproject.toml"
+ path.write_text(textwrap.dedent(content))
+ return path
+
+ def test_reports_every_unbounded_dependency(self, tmp_path):
+ path = self._write(
+ tmp_path,
+ """
+ [project]
+ name = "apache-airflow-providers-samba"
+ dependencies = ["smbprotocol>=1.5.0"]
+
+ [project.optional-dependencies]
+ "kerberos" = ["krb5", "apache-airflow-core"]
+
+ [dependency-groups]
+ dev = ["pytest"]
+ """,
+ )
+ errors = check_pyproject_file(path, WORKSPACE_NAMES)
+ assert len(errors) == 2
+ assert "krb5" in errors[0]
+ assert "pytest" in errors[1]
+
+ def test_no_errors_when_all_bounded(self, tmp_path):
+ path = self._write(
+ tmp_path,
+ """
+ [project]
+ name = "apache-airflow-providers-samba"
+ dependencies = ["smbprotocol>=1.5.0"]
+
+ [dependency-groups]
+ dev = ["pytest>=9.1.1"]
+ """,
+ )
+ assert check_pyproject_file(path, WORKSPACE_NAMES) == []
diff --git a/uv.lock b/uv.lock
index 76b7408a20f..940a928dcab 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2958,7 +2958,7 @@ docs = [
[package.metadata]
requires-dist = [
{ name = "akeyless", specifier = ">=5.0.0" },
- { name = "akeyless-cloud-id", marker = "extra == 'cloud-id'" },
+ { name = "akeyless-cloud-id", marker = "extra == 'cloud-id'", specifier =
">=0.3.0" },
{ name = "apache-airflow", editable = "." },
{ name = "apache-airflow-providers-common-compat", editable =
"providers/common/compat" },
]
@@ -3968,7 +3968,7 @@ dev = [
{ name = "apache-airflow-providers-common-compat", editable =
"providers/common/compat" },
{ name = "apache-airflow-providers-openlineage", editable =
"providers/openlineage" },
{ name = "apache-airflow-task-sdk", editable = "task-sdk" },
- { name = "pyspark" },
+ { name = "pyspark", specifier = ">=4.0.0" },
]
docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable =
"devel-common" }]
@@ -4484,11 +4484,11 @@ requires-dist = [
{ name = "pydantic-ai-shields", marker = "extra == 'shields'", specifier =
">=0.3.4" },
{ name = "pydantic-ai-skills", marker = "extra == 'skills'", specifier =
">=1.2.0" },
{ name = "pydantic-ai-slim", specifier = ">=2.0.0" },
- { name = "pydantic-ai-slim", extras = ["anthropic"], marker = "extra ==
'anthropic'" },
- { name = "pydantic-ai-slim", extras = ["bedrock"], marker = "extra ==
'bedrock'" },
- { name = "pydantic-ai-slim", extras = ["google"], marker = "extra ==
'google'" },
- { name = "pydantic-ai-slim", extras = ["mcp"], marker = "extra == 'mcp'" },
- { name = "pydantic-ai-slim", extras = ["openai"], marker = "extra ==
'openai'" },
+ { name = "pydantic-ai-slim", extras = ["anthropic"], marker = "extra ==
'anthropic'", specifier = ">=2.0.0" },
+ { name = "pydantic-ai-slim", extras = ["bedrock"], marker = "extra ==
'bedrock'", specifier = ">=2.0.0" },
+ { name = "pydantic-ai-slim", extras = ["google"], marker = "extra ==
'google'", specifier = ">=2.0.0" },
+ { name = "pydantic-ai-slim", extras = ["mcp"], marker = "extra == 'mcp'",
specifier = ">=2.0.0" },
+ { name = "pydantic-ai-slim", extras = ["openai"], marker = "extra ==
'openai'", specifier = ">=2.0.0" },
{ name = "pypdf", marker = "extra == 'pdf'", specifier = ">=4.0.0" },
{ name = "python-docx", marker = "extra == 'docx'", specifier = ">=1.0.0"
},
{ name = "sqlglot", marker = "extra == 'sql'", specifier = ">=30.0.0" },
@@ -4510,7 +4510,7 @@ dev = [
{ name = "llama-index-embeddings-openai", specifier = ">=0.6.0" },
{ name = "llama-index-llms-openai", specifier = ">=0.6.0" },
{ name = "pydantic-ai-skills", specifier = ">=1.2.0" },
- { name = "pydantic-ai-slim", extras = ["mcp"] },
+ { name = "pydantic-ai-slim", extras = ["mcp"], specifier = ">=2.0.0" },
{ name = "sqlglot", specifier = ">=30.0.0" },
]
docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable =
"devel-common" }]
@@ -7513,7 +7513,7 @@ requires-dist = [
{ name = "apache-airflow", editable = "." },
{ name = "apache-airflow-providers-common-compat", editable =
"providers/common/compat" },
{ name = "apache-airflow-providers-google", marker = "extra == 'google'",
editable = "providers/google" },
- { name = "krb5", marker = "extra == 'kerberos'" },
+ { name = "krb5", marker = "extra == 'kerberos'", specifier = ">=0.8.0" },
{ name = "smbprotocol", specifier = ">=1.5.0" },
{ name = "smbprotocol", extras = ["kerberos"], marker = "extra ==
'kerberos'", specifier = ">=1.5.0" },
]
@@ -8474,7 +8474,7 @@ requires-dist = [
]
[package.metadata.requires-dev]
-dev = [{ name = "pytest" }]
+dev = [{ name = "pytest", specifier = ">=9.1.1" }]
[[package]]
name = "apache-airflow-scripts"
