This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch ci-sync-v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a65cd843135575945b0c26ef95e59d8f0681334d Author: Dev-iL <[email protected]> AuthorDate: Tue Oct 13 16:00:27 2026 +0300 Bootstrap constraints generation for a Python without a baseline file (#74596) The generator always downloads the previous constraints file of the same Python from the constraints branch to diff against it, and failed with a 404 for the first generation of a new Python. When the branch carries the same constraints mode for other Pythons, a missing file means a first publication and the generator diffs against an empty baseline instead. A 404 on a branch without any file of that mode, and every other HTTP error, still fails. (cherry picked from commit d267fbd5caee385b0568c60d3695b3cf85b70460) --- scripts/in_container/run_generate_constraints.py | 45 +++++++++++--- .../in_container/test_run_generate_constraints.py | 69 ++++++++++++++++++++++ 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index daf0b43c2bf..5ca30fc59f1 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -20,6 +20,7 @@ from __future__ import annotations import ast import json import os +import re import sys import tomllib from concurrent.futures import ThreadPoolExecutor @@ -233,21 +234,49 @@ def freeze_distributions_to_file( console.print(f"[green]Constraints generated to file: {file.name}. Wrote {count_lines} lines") +def _github_headers(accept: str) -> dict[str, str]: + headers = {"Accept": accept} + if os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {os.environ.get('GITHUB_TOKEN')}" + else: + console.print("[bright_blue]No GITHUB_TOKEN - using non-authenticated request.") + return headers + + +def _constraints_branch_has_other_python_files(config_params: ConfigParams) -> bool: + """Tell a first publication for a new Python from a baseline that should exist but is missing.""" + listing_url = ( + f"https://api.github.com/repos/{config_params.constraints_github_repository}/contents/"; + f"?ref={config_params.default_constraints_branch}" + ) + r = requests.get(listing_url, timeout=60, headers=_github_headers("application/vnd.github.v3+json")) + r.raise_for_status() + other_python_file = re.compile(rf"{re.escape(config_params.airflow_constraints_mode)}-3\.\d+\.txt") + return any( + other_python_file.fullmatch(entry["name"]) + and entry["name"] != f"{config_params.airflow_constraints_mode}-{config_params.python}.txt" + for entry in r.json() + ) + + def download_latest_constraint_file(config_params: ConfigParams): constraints_url = ( "https://api.github.com/repos/"; f"{config_params.constraints_github_repository}/contents/" f"{config_params.airflow_constraints_mode}-{config_params.python}.txt?ref={config_params.default_constraints_branch}" ) - # download the latest constraints file - # download using requests - headers = {"Accept": "application/vnd.github.v3.raw"} - if os.environ.get("GITHUB_TOKEN"): - headers["Authorization"] = f"Bearer {os.environ.get('GITHUB_TOKEN')}" - else: - console.print("[bright_blue]No GITHUB_TOKEN - using non-authenticated request.") console.print(f"[bright_blue]Downloading constraints file from {constraints_url}") - r = requests.get(constraints_url, timeout=60, headers=headers) + r = requests.get(constraints_url, timeout=60, headers=_github_headers("application/vnd.github.v3.raw")) + if r.status_code == 404 and _constraints_branch_has_other_python_files(config_params): + # The first generation for a new Python has no file to compare with: diff against an empty + # baseline. A 404 on a branch that carries no file of this mode at all is still an error. + console.print( + f"[yellow]No {config_params.airflow_constraints_mode} file for Python {config_params.python} on " + f"{config_params.default_constraints_branch}, but other Pythons have one - first publication, " + "using an empty baseline." + ) + config_params.latest_constraints_file.write_text("") + return r.raise_for_status() with config_params.latest_constraints_file.open("w") as constraints_file: constraints_file.write(r.text) diff --git a/scripts/tests/in_container/test_run_generate_constraints.py b/scripts/tests/in_container/test_run_generate_constraints.py index e8cc482acc4..f01653a422d 100644 --- a/scripts/tests/in_container/test_run_generate_constraints.py +++ b/scripts/tests/in_container/test_run_generate_constraints.py @@ -272,3 +272,72 @@ class TestBuildPinnedProviderRequirements: assert seen_for_distributions == ["3.14"] assert seen_for_lookup == [("3.14", False)] + + +class TestDownloadLatestConstraintFile: + @staticmethod + def _download_config(tmp_path: Path, mode: str = "constraints-source-providers") -> SimpleNamespace: + return SimpleNamespace( + latest_constraints_file=tmp_path / "original.txt", + airflow_constraints_mode=mode, + constraints_github_repository="apache/airflow", + default_constraints_branch="constraints-main", + python="3.15", + ) + + @staticmethod + def _response(status_code: int = 200, text: str = "", payload: list | None = None) -> SimpleNamespace: + def raise_for_status(): + if status_code >= 400: + raise RuntimeError(f"HTTP {status_code}") + + return SimpleNamespace( + status_code=status_code, + text=text, + json=lambda: payload or [], + raise_for_status=raise_for_status, + ) + + def test_writes_the_downloaded_file(self, tmp_path, monkeypatch): + config = self._download_config(tmp_path) + monkeypatch.setattr(m.requests, "get", lambda *args, **kwargs: self._response(text="a==1\n")) + + m.download_latest_constraint_file(config) + + assert config.latest_constraints_file.read_text() == "a==1\n" + + def test_missing_file_for_a_new_python_bootstraps_an_empty_baseline(self, tmp_path, monkeypatch): + config = self._download_config(tmp_path) + listing = [ + {"name": "constraints-source-providers-3.14.txt"}, + {"name": "constraints-3.15.txt"}, + {"name": "README.md"}, + ] + responses = iter([self._response(404), self._response(payload=listing)]) + monkeypatch.setattr(m.requests, "get", lambda *args, **kwargs: next(responses)) + + m.download_latest_constraint_file(config) + + assert config.latest_constraints_file.read_text() == "" + + def test_missing_file_is_an_error_when_no_other_python_has_one(self, tmp_path, monkeypatch): + config = self._download_config(tmp_path) + listing = [ + {"name": "constraints-3.14.txt"}, + {"name": "constraints-no-providers-3.14.txt"}, + {"name": "constraints-source-providers-3.15.txt"}, + ] + responses = iter([self._response(404), self._response(payload=listing)]) + monkeypatch.setattr(m.requests, "get", lambda *args, **kwargs: next(responses)) + + with pytest.raises(RuntimeError, match="HTTP 404"): + m.download_latest_constraint_file(config) + + assert not config.latest_constraints_file.exists() + + def test_other_http_errors_are_not_treated_as_a_first_publication(self, tmp_path, monkeypatch): + config = self._download_config(tmp_path) + monkeypatch.setattr(m.requests, "get", lambda *args, **kwargs: self._response(500)) + + with pytest.raises(RuntimeError, match="HTTP 500"): + m.download_latest_constraint_file(config)
