This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new d59c7a585ba [v3-0-test] Improve dependency report and uppgrading
(#52619) (#52626)
d59c7a585ba is described below
commit d59c7a585ba3ba104205ec0f928ad3d7bd36080c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Jul 1 10:36:44 2025 +0200
[v3-0-test] Improve dependency report and uppgrading (#52619) (#52626)
Our dependencies should be set in "upgrade to newer dependencies"
mode every time every single pyproject.toml changes - this is slower
as it triggers full builds with all versions but it also prevents some
errors when dependencies from one provider are impacting what will
be resolved in the CI image. As part of it - whenever we run the
dependency report with "source constraints" we use exactly the same
`uv sync` command as used during image build with "ugprade to
newer dependencies" - this way the report is more accurate as it
includes some dependencies from dev dependency groups that have
not been included in the current reports.
(cherry picked from commit c489678571e27961d6865ed802f0f95a5fe83b4f)
---
dev/breeze/doc/ci/04_selective_checks.md | 3 +-
.../src/airflow_breeze/utils/selective_checks.py | 50 ++------
dev/breeze/tests/test_selective_checks.py | 136 +--------------------
dev/constraints-updated-version-check.py | 21 +++-
4 files changed, 30 insertions(+), 180 deletions(-)
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 615bee819b7..7f0b64dcbc7 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -170,8 +170,7 @@ GitHub Actions to pass the list of parameters to a command
to execute
| all-python-versions-list-as-string | List of all python
versions there are available in the form of space separated string
| 3.9 3.10 | * |
| all-versions | If set to true,
then all python, k8s, DB versions are used for tests.
| false | |
| basic-checks-only | Whether to run all
static checks ("false") or only basic set of static checks ("true")
| false | |
-| build_system_changed_in_pyproject_toml | When builds system
dependencies changed in pyproject.toml changed in the PR.
| false | |
-| ci-image-build | Whether CI image
build is needed
| true | |
+| ci-image-build | Whether CI image
build is needed
| true | |
| core-test-types-list-as-strings-in-json | Which test types
should be run for unit tests for core
| API Always Providers | * |
| debug-resources | Whether resources
usage should be printed during parallel job execution ("true"/ "false")
| false | |
| default-branch | Which branch is
default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.)
| main | |
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index e41cf992cdb..68b5782262b 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -111,6 +111,7 @@ class FileGroupForCi(Enum):
TASK_SDK_FILES = "task_sdk_files"
GO_SDK_FILES = "go_sdk_files"
AIRFLOW_CTL_FILES = "airflow_ctl_files"
+ ALL_PYPROJECT_TOML_FILES = "all_pyproject_toml_files"
ALL_PYTHON_FILES = "all_python_files"
ALL_SOURCE_FILES = "all_sources_for_tests"
ALL_AIRFLOW_PYTHON_FILES = "all_airflow_python_files"
@@ -186,9 +187,6 @@ CI_FILE_GROUP_MATCHES = HashableDict(
r"^airflow-core/tests/unit/kubernetes",
r"^helm-tests",
],
- FileGroupForCi.DEPENDENCY_FILES: [
- r"^generated/provider_dependencies.json$",
- ],
FileGroupForCi.DOC_FILES: [
r"^docs",
r"^devel-common/src/docs",
@@ -266,6 +264,9 @@ CI_FILE_GROUP_MATCHES = HashableDict(
FileGroupForCi.ALL_PROVIDER_YAML_FILES: [
r".*/provider\.yaml$",
],
+ FileGroupForCi.ALL_PYPROJECT_TOML_FILES: [
+ r".*pyproject\.toml$",
+ ],
FileGroupForCi.TESTS_UTILS_FILES: [
r"^airflow-core/tests/unit/utils/",
r"^devel-common/.*\.py$",
@@ -505,13 +506,8 @@ class SelectiveChecks:
if not self._commit_ref:
get_console().print("[warning]Running everything in all versions
as commit is missing[/]")
return True
- if self.hatch_build_changed:
- get_console().print("[warning]Running everything with all
versions: hatch_build.py changed[/]")
- return True
- if self.pyproject_toml_changed and
self.build_system_changed_in_pyproject_toml:
- get_console().print(
- "[warning]Running everything with all versions: build-system
changed in pyproject.toml[/]"
- )
+ if self.pyproject_toml_changed:
+ get_console().print("[warning]Running everything with all
versions: changed pyproject.toml[/]")
return True
if self.generated_dependencies_changed:
get_console().print(
@@ -926,6 +922,8 @@ class SelectiveChecks:
def _get_providers_test_types_to_run(self, split_to_individual_providers:
bool = False) -> list[str]:
if self._default_branch != "main":
return []
+ if self.upgrade_to_newer_dependencies:
+ return ["Providers"]
if self.full_tests_needed or self.run_task_sdk_tests:
if split_to_individual_providers:
return list(providers_test_type())
@@ -1069,10 +1067,6 @@ class SelectiveChecks:
def generated_dependencies_changed(self) -> bool:
return "generated/provider_dependencies.json" in self._files
- @cached_property
- def hatch_build_changed(self) -> bool:
- return "hatch_build.py" in self._files
-
@cached_property
def any_provider_yaml_or_pyproject_toml_changed(self) -> bool:
if not self._commit_ref:
@@ -1126,37 +1120,11 @@ class SelectiveChecks:
self._old_toml = tomllib.loads(old_result.stdout)
return True
- @cached_property
- def build_system_changed_in_pyproject_toml(self) -> bool:
- if not self.pyproject_toml_changed:
- return False
- new_build_backend = self._new_toml["build-system"]["build-backend"]
- old_build_backend = self._old_toml["build-system"]["build-backend"]
- if new_build_backend != old_build_backend:
- get_console().print("[warning]Build backend changed in
pyproject.toml [/]")
- self._print_diff([old_build_backend], [new_build_backend])
- return True
- new_requires = self._new_toml["build-system"]["requires"]
- old_requires = self._old_toml["build-system"]["requires"]
- if new_requires != old_requires:
- get_console().print("[warning]Build system changed in
pyproject.toml [/]")
- self._print_diff(old_requires, new_requires)
- return True
- return False
-
@cached_property
def upgrade_to_newer_dependencies(self) -> bool:
- if len(self._matching_files(FileGroupForCi.DEPENDENCY_FILES,
CI_FILE_GROUP_MATCHES)) > 0:
+ if len(self._matching_files(FileGroupForCi.ALL_PYPROJECT_TOML_FILES,
CI_FILE_GROUP_MATCHES)) > 0:
get_console().print("[warning]Upgrade to newer dependencies:
Dependency files changed[/]")
return True
- if self.hatch_build_changed:
- get_console().print("[warning]Upgrade to newer dependencies:
hatch_build.py changed[/]")
- return True
- if self.build_system_changed_in_pyproject_toml:
- get_console().print(
- "[warning]Upgrade to newer dependencies: Build system changed
in pyproject.toml[/]"
- )
- return True
if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]:
get_console().print("[warning]Upgrade to newer dependencies: Push
or Schedule event[/]")
return True
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index fa2d3b53070..f41eabd3611 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -813,34 +813,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
),
(
pytest.param(
- ("generated/provider_dependencies.json",),
- {
- "selected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.9', '3.10', '3.11', '3.12']",
- "all-python-versions-list-as-string":
ALL_PYTHON_VERSIONS_AS_STRING,
- "python-versions": "['3.9', '3.10', '3.11', '3.12']",
- "python-versions-list-as-string":
ALL_PYTHON_VERSIONS_AS_STRING,
- "ci-image-build": "true",
- "prod-image-build": "true",
- "needs-helm-tests": "true",
- "run-tests": "true",
- "run-amazon-tests": "true",
- "docs-build": "true",
- "full-tests-needed": "true",
- "skip-pre-commits":
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED,
- "upgrade-to-newer-dependencies": "true",
- "core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "needs-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
- },
- id="Everything should run - including all providers and
upgrading to "
- "newer requirements as pyproject.toml changed and all Python
versions",
- )
- ),
- (
- pytest.param(
- ("generated/provider_dependencies.json",),
+ ("pyproject.toml",),
{
"selected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
"all-python-versions": "['3.9', '3.10', '3.11', '3.12']",
@@ -864,83 +837,6 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
id="Everything should run and upgrading to newer requirements
as dependencies change",
)
),
- pytest.param(
- ("providers/amazon/src/airflow/providers/amazon/__init__.py",),
- {
- "selected-providers-list-as-string": "amazon apache.hive
cncf.kubernetes "
- "common.compat common.messaging common.sql exasol ftp google
http imap microsoft.azure "
- "mongo mysql openlineage postgres salesforce ssh teradata",
- "all-python-versions":
f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
- "all-python-versions-list-as-string":
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
- "python-versions-list-as-string":
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- "ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
- "run-tests": "true",
- "docs-build": "true",
- "skip-pre-commits":
ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
- "run-kubernetes-tests": "false",
- "upgrade-to-newer-dependencies": "false",
- "run-amazon-tests": "true",
- "core-test-types-list-as-strings-in-json": json.dumps(
- [{"description": "Always", "test_types": "Always"}]
- ),
- "providers-test-types-list-as-strings-in-json": json.dumps(
- [
- {
- "description": "amazon...google",
- "test_types": "Providers[amazon]
Providers[apache.hive,cncf.kubernetes,"
-
"common.compat,common.messaging,common.sql,exasol,ftp,http,imap,"
-
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
- "Providers[google]",
- }
- ]
- ),
- "needs-mypy": "true",
- "mypy-checks": "['mypy-providers']",
- },
- id="Providers tests run including amazon tests if amazon provider
files changed",
- ),
- pytest.param(
- ("providers/amazon/src/airflow/providers/amazon/pyproject.toml",),
- {
- "selected-providers-list-as-string": "amazon apache.hive
cncf.kubernetes "
- "common.compat common.messaging common.sql exasol ftp google
http imap microsoft.azure "
- "mongo mysql openlineage postgres salesforce ssh teradata",
- "all-python-versions":
f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
- "all-python-versions-list-as-string":
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
- "python-versions-list-as-string":
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- "ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
- "run-tests": "true",
- "docs-build": "true",
- # no python files changed so flynt should not run
- "skip-pre-commits": "flynt," +
ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
- "run-kubernetes-tests": "false",
- "upgrade-to-newer-dependencies": "false",
- "run-amazon-tests": "true",
- "core-test-types-list-as-strings-in-json": json.dumps(
- [{"description": "Always", "test_types": "Always"}]
- ),
- "providers-test-types-list-as-strings-in-json": json.dumps(
- [
- {
- "description": "amazon...google",
- "test_types": "Providers[amazon]
Providers[apache.hive,cncf.kubernetes,"
-
"common.compat,common.messaging,common.sql,exasol,ftp,http,imap,"
-
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
- "Providers[google]",
- }
- ]
- ),
- "needs-mypy": "true",
- "mypy-checks": "['mypy-providers']",
- },
- id="Providers tests run including amazon tests if only amazon
pyproject.toml files changed",
- ),
pytest.param(
("providers/amazon/src/airflow/providers/amazon/provider.yaml",),
{
@@ -1284,17 +1180,6 @@ def test_expected_output_pull_request_main(
@pytest.mark.parametrize(
"files, commit_ref, expected_outputs",
[
- (
- pytest.param(
- ("pyproject.toml",),
- "2bc8e175b3a4cc84fe33e687f1a00d2a49563090",
- {
- "full-tests-needed": "false",
- "all-versions": "false",
- },
- id="No full tests needed / all versions when pyproject.toml
changes in insignificant way",
- )
- ),
(
pytest.param(
("pyproject.toml",),
@@ -2188,16 +2073,6 @@ def
test_files_provided_trigger_full_build_for_any_event_type(github_event):
None,
id="Regular source changed",
),
- pytest.param(
- ("pyproject.toml",),
- {
- "upgrade-to-newer-dependencies": "false",
- },
- (),
- # In this commit only ruff configuration changed
- "2bc8e175b3a4cc84fe33e687f1a00d2a49563090",
- id="pyproject.toml changed but no dependency change",
- ),
pytest.param(
("providers/microsoft/azure/src/airflow/providers/microsoft/azure/provider.yaml",),
{
@@ -2207,15 +2082,6 @@ def
test_files_provided_trigger_full_build_for_any_event_type(github_event):
None,
id="Provider.yaml changed",
),
- pytest.param(
- ("generated/provider_dependencies.json",),
- {
- "upgrade-to-newer-dependencies": "true",
- },
- (),
- "None",
- id="Generated provider_dependencies changed",
- ),
pytest.param(
("airflow-core/src/airflow/models/dag.py",),
{
diff --git a/dev/constraints-updated-version-check.py
b/dev/constraints-updated-version-check.py
index 35630854b13..52476f26e9a 100755
--- a/dev/constraints-updated-version-check.py
+++ b/dev/constraints-updated-version-check.py
@@ -188,7 +188,15 @@ def main(
print_table_header(format_str, headers, total_width)
outdated_count, skipped_count, explanations = process_packages(
- packages, constraints_date, mode, explain_why, verbose, col_widths,
format_str, python_version
+ packages,
+ constraints_date,
+ mode,
+ explain_why,
+ verbose,
+ col_widths,
+ format_str,
+ python_version,
+ airflow_constraints_mode,
)
print_table_footer(total_width, len(packages), outdated_count,
skipped_count, mode)
@@ -275,6 +283,7 @@ def process_packages(
col_widths: dict,
format_str: str,
python_version: str,
+ airflow_constraints_mode: str,
) -> tuple[int, int, list[str]]:
import subprocess
import tempfile
@@ -382,6 +391,7 @@ def process_packages(
update_pyproject_dependency,
verbose,
python_version,
+ airflow_constraints_mode,
)
explanations.append(explanation)
except HTTPError as e:
@@ -444,6 +454,7 @@ def explain_package_upgrade(
update_pyproject_dependency,
verbose: bool,
python_version: str,
+ airflow_constraints_mode: str,
) -> str:
explanation = (
f"[bold blue]\n--- Explaining for {pkg} (current: {pinned_version},
latest: {latest_version}) ---[/]"
@@ -457,12 +468,18 @@ def explain_package_upgrade(
finally:
pyproject_path.write_text(original_content)
+ additional_args = []
+ if airflow_constraints_mode == "constraints-source-providers":
+ # In case of source constraints we also need to add all development
dependencies
+ # to reflect exactly what is installed in the CI image by default
+ additional_args.extend(["--group dev", "--group docs", "--group
docs-gen", "--group leveldb"])
with preserve_pyproject_file(airflow_pyproject):
before_result = run_uv_sync(
[
"uv",
"sync",
"--all-packages",
+ *additional_args,
"--resolution",
"highest",
"--refresh",
@@ -472,7 +489,7 @@ def explain_package_upgrade(
cwd=repo_root,
)
(temp_dir_path / "uv_sync_before.txt").write_text(before_result.stdout
+ before_result.stderr)
- update_pyproject_dependency(airflow_pyproject, pkg, latest_version)
+ update_pyproject_dependency(airflow_pyproject, pkg, latest_version,
airflow_constraints_mode)
if verbose:
syntax = Syntax(
airflow_pyproject.read_text(), "toml", theme="monokai",
line_numbers=True, word_wrap=False
