This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a223eb77424 All tests should run when API files change (#47417)
a223eb77424 is described below
commit a223eb77424f886ecbba98ca0433fe30863f5b92
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Mar 5 23:05:43 2025 +0100
All tests should run when API files change (#47417)
We are now more API-centric and a lot of functionality depens on API
changes and we keep on breaking stuff when changing APIs - so we
need to trigger full tests when API files change.
This PR does this.
---
.../src/airflow_breeze/utils/selective_checks.py | 17 +++-
dev/breeze/tests/test_selective_checks.py | 105 +++++++++------------
2 files changed, 60 insertions(+), 62 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 76f630dcc20..b4a0d2c4310 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -100,7 +100,7 @@ class FileGroupForCi(Enum):
PYTHON_PRODUCTION_FILES = "python_scans"
JAVASCRIPT_PRODUCTION_FILES = "javascript_scans"
ALWAYS_TESTS_FILES = "always_test_files"
- API_TEST_FILES = "api_test_files"
+ API_FILES = "api_files"
API_CODEGEN_FILES = "api_codegen_files"
LEGACY_API_FILES = "legacy_api_files"
HELM_FILES = "helm_files"
@@ -161,9 +161,11 @@ CI_FILE_GROUP_MATCHES = HashableDict(
r"^airflow/ui/.*\.yaml$",
r"^airflow/auth/managers/simple/ui/.*\.yaml$",
],
- FileGroupForCi.API_TEST_FILES: [
+ FileGroupForCi.API_FILES: [
r"^airflow/api/",
- r"^airflow/api_connexion/",
+ r"^airflow/api_fastapi/",
+ r"^tests/api/",
+ r"^tests/api_fastapi/",
],
FileGroupForCi.API_CODEGEN_FILES: [
r"^airflow/api_fastapi/core_api/openapi/v1-generated\.yaml",
@@ -496,6 +498,13 @@ class SelectiveChecks:
):
get_console().print("[warning]Running full set of tests because
env files changed[/]")
return True
+ if self._matching_files(
+ FileGroupForCi.API_FILES,
+ CI_FILE_GROUP_MATCHES,
+ CI_FILE_GROUP_EXCLUDES,
+ ):
+ get_console().print("[warning]Running full set of tests because
api files changed[/]")
+ return True
if self._matching_files(
FileGroupForCi.TESTS_UTILS_FILES,
CI_FILE_GROUP_MATCHES,
@@ -692,7 +701,7 @@ class SelectiveChecks:
@cached_property
def needs_api_tests(self) -> bool:
- return self._should_be_run(FileGroupForCi.API_TEST_FILES)
+ return self._should_be_run(FileGroupForCi.API_FILES)
@cached_property
def needs_ol_tests(self) -> bool:
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 47f80a3d28a..fd2afabf6ae 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -153,28 +153,25 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
pytest.param(
("airflow/api/file.py",),
{
- "selected-providers-list-as-string": "common.compat
databricks edge fab",
+ "selected-providers-list-as-string": "",
"all-python-versions": "['3.9']",
"all-python-versions-list-as-string": "3.9",
"python-versions": "['3.9']",
"python-versions-list-as-string": "3.9",
"ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
+ "prod-image-build": "true",
+ "needs-helm-tests": "true",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,"
-
"mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui",
+ "skip-pre-commits":
"identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
"upgrade-to-newer-dependencies": "false",
- "core-test-types-list-as-string": "API Always",
- "providers-test-types-list-as-string":
"Providers[common.compat,databricks,edge,fab]",
- "individual-providers-test-types-list-as-string":
"Providers[common.compat] Providers[databricks] Providers[edge] Providers[fab]",
- "testable-core-integrations": "['celery', 'kerberos']",
- "testable-providers-integrations": "['cassandra', 'drill',
'kafka', 'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']",
+ "core-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "providers-test-types-list-as-string":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES,
+ "individual-providers-test-types-list-as-string":
LIST_OF_ALL_PROVIDER_TESTS,
"needs-mypy": "true",
- "mypy-checks": "['mypy-airflow']",
+ "mypy-checks": "['mypy-airflow', 'mypy-providers',
'mypy-docs', 'mypy-dev', 'mypy-task-sdk']",
},
- id="Only API tests and DOCS and common.compat, FAB providers
should run",
+ id="All tests should be run when API file changed",
)
),
(
@@ -186,21 +183,20 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"python-versions": "['3.9']",
"python-versions-list-as-string": "3.9",
"ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
+ "prod-image-build": "true",
+ "needs-helm-tests": "true",
"run-tests": "true",
- "run-amazon-tests": "false",
+ "run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,"
-
"mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui",
+ "skip-pre-commits":
"identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
"upgrade-to-newer-dependencies": "false",
- "core-test-types-list-as-string": "API Always",
- "providers-test-types-list-as-string": "",
- "individual-providers-test-types-list-as-string": "",
+ "core-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "providers-test-types-list-as-string":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES,
+ "individual-providers-test-types-list-as-string":
LIST_OF_ALL_PROVIDER_TESTS,
"needs-mypy": "true",
- "mypy-checks": "['mypy-airflow']",
+ "mypy-checks": "['mypy-airflow', 'mypy-providers',
'mypy-docs', 'mypy-dev', 'mypy-task-sdk']",
},
- id="Only API tests and DOCS should run (no provider tests)
when only ui api changed",
+ id="All tests should be run when fastapi files change",
)
),
(
@@ -212,21 +208,20 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"python-versions": "['3.9']",
"python-versions-list-as-string": "3.9",
"ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
+ "prod-image-build": "true",
+ "needs-helm-tests": "true",
"run-tests": "true",
- "run-amazon-tests": "false",
- "docs-build": "false",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,"
-
"mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui",
+ "run-amazon-tests": "true",
+ "docs-build": "true",
+ "skip-pre-commits":
"identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
"upgrade-to-newer-dependencies": "false",
- "core-test-types-list-as-string": "API Always",
- "providers-test-types-list-as-string": "",
- "individual-providers-test-types-list-as-string": "",
+ "core-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "providers-test-types-list-as-string":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES,
+ "individual-providers-test-types-list-as-string":
LIST_OF_ALL_PROVIDER_TESTS,
"needs-mypy": "true",
- "mypy-checks": "['mypy-airflow']",
+ "mypy-checks": "['mypy-airflow', 'mypy-providers',
'mypy-docs', 'mypy-dev', 'mypy-task-sdk']",
},
- id="Only API tests should run (no provider tests) and no DOCs
build when only test API files changed",
+ id="All tests should run when API test files change",
)
),
(
@@ -317,31 +312,26 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"providers/postgres/tests/unit/postgres/file.py",
),
{
- "selected-providers-list-as-string": "amazon common.compat
common.sql databricks edge fab google openlineage "
- "pgvector postgres",
+ "selected-providers-list-as-string": "",
"all-python-versions": "['3.9']",
"all-python-versions-list-as-string": "3.9",
"python-versions": "['3.9']",
"python-versions-list-as-string": "3.9",
"ci-image-build": "true",
- "prod-image-build": "false",
- "needs-helm-tests": "false",
+ "prod-image-build": "true",
+ "needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,"
- "ts-compile-format-lint-ui",
+ "skip-pre-commits":
"identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
"upgrade-to-newer-dependencies": "false",
- "core-test-types-list-as-string": "API Always",
- "providers-test-types-list-as-string": "Providers[amazon] "
-
"Providers[common.compat,common.sql,databricks,edge,fab,openlineage,pgvector,postgres]
Providers[google]",
- "individual-providers-test-types-list-as-string":
"Providers[amazon] Providers[common.compat] Providers[common.sql] "
- "Providers[databricks] Providers[edge] Providers[fab]
Providers[google] Providers[openlineage] Providers[pgvector] "
- "Providers[postgres]",
+ "core-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "providers-test-types-list-as-string":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES,
+ "individual-providers-test-types-list-as-string":
LIST_OF_ALL_PROVIDER_TESTS,
"needs-mypy": "true",
- "mypy-checks": "['mypy-airflow', 'mypy-providers']",
+ "mypy-checks": "['mypy-airflow', 'mypy-providers',
'mypy-docs', 'mypy-dev', 'mypy-task-sdk']",
},
- id="API and providers tests and docs should run",
+ id="All tests and docs should run on API change",
)
),
(
@@ -1676,22 +1666,21 @@ def test_expected_output_push(
pytest.param(
("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",),
{
- "selected-providers-list-as-string": None,
+ "selected-providers-list-as-string": "",
"all-python-versions": "['3.9']",
"all-python-versions-list-as-string": "3.9",
"ci-image-build": "true",
- "needs-helm-tests": "false",
+ "needs-helm-tests": "true",
"run-tests": "true",
- "skip-providers-tests": "true",
- "test-groups": "['core']",
- "docs-build": "false",
- "docs-list-as-string": None,
+ "skip-providers-tests": "false",
+ "test-groups": "['core', 'providers']",
+ "docs-build": "true",
+ "docs-list-as-string": "",
"upgrade-to-newer-dependencies": "false",
- "skip-pre-commits":
"check-provider-yaml-valid,flynt,identity,lint-helm-chart,"
- "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
- "core-test-types-list-as-string": None,
- "needs-mypy": "false",
- "mypy-checks": "[]",
+ "skip-pre-commits":
"identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk",
+ "core-test-types-list-as-string": "API Always CLI Core
Operators Other Serialization",
+ "needs-mypy": "true",
+ "mypy-checks": "['mypy-airflow', 'mypy-providers',
'mypy-docs', 'mypy-dev', 'mypy-task-sdk']",
},
id="pre commit ts-compile-format-lint should not be ignored if
openapi spec changed.",
),
