This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b0db1f94ed Fix problems with missing selective checks on new types of
unit tests (#36372)
b0db1f94ed is described below
commit b0db1f94ede7d316b3f63a924176e5e1eefa89c1
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Dec 22 20:12:54 2023 +0100
Fix problems with missing selective checks on new types of unit tests
(#36372)
When the DB/NonDB tests were introduced (#35160) new test types have
been added (separating various Python test types from generic
Operator test type). However we have not added matching of the python
operator and test files into the right selective unit test type. This
caused that when only `operators/python.py` and `tests/test_python` were
changed, then `Operators` test type was run but the specific Python *
test types were not run.
This PR fixes it for current test type (including also separated
Serialization test type) and for the future - instead of matching
selected test type we match all of them except the few that we
now are "special" ("Always, Core, Other, PlainAsserts").
---
.../src/airflow_breeze/utils/selective_checks.py | 58 +++++++++++-----------
dev/breeze/tests/test_selective_checks.py | 47 ++++++++++++++++++
2 files changed, 77 insertions(+), 28 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index dba6000902..2cf57200a4 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -223,33 +223,42 @@ CI_FILE_GROUP_EXCLUDES = HashableDict(
}
)
+PYTHON_OPERATOR_FILES = [
+ r"^airflow/operators/python.py",
+ r"^tests/operators/test_python.py",
+]
+
TEST_TYPE_MATCHES = HashableDict(
{
SelectiveUnitTestTypes.API: [
- r"^airflow/api",
- r"^airflow/api_connexion",
- r"^tests/api",
- r"^tests/api_connexion",
+ r"^airflow/api/",
+ r"^airflow/api_connexion/",
+ r"^airflow/api_internal/",
+ r"^tests/api/",
+ r"^tests/api_connexion/",
+ r"^tests/api_internal/",
],
SelectiveUnitTestTypes.CLI: [
- r"^airflow/cli",
- r"^tests/cli",
+ r"^airflow/cli/",
+ r"^tests/cli/",
],
SelectiveUnitTestTypes.OPERATORS: [
- r"^airflow/operators",
- r"^tests/operators",
+ r"^airflow/operators/",
+ r"^tests/operators/",
],
SelectiveUnitTestTypes.PROVIDERS: [
r"^airflow/providers/",
r"^tests/system/providers/",
r"^tests/providers/",
],
- SelectiveUnitTestTypes.PYTHON_VENV: [
- r"^tests/operators/test_python.py",
- ],
- SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: [
- r"^tests/operators/test_python.py",
+ SelectiveUnitTestTypes.SERIALIZATION: [
+ r"^airflow/serialization/",
+ r"^tests/serialization/",
],
+ SelectiveUnitTestTypes.PYTHON_VENV: PYTHON_OPERATOR_FILES,
+ SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: PYTHON_OPERATOR_FILES,
+ SelectiveUnitTestTypes.EXTERNAL_PYTHON: PYTHON_OPERATOR_FILES,
+ SelectiveUnitTestTypes.EXTERNAL_BRANCH_PYTHON: PYTHON_OPERATOR_FILES,
SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
}
)
@@ -652,21 +661,14 @@ class SelectiveChecks:
candidate_test_types: set[str] = {"Always"}
matched_files: set[str] = set()
- matched_files.update(
- self._select_test_type_if_matching(candidate_test_types,
SelectiveUnitTestTypes.WWW)
- )
- matched_files.update(
- self._select_test_type_if_matching(candidate_test_types,
SelectiveUnitTestTypes.PROVIDERS)
- )
- matched_files.update(
- self._select_test_type_if_matching(candidate_test_types,
SelectiveUnitTestTypes.CLI)
- )
- matched_files.update(
- self._select_test_type_if_matching(candidate_test_types,
SelectiveUnitTestTypes.OPERATORS)
- )
- matched_files.update(
- self._select_test_type_if_matching(candidate_test_types,
SelectiveUnitTestTypes.API)
- )
+ for test_type in SelectiveUnitTestTypes:
+ if test_type not in [
+ SelectiveUnitTestTypes.ALWAYS,
+ SelectiveUnitTestTypes.CORE,
+ SelectiveUnitTestTypes.OTHER,
+ SelectiveUnitTestTypes.PLAIN_ASSERTS,
+ ]:
+
matched_files.update(self._select_test_type_if_matching(candidate_test_types,
test_type))
kubernetes_files = self._matching_files(
FileGroupForCi.KUBERNETES_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 6198cc59cd..495c98fd87 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -161,6 +161,53 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
id="Only Operator tests and DOCS should run",
)
),
+ (
+ pytest.param(
+ ("airflow/operators/python.py",),
+ {
+ "affected-providers-list-as-string": None,
+ "all-python-versions": "['3.8']",
+ "all-python-versions-list-as-string": "3.8",
+ "python-versions": "['3.8']",
+ "python-versions-list-as-string": "3.8",
+ "ci-image-build": "true",
+ "prod-image-build": "false",
+ "needs-helm-tests": "false",
+ "run-tests": "true",
+ "run-amazon-tests": "false",
+ "docs-build": "true",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "upgrade-to-newer-dependencies": "false",
+ "parallel-test-types-list-as-string": "Always
BranchExternalPython BranchPythonVenv "
+ "ExternalPython Operators PythonVenv",
+ },
+ id="Only Python tests",
+ )
+ ),
+ (
+ pytest.param(
+ ("airflow/serialization/python.py",),
+ {
+ "affected-providers-list-as-string": None,
+ "all-python-versions": "['3.8']",
+ "all-python-versions-list-as-string": "3.8",
+ "python-versions": "['3.8']",
+ "python-versions-list-as-string": "3.8",
+ "ci-image-build": "true",
+ "prod-image-build": "false",
+ "needs-helm-tests": "false",
+ "run-tests": "true",
+ "run-amazon-tests": "false",
+ "docs-build": "true",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "upgrade-to-newer-dependencies": "false",
+ "parallel-test-types-list-as-string": "Always
Serialization",
+ },
+ id="Only Serialization tests",
+ )
+ ),
(
pytest.param(
(
