This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d79f7a4027fd9b252005fa98f4a31d171f65c45b Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Dec 22 20:12:54 2023 +0100 Fix problems with missing selective checks on new types of unit tests (#36372) When the DB/NonDB tests were introduced (#35160) new test types have been added (separating various Python test types from generic Operator test type). However we have not added matching of the python operator and test files into the right selective unit test type. This caused that when only `operators/python.py` and `tests/test_python` were changed, then `Operators` test type was run but the specific Python * test types were not run. This PR fixes it for current test type (including also separated Serialization test type) and for the future - instead of matching selected test type we match all of them except the few that we now are "special" ("Always, Core, Other, PlainAsserts"). (cherry picked from commit b0db1f94ede7d316b3f63a924176e5e1eefa89c1) --- .../src/airflow_breeze/utils/selective_checks.py | 58 +++++++++++----------- dev/breeze/tests/test_selective_checks.py | 47 ++++++++++++++++++ 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index dba6000902..2cf57200a4 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -223,33 +223,42 @@ CI_FILE_GROUP_EXCLUDES = HashableDict( } ) +PYTHON_OPERATOR_FILES = [ + r"^airflow/operators/python.py", + r"^tests/operators/test_python.py", +] + TEST_TYPE_MATCHES = HashableDict( { SelectiveUnitTestTypes.API: [ - r"^airflow/api", - r"^airflow/api_connexion", - r"^tests/api", - r"^tests/api_connexion", + r"^airflow/api/", + r"^airflow/api_connexion/", + r"^airflow/api_internal/", + r"^tests/api/", + r"^tests/api_connexion/", + r"^tests/api_internal/", ], SelectiveUnitTestTypes.CLI: [ - r"^airflow/cli", - r"^tests/cli", + r"^airflow/cli/", + r"^tests/cli/", ], SelectiveUnitTestTypes.OPERATORS: [ - r"^airflow/operators", - r"^tests/operators", + r"^airflow/operators/", + r"^tests/operators/", ], SelectiveUnitTestTypes.PROVIDERS: [ r"^airflow/providers/", r"^tests/system/providers/", r"^tests/providers/", ], - SelectiveUnitTestTypes.PYTHON_VENV: [ - r"^tests/operators/test_python.py", - ], - SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: [ - r"^tests/operators/test_python.py", + SelectiveUnitTestTypes.SERIALIZATION: [ + r"^airflow/serialization/", + r"^tests/serialization/", ], + SelectiveUnitTestTypes.PYTHON_VENV: PYTHON_OPERATOR_FILES, + SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: PYTHON_OPERATOR_FILES, + SelectiveUnitTestTypes.EXTERNAL_PYTHON: PYTHON_OPERATOR_FILES, + SelectiveUnitTestTypes.EXTERNAL_BRANCH_PYTHON: PYTHON_OPERATOR_FILES, SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"], } ) @@ -652,21 +661,14 @@ class SelectiveChecks: candidate_test_types: set[str] = {"Always"} matched_files: set[str] = set() - matched_files.update( - self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.WWW) - ) - matched_files.update( - self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.PROVIDERS) - ) - matched_files.update( - self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.CLI) - ) - matched_files.update( - self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.OPERATORS) - ) - matched_files.update( - self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.API) - ) + for test_type in SelectiveUnitTestTypes: + if test_type not in [ + SelectiveUnitTestTypes.ALWAYS, + SelectiveUnitTestTypes.CORE, + SelectiveUnitTestTypes.OTHER, + SelectiveUnitTestTypes.PLAIN_ASSERTS, + ]: + matched_files.update(self._select_test_type_if_matching(candidate_test_types, test_type)) kubernetes_files = self._matching_files( FileGroupForCi.KUBERNETES_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 6198cc59cd..495c98fd87 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -161,6 +161,53 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Operator tests and DOCS should run", ) ), + ( + pytest.param( + ("airflow/operators/python.py",), + { + "affected-providers-list-as-string": None, + "all-python-versions": "['3.8']", + "all-python-versions-list-as-string": "3.8", + "python-versions": "['3.8']", + "python-versions-list-as-string": "3.8", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "false", + "docs-build": "true", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev," + "mypy-docs,mypy-providers,ts-compile-format-lint-www", + "upgrade-to-newer-dependencies": "false", + "parallel-test-types-list-as-string": "Always BranchExternalPython BranchPythonVenv " + "ExternalPython Operators PythonVenv", + }, + id="Only Python tests", + ) + ), + ( + pytest.param( + ("airflow/serialization/python.py",), + { + "affected-providers-list-as-string": None, + "all-python-versions": "['3.8']", + "all-python-versions-list-as-string": "3.8", + "python-versions": "['3.8']", + "python-versions-list-as-string": "3.8", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "false", + "docs-build": "true", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev," + "mypy-docs,mypy-providers,ts-compile-format-lint-www", + "upgrade-to-newer-dependencies": "false", + "parallel-test-types-list-as-string": "Always Serialization", + }, + id="Only Serialization tests", + ) + ), ( pytest.param( (
