This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8bf64109ac0f839c8c0c39f00f9424d85d1eabca Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 27 16:16:27 2023 +0100 Move check provider yaml file to breeze shell (#35879) (cherry picked from commit 7f8ba9deed62a66f07deb22e52b19628edd111f9) --- .pre-commit-config.yaml | 2 +- dev/breeze/tests/test_shell_params.py | 16 ++++++ .../pre_commit_check_provider_yaml_files.py | 63 ++++++---------------- 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 805cc72ebd..53d655ae12 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1063,7 +1063,7 @@ repos: entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py language: python files: ^airflow/providers/.*/provider\.yaml$ - additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py'] + additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: update-migration-references name: Update migration ref doc diff --git a/dev/breeze/tests/test_shell_params.py b/dev/breeze/tests/test_shell_params.py index 2ce10ba66a..987884f4c8 100644 --- a/dev/breeze/tests/test_shell_params.py +++ b/dev/breeze/tests/test_shell_params.py @@ -170,6 +170,22 @@ console = Console(width=400, color_system="standard") }, id="ENABLED_SYSTEMS empty by default even if they are None in ShellParams", ), + pytest.param( + {}, + {}, + { + "PYTHONWARNINGS": None, + }, + id="PYTHONWARNINGS should not be set by default", + ), + pytest.param( + {"PYTHONWARNINGS": "default"}, + {}, + { + "PYTHONWARNINGS": "default", + }, + id="PYTHONWARNINGS should be set when specified in environment", + ), ], ) def test_shell_params_to_env_var_conversion( diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py index c5126237d3..7c934c60fb 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py @@ -17,57 +17,24 @@ # under the License. from __future__ import annotations -import os -import shlex import sys from pathlib import Path -if __name__ not in ("__main__", "__mp_main__"): - raise SystemExit( - "This file is intended to be executed as an executable program. You cannot use it as a module." - f"To run this script, run the ./{__file__} command" - ) - -AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow") -os.environ["SKIP_GROUP_OUTPUT"] = "true" - -if __name__ == "__main__": - sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) - from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED - from airflow_breeze.params.shell_params import ShellParams - from airflow_breeze.utils.console import get_console - from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags - from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits, run_command +sys.path.insert(0, str(Path(__file__).parent.resolve())) +from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell - shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) +initialize_breeze_precommit(__name__, __file__) - cmd = "python3 /opt/airflow/scripts/in_container/run_provider_yaml_files_check.py" - if len(sys.argv) > 1: - cmd = cmd + " " + " ".join([shlex.quote(f) for f in sys.argv[1:]]) - airflow_image = get_ci_image_for_pre_commits() - cmd_result = run_command( - [ - "docker", - "run", - "-t", - *get_extra_docker_flags(mount_sources=MOUNT_SELECTED), - "-e", - "SKIP_ENVIRONMENT_INITIALIZATION=true", - "-e", - "PYTHONWARNINGS=default", - "--pull", - "never", - airflow_image, - "-c", - cmd, - ], - check=False, - env=shell_params.env_variables_for_docker_commands, +files_to_test = sys.argv[1:] +cmd_result = run_command_via_breeze_shell( + ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], + backend="sqlite", + warn_image_upgrade_needed=True, + extra_env={"PYTHONWARNINGS": "default"}, +) +if cmd_result.returncode != 0: + console.print( + "[warning]\nIf you see strange stacktraces above, " + "run `breeze ci-image build --python 3.8` and try again." ) - if cmd_result.returncode != 0: - get_console().print( - "[warning]If you see strange stacktraces above, " - "run `breeze ci-image build --python 3.8` and try again." - ) - sys.exit(cmd_result.returncode) +sys.exit(cmd_result.returncode)
