This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7f8ba9deed Move check provider yaml file to breeze shell (#35879)
7f8ba9deed is described below
commit 7f8ba9deed62a66f07deb22e52b19628edd111f9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Nov 27 16:16:27 2023 +0100
Move check provider yaml file to breeze shell (#35879)
---
.pre-commit-config.yaml | 2 +-
dev/breeze/tests/test_shell_params.py | 16 ++++++
.../pre_commit_check_provider_yaml_files.py | 63 ++++++----------------
3 files changed, 32 insertions(+), 49 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 85a7cf96c7..097608aa8f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1056,7 +1056,7 @@ repos:
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
language: python
files: ^airflow/providers/.*/provider\.yaml$
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
+ additional_dependencies: ['rich>=12.4.4']
require_serial: true
- id: update-migration-references
name: Update migration ref doc
diff --git a/dev/breeze/tests/test_shell_params.py
b/dev/breeze/tests/test_shell_params.py
index 2ce10ba66a..987884f4c8 100644
--- a/dev/breeze/tests/test_shell_params.py
+++ b/dev/breeze/tests/test_shell_params.py
@@ -170,6 +170,22 @@ console = Console(width=400, color_system="standard")
},
id="ENABLED_SYSTEMS empty by default even if they are None in
ShellParams",
),
+ pytest.param(
+ {},
+ {},
+ {
+ "PYTHONWARNINGS": None,
+ },
+ id="PYTHONWARNINGS should not be set by default",
+ ),
+ pytest.param(
+ {"PYTHONWARNINGS": "default"},
+ {},
+ {
+ "PYTHONWARNINGS": "default",
+ },
+ id="PYTHONWARNINGS should be set when specified in environment",
+ ),
],
)
def test_shell_params_to_env_var_conversion(
diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
index c5126237d3..7c934c60fb 100755
--- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
+++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
@@ -17,57 +17,24 @@
# under the License.
from __future__ import annotations
-import os
-import shlex
import sys
from pathlib import Path
-if __name__ not in ("__main__", "__mp_main__"):
- raise SystemExit(
- "This file is intended to be executed as an executable program. You
cannot use it as a module."
- f"To run this script, run the ./{__file__} command"
- )
-
-AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
-GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow")
-os.environ["SKIP_GROUP_OUTPUT"] = "true"
-
-if __name__ == "__main__":
- sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
- from airflow_breeze.global_constants import
DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED
- from airflow_breeze.params.shell_params import ShellParams
- from airflow_breeze.utils.console import get_console
- from airflow_breeze.utils.docker_command_utils import
get_extra_docker_flags
- from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits,
run_command
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+from common_precommit_utils import console, initialize_breeze_precommit,
run_command_via_breeze_shell
- shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
+initialize_breeze_precommit(__name__, __file__)
- cmd = "python3
/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py"
- if len(sys.argv) > 1:
- cmd = cmd + " " + " ".join([shlex.quote(f) for f in sys.argv[1:]])
- airflow_image = get_ci_image_for_pre_commits()
- cmd_result = run_command(
- [
- "docker",
- "run",
- "-t",
- *get_extra_docker_flags(mount_sources=MOUNT_SELECTED),
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- "PYTHONWARNINGS=default",
- "--pull",
- "never",
- airflow_image,
- "-c",
- cmd,
- ],
- check=False,
- env=shell_params.env_variables_for_docker_commands,
+files_to_test = sys.argv[1:]
+cmd_result = run_command_via_breeze_shell(
+ ["python3",
"/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py",
*files_to_test],
+ backend="sqlite",
+ warn_image_upgrade_needed=True,
+ extra_env={"PYTHONWARNINGS": "default"},
+)
+if cmd_result.returncode != 0:
+ console.print(
+ "[warning]\nIf you see strange stacktraces above, "
+ "run `breeze ci-image build --python 3.8` and try again."
)
- if cmd_result.returncode != 0:
- get_console().print(
- "[warning]If you see strange stacktraces above, "
- "run `breeze ci-image build --python 3.8` and try again."
- )
- sys.exit(cmd_result.returncode)
+sys.exit(cmd_result.returncode)
