This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 86c4540dedef05f0fd751f79feac5efd86cfbec3 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Jun 5 11:51:04 2022 +0200 Remove warnings when starting breeze (#24183) Breeze when started produced three warnings that were harmless, but we should fix them to remove "false positives". (cherry picked from commit ac8a790ab9dabfa747964b65aa9b93451b735ede) --- dev/breeze/src/airflow_breeze/breeze.py | 6 +++++- .../airflow_breeze/commands/developer_commands.py | 3 ++- .../src/airflow_breeze/commands/main_command.py | 2 -- .../src/airflow_breeze/params/build_ci_params.py | 1 + .../src/airflow_breeze/params/build_prod_params.py | 3 ++- dev/breeze/src/airflow_breeze/params/shell_params.py | 5 +++++ .../src/airflow_breeze/utils/docker_command_utils.py | 16 ++++------------ dev/breeze/src/airflow_breeze/utils/path_utils.py | 14 ++++++++++++++ scripts/ci/docker-compose/local.yml | 7 +------ .../breeze.py => scripts/ci/docker-compose/mypy.yml | 20 +++++++++++--------- scripts/ci/pre_commit/pre_commit_flake8.py | 3 ++- .../ci/pre_commit/pre_commit_migration_reference.py | 3 ++- scripts/ci/pre_commit/pre_commit_mypy.py | 3 ++- scripts/ci/pre_commit/pre_commit_ui_lint.py | 3 ++- scripts/ci/pre_commit/pre_commit_www_lint.py | 3 ++- 15 files changed, 55 insertions(+), 37 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index a596c2be49..608619ae8e 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -17,9 +17,13 @@ # under the License. from airflow_breeze.configure_rich_click import click # isort: skip # noqa from airflow_breeze.commands.main_command import main -from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on +from airflow_breeze.utils.path_utils import ( + create_directories_and_files, + find_airflow_sources_root_to_operate_on, +) find_airflow_sources_root_to_operate_on() +create_directories_and_files() if __name__ == '__main__': main() diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 1719bab5fc..e10e79d0a8 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -524,7 +524,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool): command_to_execute = ['docker-compose', 'down', "--remove-orphans"] if not preserve_volumes: command_to_execute.append("--volumes") - shell_params = ShellParams(verbose=verbose, backend="all") + shell_params = ShellParams(verbose=verbose, backend="all", include_mypy_volume=True) env_variables = get_env_variables_for_docker_commands(shell_params) run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables) @@ -579,6 +579,7 @@ def enter_shell(**kwargs) -> RunCommandResult: if read_from_cache_file('suppress_cheatsheet') is None: get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE) enter_shell_params = ShellParams(**filter_out_none(**kwargs)) + enter_shell_params.include_mypy_volume = True rebuild_ci_image_if_needed(build_params=enter_shell_params, dry_run=dry_run, verbose=verbose) return run_shell(verbose, dry_run, enter_shell_params) diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index 8ed451b0e5..c62101a44f 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -37,7 +37,6 @@ from airflow_breeze.utils.common_options import ( option_use_packages_from_dist, option_verbose, ) -from airflow_breeze.utils.path_utils import create_directories_and_files @click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']}) @@ -64,6 +63,5 @@ from airflow_breeze.utils.path_utils import create_directories_and_files def main(ctx: click.Context, **kwargs): from airflow_breeze.commands.developer_commands import shell - create_directories_and_files() if not ctx.invoked_subcommand: ctx.forward(shell, extra_args={}) diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index aa9681fc95..52c242b65a 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -31,6 +31,7 @@ class BuildCiParams(CommonBuildParams): """ airflow_constraints_mode: str = "constraints-source-providers" + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "devel_ci" airflow_pre_cached_pip_packages: bool = True diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index e81a5d6739..c37569f85f 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -21,7 +21,7 @@ import sys from dataclasses import dataclass from typing import List -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( AIRFLOW_SOURCES_FROM, AIRFLOW_SOURCES_TO, @@ -41,6 +41,7 @@ class BuildProdParams(CommonBuildParams): """ airflow_constraints_mode: str = "constraints" + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = "" airflow_is_in_context: bool = False cleanup_context: bool = False diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 58107b9e27..4ba4022651 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -48,6 +48,7 @@ class ShellParams: """ airflow_branch: str = AIRFLOW_BRANCH + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "" answer: Optional[str] = None @@ -64,6 +65,7 @@ class ShellParams: github_repository: str = "apache/airflow" github_token: str = os.environ.get('GITHUB_TOKEN', "") image_tag: str = "latest" + include_mypy_volume: bool = False install_airflow_version: str = "" install_providers_from_sources: bool = True integration: Tuple[str, ...] = () @@ -189,6 +191,7 @@ class ShellParams: local_all_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/local-all-sources.yml" files_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/files.yml" remove_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/remove-sources.yml" + mypy_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/mypy.yml" forward_credentials_docker_compose_file = ( f"{str(SCRIPTS_CI_DIR)}/docker-compose/forward-credentials.yml" ) @@ -213,6 +216,8 @@ class ShellParams: compose_ci_file.append(forward_credentials_docker_compose_file) if self.use_airflow_version is not None: compose_ci_file.append(remove_sources_docker_compose_file) + if self.include_mypy_volume: + compose_ci_file.append(mypy_docker_compose_file) if "all" in self.integration: integrations = AVAILABLE_INTEGRATIONS else: diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 05bcc11d6c..1f30642edd 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -17,7 +17,6 @@ """Various utils to prepare docker and docker compose commands.""" import os import re -import subprocess import sys from copy import deepcopy from random import randint @@ -97,16 +96,6 @@ NECESSARY_HOST_VOLUMES = [ ] -def create_volume_if_missing(volume_name: str): - res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False) - if res_inspect.returncode != 0: - run_command(cmd=["docker", "volume", "create", volume_name], check=True) - - -def create_static_check_volumes(): - create_volume_if_missing("docker-compose_mypy-cache-volume") - - def get_extra_docker_flags(mount_sources: str) -> List[str]: """ Returns extra docker flags based on the type of mounting we want to do for sources. @@ -124,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] ) extra_docker_flags.extend( - ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] + ['--mount', "type=volume,src=mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] ) else: # none extra_docker_flags.extend( @@ -512,6 +501,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: :param env: environment variables to update with missing values if not set. """ set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers") + set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', "constraints-source-providers") set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "") set_value_to_default_if_not_set(env, 'ANSWER', "") set_value_to_default_if_not_set(env, 'BREEZE', "true") @@ -557,7 +547,9 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = { "AIRFLOW_CI_IMAGE": "airflow_image_name", "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag", "AIRFLOW_EXTRAS": "airflow_extras", + "DEFAULT_CONSTRAINTS_BRANCH": "default-constraints-branch", "AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode", + "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference", "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", "AIRFLOW_PROD_IMAGE": "airflow_image_name", "AIRFLOW_SOURCES": "airflow_sources", diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 759c3e09a7..e4e79d5520 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -19,6 +19,7 @@ Useful tools for various Paths used inside Airflow Sources. """ import hashlib import os +import subprocess import sys import tempfile from functools import lru_cache @@ -249,6 +250,18 @@ OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, 'out.log') BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze" +def create_volume_if_missing(volume_name: str): + from airflow_breeze.utils.run_utils import run_command + + res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False) + if res_inspect.returncode != 0: + run_command(cmd=["docker", "volume", "create", volume_name], check=True) + + +def create_static_check_volumes(): + create_volume_if_missing("mypy-cache-volume") + + def create_directories_and_files() -> None: """ Creates all directories and files that are needed for Breeze to work via docker-compose. @@ -264,3 +277,4 @@ def create_directories_and_files() -> None: (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch() (AIRFLOW_SOURCES_ROOT / ".bash_history").touch() (AIRFLOW_SOURCES_ROOT / ".inputrc").touch() + create_static_check_volumes() diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index e356cb8736..9e63ffa467 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -26,10 +26,7 @@ services: # or those that might be useful to see in the host as output of the # tests (such as logs) volumes: - - type: volume - source: mypy-cache-volume - target: /opt/airflow/.mypy_cache/ - # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py + # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py - type: bind source: ../../../.bash_aliases target: /root/.bash_aliases @@ -121,5 +118,3 @@ services: source: ../../../metastore_browser target: /opt/airflow/metastore_browser # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py -volumes: - mypy-cache-volume: diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/scripts/ci/docker-compose/mypy.yml old mode 100755 new mode 100644 similarity index 71% copy from dev/breeze/src/airflow_breeze/breeze.py copy to scripts/ci/docker-compose/mypy.yml index a596c2be49..7287738690 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/scripts/ci/docker-compose/mypy.yml @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,11 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from airflow_breeze.configure_rich_click import click # isort: skip # noqa -from airflow_breeze.commands.main_command import main -from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on - -find_airflow_sources_root_to_operate_on() - -if __name__ == '__main__': - main() +--- +version: "3.7" +services: + airflow: + volumes: + - type: volume + source: mypy-cache-volume + target: /opt/airflow/.mypy_cache/ +volumes: + mypy-cache-volume: + external: true diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 59bd356d24..22b0997893 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index 154a795f3e..25ac796fdf 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 74d511a65b..965bed9a74 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -35,7 +35,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index 7755e3d204..5812344a45 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -33,7 +33,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index 65553c3d81..b22750c56d 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)
