This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 84554b57375eefe724c67e1dd2aeb52ccbf65278 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat May 7 15:56:34 2022 +0200 Refactor Breeze to group related methods and classes together (#23556) This change refactors Breeze classes to more consistent approach. * The "commands" package only contains commands * All Parameters (BuildCi, BuildProd, BuildDoc, Shell) are now in "params" package * Required/Optional Build args are now members of the BuildCiParams, BuildProdParams which makes the params much more self-contained.. * All utils are in "utils" package This helps with avoiding circular imports (all utios are now standalone and do not use any of the commands. Co-authored-by: eladkal <[email protected]> (cherry picked from commit 35b917396fa35e70c47e114da8f178b1afeca5bc) --- dev/breeze/src/airflow_breeze/breeze.py | 5 +- .../src/airflow_breeze/build_image/ci/__init__.py | 17 -- .../build_image/ci/build_ci_image.py | 207 -------------------- .../airflow_breeze/build_image/prod/__init__.py | 17 -- .../build_image/prod/build_prod_image.py | 193 ------------------ .../{ci_image_tools.py => ci_image_commands.py} | 215 ++++++++++++++++++++- ...y => configuration_and_maintenance_commands.py} | 10 +- .../airflow_breeze/commands/developer_commands.py | 162 ++++++++++++---- .../commands/{main.py => main_command.py} | 4 +- ...image_tools.py => production_image_commands.py} | 144 +++++++++++++- ...anagement.py => release_management_commands.py} | 22 +-- .../commands/{testing.py => testing_commands.py} | 14 +- .../{commands => }/configure_rich_click.py | 12 +- .../{build_image/ci => params}/build_ci_params.py | 59 +++--- .../prod => params}/build_prod_params.py | 39 +++- .../src/airflow_breeze/params/doc_build_params.py | 44 +++++ .../{shell => params}/shell_params.py | 2 +- dev/breeze/src/airflow_breeze/shell/__init__.py | 17 -- dev/breeze/src/airflow_breeze/shell/enter_shell.py | 132 ------------- .../{commands => utils}/common_options.py | 14 +- .../{commands => utils}/custom_param_types.py | 0 .../airflow_breeze/utils/docker_command_utils.py | 22 +-- .../mark_image_as_refreshed.py} | 13 +- dev/breeze/src/airflow_breeze/utils/pulll_image.py | 6 +- .../utils/rebuild_image_if_needed.py | 58 ------ dev/breeze/src/airflow_breeze/utils/registry.py | 4 +- 26 files changed, 641 insertions(+), 791 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 276021fd40..a596c2be49 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -15,9 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -from airflow_breeze.commands.configure_rich_click import click # noqa -from airflow_breeze.commands.main import main +from airflow_breeze.configure_rich_click import click # isort: skip # noqa +from airflow_breeze.commands.main_command import main from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on find_airflow_sources_root_to_operate_on() diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py b/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py deleted file mode 100644 index 7a633ffa0b..0000000000 --- a/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Building CI image.""" diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py deleted file mode 100644 index c962027ddf..0000000000 --- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py +++ /dev/null @@ -1,207 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import multiprocessing as mp -import os -import sys -from typing import List, Tuple - -from airflow_breeze.build_image.ci.build_ci_params import ( - OPTIONAL_CI_IMAGE_ARGS, - REQUIRED_CI_IMAGE_ARGS, - BuildCiParams, -) -from airflow_breeze.utils.cache import touch_cache_file -from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm -from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.docker_command_utils import ( - construct_docker_build_command, - construct_empty_docker_build_command, - tag_and_push_image, -) -from airflow_breeze.utils.md5_build_check import ( - calculate_md5_checksum_for_files, - md5sum_check_if_build_is_needed, -) -from airflow_breeze.utils.parallel import check_async_run_results -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR -from airflow_breeze.utils.registry import login_to_docker_registry -from airflow_breeze.utils.run_utils import ( - fix_group_permissions, - instruct_build_image, - is_repo_rebased, - run_command, -) - - -def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) -> bool: - """ - Check if we should run the build based on what files have been modified since last build and answer from - the user. - - * If build is needed, the user is asked for confirmation - * If the branch is not rebased it warns the user to rebase (to make sure latest remote cache is useful) - * Builds Image/Skips/Quits depending on the answer - - :param build_ci_params: parameters for the build - :param verbose: should we get verbose information - """ - # We import those locally so that click autocomplete works - from inputimeout import TimeoutOccurred - - if not md5sum_check_if_build_is_needed( - md5sum_cache_dir=build_ci_params.md5sum_cache_dir, verbose=verbose - ): - return False - try: - answer = user_confirm( - message="Do you want to build the image?", timeout=STANDARD_TIMEOUT, default_answer=Answer.NO - ) - if answer == answer.YES: - if is_repo_rebased(build_ci_params.github_repository, build_ci_params.airflow_branch): - return True - else: - get_console().print( - "\n[warning]This might take a lot of time, we think you should rebase first.[/]\n" - ) - answer = user_confirm( - "But if you really, really want - you can do it. Are you really sure?", - timeout=STANDARD_TIMEOUT, - default_answer=Answer.NO, - ) - if answer == Answer.YES: - return True - else: - get_console().print( - "[info]Please rebase your code before continuing.[/]\n" - "Check this link to know more " - "https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n"; - ) - get_console().print('[error]Exiting the process[/]\n') - sys.exit(1) - elif answer == Answer.NO: - instruct_build_image(build_ci_params.python) - return False - else: # users_status == Answer.QUIT: - get_console().print('\n[warning]Quitting the process[/]\n') - sys.exit() - except TimeoutOccurred: - get_console().print('\nTimeout. Considering your response as No\n') - instruct_build_image(build_ci_params.python) - return False - except Exception as e: - get_console().print(f'\nTerminating the process on {e}') - sys.exit(1) - - -def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) -> Tuple[int, str]: - """ - Builds CI image: - - * fixes group permissions for files (to improve caching when umask is 002) - * converts all the parameters received via kwargs into BuildCIParams (including cache) - * prints info about the image to build - * logs int to docker registry on CI if build cache is being executed - * removes "tag" for previously build image so that inline cache uses only remote image - * constructs docker-compose command to run based on parameters passed - * run the build command - * update cached information that the build completed and saves checksums of all files - for quick future check if the build is needed - - :param verbose: print commands when running - :param dry_run: do not execute "write" commands - just print what would happen - :param ci_image_params: CI image parameters - """ - fix_group_permissions(verbose=verbose) - if verbose or dry_run: - get_console().print( - f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} " - f"python version: {ci_image_params.python}[/]\n" - ) - if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: - if not should_we_run_the_build(build_ci_params=ci_image_params, verbose=verbose): - return 0, f"Image build: {ci_image_params.python}" - run_command( - ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name], - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - check=False, - ) - if ci_image_params.prepare_buildx_cache: - login_to_docker_registry(ci_image_params, dry_run=dry_run) - cmd = construct_docker_build_command( - image_params=ci_image_params, - verbose=verbose, - required_args=REQUIRED_CI_IMAGE_ARGS, - optional_args=OPTIONAL_CI_IMAGE_ARGS, - production_image=False, - ) - if ci_image_params.empty_image: - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = "1" - get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n") - cmd = construct_empty_docker_build_command(image_params=ci_image_params) - build_result = run_command( - cmd, - input="FROM scratch\n", - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - env=env, - ) - else: - get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") - build_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False - ) - if not dry_run: - if build_result.returncode == 0: - mark_image_as_refreshed(ci_image_params) - else: - get_console().print("[error]Error when building image![/]") - return ( - build_result.returncode, - f"Image build: {ci_image_params.python}", - ) - else: - get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") - if ci_image_params.push_image: - return tag_and_push_image(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) - return build_result.returncode, f"Image build: {ci_image_params.python}" - - -def mark_image_as_refreshed(ci_image_params: BuildCiParams): - ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch - ci_image_cache_dir.mkdir(parents=True, exist_ok=True) - touch_cache_file(f"built_{ci_image_params.python}", root_dir=ci_image_cache_dir) - calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True) - - -def build_ci_image_in_parallel( - verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs -): - """Run CI image builds in parallel.""" - get_console().print( - f"\n[info]Running with parallelism = {parallelism} for the images: {python_version_list}:" - ) - pool = mp.Pool(parallelism) - results = [pool.apply_async(build_ci_image, args=(verbose, dry_run, False), kwds=kwargs)] - check_async_run_results(results) - pool.close() diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py b/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py deleted file mode 100644 index 141dd81f21..0000000000 --- a/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Building PROD Image.""" diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py deleted file mode 100644 index abdc873628..0000000000 --- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py +++ /dev/null @@ -1,193 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Command to build PROD image.""" -import contextlib -import os -import sys -from typing import Tuple - -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams -from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.docker_command_utils import ( - construct_docker_build_command, - construct_empty_docker_build_command, - tag_and_push_image, -) -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR -from airflow_breeze.utils.registry import login_to_docker_registry -from airflow_breeze.utils.run_utils import fix_group_permissions, run_command - -REQUIRED_PROD_IMAGE_ARGS = [ - "python_base_image", - "install_mysql_client", - "install_mssql_client", - "install_postgres_client", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "docker_context_files", - "extras", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", - "airflow_image_readme_url", - "install_providers_from_sources", - "airflow_is_in_context", - "install_packages_from_context", -] - -OPTIONAL_PROD_IMAGE_ARGS = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", -] - - -def clean_docker_context_files(verbose: bool, dry_run: bool): - """ - Cleans up docker context files folder - leaving only .README.md there. - """ - if verbose or dry_run: - get_console().print("[info]Cleaning docker-context-files[/]") - if dry_run: - return - with contextlib.suppress(FileNotFoundError): - context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*') - for file_to_delete in context_files_to_delete: - if file_to_delete.name != '.README.md': - file_to_delete.unlink() - - -def check_docker_context_files(install_packages_from_context: bool): - """ - Sanity check - if we want to install from docker-context-files we expect some packages there but if - we don't - we don't expect them, and they might invalidate Docker cache. - - This method exits with an error if what we see is unexpected for given operation. - - :param install_packages_from_context: whether we want to install from docker-context-files - """ - context_file = DOCKER_CONTEXT_DIR.glob('**/*') - number_of_context_files = len( - [context for context in context_file if context.is_file() and context.name != '.README.md'] - ) - if number_of_context_files == 0: - if install_packages_from_context: - get_console().print('[warning]\nERROR! You want to install packages from docker-context-files') - get_console().print('[warning]\n but there are no packages to install in this folder.') - sys.exit(1) - else: - if not install_packages_from_context: - get_console().print( - '[warning]\n ERROR! There are some extra files in docker-context-files except README.md' - ) - get_console().print('[warning]\nAnd you did not choose --install-packages-from-context flag') - get_console().print( - '[warning]\nThis might result in unnecessary cache invalidation and long build times' - ) - get_console().print( - '[warning]\nExiting now \ - - please restart the command with --cleanup-context switch' - ) - sys.exit(1) - - -def build_production_image( - verbose: bool, dry_run: bool, prod_image_params: BuildProdParams -) -> Tuple[int, str]: - """ - Builds PROD image: - - * fixes group permissions for files (to improve caching when umask is 002) - * converts all the parameters received via kwargs into BuildProdParams (including cache) - * prints info about the image to build - * removes docker-context-files if requested - * performs sanity check if the files are present in docker-context-files if expected - * logs int to docker registry on CI if build cache is being executed - * removes "tag" for previously build image so that inline cache uses only remote image - * constructs docker-compose command to run based on parameters passed - * run the build command - * update cached information that the build completed and saves checksums of all files - for quick future check if the build is needed - - :param verbose: print commands when running - :param dry_run: do not execute "write" commands - just print what would happen - :param prod_image_params: PROD image parameters - """ - fix_group_permissions(verbose=verbose) - if verbose or dry_run: - get_console().print( - f"\n[info]Building PROD image of airflow from {AIRFLOW_SOURCES_ROOT} " - f"python version: {prod_image_params.python}[/]\n" - ) - if prod_image_params.cleanup_context: - clean_docker_context_files(verbose=verbose, dry_run=dry_run) - check_docker_context_files(prod_image_params.install_packages_from_context) - if prod_image_params.prepare_buildx_cache: - login_to_docker_registry(prod_image_params, dry_run=dry_run) - run_command( - ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name], - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - check=False, - ) - get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") - if prod_image_params.empty_image: - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = "1" - get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n") - cmd = construct_empty_docker_build_command(image_params=prod_image_params) - build_command_result = run_command( - cmd, - input="FROM scratch\n", - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - check=False, - text=True, - env=env, - ) - else: - cmd = construct_docker_build_command( - image_params=prod_image_params, - verbose=verbose, - required_args=REQUIRED_PROD_IMAGE_ARGS, - optional_args=OPTIONAL_PROD_IMAGE_ARGS, - production_image=True, - ) - build_command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True - ) - if build_command_result.returncode == 0: - if prod_image_params.push_image: - return tag_and_push_image(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) - return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py similarity index 52% rename from dev/breeze/src/airflow_breeze/commands/ci_image_tools.py rename to dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 9bf08ef146..6ec528406b 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -14,15 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import multiprocessing as mp +import os import sys -from typing import Optional, Tuple +from pathlib import Path +from typing import List, Optional, Tuple, Union import click -from airflow_breeze.build_image.ci.build_ci_image import build_ci_image -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.main_command import main +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.common_options import ( option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, @@ -62,12 +65,31 @@ from airflow_breeze.commands.common_options import ( option_verify_image, option_wait_for_image, ) -from airflow_breeze.commands.main import main +from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.docker_command_utils import ( + check_docker_compose_version, + check_docker_resources, + check_docker_version, + construct_docker_build_command, + construct_empty_docker_build_command, + tag_and_push_image, +) +from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed +from airflow_breeze.utils.md5_build_check import md5sum_check_if_build_is_needed +from airflow_breeze.utils.parallel import check_async_run_results +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel from airflow_breeze.utils.python_versions import get_python_version_list +from airflow_breeze.utils.registry import login_to_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import filter_out_none +from airflow_breeze.utils.run_utils import ( + filter_out_none, + fix_group_permissions, + instruct_build_image, + is_repo_rebased, + run_command, +) CI_IMAGE_TOOLS_COMMANDS = { "name": "CI Image tools", @@ -331,3 +353,182 @@ def verify_image( extra_pytest_args=extra_pytest_args, ) sys.exit(return_code) + + +def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) -> bool: + """ + Check if we should run the build based on what files have been modified since last build and answer from + the user. + + * If build is needed, the user is asked for confirmation + * If the branch is not rebased it warns the user to rebase (to make sure latest remote cache is useful) + * Builds Image/Skips/Quits depending on the answer + + :param build_ci_params: parameters for the build + :param verbose: should we get verbose information + """ + # We import those locally so that click autocomplete works + from inputimeout import TimeoutOccurred + + if not md5sum_check_if_build_is_needed( + md5sum_cache_dir=build_ci_params.md5sum_cache_dir, verbose=verbose + ): + return False + try: + answer = user_confirm( + message="Do you want to build the image?", timeout=STANDARD_TIMEOUT, default_answer=Answer.NO + ) + if answer == answer.YES: + if is_repo_rebased(build_ci_params.github_repository, build_ci_params.airflow_branch): + return True + else: + get_console().print( + "\n[warning]This might take a lot of time, we think you should rebase first.[/]\n" + ) + answer = user_confirm( + "But if you really, really want - you can do it. Are you really sure?", + timeout=STANDARD_TIMEOUT, + default_answer=Answer.NO, + ) + if answer == Answer.YES: + return True + else: + get_console().print( + "[info]Please rebase your code before continuing.[/]\n" + "Check this link to know more " + "https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n"; + ) + get_console().print('[error]Exiting the process[/]\n') + sys.exit(1) + elif answer == Answer.NO: + instruct_build_image(build_ci_params.python) + return False + else: # users_status == Answer.QUIT: + get_console().print('\n[warning]Quitting the process[/]\n') + sys.exit() + except TimeoutOccurred: + get_console().print('\nTimeout. Considering your response as No\n') + instruct_build_image(build_ci_params.python) + return False + except Exception as e: + get_console().print(f'\nTerminating the process on {e}') + sys.exit(1) + + +def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) -> Tuple[int, str]: + """ + Builds CI image: + + * fixes group permissions for files (to improve caching when umask is 002) + * converts all the parameters received via kwargs into BuildCIParams (including cache) + * prints info about the image to build + * logs int to docker registry on CI if build cache is being executed + * removes "tag" for previously build image so that inline cache uses only remote image + * constructs docker-compose command to run based on parameters passed + * run the build command + * update cached information that the build completed and saves checksums of all files + for quick future check if the build is needed + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param ci_image_params: CI image parameters + """ + + fix_group_permissions(verbose=verbose) + if verbose or dry_run: + get_console().print( + f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} " + f"python version: {ci_image_params.python}[/]\n" + ) + if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: + if not should_we_run_the_build(build_ci_params=ci_image_params, verbose=verbose): + return 0, f"Image build: {ci_image_params.python}" + run_command( + ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name], + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + check=False, + ) + if ci_image_params.prepare_buildx_cache: + login_to_docker_registry(ci_image_params, dry_run=dry_run) + cmd = construct_docker_build_command( + image_params=ci_image_params, + verbose=verbose, + production_image=False, + ) + if ci_image_params.empty_image: + env = os.environ.copy() + env['DOCKER_BUILDKIT'] = "1" + get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n") + cmd = construct_empty_docker_build_command(image_params=ci_image_params) + build_result = run_command( + cmd, + input="FROM scratch\n", + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + env=env, + ) + else: + get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") + build_result = run_command( + cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False + ) + if not dry_run: + if build_result.returncode == 0: + mark_image_as_refreshed(ci_image_params) + else: + get_console().print("[error]Error when building image![/]") + return ( + build_result.returncode, + f"Image build: {ci_image_params.python}", + ) + else: + get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") + if ci_image_params.push_image: + return tag_and_push_image(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) + return build_result.returncode, f"Image build: {ci_image_params.python}" + + +def build_ci_image_in_parallel( + verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs +): + """Run CI image builds in parallel.""" + get_console().print( + f"\n[info]Running with parallelism = {parallelism} for the images: {python_version_list}:" + ) + pool = mp.Pool(parallelism) + results = [pool.apply_async(build_ci_image, args=(verbose, dry_run, False), kwds=kwargs)] + check_async_run_results(results) + pool.close() + + +def rebuild_ci_image_if_needed( + build_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool +) -> None: + """ + Rebuilds CI image if needed and user confirms it. + + :param build_params: parameters of the shell + :param dry_run: whether it's a dry_run + :param verbose: should we print verbose messages + """ + check_docker_version(verbose=verbose) + check_docker_compose_version(verbose=verbose) + check_docker_resources(build_params.airflow_image_name, verbose=verbose, dry_run=dry_run) + build_ci_image_check_cache = Path( + BUILD_CACHE_DIR, build_params.airflow_branch, f".built_{build_params.python}" + ) + ci_image_params = BuildCiParams(python=build_params.python, upgrade_to_newer_dependencies=False) + if build_ci_image_check_cache.exists(): + if verbose: + get_console().print(f'[info]{build_params.the_image_type} image already built locally.[/]') + else: + get_console().print( + f'[warning]{build_params.the_image_type} image not built locally. Forcing build.[/]' + ) + ci_image_params.force_build = True + build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params) diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py similarity index 99% rename from dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py rename to dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py index 39b150b8f4..4c32f6b3b9 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py +++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py @@ -25,7 +25,11 @@ from typing import Optional import click from airflow_breeze import NAME, VERSION -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file +from airflow_breeze.utils.common_options import ( option_answer, option_backend, option_dry_run, @@ -36,10 +40,6 @@ from airflow_breeze.commands.common_options import ( option_python, option_verbose, ) -from airflow_breeze.commands.main import main -from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL -from airflow_breeze.shell.shell_params import ShellParams -from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import get_console from airflow_breeze.utils.docker_command_utils import ( diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 8404337f62..2e36a422ec 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -16,14 +16,25 @@ # under the License. import os +import subprocess import sys -from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import Optional, Tuple, Union import rich_click as click -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.ci_image_commands import rebuild_ci_image_if_needed +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ( + DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + MOUNT_SELECTED, + get_available_packages, +) +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.doc_build_params import DocBuildParams +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST +from airflow_breeze.utils.cache import read_from_cache_file +from airflow_breeze.utils.common_options import ( option_airflow_constraints_reference, option_airflow_extras, option_answer, @@ -47,24 +58,19 @@ from airflow_breeze.commands.common_options import ( option_use_packages_from_dist, option_verbose, ) -from airflow_breeze.commands.custom_param_types import BetterChoice -from airflow_breeze.commands.main import main -from airflow_breeze.global_constants import ( - DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - MOUNT_SELECTED, - get_available_packages, -) -from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST -from airflow_breeze.shell.enter_shell import enter_shell, find_airflow_container -from airflow_breeze.shell.shell_params import ShellParams from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( + check_docker_compose_version, + check_docker_is_running, + check_docker_resources, + check_docker_version, get_env_variables_for_docker_commands, get_extra_docker_flags, ) from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT -from airflow_breeze.utils.rebuild_image_if_needed import rebuild_ci_image_if_needed -from airflow_breeze.utils.run_utils import assert_pre_commit_installed, run_command +from airflow_breeze.utils.run_utils import assert_pre_commit_installed, filter_out_none, run_command +from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE DEVELOPER_COMMANDS = { "name": "Developer tools", @@ -207,6 +213,8 @@ DEVELOPER_PARAMETERS = { # Is used for a shorthand of shell and except the extra # Args it should have the same parameters. # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + @main.command() @option_verbose @option_dry_run @@ -516,30 +524,6 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool): run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables) -@dataclass -class DocBuildParams: - package_filter: Tuple[str] - docs_only: bool - spellcheck_only: bool - for_production: bool - skip_environment_initialization: bool = False - github_actions = os.environ.get('GITHUB_ACTIONS', "false") - - @property - def args_doc_builder(self) -> List[str]: - doc_args = [] - if self.docs_only: - doc_args.append("--docs-only") - if self.spellcheck_only: - doc_args.append("--spellcheck-only") - if self.for_production: - doc_args.append("--for-production") - if self.package_filter and len(self.package_filter) > 0: - for single_filter in self.package_filter: - doc_args.extend(["--package-filter", single_filter]) - return doc_args - - @main.command(name='exec', help='Joins the interactive shell of running airflow container') @option_verbose @option_dry_run @@ -567,3 +551,101 @@ def exec(verbose: bool, dry_run: bool, exec_args: Tuple): if not process: sys.exit(1) sys.exit(process.returncode) + + +def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: + """ + Executes entering shell using the parameters passed as kwargs: + + * checks if docker version is good + * checks if docker-compose version is good + * updates kwargs with cached parameters + * displays ASCIIART and CHEATSHEET unless disabled + * build ShellParams from the updated kwargs + * executes the command to drop the user to Breeze shell + + """ + verbose = kwargs['verbose'] + dry_run = kwargs['dry_run'] + check_docker_is_running(verbose) + check_docker_version(verbose) + check_docker_compose_version(verbose) + if read_from_cache_file('suppress_asciiart') is None: + get_console().print(ASCIIART, style=ASCIIART_STYLE) + if read_from_cache_file('suppress_cheatsheet') is None: + get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE) + enter_shell_params = ShellParams(**filter_out_none(**kwargs)) + return run_shell_with_build_image_checks(verbose, dry_run, enter_shell_params) + + +def run_shell_with_build_image_checks( + verbose: bool, dry_run: bool, shell_params: ShellParams +) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: + """ + Executes a shell command built from params passed, checking if build is not needed. + * checks if there are enough resources to run shell + * checks if image was built at least once (if not - forces the build) + * if not forces, checks if build is needed and asks the user if so + * builds the image if needed + * prints information about the build + * constructs docker compose command to enter shell + * executes it + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param shell_params: parameters of the execution + """ + rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) + shell_params.print_badge_info() + cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow'] + cmd_added = shell_params.command_passed + env_variables = get_env_variables_for_docker_commands(shell_params) + if cmd_added is not None: + cmd.extend(['-c', cmd_added]) + + command_result = run_command( + cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True, check=False + ) + if command_result.returncode == 0: + return command_result + else: + get_console().print(f"[red]Error {command_result.returncode} returned[/]") + if verbose: + get_console().print(command_result.stderr) + return command_result + + +def stop_exec_on_error(returncode: int): + get_console().print('\n[error]ERROR in finding the airflow docker-compose process id[/]\n') + sys.exit(returncode) + + +def find_airflow_container(verbose, dry_run) -> Optional[str]: + exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run) + check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run) + exec_shell_params.print_badge_info() + env_variables = get_env_variables_for_docker_commands(exec_shell_params) + cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running', 'airflow'] + docker_compose_ps_command = run_command( + cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True, env=env_variables, check=False + ) + if dry_run: + return "CONTAINER_ID" + if docker_compose_ps_command.returncode != 0: + if verbose: + get_console().print(docker_compose_ps_command.stdout) + get_console().print(docker_compose_ps_command.stderr) + stop_exec_on_error(docker_compose_ps_command.returncode) + return None + + output = docker_compose_ps_command.stdout + container_info = output.strip().split('\n') + if container_info: + container_running = container_info[-1].split(' ')[0] + if container_running.startswith('-'): + # On docker-compose v1 we get '--------' as output here + stop_exec_on_error(docker_compose_ps_command.returncode) + return container_running + else: + stop_exec_on_error(1) + return None diff --git a/dev/breeze/src/airflow_breeze/commands/main.py b/dev/breeze/src/airflow_breeze/commands/main_command.py similarity index 94% rename from dev/breeze/src/airflow_breeze/commands/main.py rename to dev/breeze/src/airflow_breeze/commands/main_command.py index 10a12359be..4d805e7b7a 100644 --- a/dev/breeze/src/airflow_breeze/commands/main.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -from airflow_breeze.commands.common_options import ( +from airflow_breeze.configure_rich_click import click +from airflow_breeze.utils.common_options import ( option_airflow_extras, option_answer, option_backend, @@ -36,7 +37,6 @@ from airflow_breeze.commands.common_options import ( option_use_packages_from_dist, option_verbose, ) -from airflow_breeze.commands.configure_rich_click import click from airflow_breeze.utils.path_utils import create_directories diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py similarity index 67% rename from dev/breeze/src/airflow_breeze/commands/production_image_tools.py rename to dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 11f253f1c9..23e68e884a 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -14,15 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import contextlib +import os import sys from typing import Optional, Tuple import click -from airflow_breeze.build_image.prod.build_prod_image import build_production_image -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.utils.common_options import ( option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, @@ -61,14 +63,19 @@ from airflow_breeze.commands.common_options import ( option_verify_image, option_wait_for_image, ) -from airflow_breeze.commands.custom_param_types import BetterChoice -from airflow_breeze.commands.main import main -from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.docker_command_utils import ( + construct_docker_build_command, + construct_empty_docker_build_command, + tag_and_push_image, +) +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel from airflow_breeze.utils.python_versions import get_python_version_list +from airflow_breeze.utils.registry import login_to_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import filter_out_none +from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command PRODUCTION_IMAGE_TOOLS_COMMANDS = { "name": "Production Image tools", @@ -387,3 +394,124 @@ def verify_prod_image( extra_pytest_args=extra_pytest_args, ) sys.exit(return_code) + + +def clean_docker_context_files(verbose: bool, dry_run: bool): + """ + Cleans up docker context files folder - leaving only .README.md there. + """ + if verbose or dry_run: + get_console().print("[info]Cleaning docker-context-files[/]") + if dry_run: + return + with contextlib.suppress(FileNotFoundError): + context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*') + for file_to_delete in context_files_to_delete: + if file_to_delete.name != '.README.md': + file_to_delete.unlink() + + +def check_docker_context_files(install_packages_from_context: bool): + """ + Sanity check - if we want to install from docker-context-files we expect some packages there but if + we don't - we don't expect them, and they might invalidate Docker cache. + + This method exits with an error if what we see is unexpected for given operation. + + :param install_packages_from_context: whether we want to install from docker-context-files + """ + context_file = DOCKER_CONTEXT_DIR.glob('**/*') + number_of_context_files = len( + [context for context in context_file if context.is_file() and context.name != '.README.md'] + ) + if number_of_context_files == 0: + if install_packages_from_context: + get_console().print('[warning]\nERROR! You want to install packages from docker-context-files') + get_console().print('[warning]\n but there are no packages to install in this folder.') + sys.exit(1) + else: + if not install_packages_from_context: + get_console().print( + '[warning]\n ERROR! There are some extra files in docker-context-files except README.md' + ) + get_console().print('[warning]\nAnd you did not choose --install-packages-from-context flag') + get_console().print( + '[warning]\nThis might result in unnecessary cache invalidation and long build times' + ) + get_console().print( + '[warning]\nExiting now \ + - please restart the command with --cleanup-context switch' + ) + sys.exit(1) + + +def build_production_image( + verbose: bool, dry_run: bool, prod_image_params: BuildProdParams +) -> Tuple[int, str]: + """ + Builds PROD image: + + * fixes group permissions for files (to improve caching when umask is 002) + * converts all the parameters received via kwargs into BuildProdParams (including cache) + * prints info about the image to build + * removes docker-context-files if requested + * performs sanity check if the files are present in docker-context-files if expected + * logs int to docker registry on CI if build cache is being executed + * removes "tag" for previously build image so that inline cache uses only remote image + * constructs docker-compose command to run based on parameters passed + * run the build command + * update cached information that the build completed and saves checksums of all files + for quick future check if the build is needed + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param prod_image_params: PROD image parameters + """ + fix_group_permissions(verbose=verbose) + if verbose or dry_run: + get_console().print( + f"\n[info]Building PROD image of airflow from {AIRFLOW_SOURCES_ROOT} " + f"python version: {prod_image_params.python}[/]\n" + ) + if prod_image_params.cleanup_context: + clean_docker_context_files(verbose=verbose, dry_run=dry_run) + check_docker_context_files(prod_image_params.install_packages_from_context) + if prod_image_params.prepare_buildx_cache: + login_to_docker_registry(prod_image_params, dry_run=dry_run) + run_command( + ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name], + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + check=False, + ) + get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") + if prod_image_params.empty_image: + env = os.environ.copy() + env['DOCKER_BUILDKIT'] = "1" + get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n") + cmd = construct_empty_docker_build_command(image_params=prod_image_params) + build_command_result = run_command( + cmd, + input="FROM scratch\n", + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + check=False, + text=True, + env=env, + ) + else: + cmd = construct_docker_build_command( + image_params=prod_image_params, + verbose=verbose, + production_image=True, + ) + build_command_result = run_command( + cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True + ) + if build_command_result.returncode == 0: + if prod_image_params.push_image: + return tag_and_push_image(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) + return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py similarity index 98% rename from dev/breeze/src/airflow_breeze/commands/release_management.py rename to dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 842759693f..1b4602c450 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -22,7 +22,16 @@ from typing import IO, List, Optional, Tuple, Union import click -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ( + ALLOWED_GENERATE_CONSTRAINTS_MODES, + DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + MOUNT_ALL, + MOUNT_NONE, + MOUNT_SELECTED, +) +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.common_options import ( argument_packages, option_airflow_constraints_reference, option_airflow_extras, @@ -44,18 +53,9 @@ from airflow_breeze.commands.common_options import ( option_verbose, option_version_suffix_for_pypi, ) -from airflow_breeze.commands.custom_param_types import BetterChoice -from airflow_breeze.commands.main import main -from airflow_breeze.global_constants import ( - ALLOWED_GENERATE_CONSTRAINTS_MODES, - DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - MOUNT_ALL, - MOUNT_NONE, - MOUNT_SELECTED, -) -from airflow_breeze.shell.shell_params import ShellParams from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( get_env_variables_for_docker_commands, get_extra_docker_flags, diff --git a/dev/breeze/src/airflow_breeze/commands/testing.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py similarity index 91% rename from dev/breeze/src/airflow_breeze/commands/testing.py rename to dev/breeze/src/airflow_breeze/commands/testing_commands.py index d51093ef72..9115ee8896 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -21,8 +21,12 @@ from typing import Tuple import click -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams -from airflow_breeze.commands.common_options import ( +from airflow_breeze.commands.developer_commands import check_docker_is_running, check_docker_resources +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ALLOWED_TEST_TYPES +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.common_options import ( option_db_reset, option_dry_run, option_github_repository, @@ -32,12 +36,8 @@ from airflow_breeze.commands.common_options import ( option_python, option_verbose, ) -from airflow_breeze.commands.custom_param_types import BetterChoice -from airflow_breeze.commands.main import main -from airflow_breeze.global_constants import ALLOWED_TEST_TYPES -from airflow_breeze.shell.enter_shell import check_docker_is_running, check_docker_resources -from airflow_breeze.shell.shell_params import ShellParams from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import get_env_variables_for_docker_commands from airflow_breeze.utils.run_tests import run_docker_compose_tests from airflow_breeze.utils.run_utils import run_command diff --git a/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py b/dev/breeze/src/airflow_breeze/configure_rich_click.py similarity index 85% rename from dev/breeze/src/airflow_breeze/commands/configure_rich_click.py rename to dev/breeze/src/airflow_breeze/configure_rich_click.py index 1807ad3fb0..f9010dd28f 100644 --- a/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py +++ b/dev/breeze/src/airflow_breeze/configure_rich_click.py @@ -22,21 +22,21 @@ try: # We handle ImportError so that click autocomplete works import rich_click as click - from airflow_breeze.commands.ci_image_tools import CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS - from airflow_breeze.commands.configuration_and_maintenance import ( + from airflow_breeze.commands.ci_image_commands import CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS + from airflow_breeze.commands.configuration_and_maintenance_commands import ( CONFIGURATION_AND_MAINTENANCE_COMMANDS, CONFIGURATION_AND_MAINTENANCE_PARAMETERS, ) from airflow_breeze.commands.developer_commands import DEVELOPER_COMMANDS, DEVELOPER_PARAMETERS - from airflow_breeze.commands.production_image_tools import ( + from airflow_breeze.commands.production_image_commands import ( PRODUCTION_IMAGE_TOOLS_COMMANDS, PRODUCTION_IMAGE_TOOLS_PARAMETERS, ) - from airflow_breeze.commands.release_management import ( + from airflow_breeze.commands.release_management_commands import ( RELEASE_MANAGEMENT_COMMANDS, RELEASE_MANAGEMENT_PARAMETERS, ) - from airflow_breeze.commands.testing import TESTING_COMMANDS, TESTING_PARAMETERS + from airflow_breeze.commands.testing_commands import TESTING_COMMANDS, TESTING_PARAMETERS click.rich_click.SHOW_METAVARS_COLUMN = False click.rich_click.SHOW_ARGUMENTS = False @@ -64,7 +64,5 @@ try: RELEASE_MANAGEMENT_COMMANDS, ] } - - except ImportError: import click # type: ignore[no-redef] diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py similarity index 86% rename from dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py rename to dev/breeze/src/airflow_breeze/params/build_ci_params.py index b744ce288d..ce244162fc 100644 --- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Parameters for Build CI Image.""" + import os from dataclasses import dataclass from datetime import datetime @@ -148,32 +148,31 @@ class BuildCiParams: def md5sum_cache_dir(self) -> Path: return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI") - -REQUIRED_CI_IMAGE_ARGS = [ - "python_base_image", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints_reference", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", -] -OPTIONAL_CI_IMAGE_ARGS = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", -] + required_image_args = [ + "python_base_image", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints_reference", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + ] + optional_image_args = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", + ] diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py similarity index 91% rename from dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py rename to dev/breeze/src/airflow_breeze/params/build_prod_params.py index 8372af7233..9fe08e35fa 100644 --- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Parameters to build PROD image.""" + import json import os import re @@ -311,3 +311,40 @@ class BuildProdParams: """Construct PROD image link""" image = f'{self.airflow_base_image_name}/{self.airflow_branch}/prod/python{self.python}' return image if self.image_tag is None else image + f":{self.image_tag}" + + required_image_args = [ + "python_base_image", + "install_mysql_client", + "install_mssql_client", + "install_postgres_client", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "docker_context_files", + "extras", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + "airflow_image_readme_url", + "install_providers_from_sources", + "airflow_is_in_context", + "install_packages_from_context", + ] + optional_image_args = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", + ] diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py b/dev/breeze/src/airflow_breeze/params/doc_build_params.py new file mode 100644 index 0000000000..25350db93b --- /dev/null +++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from dataclasses import dataclass +from typing import List, Tuple + + +@dataclass +class DocBuildParams: + package_filter: Tuple[str] + docs_only: bool + spellcheck_only: bool + for_production: bool + skip_environment_initialization: bool = False + github_actions = os.environ.get('GITHUB_ACTIONS', "false") + + @property + def args_doc_builder(self) -> List[str]: + doc_args = [] + if self.docs_only: + doc_args.append("--docs-only") + if self.spellcheck_only: + doc_args.append("--spellcheck-only") + if self.for_production: + doc_args.append("--for-production") + if self.package_filter and len(self.package_filter) > 0: + for single_filter in self.package_filter: + doc_args.extend(["--package-filter", single_filter]) + return doc_args diff --git a/dev/breeze/src/airflow_breeze/shell/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py similarity index 99% rename from dev/breeze/src/airflow_breeze/shell/shell_params.py rename to dev/breeze/src/airflow_breeze/params/shell_params.py index e5a20fb232..61eb576b60 100644 --- a/dev/breeze/src/airflow_breeze/shell/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Breeze shell paameters.""" + import os from dataclasses import dataclass from pathlib import Path diff --git a/dev/breeze/src/airflow_breeze/shell/__init__.py b/dev/breeze/src/airflow_breeze/shell/__init__.py deleted file mode 100644 index 209c3a09f9..0000000000 --- a/dev/breeze/src/airflow_breeze/shell/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Entering Shell""" diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py deleted file mode 100644 index 2249356e8f..0000000000 --- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Command to enter container shell for Breeze.""" -import subprocess -import sys -from typing import Optional, Union - -from airflow_breeze.shell.shell_params import ShellParams -from airflow_breeze.utils.cache import read_from_cache_file -from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.docker_command_utils import ( - check_docker_compose_version, - check_docker_is_running, - check_docker_resources, - check_docker_version, - get_env_variables_for_docker_commands, -) -from airflow_breeze.utils.rebuild_image_if_needed import rebuild_ci_image_if_needed -from airflow_breeze.utils.run_utils import filter_out_none, run_command -from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE - - -def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: - """ - Executes entering shell using the parameters passed as kwargs: - - * checks if docker version is good - * checks if docker-compose version is good - * updates kwargs with cached parameters - * displays ASCIIART and CHEATSHEET unless disabled - * build ShellParams from the updated kwargs - * executes the command to drop the user to Breeze shell - - """ - verbose = kwargs['verbose'] - dry_run = kwargs['dry_run'] - check_docker_is_running(verbose) - check_docker_version(verbose) - check_docker_compose_version(verbose) - if read_from_cache_file('suppress_asciiart') is None: - get_console().print(ASCIIART, style=ASCIIART_STYLE) - if read_from_cache_file('suppress_cheatsheet') is None: - get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE) - enter_shell_params = ShellParams(**filter_out_none(**kwargs)) - return run_shell_with_build_image_checks(verbose, dry_run, enter_shell_params) - - -def run_shell_with_build_image_checks( - verbose: bool, dry_run: bool, shell_params: ShellParams -) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: - """ - Executes a shell command built from params passed, checking if build is not needed. - * checks if there are enough resources to run shell - * checks if image was built at least once (if not - forces the build) - * if not forces, checks if build is needed and asks the user if so - * builds the image if needed - * prints information about the build - * constructs docker compose command to enter shell - * executes it - - :param verbose: print commands when running - :param dry_run: do not execute "write" commands - just print what would happen - :param shell_params: parameters of the execution - """ - rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) - shell_params.print_badge_info() - cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow'] - cmd_added = shell_params.command_passed - env_variables = get_env_variables_for_docker_commands(shell_params) - if cmd_added is not None: - cmd.extend(['-c', cmd_added]) - - command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True, check=False - ) - if command_result.returncode == 0: - return command_result - else: - get_console().print(f"[red]Error {command_result.returncode} returned[/]") - if verbose: - get_console().print(command_result.stderr) - return command_result - - -def stop_exec_on_error(returncode: int): - get_console().print('\n[error]ERROR in finding the airflow docker-compose process id[/]\n') - sys.exit(returncode) - - -def find_airflow_container(verbose, dry_run) -> Optional[str]: - exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run) - check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run) - exec_shell_params.print_badge_info() - env_variables = get_env_variables_for_docker_commands(exec_shell_params) - cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running', 'airflow'] - docker_compose_ps_command = run_command( - cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True, env=env_variables, check=False - ) - if dry_run: - return "CONTAINER_ID" - if docker_compose_ps_command.returncode != 0: - if verbose: - get_console().print(docker_compose_ps_command.stdout) - get_console().print(docker_compose_ps_command.stderr) - stop_exec_on_error(docker_compose_ps_command.returncode) - return None - - output = docker_compose_ps_command.stdout - container_info = output.strip().split('\n') - if container_info: - container_running = container_info[-1].split(' ')[0] - if container_running.startswith('-'): - # On docker-compose v1 we get '--------' as output here - stop_exec_on_error(docker_compose_ps_command.returncode) - return container_running - else: - stop_exec_on_error(1) - return None diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py similarity index 99% rename from dev/breeze/src/airflow_breeze/commands/common_options.py rename to dev/breeze/src/airflow_breeze/utils/common_options.py index 474ba81b1c..0857a1f71e 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -20,13 +20,6 @@ import multiprocessing as mp import click from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH -from airflow_breeze.commands.custom_param_types import ( - AnswerChoice, - BetterChoice, - CacheableChoice, - CacheableDefault, - UseAirflowVersionType, -) from airflow_breeze.global_constants import ( ALLOWED_BACKENDS, ALLOWED_BUILD_CACHE, @@ -44,6 +37,13 @@ from airflow_breeze.global_constants import ( ALLOWED_USE_AIRFLOW_VERSIONS, get_available_packages, ) +from airflow_breeze.utils.custom_param_types import ( + AnswerChoice, + BetterChoice, + CacheableChoice, + CacheableDefault, + UseAirflowVersionType, +) from airflow_breeze.utils.recording import output_file_for_recording option_verbose = click.option( diff --git a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py similarity index 100% rename from dev/breeze/src/airflow_breeze/commands/custom_param_types.py rename to dev/breeze/src/airflow_breeze/utils/custom_param_types.py diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 069c0a3c9d..0478a2f323 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -22,9 +22,9 @@ import sys from random import randint from typing import Dict, List, Tuple, Union -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams -from airflow_breeze.shell.shell_params import ShellParams +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.host_info_utils import get_host_group_id, get_host_os, get_host_user_id from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT from airflow_breeze.utils.registry import login_to_docker_registry @@ -297,7 +297,7 @@ def get_env_variable_value(arg_name: str, params: Union[BuildCiParams, BuildProd def construct_arguments_for_docker_build_command( - image_params: Union[BuildCiParams, BuildProdParams], required_args: List[str], optional_args: List[str] + image_params: Union[BuildCiParams, BuildProdParams] ) -> List[str]: """ Constructs docker compose command arguments list based on parameters passed. Maps arguments to @@ -309,18 +309,16 @@ def construct_arguments_for_docker_build_command( for the need of always triggering upgrade for docker build. :param image_params: parameters of the image - :param required_args: build argument that are required - :param optional_args: build arguments that are optional (should not be used if missing or empty) :return: list of `--build-arg` commands to use for the parameters passed """ args_command = [] - for required_arg in required_args: + for required_arg in image_params.required_image_args: args_command.append("--build-arg") args_command.append( required_arg.upper() + "=" + get_env_variable_value(arg_name=required_arg, params=image_params) ) - for optional_arg in optional_args: + for optional_arg in image_params.optional_image_args: param_value = get_env_variable_value(optional_arg, params=image_params) if len(param_value) > 0: args_command.append("--build-arg") @@ -332,22 +330,16 @@ def construct_arguments_for_docker_build_command( def construct_docker_build_command( image_params: Union[BuildProdParams, BuildCiParams], verbose: bool, - required_args: List[str], - optional_args: List[str], production_image: bool, ) -> List[str]: """ Constructs docker build command based on the parameters passed. :param image_params: parameters of the image :param verbose: print commands when running - :param required_args: build argument that are required - :param optional_args: build arguments that are optional (should not be used if missing or empty) :param production_image: whether this is production image or ci image :return: Command to run as list of string """ - arguments = construct_arguments_for_docker_build_command( - image_params, required_args=required_args, optional_args=optional_args - ) + arguments = construct_arguments_for_docker_build_command(image_params) build_command = prepare_build_command( prepare_buildx_cache=image_params.prepare_buildx_cache, verbose=verbose ) diff --git a/dev/breeze/src/airflow_breeze/build_image/__init__.py b/dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py similarity index 55% rename from dev/breeze/src/airflow_breeze/build_image/__init__.py rename to dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py index 7b70ea1ee8..b62880841d 100644 --- a/dev/breeze/src/airflow_breeze/build_image/__init__.py +++ b/dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py @@ -14,4 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Build CI and PROD images.""" + +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.utils.cache import touch_cache_file +from airflow_breeze.utils.md5_build_check import calculate_md5_checksum_for_files +from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR + + +def mark_image_as_refreshed(ci_image_params: BuildCiParams): + ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch + ci_image_cache_dir.mkdir(parents=True, exist_ok=True) + touch_cache_file(f"built_{ci_image_params.python}", root_dir=ci_image_cache_dir) + calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True) diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py b/dev/breeze/src/airflow_breeze/utils/pulll_image.py index a272f65d29..0efe2cb154 100644 --- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py +++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py @@ -19,10 +19,10 @@ import multiprocessing as mp import time from typing import List, Tuple, Union -from airflow_breeze.build_image.ci.build_ci_image import mark_image_as_refreshed -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.registry import login_to_docker_registry from airflow_breeze.utils.run_tests import verify_an_image diff --git a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py b/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py deleted file mode 100644 index 158361997a..0000000000 --- a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from pathlib import Path -from typing import Union - -from airflow_breeze.build_image.ci.build_ci_image import build_ci_image -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.shell.shell_params import ShellParams -from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.docker_command_utils import ( - check_docker_compose_version, - check_docker_resources, - check_docker_version, -) -from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR - - -def rebuild_ci_image_if_needed( - build_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool -) -> None: - """ - Rebuilds CI image if needed and user confirms it. - - :param build_params: parameters of the shell - :param dry_run: whether it's a dry_run - :param verbose: should we print verbose messages - """ - check_docker_version(verbose=verbose) - check_docker_compose_version(verbose=verbose) - check_docker_resources(build_params.airflow_image_name, verbose=verbose, dry_run=dry_run) - build_ci_image_check_cache = Path( - BUILD_CACHE_DIR, build_params.airflow_branch, f".built_{build_params.python}" - ) - ci_image_params = BuildCiParams(python=build_params.python, upgrade_to_newer_dependencies=False) - if build_ci_image_check_cache.exists(): - if verbose: - get_console().print(f'[info]{build_params.the_image_type} image already built locally.[/]') - else: - get_console().print( - f'[warning]{build_params.the_image_type} image not built locally. Forcing build.[/]' - ) - ci_image_params.force_build = True - build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params) diff --git a/dev/breeze/src/airflow_breeze/utils/registry.py b/dev/breeze/src/airflow_breeze/utils/registry.py index 7c2e8a6fdb..af5126934e 100644 --- a/dev/breeze/src/airflow_breeze/utils/registry.py +++ b/dev/breeze/src/airflow_breeze/utils/registry.py @@ -18,8 +18,8 @@ import os from typing import Tuple, Union -from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams -from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.utils.console import get_console from airflow_breeze.utils.run_utils import run_command
