This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ec92aa9707aa88f3482e86ed5964efd6c1e1b56c Author: Jarek Potiuk <[email protected]> AuthorDate: Thu May 19 17:43:26 2022 +0200 Further speed up fixing ownership in CI (#23782) After #23775 I noticed that there is yet another small improvement area in the CI buld speed. Currently build-ci-image builds and push only "commit-tagged" images, but "fix-ownership" requires the "latest" image to run. This PR adds --tag-as-latest option also to build-image and build-prod-image commands - similarly as for the pull-image and pull-prod-image. This will retag the "commit" images as latest in the build-ci-images step and allow to save 1m on pulling the latest image before fix-ownership (bringing it back to 1s overhead) (cherry picked from commit 252ef66438ecda87a8aac4beed1f689f14ee8bec) --- .github/workflows/build-images.yml | 3 +- .github/workflows/ci.yml | 3 +- .../airflow_breeze/commands/ci_image_commands.py | 15 ++++-- .../configuration_and_maintenance_commands.py | 5 +- .../airflow_breeze/commands/developer_commands.py | 16 ++++--- .../commands/production_image_commands.py | 7 ++- .../commands/release_management_commands.py | 7 ++- .../airflow_breeze/params/_common_build_params.py | 1 + .../src/airflow_breeze/utils/common_options.py | 2 +- .../airflow_breeze/utils/docker_command_utils.py | 9 ++-- dev/breeze/src/airflow_breeze/utils/image.py | 55 ++++++++++++++-------- dev/breeze/src/airflow_breeze/utils/run_utils.py | 4 +- images/breeze/output-commands-hash.txt | 2 +- 13 files changed, 81 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f95b404e7f..9970a82e6c 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -227,7 +227,7 @@ jobs: - name: "Free space" run: breeze free-space - name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image + run: breeze build-image --push-image --tag-as-latest env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} @@ -341,6 +341,7 @@ jobs: - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: > breeze build-prod-image + --tag-as-latest --push-image --install-packages-from-context --disable-airflow-repo-cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3456c43f1e..cb24723b10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,7 +332,7 @@ jobs: run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image + run: breeze build-image --push-image --tag-as-latest env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} @@ -427,6 +427,7 @@ jobs: - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: > breeze build-prod-image + --tag-as-latest --push-image --install-packages-from-context --disable-airflow-repo-cache diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 1bf3bb461d..7a04670c52 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -73,7 +73,7 @@ from airflow_breeze.utils.docker_command_utils import ( prepare_docker_build_command, prepare_empty_docker_build_command, ) -from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel +from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed from airflow_breeze.utils.md5_build_check import md5sum_check_if_build_is_needed from airflow_breeze.utils.parallel import check_async_run_results @@ -107,6 +107,7 @@ CI_IMAGE_TOOLS_PARAMETERS = { "--upgrade-to-newer-dependencies", "--debian-version", "--image-tag", + "--tag-as-latest", "--docker-cache", "--force-build", ], @@ -218,6 +219,7 @@ CI_IMAGE_TOOLS_PARAMETERS = { @option_runtime_apt_deps @option_force_build @option_airflow_constraints_mode_ci +@option_tag_as_latest def build_image( verbose: bool, dry_run: bool, @@ -259,8 +261,8 @@ def build_image( @option_github_token @option_verify_image @option_wait_for_image -@option_tag_as_latest @option_image_tag +@option_tag_as_latest @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def pull_image( verbose: bool, @@ -483,7 +485,14 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) if not ci_image_params.prepare_buildx_cache: if not dry_run: if build_command_result.returncode == 0: - mark_image_as_refreshed(ci_image_params) + if ci_image_params.tag_as_latest: + build_command_result = tag_image_as_latest(ci_image_params, dry_run, verbose) + if ( + ci_image_params.airflow_image_name == ci_image_params.airflow_image_name_with_tag + or ci_image_params.tag_as_latest + and build_command_result.returncode == 0 + ): + mark_image_as_refreshed(ci_image_params) else: get_console().print("[error]Error when building image![/]") return ( diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py index c1ba93b868..032b5f0380 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py @@ -437,11 +437,12 @@ def command_hash_export(verbose: bool, output: IO): @main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.") +@option_github_repository @option_verbose @option_dry_run -def fix_ownership(verbose: bool, dry_run: bool): +def fix_ownership(github_repository: str, verbose: bool, dry_run: bool): perform_environment_checks(verbose=verbose) - shell_params = find_available_ci_image(dry_run, verbose) + shell_params = find_available_ci_image(github_repository, dry_run, verbose) extra_docker_flags = get_extra_docker_flags(MOUNT_ALL) env = get_env_variables_for_docker_commands(shell_params) cmd = [ diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 311fd3f726..1cc9fba59c 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -16,9 +16,8 @@ # under the License. import os -import subprocess import sys -from typing import Optional, Tuple, Union +from typing import Optional, Tuple import rich_click as click @@ -67,7 +66,12 @@ from airflow_breeze.utils.docker_command_utils import ( perform_environment_checks, ) from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT -from airflow_breeze.utils.run_utils import assert_pre_commit_installed, filter_out_none, run_command +from airflow_breeze.utils.run_utils import ( + RunCommandResult, + assert_pre_commit_installed, + filter_out_none, + run_command, +) from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE DEVELOPER_COMMANDS = { @@ -554,7 +558,7 @@ def exec(verbose: bool, dry_run: bool, exec_args: Tuple): sys.exit(process.returncode) -def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: +def enter_shell(**kwargs) -> RunCommandResult: """ Executes entering shell using the parameters passed as kwargs: @@ -578,9 +582,7 @@ def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.Calle return run_shell(verbose, dry_run, enter_shell_params) -def run_shell( - verbose: bool, dry_run: bool, shell_params: ShellParams -) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: +def run_shell(verbose: bool, dry_run: bool, shell_params: ShellParams) -> RunCommandResult: """ Executes a shell command built from params passed. * prints information about the build diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 5fced6abda..53349e97c2 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -71,7 +71,7 @@ from airflow_breeze.utils.docker_command_utils import ( prepare_docker_build_command, prepare_empty_docker_build_command, ) -from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel +from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry @@ -96,6 +96,7 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = { "--upgrade-to-newer-dependencies", "--debian-version", "--image-tag", + "--tag-as-latest", "--docker-cache", ], }, @@ -259,6 +260,7 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = { @option_dev_apt_deps @option_runtime_apt_command @option_runtime_apt_deps +@option_tag_as_latest def build_prod_image( verbose: bool, dry_run: bool, @@ -520,5 +522,8 @@ def build_production_image( build_command_result = build_cache( image_params=prod_image_params, dry_run=dry_run, verbose=verbose ) + else: + if prod_image_params.tag_as_latest: + build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose) return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index e5851d0c92..1e5b8286ff 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -20,8 +20,7 @@ import sys import time from copy import deepcopy from re import match -from subprocess import CalledProcessError, CompletedProcess -from typing import IO, Dict, List, Optional, Tuple, Union +from typing import IO, Dict, List, Optional, Tuple import click @@ -70,7 +69,7 @@ from airflow_breeze.utils.docker_command_utils import ( from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.python_versions import get_python_version_list -from airflow_breeze.utils.run_utils import run_command +from airflow_breeze.utils.run_utils import RunCommandResult, run_command RELEASE_MANAGEMENT_PARAMETERS = { "breeze prepare-airflow-package": [ @@ -182,7 +181,7 @@ def run_with_debug( dry_run: bool, debug: bool, enable_input: bool = False, -) -> Union[CompletedProcess, CalledProcessError]: +) -> RunCommandResult: env_variables = get_env_variables_for_docker_commands(params) extra_docker_flags = get_extra_docker_flags(mount_sources=params.mount_sources) if enable_input or debug: diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/_common_build_params.py index 9b510a06f8..2fda00fbd5 100644 --- a/dev/breeze/src/airflow_breeze/params/_common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/_common_build_params.py @@ -64,6 +64,7 @@ class _CommonBuildParams: python: str = "3.7" runtime_apt_command: str = "" runtime_apt_deps: str = "" + tag_as_latest: bool = False upgrade_to_newer_dependencies: bool = False @property diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 615b7a2565..4d0fcfa21b 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -310,7 +310,7 @@ option_wait_for_image = click.option( option_tag_as_latest = click.option( '--tag-as-latest', help='Tags the image as latest and update checksum of all files after pulling. ' - 'Used in CI to pull the image built in another job.', + 'Useful when you build or pull image with --image-tag.', is_flag=True, envvar='TAG_AS_LATEST', ) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 1efee4cf1c..85361c8d2a 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -56,6 +56,7 @@ from airflow_breeze.global_constants import ( ) from airflow_breeze.utils.console import get_console from airflow_breeze.utils.run_utils import ( + RunCommandResult, commit_sha, prepare_base_build_command, prepare_build_cache_command, @@ -118,9 +119,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: return extra_docker_flags -def check_docker_resources( - airflow_image_name: str, verbose: bool, dry_run: bool -) -> Union[CompletedProcess, CalledProcessError]: +def check_docker_resources(airflow_image_name: str, verbose: bool, dry_run: bool) -> RunCommandResult: """ Check if we have enough resources to run docker. This is done via running script embedded in our image. :param verbose: print commands when running @@ -418,9 +417,7 @@ def prepare_empty_docker_build_command( return ["docker", "build", "-t", image_params.airflow_image_name_with_tag, "-"] -def build_cache( - image_params: _CommonBuildParams, dry_run: bool, verbose: bool -) -> Union[CompletedProcess, CalledProcessError]: +def build_cache(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: build_command_result: Union[CompletedProcess, CalledProcessError] = CompletedProcess( args=[], returncode=0 ) diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index c5ec0cde66..4131b1c0bb 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -34,7 +34,7 @@ from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import run_command +from airflow_breeze.utils.run_utils import RunCommandResult, run_command def run_pull_in_parallel( @@ -137,18 +137,7 @@ def run_pull_image( f"Image Python {image_params.python}", ) if tag_as_latest: - command_result = run_command( - [ - "docker", - "tag", - image_params.airflow_image_name_with_tag, - image_params.airflow_image_name, - ], - capture_output=True, - verbose=verbose, - dry_run=dry_run, - check=False, - ) + command_result = tag_image_as_latest(image_params, dry_run, verbose) if command_result.returncode == 0 and isinstance(image_params, BuildCiParams): mark_image_as_refreshed(image_params) return command_result.returncode, f"Image Python {image_params.python}" @@ -167,6 +156,26 @@ def run_pull_image( return command_result.returncode, f"Image Python {image_params.python}" +def tag_image_as_latest(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: + if image_params.airflow_image_name_with_tag == image_params.airflow_image_name: + get_console().print( + f"[info]Skip tagging {image_params.airflow_image_name} " "as latest as it is already 'latest'[/]" + ) + return subprocess.CompletedProcess(returncode=0, args=[]) + return run_command( + [ + "docker", + "tag", + image_params.airflow_image_name_with_tag, + image_params.airflow_image_name, + ], + capture_output=True, + verbose=verbose, + dry_run=dry_run, + check=False, + ) + + def run_pull_and_verify_image( image_params: _CommonBuildParams, dry_run: bool, @@ -193,12 +202,13 @@ def run_pull_and_verify_image( def just_pull_ci_image( - python_version: str, dry_run: bool, verbose: bool -) -> Tuple[ShellParams, Union[subprocess.CompletedProcess, subprocess.CalledProcessError]]: + github_repository, python_version: str, dry_run: bool, verbose: bool +) -> Tuple[ShellParams, RunCommandResult]: shell_params = ShellParams( verbose=verbose, mount_sources=MOUNT_ALL, python=python_version, + github_repository=github_repository, skip_environment_initialization=True, ) get_console().print(f"[info]Pulling {shell_params.airflow_image_name_with_tag}.[/]") @@ -212,12 +222,13 @@ def just_pull_ci_image( def check_if_ci_image_available( - python_version: str, dry_run: bool, verbose: bool -) -> Tuple[ShellParams, Union[subprocess.CompletedProcess, subprocess.CalledProcessError]]: + github_repository: str, python_version: str, dry_run: bool, verbose: bool +) -> Tuple[ShellParams, RunCommandResult]: shell_params = ShellParams( verbose=verbose, mount_sources=MOUNT_ALL, python=python_version, + github_repository=github_repository, skip_environment_initialization=True, ) inspect_command_result = run_command( @@ -233,13 +244,17 @@ def check_if_ci_image_available( ) -def find_available_ci_image(dry_run: bool, verbose: bool) -> ShellParams: +def find_available_ci_image(github_repository: str, dry_run: bool, verbose: bool) -> ShellParams: for python_version in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS: - shell_params, inspect_command_result = check_if_ci_image_available(python_version, dry_run, verbose) + shell_params, inspect_command_result = check_if_ci_image_available( + github_repository, python_version, dry_run, verbose + ) if inspect_command_result.returncode == 0: get_console().print( "[info]Running fix_ownership " f"with {shell_params.airflow_image_name_with_tag}.[/]" ) return shell_params - shell_params, _ = just_pull_ci_image(DEFAULT_PYTHON_MAJOR_MINOR_VERSION, dry_run, verbose) + shell_params, _ = just_pull_ci_image( + github_repository, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, dry_run, verbose + ) return shell_params diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index d0d0d535d5..241407a1af 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -32,6 +32,8 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +RunCommandResult = Union[subprocess.CompletedProcess, subprocess.CalledProcessError] + def run_command( cmd: List[str], @@ -45,7 +47,7 @@ def run_command( cwd: Optional[Path] = None, input: Optional[str] = None, **kwargs, -) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: +) -> RunCommandResult: """ Runs command passed as list of strings with some extra functionality over POpen (kwargs from PoPen can be used in this command even if not explicitly specified). diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index eddf757dea..9eeb12fd47 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -8beb1fd4bb08181701933ace04eb0911 +bc274910e868b80f476c1048dea1f957
