This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-3-test by this push:
new 254cb727e0 Refactor Breeze to group related methods and classes
together (#23556)
254cb727e0 is described below
commit 254cb727e0641d6816e7c8b5978bcb96ac45a38f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat May 7 15:56:34 2022 +0200
Refactor Breeze to group related methods and classes together (#23556)
This change refactors Breeze classes to more consistent approach.
* The "commands" package only contains commands
* All Parameters (BuildCi, BuildProd, BuildDoc, Shell) are now
in "params" package
* Required/Optional Build args are now members of the
BuildCiParams, BuildProdParams which makes the params
much more self-contained..
* All utils are in "utils" package
This helps with avoiding circular imports (all utios are now
standalone and do not use any of the commands.
Co-authored-by: eladkal <[email protected]>
(cherry picked from commit 35b917396fa35e70c47e114da8f178b1afeca5bc)
---
dev/breeze/src/airflow_breeze/breeze.py | 5 +-
.../src/airflow_breeze/build_image/ci/__init__.py | 17 --
.../build_image/ci/build_ci_image.py | 207 --------------------
.../airflow_breeze/build_image/prod/__init__.py | 17 --
.../build_image/prod/build_prod_image.py | 193 ------------------
.../{ci_image_tools.py => ci_image_commands.py} | 215 ++++++++++++++++++++-
...y => configuration_and_maintenance_commands.py} | 10 +-
.../airflow_breeze/commands/developer_commands.py | 162 ++++++++++++----
.../commands/{main.py => main_command.py} | 4 +-
...image_tools.py => production_image_commands.py} | 144 +++++++++++++-
...anagement.py => release_management_commands.py} | 22 +--
.../commands/{testing.py => testing_commands.py} | 14 +-
.../{commands => }/configure_rich_click.py | 12 +-
.../{build_image/ci => params}/build_ci_params.py | 59 +++---
.../prod => params}/build_prod_params.py | 39 +++-
.../src/airflow_breeze/params/doc_build_params.py | 44 +++++
.../{shell => params}/shell_params.py | 2 +-
dev/breeze/src/airflow_breeze/shell/__init__.py | 17 --
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 132 -------------
.../{commands => utils}/common_options.py | 14 +-
.../{commands => utils}/custom_param_types.py | 0
.../airflow_breeze/utils/docker_command_utils.py | 22 +--
.../mark_image_as_refreshed.py} | 13 +-
dev/breeze/src/airflow_breeze/utils/pulll_image.py | 6 +-
.../utils/rebuild_image_if_needed.py | 58 ------
dev/breeze/src/airflow_breeze/utils/registry.py | 4 +-
26 files changed, 641 insertions(+), 791 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/breeze.py
b/dev/breeze/src/airflow_breeze/breeze.py
index 276021fd40..a596c2be49 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -15,9 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from airflow_breeze.commands.configure_rich_click import click # noqa
-from airflow_breeze.commands.main import main
+from airflow_breeze.configure_rich_click import click # isort: skip # noqa
+from airflow_breeze.commands.main_command import main
from airflow_breeze.utils.path_utils import
find_airflow_sources_root_to_operate_on
find_airflow_sources_root_to_operate_on()
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py
b/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py
deleted file mode 100644
index 7a633ffa0b..0000000000
--- a/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Building CI image."""
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
deleted file mode 100644
index c962027ddf..0000000000
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ /dev/null
@@ -1,207 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import multiprocessing as mp
-import os
-import sys
-from typing import List, Tuple
-
-from airflow_breeze.build_image.ci.build_ci_params import (
- OPTIONAL_CI_IMAGE_ARGS,
- REQUIRED_CI_IMAGE_ARGS,
- BuildCiParams,
-)
-from airflow_breeze.utils.cache import touch_cache_file
-from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
-from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import (
- construct_docker_build_command,
- construct_empty_docker_build_command,
- tag_and_push_image,
-)
-from airflow_breeze.utils.md5_build_check import (
- calculate_md5_checksum_for_files,
- md5sum_check_if_build_is_needed,
-)
-from airflow_breeze.utils.parallel import check_async_run_results
-from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
BUILD_CACHE_DIR
-from airflow_breeze.utils.registry import login_to_docker_registry
-from airflow_breeze.utils.run_utils import (
- fix_group_permissions,
- instruct_build_image,
- is_repo_rebased,
- run_command,
-)
-
-
-def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) ->
bool:
- """
- Check if we should run the build based on what files have been modified
since last build and answer from
- the user.
-
- * If build is needed, the user is asked for confirmation
- * If the branch is not rebased it warns the user to rebase (to make sure
latest remote cache is useful)
- * Builds Image/Skips/Quits depending on the answer
-
- :param build_ci_params: parameters for the build
- :param verbose: should we get verbose information
- """
- # We import those locally so that click autocomplete works
- from inputimeout import TimeoutOccurred
-
- if not md5sum_check_if_build_is_needed(
- md5sum_cache_dir=build_ci_params.md5sum_cache_dir, verbose=verbose
- ):
- return False
- try:
- answer = user_confirm(
- message="Do you want to build the image?",
timeout=STANDARD_TIMEOUT, default_answer=Answer.NO
- )
- if answer == answer.YES:
- if is_repo_rebased(build_ci_params.github_repository,
build_ci_params.airflow_branch):
- return True
- else:
- get_console().print(
- "\n[warning]This might take a lot of time, we think you
should rebase first.[/]\n"
- )
- answer = user_confirm(
- "But if you really, really want - you can do it. Are you
really sure?",
- timeout=STANDARD_TIMEOUT,
- default_answer=Answer.NO,
- )
- if answer == Answer.YES:
- return True
- else:
- get_console().print(
- "[info]Please rebase your code before continuing.[/]\n"
- "Check this link to know more "
-
"https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n";
- )
- get_console().print('[error]Exiting the process[/]\n')
- sys.exit(1)
- elif answer == Answer.NO:
- instruct_build_image(build_ci_params.python)
- return False
- else: # users_status == Answer.QUIT:
- get_console().print('\n[warning]Quitting the process[/]\n')
- sys.exit()
- except TimeoutOccurred:
- get_console().print('\nTimeout. Considering your response as No\n')
- instruct_build_image(build_ci_params.python)
- return False
- except Exception as e:
- get_console().print(f'\nTerminating the process on {e}')
- sys.exit(1)
-
-
-def build_ci_image(verbose: bool, dry_run: bool, ci_image_params:
BuildCiParams) -> Tuple[int, str]:
- """
- Builds CI image:
-
- * fixes group permissions for files (to improve caching when umask is
002)
- * converts all the parameters received via kwargs into BuildCIParams
(including cache)
- * prints info about the image to build
- * logs int to docker registry on CI if build cache is being executed
- * removes "tag" for previously build image so that inline cache uses
only remote image
- * constructs docker-compose command to run based on parameters passed
- * run the build command
- * update cached information that the build completed and saves checksums
of all files
- for quick future check if the build is needed
-
- :param verbose: print commands when running
- :param dry_run: do not execute "write" commands - just print what would
happen
- :param ci_image_params: CI image parameters
- """
- fix_group_permissions(verbose=verbose)
- if verbose or dry_run:
- get_console().print(
- f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT}
"
- f"python version: {ci_image_params.python}[/]\n"
- )
- if not ci_image_params.force_build and not
ci_image_params.upgrade_to_newer_dependencies:
- if not should_we_run_the_build(build_ci_params=ci_image_params,
verbose=verbose):
- return 0, f"Image build: {ci_image_params.python}"
- run_command(
- ["docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name],
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- text=True,
- check=False,
- )
- if ci_image_params.prepare_buildx_cache:
- login_to_docker_registry(ci_image_params, dry_run=dry_run)
- cmd = construct_docker_build_command(
- image_params=ci_image_params,
- verbose=verbose,
- required_args=REQUIRED_CI_IMAGE_ARGS,
- optional_args=OPTIONAL_CI_IMAGE_ARGS,
- production_image=False,
- )
- if ci_image_params.empty_image:
- env = os.environ.copy()
- env['DOCKER_BUILDKIT'] = "1"
- get_console().print(f"\n[info]Building empty CI Image for Python
{ci_image_params.python}\n")
- cmd =
construct_empty_docker_build_command(image_params=ci_image_params)
- build_result = run_command(
- cmd,
- input="FROM scratch\n",
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- text=True,
- env=env,
- )
- else:
- get_console().print(f"\n[info]Building CI Image for Python
{ci_image_params.python}\n")
- build_result = run_command(
- cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT,
text=True, check=False
- )
- if not dry_run:
- if build_result.returncode == 0:
- mark_image_as_refreshed(ci_image_params)
- else:
- get_console().print("[error]Error when building image![/]")
- return (
- build_result.returncode,
- f"Image build: {ci_image_params.python}",
- )
- else:
- get_console().print("[info]Not updating build cache because we are in
`dry_run` mode.[/]")
- if ci_image_params.push_image:
- return tag_and_push_image(image_params=ci_image_params,
dry_run=dry_run, verbose=verbose)
- return build_result.returncode, f"Image build: {ci_image_params.python}"
-
-
-def mark_image_as_refreshed(ci_image_params: BuildCiParams):
- ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
- ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
- touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
- calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
-
-
-def build_ci_image_in_parallel(
- verbose: bool, dry_run: bool, parallelism: int, python_version_list:
List[str], **kwargs
-):
- """Run CI image builds in parallel."""
- get_console().print(
- f"\n[info]Running with parallelism = {parallelism} for the images:
{python_version_list}:"
- )
- pool = mp.Pool(parallelism)
- results = [pool.apply_async(build_ci_image, args=(verbose, dry_run,
False), kwds=kwargs)]
- check_async_run_results(results)
- pool.close()
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py
b/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py
deleted file mode 100644
index 141dd81f21..0000000000
--- a/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Building PROD Image."""
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
deleted file mode 100644
index abdc873628..0000000000
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Command to build PROD image."""
-import contextlib
-import os
-import sys
-from typing import Tuple
-
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import (
- construct_docker_build_command,
- construct_empty_docker_build_command,
- tag_and_push_image,
-)
-from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
DOCKER_CONTEXT_DIR
-from airflow_breeze.utils.registry import login_to_docker_registry
-from airflow_breeze.utils.run_utils import fix_group_permissions, run_command
-
-REQUIRED_PROD_IMAGE_ARGS = [
- "python_base_image",
- "install_mysql_client",
- "install_mssql_client",
- "install_postgres_client",
- "airflow_version",
- "airflow_branch",
- "airflow_extras",
- "airflow_pre_cached_pip_packages",
- "docker_context_files",
- "extras",
- "additional_airflow_extras",
- "additional_python_deps",
- "additional_dev_apt_command",
- "additional_dev_apt_deps",
- "additional_dev_apt_env",
- "additional_runtime_apt_command",
- "additional_runtime_apt_deps",
- "additional_runtime_apt_env",
- "upgrade_to_newer_dependencies",
- "constraints_github_repository",
- "airflow_constraints",
- "airflow_image_repository",
- "airflow_image_date_created",
- "build_id",
- "airflow_image_readme_url",
- "install_providers_from_sources",
- "airflow_is_in_context",
- "install_packages_from_context",
-]
-
-OPTIONAL_PROD_IMAGE_ARGS = [
- "dev_apt_command",
- "dev_apt_deps",
- "runtime_apt_command",
- "runtime_apt_deps",
-]
-
-
-def clean_docker_context_files(verbose: bool, dry_run: bool):
- """
- Cleans up docker context files folder - leaving only .README.md there.
- """
- if verbose or dry_run:
- get_console().print("[info]Cleaning docker-context-files[/]")
- if dry_run:
- return
- with contextlib.suppress(FileNotFoundError):
- context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*')
- for file_to_delete in context_files_to_delete:
- if file_to_delete.name != '.README.md':
- file_to_delete.unlink()
-
-
-def check_docker_context_files(install_packages_from_context: bool):
- """
- Sanity check - if we want to install from docker-context-files we expect
some packages there but if
- we don't - we don't expect them, and they might invalidate Docker cache.
-
- This method exits with an error if what we see is unexpected for given
operation.
-
- :param install_packages_from_context: whether we want to install from
docker-context-files
- """
- context_file = DOCKER_CONTEXT_DIR.glob('**/*')
- number_of_context_files = len(
- [context for context in context_file if context.is_file() and
context.name != '.README.md']
- )
- if number_of_context_files == 0:
- if install_packages_from_context:
- get_console().print('[warning]\nERROR! You want to install
packages from docker-context-files')
- get_console().print('[warning]\n but there are no packages to
install in this folder.')
- sys.exit(1)
- else:
- if not install_packages_from_context:
- get_console().print(
- '[warning]\n ERROR! There are some extra files in
docker-context-files except README.md'
- )
- get_console().print('[warning]\nAnd you did not choose
--install-packages-from-context flag')
- get_console().print(
- '[warning]\nThis might result in unnecessary cache
invalidation and long build times'
- )
- get_console().print(
- '[warning]\nExiting now \
- - please restart the command with --cleanup-context switch'
- )
- sys.exit(1)
-
-
-def build_production_image(
- verbose: bool, dry_run: bool, prod_image_params: BuildProdParams
-) -> Tuple[int, str]:
- """
- Builds PROD image:
-
- * fixes group permissions for files (to improve caching when umask is
002)
- * converts all the parameters received via kwargs into BuildProdParams
(including cache)
- * prints info about the image to build
- * removes docker-context-files if requested
- * performs sanity check if the files are present in docker-context-files
if expected
- * logs int to docker registry on CI if build cache is being executed
- * removes "tag" for previously build image so that inline cache uses
only remote image
- * constructs docker-compose command to run based on parameters passed
- * run the build command
- * update cached information that the build completed and saves checksums
of all files
- for quick future check if the build is needed
-
- :param verbose: print commands when running
- :param dry_run: do not execute "write" commands - just print what would
happen
- :param prod_image_params: PROD image parameters
- """
- fix_group_permissions(verbose=verbose)
- if verbose or dry_run:
- get_console().print(
- f"\n[info]Building PROD image of airflow from
{AIRFLOW_SOURCES_ROOT} "
- f"python version: {prod_image_params.python}[/]\n"
- )
- if prod_image_params.cleanup_context:
- clean_docker_context_files(verbose=verbose, dry_run=dry_run)
- check_docker_context_files(prod_image_params.install_packages_from_context)
- if prod_image_params.prepare_buildx_cache:
- login_to_docker_registry(prod_image_params, dry_run=dry_run)
- run_command(
- ["docker", "rmi", "--no-prune", "--force",
prod_image_params.airflow_image_name],
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- text=True,
- check=False,
- )
- get_console().print(f"\n[info]Building PROD Image for Python
{prod_image_params.python}\n")
- if prod_image_params.empty_image:
- env = os.environ.copy()
- env['DOCKER_BUILDKIT'] = "1"
- get_console().print(f"\n[info]Building empty PROD Image for Python
{prod_image_params.python}\n")
- cmd =
construct_empty_docker_build_command(image_params=prod_image_params)
- build_command_result = run_command(
- cmd,
- input="FROM scratch\n",
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- check=False,
- text=True,
- env=env,
- )
- else:
- cmd = construct_docker_build_command(
- image_params=prod_image_params,
- verbose=verbose,
- required_args=REQUIRED_PROD_IMAGE_ARGS,
- optional_args=OPTIONAL_PROD_IMAGE_ARGS,
- production_image=True,
- )
- build_command_result = run_command(
- cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT,
check=False, text=True
- )
- if build_command_result.returncode == 0:
- if prod_image_params.push_image:
- return tag_and_push_image(image_params=prod_image_params,
dry_run=dry_run, verbose=verbose)
- return build_command_result.returncode, f"Image build:
{prod_image_params.python}"
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
similarity index 52%
rename from dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
rename to dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index f6f01d3dba..bfdc695dc9 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -14,15 +14,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+import multiprocessing as mp
+import os
import sys
-from typing import Optional, Tuple
+from pathlib import Path
+from typing import List, Optional, Tuple, Union
import click
-from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.utils.common_options import (
option_additional_dev_apt_command,
option_additional_dev_apt_deps,
option_additional_dev_apt_env,
@@ -62,12 +65,31 @@ from airflow_breeze.commands.common_options import (
option_verify_image,
option_wait_for_image,
)
-from airflow_breeze.commands.main import main
+from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.docker_command_utils import (
+ check_docker_compose_version,
+ check_docker_resources,
+ check_docker_version,
+ construct_docker_build_command,
+ construct_empty_docker_build_command,
+ tag_and_push_image,
+)
+from airflow_breeze.utils.mark_image_as_refreshed import
mark_image_as_refreshed
+from airflow_breeze.utils.md5_build_check import
md5sum_check_if_build_is_needed
+from airflow_breeze.utils.parallel import check_async_run_results
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
BUILD_CACHE_DIR
from airflow_breeze.utils.pulll_image import run_pull_image,
run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
+from airflow_breeze.utils.registry import login_to_docker_registry
from airflow_breeze.utils.run_tests import verify_an_image
-from airflow_breeze.utils.run_utils import filter_out_none
+from airflow_breeze.utils.run_utils import (
+ filter_out_none,
+ fix_group_permissions,
+ instruct_build_image,
+ is_repo_rebased,
+ run_command,
+)
CI_IMAGE_TOOLS_COMMANDS = {
"name": "CI Image tools",
@@ -321,3 +343,182 @@ def verify_image(
extra_pytest_args=extra_pytest_args,
)
sys.exit(return_code)
+
+
+def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) ->
bool:
+ """
+ Check if we should run the build based on what files have been modified
since last build and answer from
+ the user.
+
+ * If build is needed, the user is asked for confirmation
+ * If the branch is not rebased it warns the user to rebase (to make sure
latest remote cache is useful)
+ * Builds Image/Skips/Quits depending on the answer
+
+ :param build_ci_params: parameters for the build
+ :param verbose: should we get verbose information
+ """
+ # We import those locally so that click autocomplete works
+ from inputimeout import TimeoutOccurred
+
+ if not md5sum_check_if_build_is_needed(
+ md5sum_cache_dir=build_ci_params.md5sum_cache_dir, verbose=verbose
+ ):
+ return False
+ try:
+ answer = user_confirm(
+ message="Do you want to build the image?",
timeout=STANDARD_TIMEOUT, default_answer=Answer.NO
+ )
+ if answer == answer.YES:
+ if is_repo_rebased(build_ci_params.github_repository,
build_ci_params.airflow_branch):
+ return True
+ else:
+ get_console().print(
+ "\n[warning]This might take a lot of time, we think you
should rebase first.[/]\n"
+ )
+ answer = user_confirm(
+ "But if you really, really want - you can do it. Are you
really sure?",
+ timeout=STANDARD_TIMEOUT,
+ default_answer=Answer.NO,
+ )
+ if answer == Answer.YES:
+ return True
+ else:
+ get_console().print(
+ "[info]Please rebase your code before continuing.[/]\n"
+ "Check this link to know more "
+
"https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n";
+ )
+ get_console().print('[error]Exiting the process[/]\n')
+ sys.exit(1)
+ elif answer == Answer.NO:
+ instruct_build_image(build_ci_params.python)
+ return False
+ else: # users_status == Answer.QUIT:
+ get_console().print('\n[warning]Quitting the process[/]\n')
+ sys.exit()
+ except TimeoutOccurred:
+ get_console().print('\nTimeout. Considering your response as No\n')
+ instruct_build_image(build_ci_params.python)
+ return False
+ except Exception as e:
+ get_console().print(f'\nTerminating the process on {e}')
+ sys.exit(1)
+
+
+def build_ci_image(verbose: bool, dry_run: bool, ci_image_params:
BuildCiParams) -> Tuple[int, str]:
+ """
+ Builds CI image:
+
+ * fixes group permissions for files (to improve caching when umask is
002)
+ * converts all the parameters received via kwargs into BuildCIParams
(including cache)
+ * prints info about the image to build
+ * logs int to docker registry on CI if build cache is being executed
+ * removes "tag" for previously build image so that inline cache uses
only remote image
+ * constructs docker-compose command to run based on parameters passed
+ * run the build command
+ * update cached information that the build completed and saves checksums
of all files
+ for quick future check if the build is needed
+
+ :param verbose: print commands when running
+ :param dry_run: do not execute "write" commands - just print what would
happen
+ :param ci_image_params: CI image parameters
+ """
+
+ fix_group_permissions(verbose=verbose)
+ if verbose or dry_run:
+ get_console().print(
+ f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT}
"
+ f"python version: {ci_image_params.python}[/]\n"
+ )
+ if not ci_image_params.force_build and not
ci_image_params.upgrade_to_newer_dependencies:
+ if not should_we_run_the_build(build_ci_params=ci_image_params,
verbose=verbose):
+ return 0, f"Image build: {ci_image_params.python}"
+ run_command(
+ ["docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name],
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ text=True,
+ check=False,
+ )
+ if ci_image_params.prepare_buildx_cache:
+ login_to_docker_registry(ci_image_params, dry_run=dry_run)
+ cmd = construct_docker_build_command(
+ image_params=ci_image_params,
+ verbose=verbose,
+ production_image=False,
+ )
+ if ci_image_params.empty_image:
+ env = os.environ.copy()
+ env['DOCKER_BUILDKIT'] = "1"
+ get_console().print(f"\n[info]Building empty CI Image for Python
{ci_image_params.python}\n")
+ cmd =
construct_empty_docker_build_command(image_params=ci_image_params)
+ build_result = run_command(
+ cmd,
+ input="FROM scratch\n",
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ text=True,
+ env=env,
+ )
+ else:
+ get_console().print(f"\n[info]Building CI Image for Python
{ci_image_params.python}\n")
+ build_result = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT,
text=True, check=False
+ )
+ if not dry_run:
+ if build_result.returncode == 0:
+ mark_image_as_refreshed(ci_image_params)
+ else:
+ get_console().print("[error]Error when building image![/]")
+ return (
+ build_result.returncode,
+ f"Image build: {ci_image_params.python}",
+ )
+ else:
+ get_console().print("[info]Not updating build cache because we are in
`dry_run` mode.[/]")
+ if ci_image_params.push_image:
+ return tag_and_push_image(image_params=ci_image_params,
dry_run=dry_run, verbose=verbose)
+ return build_result.returncode, f"Image build: {ci_image_params.python}"
+
+
+def build_ci_image_in_parallel(
+ verbose: bool, dry_run: bool, parallelism: int, python_version_list:
List[str], **kwargs
+):
+ """Run CI image builds in parallel."""
+ get_console().print(
+ f"\n[info]Running with parallelism = {parallelism} for the images:
{python_version_list}:"
+ )
+ pool = mp.Pool(parallelism)
+ results = [pool.apply_async(build_ci_image, args=(verbose, dry_run,
False), kwds=kwargs)]
+ check_async_run_results(results)
+ pool.close()
+
+
+def rebuild_ci_image_if_needed(
+ build_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose:
bool
+) -> None:
+ """
+ Rebuilds CI image if needed and user confirms it.
+
+ :param build_params: parameters of the shell
+ :param dry_run: whether it's a dry_run
+ :param verbose: should we print verbose messages
+ """
+ check_docker_version(verbose=verbose)
+ check_docker_compose_version(verbose=verbose)
+ check_docker_resources(build_params.airflow_image_name, verbose=verbose,
dry_run=dry_run)
+ build_ci_image_check_cache = Path(
+ BUILD_CACHE_DIR, build_params.airflow_branch,
f".built_{build_params.python}"
+ )
+ ci_image_params = BuildCiParams(python=build_params.python,
upgrade_to_newer_dependencies=False)
+ if build_ci_image_check_cache.exists():
+ if verbose:
+ get_console().print(f'[info]{build_params.the_image_type} image
already built locally.[/]')
+ else:
+ get_console().print(
+ f'[warning]{build_params.the_image_type} image not built locally.
Forcing build.[/]'
+ )
+ ci_image_params.force_build = True
+ build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params)
diff --git
a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
similarity index 99%
rename from
dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
rename to
dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
index 39b150b8f4..4c32f6b3b9 100644
--- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
+++
b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
@@ -25,7 +25,11 @@ from typing import Optional
import click
from airflow_breeze import NAME, VERSION
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.global_constants import
DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL
+from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache,
touch_cache_file
+from airflow_breeze.utils.common_options import (
option_answer,
option_backend,
option_dry_run,
@@ -36,10 +40,6 @@ from airflow_breeze.commands.common_options import (
option_python,
option_verbose,
)
-from airflow_breeze.commands.main import main
-from airflow_breeze.global_constants import
DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL
-from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache,
touch_cache_file
from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 8404337f62..2e36a422ec 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -16,14 +16,25 @@
# under the License.
import os
+import subprocess
import sys
-from dataclasses import dataclass
-from typing import List, Optional, Tuple
+from typing import Optional, Tuple, Union
import rich_click as click
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.ci_image_commands import
rebuild_ci_image_if_needed
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.global_constants import (
+ DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ MOUNT_SELECTED,
+ get_available_packages,
+)
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.params.doc_build_params import DocBuildParams
+from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
+from airflow_breeze.utils.cache import read_from_cache_file
+from airflow_breeze.utils.common_options import (
option_airflow_constraints_reference,
option_airflow_extras,
option_answer,
@@ -47,24 +58,19 @@ from airflow_breeze.commands.common_options import (
option_use_packages_from_dist,
option_verbose,
)
-from airflow_breeze.commands.custom_param_types import BetterChoice
-from airflow_breeze.commands.main import main
-from airflow_breeze.global_constants import (
- DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- MOUNT_SELECTED,
- get_available_packages,
-)
-from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
-from airflow_breeze.shell.enter_shell import enter_shell,
find_airflow_container
-from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.custom_param_types import BetterChoice
from airflow_breeze.utils.docker_command_utils import (
+ check_docker_compose_version,
+ check_docker_is_running,
+ check_docker_resources,
+ check_docker_version,
get_env_variables_for_docker_commands,
get_extra_docker_flags,
)
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
-from airflow_breeze.utils.rebuild_image_if_needed import
rebuild_ci_image_if_needed
-from airflow_breeze.utils.run_utils import assert_pre_commit_installed,
run_command
+from airflow_breeze.utils.run_utils import assert_pre_commit_installed,
filter_out_none, run_command
+from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
DEVELOPER_COMMANDS = {
"name": "Developer tools",
@@ -207,6 +213,8 @@ DEVELOPER_PARAMETERS = {
# Is used for a shorthand of shell and except the extra
# Args it should have the same parameters.
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
@main.command()
@option_verbose
@option_dry_run
@@ -516,30 +524,6 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes:
bool):
run_command(command_to_execute, verbose=verbose, dry_run=dry_run,
env=env_variables)
-@dataclass
-class DocBuildParams:
- package_filter: Tuple[str]
- docs_only: bool
- spellcheck_only: bool
- for_production: bool
- skip_environment_initialization: bool = False
- github_actions = os.environ.get('GITHUB_ACTIONS', "false")
-
- @property
- def args_doc_builder(self) -> List[str]:
- doc_args = []
- if self.docs_only:
- doc_args.append("--docs-only")
- if self.spellcheck_only:
- doc_args.append("--spellcheck-only")
- if self.for_production:
- doc_args.append("--for-production")
- if self.package_filter and len(self.package_filter) > 0:
- for single_filter in self.package_filter:
- doc_args.extend(["--package-filter", single_filter])
- return doc_args
-
-
@main.command(name='exec', help='Joins the interactive shell of running
airflow container')
@option_verbose
@option_dry_run
@@ -567,3 +551,101 @@ def exec(verbose: bool, dry_run: bool, exec_args: Tuple):
if not process:
sys.exit(1)
sys.exit(process.returncode)
+
+
+def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess,
subprocess.CalledProcessError]:
+ """
+ Executes entering shell using the parameters passed as kwargs:
+
+ * checks if docker version is good
+ * checks if docker-compose version is good
+ * updates kwargs with cached parameters
+ * displays ASCIIART and CHEATSHEET unless disabled
+ * build ShellParams from the updated kwargs
+ * executes the command to drop the user to Breeze shell
+
+ """
+ verbose = kwargs['verbose']
+ dry_run = kwargs['dry_run']
+ check_docker_is_running(verbose)
+ check_docker_version(verbose)
+ check_docker_compose_version(verbose)
+ if read_from_cache_file('suppress_asciiart') is None:
+ get_console().print(ASCIIART, style=ASCIIART_STYLE)
+ if read_from_cache_file('suppress_cheatsheet') is None:
+ get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE)
+ enter_shell_params = ShellParams(**filter_out_none(**kwargs))
+ return run_shell_with_build_image_checks(verbose, dry_run,
enter_shell_params)
+
+
+def run_shell_with_build_image_checks(
+ verbose: bool, dry_run: bool, shell_params: ShellParams
+) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
+ """
+ Executes a shell command built from params passed, checking if build is
not needed.
+ * checks if there are enough resources to run shell
+ * checks if image was built at least once (if not - forces the build)
+ * if not forces, checks if build is needed and asks the user if so
+ * builds the image if needed
+ * prints information about the build
+ * constructs docker compose command to enter shell
+ * executes it
+
+ :param verbose: print commands when running
+ :param dry_run: do not execute "write" commands - just print what would
happen
+ :param shell_params: parameters of the execution
+ """
+ rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run,
verbose=verbose)
+ shell_params.print_badge_info()
+ cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm',
'airflow']
+ cmd_added = shell_params.command_passed
+ env_variables = get_env_variables_for_docker_commands(shell_params)
+ if cmd_added is not None:
+ cmd.extend(['-c', cmd_added])
+
+ command_result = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True,
check=False
+ )
+ if command_result.returncode == 0:
+ return command_result
+ else:
+ get_console().print(f"[red]Error {command_result.returncode}
returned[/]")
+ if verbose:
+ get_console().print(command_result.stderr)
+ return command_result
+
+
+def stop_exec_on_error(returncode: int):
+ get_console().print('\n[error]ERROR in finding the airflow docker-compose
process id[/]\n')
+ sys.exit(returncode)
+
+
+def find_airflow_container(verbose, dry_run) -> Optional[str]:
+ exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
+ check_docker_resources(exec_shell_params.airflow_image_name,
verbose=verbose, dry_run=dry_run)
+ exec_shell_params.print_badge_info()
+ env_variables = get_env_variables_for_docker_commands(exec_shell_params)
+ cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running',
'airflow']
+ docker_compose_ps_command = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True,
env=env_variables, check=False
+ )
+ if dry_run:
+ return "CONTAINER_ID"
+ if docker_compose_ps_command.returncode != 0:
+ if verbose:
+ get_console().print(docker_compose_ps_command.stdout)
+ get_console().print(docker_compose_ps_command.stderr)
+ stop_exec_on_error(docker_compose_ps_command.returncode)
+ return None
+
+ output = docker_compose_ps_command.stdout
+ container_info = output.strip().split('\n')
+ if container_info:
+ container_running = container_info[-1].split(' ')[0]
+ if container_running.startswith('-'):
+ # On docker-compose v1 we get '--------' as output here
+ stop_exec_on_error(docker_compose_ps_command.returncode)
+ return container_running
+ else:
+ stop_exec_on_error(1)
+ return None
diff --git a/dev/breeze/src/airflow_breeze/commands/main.py
b/dev/breeze/src/airflow_breeze/commands/main_command.py
similarity index 94%
rename from dev/breeze/src/airflow_breeze/commands/main.py
rename to dev/breeze/src/airflow_breeze/commands/main_command.py
index 10a12359be..4d805e7b7a 100644
--- a/dev/breeze/src/airflow_breeze/commands/main.py
+++ b/dev/breeze/src/airflow_breeze/commands/main_command.py
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.configure_rich_click import click
+from airflow_breeze.utils.common_options import (
option_airflow_extras,
option_answer,
option_backend,
@@ -36,7 +37,6 @@ from airflow_breeze.commands.common_options import (
option_use_packages_from_dist,
option_verbose,
)
-from airflow_breeze.commands.configure_rich_click import click
from airflow_breeze.utils.path_utils import create_directories
diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
similarity index 66%
rename from dev/breeze/src/airflow_breeze/commands/production_image_tools.py
rename to dev/breeze/src/airflow_breeze/commands/production_image_commands.py
index ac5bc7fe94..56a4a07dce 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -14,15 +14,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+import contextlib
+import os
import sys
from typing import Optional, Tuple
import click
-from airflow_breeze.build_image.prod.build_prod_image import
build_production_image
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS,
DEFAULT_EXTRAS
+from airflow_breeze.params.build_prod_params import BuildProdParams
+from airflow_breeze.utils.common_options import (
option_additional_dev_apt_command,
option_additional_dev_apt_deps,
option_additional_dev_apt_env,
@@ -61,14 +63,19 @@ from airflow_breeze.commands.common_options import (
option_verify_image,
option_wait_for_image,
)
-from airflow_breeze.commands.custom_param_types import BetterChoice
-from airflow_breeze.commands.main import main
-from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS,
DEFAULT_EXTRAS
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.custom_param_types import BetterChoice
+from airflow_breeze.utils.docker_command_utils import (
+ construct_docker_build_command,
+ construct_empty_docker_build_command,
+ tag_and_push_image,
+)
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
DOCKER_CONTEXT_DIR
from airflow_breeze.utils.pulll_image import run_pull_image,
run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
+from airflow_breeze.utils.registry import login_to_docker_registry
from airflow_breeze.utils.run_tests import verify_an_image
-from airflow_breeze.utils.run_utils import filter_out_none
+from airflow_breeze.utils.run_utils import filter_out_none,
fix_group_permissions, run_command
PRODUCTION_IMAGE_TOOLS_COMMANDS = {
"name": "Production Image tools",
@@ -379,3 +386,124 @@ def verify_prod_image(
extra_pytest_args=extra_pytest_args,
)
sys.exit(return_code)
+
+
+def clean_docker_context_files(verbose: bool, dry_run: bool):
+ """
+ Cleans up docker context files folder - leaving only .README.md there.
+ """
+ if verbose or dry_run:
+ get_console().print("[info]Cleaning docker-context-files[/]")
+ if dry_run:
+ return
+ with contextlib.suppress(FileNotFoundError):
+ context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*')
+ for file_to_delete in context_files_to_delete:
+ if file_to_delete.name != '.README.md':
+ file_to_delete.unlink()
+
+
+def check_docker_context_files(install_packages_from_context: bool):
+ """
+ Sanity check - if we want to install from docker-context-files we expect
some packages there but if
+ we don't - we don't expect them, and they might invalidate Docker cache.
+
+ This method exits with an error if what we see is unexpected for given
operation.
+
+ :param install_packages_from_context: whether we want to install from
docker-context-files
+ """
+ context_file = DOCKER_CONTEXT_DIR.glob('**/*')
+ number_of_context_files = len(
+ [context for context in context_file if context.is_file() and
context.name != '.README.md']
+ )
+ if number_of_context_files == 0:
+ if install_packages_from_context:
+ get_console().print('[warning]\nERROR! You want to install
packages from docker-context-files')
+ get_console().print('[warning]\n but there are no packages to
install in this folder.')
+ sys.exit(1)
+ else:
+ if not install_packages_from_context:
+ get_console().print(
+ '[warning]\n ERROR! There are some extra files in
docker-context-files except README.md'
+ )
+ get_console().print('[warning]\nAnd you did not choose
--install-packages-from-context flag')
+ get_console().print(
+ '[warning]\nThis might result in unnecessary cache
invalidation and long build times'
+ )
+ get_console().print(
+ '[warning]\nExiting now \
+ - please restart the command with --cleanup-context switch'
+ )
+ sys.exit(1)
+
+
+def build_production_image(
+ verbose: bool, dry_run: bool, prod_image_params: BuildProdParams
+) -> Tuple[int, str]:
+ """
+ Builds PROD image:
+
+ * fixes group permissions for files (to improve caching when umask is
002)
+ * converts all the parameters received via kwargs into BuildProdParams
(including cache)
+ * prints info about the image to build
+ * removes docker-context-files if requested
+ * performs sanity check if the files are present in docker-context-files
if expected
+ * logs int to docker registry on CI if build cache is being executed
+ * removes "tag" for previously build image so that inline cache uses
only remote image
+ * constructs docker-compose command to run based on parameters passed
+ * run the build command
+ * update cached information that the build completed and saves checksums
of all files
+ for quick future check if the build is needed
+
+ :param verbose: print commands when running
+ :param dry_run: do not execute "write" commands - just print what would
happen
+ :param prod_image_params: PROD image parameters
+ """
+ fix_group_permissions(verbose=verbose)
+ if verbose or dry_run:
+ get_console().print(
+ f"\n[info]Building PROD image of airflow from
{AIRFLOW_SOURCES_ROOT} "
+ f"python version: {prod_image_params.python}[/]\n"
+ )
+ if prod_image_params.cleanup_context:
+ clean_docker_context_files(verbose=verbose, dry_run=dry_run)
+ check_docker_context_files(prod_image_params.install_packages_from_context)
+ if prod_image_params.prepare_buildx_cache:
+ login_to_docker_registry(prod_image_params, dry_run=dry_run)
+ run_command(
+ ["docker", "rmi", "--no-prune", "--force",
prod_image_params.airflow_image_name],
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ text=True,
+ check=False,
+ )
+ get_console().print(f"\n[info]Building PROD Image for Python
{prod_image_params.python}\n")
+ if prod_image_params.empty_image:
+ env = os.environ.copy()
+ env['DOCKER_BUILDKIT'] = "1"
+ get_console().print(f"\n[info]Building empty PROD Image for Python
{prod_image_params.python}\n")
+ cmd =
construct_empty_docker_build_command(image_params=prod_image_params)
+ build_command_result = run_command(
+ cmd,
+ input="FROM scratch\n",
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ check=False,
+ text=True,
+ env=env,
+ )
+ else:
+ cmd = construct_docker_build_command(
+ image_params=prod_image_params,
+ verbose=verbose,
+ production_image=True,
+ )
+ build_command_result = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT,
check=False, text=True
+ )
+ if build_command_result.returncode == 0:
+ if prod_image_params.push_image:
+ return tag_and_push_image(image_params=prod_image_params,
dry_run=dry_run, verbose=verbose)
+ return build_command_result.returncode, f"Image build:
{prod_image_params.python}"
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
similarity index 98%
rename from dev/breeze/src/airflow_breeze/commands/release_management.py
rename to dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 842759693f..1b4602c450 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -22,7 +22,16 @@ from typing import IO, List, Optional, Tuple, Union
import click
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.global_constants import (
+ ALLOWED_GENERATE_CONSTRAINTS_MODES,
+ DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ MOUNT_ALL,
+ MOUNT_NONE,
+ MOUNT_SELECTED,
+)
+from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.utils.common_options import (
argument_packages,
option_airflow_constraints_reference,
option_airflow_extras,
@@ -44,18 +53,9 @@ from airflow_breeze.commands.common_options import (
option_verbose,
option_version_suffix_for_pypi,
)
-from airflow_breeze.commands.custom_param_types import BetterChoice
-from airflow_breeze.commands.main import main
-from airflow_breeze.global_constants import (
- ALLOWED_GENERATE_CONSTRAINTS_MODES,
- DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- MOUNT_ALL,
- MOUNT_NONE,
- MOUNT_SELECTED,
-)
-from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.confirm import Answer, user_confirm
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.custom_param_types import BetterChoice
from airflow_breeze.utils.docker_command_utils import (
get_env_variables_for_docker_commands,
get_extra_docker_flags,
diff --git a/dev/breeze/src/airflow_breeze/commands/testing.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
similarity index 91%
rename from dev/breeze/src/airflow_breeze/commands/testing.py
rename to dev/breeze/src/airflow_breeze/commands/testing_commands.py
index d51093ef72..9115ee8896 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -21,8 +21,12 @@ from typing import Tuple
import click
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.commands.common_options import (
+from airflow_breeze.commands.developer_commands import
check_docker_is_running, check_docker_resources
+from airflow_breeze.commands.main_command import main
+from airflow_breeze.global_constants import ALLOWED_TEST_TYPES
+from airflow_breeze.params.build_prod_params import BuildProdParams
+from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.utils.common_options import (
option_db_reset,
option_dry_run,
option_github_repository,
@@ -32,12 +36,8 @@ from airflow_breeze.commands.common_options import (
option_python,
option_verbose,
)
-from airflow_breeze.commands.custom_param_types import BetterChoice
-from airflow_breeze.commands.main import main
-from airflow_breeze.global_constants import ALLOWED_TEST_TYPES
-from airflow_breeze.shell.enter_shell import check_docker_is_running,
check_docker_resources
-from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.custom_param_types import BetterChoice
from airflow_breeze.utils.docker_command_utils import
get_env_variables_for_docker_commands
from airflow_breeze.utils.run_tests import run_docker_compose_tests
from airflow_breeze.utils.run_utils import run_command
diff --git a/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
b/dev/breeze/src/airflow_breeze/configure_rich_click.py
similarity index 85%
rename from dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
rename to dev/breeze/src/airflow_breeze/configure_rich_click.py
index 1807ad3fb0..f9010dd28f 100644
--- a/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
+++ b/dev/breeze/src/airflow_breeze/configure_rich_click.py
@@ -22,21 +22,21 @@ try:
# We handle ImportError so that click autocomplete works
import rich_click as click
- from airflow_breeze.commands.ci_image_tools import
CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS
- from airflow_breeze.commands.configuration_and_maintenance import (
+ from airflow_breeze.commands.ci_image_commands import
CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS
+ from airflow_breeze.commands.configuration_and_maintenance_commands import
(
CONFIGURATION_AND_MAINTENANCE_COMMANDS,
CONFIGURATION_AND_MAINTENANCE_PARAMETERS,
)
from airflow_breeze.commands.developer_commands import DEVELOPER_COMMANDS,
DEVELOPER_PARAMETERS
- from airflow_breeze.commands.production_image_tools import (
+ from airflow_breeze.commands.production_image_commands import (
PRODUCTION_IMAGE_TOOLS_COMMANDS,
PRODUCTION_IMAGE_TOOLS_PARAMETERS,
)
- from airflow_breeze.commands.release_management import (
+ from airflow_breeze.commands.release_management_commands import (
RELEASE_MANAGEMENT_COMMANDS,
RELEASE_MANAGEMENT_PARAMETERS,
)
- from airflow_breeze.commands.testing import TESTING_COMMANDS,
TESTING_PARAMETERS
+ from airflow_breeze.commands.testing_commands import TESTING_COMMANDS,
TESTING_PARAMETERS
click.rich_click.SHOW_METAVARS_COLUMN = False
click.rich_click.SHOW_ARGUMENTS = False
@@ -64,7 +64,5 @@ try:
RELEASE_MANAGEMENT_COMMANDS,
]
}
-
-
except ImportError:
import click # type: ignore[no-redef]
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
similarity index 86%
rename from dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
rename to dev/breeze/src/airflow_breeze/params/build_ci_params.py
index b744ce288d..ce244162fc 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Parameters for Build CI Image."""
+
import os
from dataclasses import dataclass
from datetime import datetime
@@ -148,32 +148,31 @@ class BuildCiParams:
def md5sum_cache_dir(self) -> Path:
return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI")
-
-REQUIRED_CI_IMAGE_ARGS = [
- "python_base_image",
- "airflow_version",
- "airflow_branch",
- "airflow_extras",
- "airflow_pre_cached_pip_packages",
- "additional_airflow_extras",
- "additional_python_deps",
- "additional_dev_apt_command",
- "additional_dev_apt_deps",
- "additional_dev_apt_env",
- "additional_runtime_apt_command",
- "additional_runtime_apt_deps",
- "additional_runtime_apt_env",
- "upgrade_to_newer_dependencies",
- "constraints_github_repository",
- "airflow_constraints_reference",
- "airflow_constraints",
- "airflow_image_repository",
- "airflow_image_date_created",
- "build_id",
-]
-OPTIONAL_CI_IMAGE_ARGS = [
- "dev_apt_command",
- "dev_apt_deps",
- "runtime_apt_command",
- "runtime_apt_deps",
-]
+ required_image_args = [
+ "python_base_image",
+ "airflow_version",
+ "airflow_branch",
+ "airflow_extras",
+ "airflow_pre_cached_pip_packages",
+ "additional_airflow_extras",
+ "additional_python_deps",
+ "additional_dev_apt_command",
+ "additional_dev_apt_deps",
+ "additional_dev_apt_env",
+ "additional_runtime_apt_command",
+ "additional_runtime_apt_deps",
+ "additional_runtime_apt_env",
+ "upgrade_to_newer_dependencies",
+ "constraints_github_repository",
+ "airflow_constraints_reference",
+ "airflow_constraints",
+ "airflow_image_repository",
+ "airflow_image_date_created",
+ "build_id",
+ ]
+ optional_image_args = [
+ "dev_apt_command",
+ "dev_apt_deps",
+ "runtime_apt_command",
+ "runtime_apt_deps",
+ ]
diff --git
a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
similarity index 91%
rename from dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
rename to dev/breeze/src/airflow_breeze/params/build_prod_params.py
index 8372af7233..9fe08e35fa 100644
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Parameters to build PROD image."""
+
import json
import os
import re
@@ -311,3 +311,40 @@ class BuildProdParams:
"""Construct PROD image link"""
image =
f'{self.airflow_base_image_name}/{self.airflow_branch}/prod/python{self.python}'
return image if self.image_tag is None else image +
f":{self.image_tag}"
+
+ required_image_args = [
+ "python_base_image",
+ "install_mysql_client",
+ "install_mssql_client",
+ "install_postgres_client",
+ "airflow_version",
+ "airflow_branch",
+ "airflow_extras",
+ "airflow_pre_cached_pip_packages",
+ "docker_context_files",
+ "extras",
+ "additional_airflow_extras",
+ "additional_python_deps",
+ "additional_dev_apt_command",
+ "additional_dev_apt_deps",
+ "additional_dev_apt_env",
+ "additional_runtime_apt_command",
+ "additional_runtime_apt_deps",
+ "additional_runtime_apt_env",
+ "upgrade_to_newer_dependencies",
+ "constraints_github_repository",
+ "airflow_constraints",
+ "airflow_image_repository",
+ "airflow_image_date_created",
+ "build_id",
+ "airflow_image_readme_url",
+ "install_providers_from_sources",
+ "airflow_is_in_context",
+ "install_packages_from_context",
+ ]
+ optional_image_args = [
+ "dev_apt_command",
+ "dev_apt_deps",
+ "runtime_apt_command",
+ "runtime_apt_deps",
+ ]
diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
new file mode 100644
index 0000000000..25350db93b
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from dataclasses import dataclass
+from typing import List, Tuple
+
+
+@dataclass
+class DocBuildParams:
+ package_filter: Tuple[str]
+ docs_only: bool
+ spellcheck_only: bool
+ for_production: bool
+ skip_environment_initialization: bool = False
+ github_actions = os.environ.get('GITHUB_ACTIONS', "false")
+
+ @property
+ def args_doc_builder(self) -> List[str]:
+ doc_args = []
+ if self.docs_only:
+ doc_args.append("--docs-only")
+ if self.spellcheck_only:
+ doc_args.append("--spellcheck-only")
+ if self.for_production:
+ doc_args.append("--for-production")
+ if self.package_filter and len(self.package_filter) > 0:
+ for single_filter in self.package_filter:
+ doc_args.extend(["--package-filter", single_filter])
+ return doc_args
diff --git a/dev/breeze/src/airflow_breeze/shell/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
similarity index 99%
rename from dev/breeze/src/airflow_breeze/shell/shell_params.py
rename to dev/breeze/src/airflow_breeze/params/shell_params.py
index e5a20fb232..61eb576b60 100644
--- a/dev/breeze/src/airflow_breeze/shell/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Breeze shell paameters."""
+
import os
from dataclasses import dataclass
from pathlib import Path
diff --git a/dev/breeze/src/airflow_breeze/shell/__init__.py
b/dev/breeze/src/airflow_breeze/shell/__init__.py
deleted file mode 100644
index 209c3a09f9..0000000000
--- a/dev/breeze/src/airflow_breeze/shell/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Entering Shell"""
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
deleted file mode 100644
index 2249356e8f..0000000000
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Command to enter container shell for Breeze."""
-import subprocess
-import sys
-from typing import Optional, Union
-
-from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.cache import read_from_cache_file
-from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import (
- check_docker_compose_version,
- check_docker_is_running,
- check_docker_resources,
- check_docker_version,
- get_env_variables_for_docker_commands,
-)
-from airflow_breeze.utils.rebuild_image_if_needed import
rebuild_ci_image_if_needed
-from airflow_breeze.utils.run_utils import filter_out_none, run_command
-from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
-
-
-def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess,
subprocess.CalledProcessError]:
- """
- Executes entering shell using the parameters passed as kwargs:
-
- * checks if docker version is good
- * checks if docker-compose version is good
- * updates kwargs with cached parameters
- * displays ASCIIART and CHEATSHEET unless disabled
- * build ShellParams from the updated kwargs
- * executes the command to drop the user to Breeze shell
-
- """
- verbose = kwargs['verbose']
- dry_run = kwargs['dry_run']
- check_docker_is_running(verbose)
- check_docker_version(verbose)
- check_docker_compose_version(verbose)
- if read_from_cache_file('suppress_asciiart') is None:
- get_console().print(ASCIIART, style=ASCIIART_STYLE)
- if read_from_cache_file('suppress_cheatsheet') is None:
- get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE)
- enter_shell_params = ShellParams(**filter_out_none(**kwargs))
- return run_shell_with_build_image_checks(verbose, dry_run,
enter_shell_params)
-
-
-def run_shell_with_build_image_checks(
- verbose: bool, dry_run: bool, shell_params: ShellParams
-) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
- """
- Executes a shell command built from params passed, checking if build is
not needed.
- * checks if there are enough resources to run shell
- * checks if image was built at least once (if not - forces the build)
- * if not forces, checks if build is needed and asks the user if so
- * builds the image if needed
- * prints information about the build
- * constructs docker compose command to enter shell
- * executes it
-
- :param verbose: print commands when running
- :param dry_run: do not execute "write" commands - just print what would
happen
- :param shell_params: parameters of the execution
- """
- rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run,
verbose=verbose)
- shell_params.print_badge_info()
- cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm',
'airflow']
- cmd_added = shell_params.command_passed
- env_variables = get_env_variables_for_docker_commands(shell_params)
- if cmd_added is not None:
- cmd.extend(['-c', cmd_added])
-
- command_result = run_command(
- cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True,
check=False
- )
- if command_result.returncode == 0:
- return command_result
- else:
- get_console().print(f"[red]Error {command_result.returncode}
returned[/]")
- if verbose:
- get_console().print(command_result.stderr)
- return command_result
-
-
-def stop_exec_on_error(returncode: int):
- get_console().print('\n[error]ERROR in finding the airflow docker-compose
process id[/]\n')
- sys.exit(returncode)
-
-
-def find_airflow_container(verbose, dry_run) -> Optional[str]:
- exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
- check_docker_resources(exec_shell_params.airflow_image_name,
verbose=verbose, dry_run=dry_run)
- exec_shell_params.print_badge_info()
- env_variables = get_env_variables_for_docker_commands(exec_shell_params)
- cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running',
'airflow']
- docker_compose_ps_command = run_command(
- cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True,
env=env_variables, check=False
- )
- if dry_run:
- return "CONTAINER_ID"
- if docker_compose_ps_command.returncode != 0:
- if verbose:
- get_console().print(docker_compose_ps_command.stdout)
- get_console().print(docker_compose_ps_command.stderr)
- stop_exec_on_error(docker_compose_ps_command.returncode)
- return None
-
- output = docker_compose_ps_command.stdout
- container_info = output.strip().split('\n')
- if container_info:
- container_running = container_info[-1].split(' ')[0]
- if container_running.startswith('-'):
- # On docker-compose v1 we get '--------' as output here
- stop_exec_on_error(docker_compose_ps_command.returncode)
- return container_running
- else:
- stop_exec_on_error(1)
- return None
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py
b/dev/breeze/src/airflow_breeze/utils/common_options.py
similarity index 99%
rename from dev/breeze/src/airflow_breeze/commands/common_options.py
rename to dev/breeze/src/airflow_breeze/utils/common_options.py
index 474ba81b1c..0857a1f71e 100644
--- a/dev/breeze/src/airflow_breeze/commands/common_options.py
+++ b/dev/breeze/src/airflow_breeze/utils/common_options.py
@@ -20,13 +20,6 @@ import multiprocessing as mp
import click
from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
-from airflow_breeze.commands.custom_param_types import (
- AnswerChoice,
- BetterChoice,
- CacheableChoice,
- CacheableDefault,
- UseAirflowVersionType,
-)
from airflow_breeze.global_constants import (
ALLOWED_BACKENDS,
ALLOWED_BUILD_CACHE,
@@ -44,6 +37,13 @@ from airflow_breeze.global_constants import (
ALLOWED_USE_AIRFLOW_VERSIONS,
get_available_packages,
)
+from airflow_breeze.utils.custom_param_types import (
+ AnswerChoice,
+ BetterChoice,
+ CacheableChoice,
+ CacheableDefault,
+ UseAirflowVersionType,
+)
from airflow_breeze.utils.recording import output_file_for_recording
option_verbose = click.option(
diff --git a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py
similarity index 100%
rename from dev/breeze/src/airflow_breeze/commands/custom_param_types.py
rename to dev/breeze/src/airflow_breeze/utils/custom_param_types.py
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 069c0a3c9d..0478a2f323 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -22,9 +22,9 @@ import sys
from random import randint
from typing import Dict, List, Tuple, Union
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.shell.shell_params import ShellParams
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.params.build_prod_params import BuildProdParams
+from airflow_breeze.params.shell_params import ShellParams
from airflow_breeze.utils.host_info_utils import get_host_group_id,
get_host_os, get_host_user_id
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
from airflow_breeze.utils.registry import login_to_docker_registry
@@ -297,7 +297,7 @@ def get_env_variable_value(arg_name: str, params:
Union[BuildCiParams, BuildProd
def construct_arguments_for_docker_build_command(
- image_params: Union[BuildCiParams, BuildProdParams], required_args:
List[str], optional_args: List[str]
+ image_params: Union[BuildCiParams, BuildProdParams]
) -> List[str]:
"""
Constructs docker compose command arguments list based on parameters
passed. Maps arguments to
@@ -309,18 +309,16 @@ def construct_arguments_for_docker_build_command(
for the need of always triggering upgrade for docker build.
:param image_params: parameters of the image
- :param required_args: build argument that are required
- :param optional_args: build arguments that are optional (should not be
used if missing or empty)
:return: list of `--build-arg` commands to use for the parameters passed
"""
args_command = []
- for required_arg in required_args:
+ for required_arg in image_params.required_image_args:
args_command.append("--build-arg")
args_command.append(
required_arg.upper() + "=" +
get_env_variable_value(arg_name=required_arg, params=image_params)
)
- for optional_arg in optional_args:
+ for optional_arg in image_params.optional_image_args:
param_value = get_env_variable_value(optional_arg, params=image_params)
if len(param_value) > 0:
args_command.append("--build-arg")
@@ -332,22 +330,16 @@ def construct_arguments_for_docker_build_command(
def construct_docker_build_command(
image_params: Union[BuildProdParams, BuildCiParams],
verbose: bool,
- required_args: List[str],
- optional_args: List[str],
production_image: bool,
) -> List[str]:
"""
Constructs docker build command based on the parameters passed.
:param image_params: parameters of the image
:param verbose: print commands when running
- :param required_args: build argument that are required
- :param optional_args: build arguments that are optional (should not be
used if missing or empty)
:param production_image: whether this is production image or ci image
:return: Command to run as list of string
"""
- arguments = construct_arguments_for_docker_build_command(
- image_params, required_args=required_args, optional_args=optional_args
- )
+ arguments = construct_arguments_for_docker_build_command(image_params)
build_command = prepare_build_command(
prepare_buildx_cache=image_params.prepare_buildx_cache, verbose=verbose
)
diff --git a/dev/breeze/src/airflow_breeze/build_image/__init__.py
b/dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py
similarity index 55%
rename from dev/breeze/src/airflow_breeze/build_image/__init__.py
rename to dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py
index 7b70ea1ee8..b62880841d 100644
--- a/dev/breeze/src/airflow_breeze/build_image/__init__.py
+++ b/dev/breeze/src/airflow_breeze/utils/mark_image_as_refreshed.py
@@ -14,4 +14,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Build CI and PROD images."""
+
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.utils.cache import touch_cache_file
+from airflow_breeze.utils.md5_build_check import
calculate_md5_checksum_for_files
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+
+
+def mark_image_as_refreshed(ci_image_params: BuildCiParams):
+ ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
+ ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
+ touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
+ calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
index adc1ecfb38..e752463ac4 100644
--- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
+++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
@@ -19,10 +19,10 @@ import multiprocessing as mp
import time
from typing import List, Tuple, Union
-from airflow_breeze.build_image.ci.build_ci_image import
mark_image_as_refreshed
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.params.build_prod_params import BuildProdParams
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.mark_image_as_refreshed import
mark_image_as_refreshed
from airflow_breeze.utils.parallel import check_async_run_results
from airflow_breeze.utils.run_tests import verify_an_image
from airflow_breeze.utils.run_utils import run_command
diff --git a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
b/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
deleted file mode 100644
index 158361997a..0000000000
--- a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pathlib import Path
-from typing import Union
-
-from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import (
- check_docker_compose_version,
- check_docker_resources,
- check_docker_version,
-)
-from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
-
-
-def rebuild_ci_image_if_needed(
- build_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose:
bool
-) -> None:
- """
- Rebuilds CI image if needed and user confirms it.
-
- :param build_params: parameters of the shell
- :param dry_run: whether it's a dry_run
- :param verbose: should we print verbose messages
- """
- check_docker_version(verbose=verbose)
- check_docker_compose_version(verbose=verbose)
- check_docker_resources(build_params.airflow_image_name, verbose=verbose,
dry_run=dry_run)
- build_ci_image_check_cache = Path(
- BUILD_CACHE_DIR, build_params.airflow_branch,
f".built_{build_params.python}"
- )
- ci_image_params = BuildCiParams(python=build_params.python,
upgrade_to_newer_dependencies=False)
- if build_ci_image_check_cache.exists():
- if verbose:
- get_console().print(f'[info]{build_params.the_image_type} image
already built locally.[/]')
- else:
- get_console().print(
- f'[warning]{build_params.the_image_type} image not built locally.
Forcing build.[/]'
- )
- ci_image_params.force_build = True
- build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params)
diff --git a/dev/breeze/src/airflow_breeze/utils/registry.py
b/dev/breeze/src/airflow_breeze/utils/registry.py
index 7c2e8a6fdb..af5126934e 100644
--- a/dev/breeze/src/airflow_breeze/utils/registry.py
+++ b/dev/breeze/src/airflow_breeze/utils/registry.py
@@ -18,8 +18,8 @@
import os
from typing import Tuple, Union
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
+from airflow_breeze.params.build_ci_params import BuildCiParams
+from airflow_breeze.params.build_prod_params import BuildProdParams
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.run_utils import run_command
