This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7fa9bba7ce6f893eb6c24df0d31a82e40f32e952 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Dec 10 19:24:21 2023 +0100 Reorganize command options in Breeze (#36152) The command options in breeze have been rather badly organized. The "common_option" in utils package was misplaced and it contained a lot of options that were not really "common" - only used in one or two commands. This PR reorganizes those options by making a bit more reasonable decisions where options are placed: * when options are used by a single command - they are defined in that command * when options are used by more comnands in the same command module, they are defined in that command module - at the tops * in cases of "image" and "package_installation" optins - they are used by several modules only and they have their own "common_"*" modules * only in case a command is used in multiple modules, they are put in "common_options". * the "common_options" module has been moved to "commands" package (it makes much more sense to be there - together with all commands * the command options are sorted alphabetically in their corresponding files. * Fixed missing default for "--platform" flag (cherry picked from commit 2563442e5fd118ea9feb3e3eb67fcf13f7954ea4) --- .../src/airflow_breeze/commands/ci_commands.py | 12 +- .../airflow_breeze/commands/ci_image_commands.py | 92 ++- .../commands/common_image_options.py | 206 +++++ .../src/airflow_breeze/commands/common_options.py | 347 +++++++++ .../common_package_installation_options.py | 107 +++ .../airflow_breeze/commands/developer_commands.py | 148 +++- .../airflow_breeze/commands/kubernetes_commands.py | 110 ++- .../src/airflow_breeze/commands/main_command.py | 10 +- .../commands/minor_release_command.py | 2 +- .../commands/production_image_commands.py | 52 +- .../commands/release_candidate_command.py | 2 +- .../src/airflow_breeze/commands/release_command.py | 2 +- .../commands/release_management_commands.py | 119 ++- .../src/airflow_breeze/commands/sbom_commands.py | 22 +- .../src/airflow_breeze/commands/setup_commands.py | 8 +- .../airflow_breeze/commands/testing_commands.py | 103 ++- .../src/airflow_breeze/utils/common_options.py | 844 --------------------- 17 files changed, 1110 insertions(+), 1076 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index ed9864b4c0..3a86869907 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -30,6 +30,12 @@ from typing import Any, Iterable, NamedTuple import click +from airflow_breeze.commands.common_options import ( + option_answer, + option_dry_run, + option_github_repository, + option_verbose, +) from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, RUNS_ON_PUBLIC_RUNNER, @@ -39,12 +45,6 @@ from airflow_breeze.global_constants import ( ) from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( - option_answer, - option_dry_run, - option_github_repository, - option_verbose, -) from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import get_console from airflow_breeze.utils.custom_param_types import BetterChoice diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index bc0ef5ac4f..b0a6d12447 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -29,58 +29,57 @@ from typing import TYPE_CHECKING, Any, Callable import click -from airflow_breeze.params.build_ci_params import BuildCiParams -from airflow_breeze.utils.ci_group import ci_group -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_image_options import ( option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, option_additional_pip_install_flags, option_additional_python_deps, - option_airflow_constraints_location, - option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, - option_answer, option_build_progress, - option_build_timeout_minutes, - option_builder, - option_commit_sha, option_debian_version, - option_debug_resources, option_dev_apt_command, option_dev_apt_deps, option_docker_cache, - option_docker_host, - option_dry_run, - option_eager_upgrade_additional_requirements, - option_github_repository, - option_github_token, - option_image_name, option_image_tag_for_building, option_image_tag_for_pulling, option_image_tag_for_verifying, - option_include_success_outputs, option_install_providers_from_sources, - option_parallelism, option_platform_multiple, option_prepare_buildx_cache, option_pull, option_push, - option_python, option_python_image, + option_tag_as_latest, + option_verify, + option_wait_for_image, +) +from airflow_breeze.commands.common_options import ( + option_answer, + option_builder, + option_commit_sha, + option_debug_resources, + option_docker_host, + option_dry_run, + option_github_repository, + option_github_token, + option_image_name, + option_include_success_outputs, + option_parallelism, + option_python, option_python_versions, option_run_in_parallel, option_skip_cleanup, - option_tag_as_latest, - option_upgrade_on_failure, - option_upgrade_to_newer_dependencies, option_verbose, - option_verify, - option_version_suffix_for_pypi_ci, - option_wait_for_image, ) +from airflow_breeze.commands.common_package_installation_options import ( + option_airflow_constraints_location, + option_airflow_constraints_mode_ci, +) +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.utils.ci_group import ci_group +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.docker_command_utils import ( @@ -235,6 +234,47 @@ def get_exitcode(status: int) -> int: return 1 +option_build_timeout_minutes = click.option( + "--build-timeout-minutes", + required=False, + type=int, + envvar="BUILD_TIMEOUT_MINUTES", + help="Optional timeout for the build in minutes. Useful to detect `pip` backtracking problems.", +) + +option_eager_upgrade_additional_requirements = click.option( + "--eager-upgrade-additional-requirements", + required=False, + type=str, + envvar="EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS", + help="Optional additional requirements to upgrade eagerly to avoid backtracking " + "(see `breeze ci find-backtracking-candidates`).", +) + +option_upgrade_to_newer_dependencies = click.option( + "-u", + "--upgrade-to-newer-dependencies", + is_flag=True, + help="When set, upgrade all PIP packages to latest.", + envvar="UPGRADE_TO_NEWER_DEPENDENCIES", +) + +option_upgrade_on_failure = click.option( + "--upgrade-on-failure", + is_flag=True, + help="When set, attempt to run upgrade to newer dependencies when regular build fails.", + envvar="UPGRADE_ON_FAILURE", +) + +option_version_suffix_for_pypi_ci = click.option( + "--version-suffix-for-pypi", + help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", + default="dev0", + show_default=True, + envvar="VERSION_SUFFIX_FOR_PYPI", +) + + @ci_image.command(name="build") @option_additional_airflow_extras @option_additional_dev_apt_command diff --git a/dev/breeze/src/airflow_breeze/commands/common_image_options.py b/dev/breeze/src/airflow_breeze/commands/common_image_options.py new file mode 100644 index 0000000000..a86e2a4431 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/common_image_options.py @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import click + +from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.global_constants import ( + ALLOWED_BUILD_CACHE, + ALLOWED_BUILD_PROGRESS, + ALLOWED_DEBIAN_VERSIONS, + ALLOWED_PLATFORMS, + DOCKER_DEFAULT_PLATFORM, +) +from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.recording import generating_command_images + +option_additional_dev_apt_command = click.option( + "--additional-dev-apt-command", + help="Additional command executed before dev apt deps are installed.", + envvar="ADDITIONAL_DEV_APT_COMMAND", +) +option_additional_dev_apt_deps = click.option( + "--additional-dev-apt-deps", + help="Additional apt dev dependencies to use when building the images.", + envvar="ADDITIONAL_DEV_APT_DEPS", +) +option_additional_dev_apt_env = click.option( + "--additional-dev-apt-env", + help="Additional environment variables set when adding dev dependencies.", + envvar="ADDITIONAL_DEV_APT_ENV", +) +option_additional_airflow_extras = click.option( + "--additional-airflow-extras", + help="Additional extra package while installing Airflow in the image.", + envvar="ADDITIONAL_AIRFLOW_EXTRAS", +) +option_additional_python_deps = click.option( + "--additional-python-deps", + help="Additional python dependencies to use when building the images.", + envvar="ADDITIONAL_PYTHON_DEPS", +) +option_additional_pip_install_flags = click.option( + "--additional-pip-install-flags", + help="Additional flags added to `pip install` commands (except reinstalling `pip` itself).", + envvar="ADDITIONAL_PIP_INSTALL_FLAGS", +) +option_additional_runtime_apt_command = click.option( + "--additional-runtime-apt-command", + help="Additional command executed before runtime apt deps are installed.", + envvar="ADDITIONAL_RUNTIME_APT_COMMAND", +) +option_additional_runtime_apt_deps = click.option( + "--additional-runtime-apt-deps", + help="Additional apt runtime dependencies to use when building the images.", + envvar="ADDITIONAL_RUNTIME_APT_DEPS", +) +option_additional_runtime_apt_env = click.option( + "--additional-runtime-apt-env", + help="Additional environment variables set when adding runtime dependencies.", + envvar="ADDITIONAL_RUNTIME_APT_ENV", +) +option_airflow_constraints_reference_build = click.option( + "--airflow-constraints-reference", + default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, + help="Constraint reference to use when building the image.", + envvar="AIRFLOW_CONSTRAINTS_REFERENCE", +) +option_build_progress = click.option( + "--build-progress", + help="Build progress.", + type=BetterChoice(ALLOWED_BUILD_PROGRESS), + envvar="BUILD_PROGRESS", + show_default=True, + default=ALLOWED_BUILD_PROGRESS[0], +) +option_debian_version = click.option( + "--debian-version", + type=BetterChoice(ALLOWED_DEBIAN_VERSIONS), + default=ALLOWED_DEBIAN_VERSIONS[0], + show_default=True, + help="Debian version used in Airflow image as base for building images.", + envvar="DEBIAN_VERSION", +) +option_dev_apt_command = click.option( + "--dev-apt-command", + help="Command executed before dev apt deps are installed.", + envvar="DEV_APT_COMMAND", +) +option_dev_apt_deps = click.option( + "--dev-apt-deps", + help="Apt dev dependencies to use when building the images.", + envvar="DEV_APT_DEPS", +) +option_docker_cache = click.option( + "-c", + "--docker-cache", + help="Cache option for image used during the build.", + default=ALLOWED_BUILD_CACHE[0], + show_default=True, + type=BetterChoice(ALLOWED_BUILD_CACHE), +) +option_image_tag_for_pulling = click.option( + "-t", + "--image-tag", + help="Tag of the image which is used to pull the image.", + show_default=True, + default="latest", + envvar="IMAGE_TAG", +) +option_image_tag_for_building = click.option( + "--image-tag", + help="Tag the image after building it.", + show_default=True, + default="latest", + envvar="IMAGE_TAG", +) +option_image_tag_for_verifying = click.option( + "-t", + "--image-tag", + help="Tag of the image when verifying it.", + show_default=True, + default="latest", + envvar="IMAGE_TAG", +) +option_install_providers_from_sources = click.option( + "--install-providers-from-sources", + help="Install providers from sources when installing.", + is_flag=True, + envvar="INSTALL_PROVIDERS_FROM_SOURCES", +) +option_platform_multiple = click.option( + "--platform", + help="Platform for Airflow image.", + default=DOCKER_DEFAULT_PLATFORM if not generating_command_images() else ALLOWED_PLATFORMS[0], + envvar="PLATFORM", + type=BetterChoice(ALLOWED_PLATFORMS), +) +option_prepare_buildx_cache = click.option( + "--prepare-buildx-cache", + help="Prepares build cache (this is done as separate per-platform steps instead of building the image).", + is_flag=True, + envvar="PREPARE_BUILDX_CACHE", +) +option_pull = click.option( + "--pull", + help="Pull image is missing before attempting to verify it.", + is_flag=True, + envvar="PULL", +) +option_push = click.option( + "--push", + help="Push image after building it.", + is_flag=True, + envvar="PUSH", +) +option_python_image = click.option( + "--python-image", + help="If specified this is the base python image used to build the image. " + "Should be something like: python:VERSION-slim-bookworm.", + envvar="PYTHON_IMAGE", +) +option_runtime_apt_command = click.option( + "--runtime-apt-command", + help="Command executed before runtime apt deps are installed.", + envvar="RUNTIME_APT_COMMAND", +) +option_runtime_apt_deps = click.option( + "--runtime-apt-deps", + help="Apt runtime dependencies to use when building the images.", + envvar="RUNTIME_APT_DEPS", +) +option_tag_as_latest = click.option( + "--tag-as-latest", + help="Tags the image as latest and update checksum of all files after pulling. " + "Useful when you build or pull image with --image-tag.", + is_flag=True, + envvar="TAG_AS_LATEST", +) +option_verify = click.option( + "--verify", + help="Verify image.", + is_flag=True, + envvar="VERIFY", +) +option_wait_for_image = click.option( + "--wait-for-image", + help="Wait until image is available.", + is_flag=True, + envvar="WAIT_FOR_IMAGE", +) diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py new file mode 100644 index 0000000000..41652b7b12 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import multiprocessing as mp + +import click + +from airflow_breeze.global_constants import ( + ALL_HISTORICAL_PYTHON_VERSIONS, + ALLOWED_BACKENDS, + ALLOWED_DOCKER_COMPOSE_PROJECTS, + ALLOWED_INSTALLATION_PACKAGE_FORMATS, + ALLOWED_MOUNT_OPTIONS, + ALLOWED_MSSQL_VERSIONS, + ALLOWED_MYSQL_VERSIONS, + ALLOWED_POSTGRES_VERSIONS, + ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, + ALLOWED_USE_AIRFLOW_VERSIONS, + APACHE_AIRFLOW_GITHUB_REPOSITORY, + AUTOCOMPLETE_INTEGRATIONS, +) +from airflow_breeze.utils.custom_param_types import ( + AnswerChoice, + BetterChoice, + CacheableChoice, + CacheableDefault, + DryRunOption, + MySQLBackendVersionType, + NotVerifiedBetterChoice, + UseAirflowVersionType, + VerboseOption, +) +from airflow_breeze.utils.packages import get_available_packages +from airflow_breeze.utils.recording import generating_command_images + + +def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, value): + from click.core import ParameterSource + + if ( + ctx.parent + and option.name in ctx.parent.params + and ctx.get_parameter_source(option.name) + in ( + ParameterSource.DEFAULT, + ParameterSource.DEFAULT_MAP, + ) + ): + # Current value is the default, use the parent's value (i.e. for `breeze + # # -v static-checks` respect the "global" option) + value = ctx.parent.params[option.name] + return value + + +argument_doc_packages = click.argument( + "doc_packages", + nargs=-1, + required=False, + type=NotVerifiedBetterChoice( + get_available_packages(include_non_provider_doc_packages=True, include_all_providers=True) + ), +) +option_airflow_extras = click.option( + "--airflow-extras", + help="Airflow extras to install when --use-airflow-version is used", + default="", + show_default=True, + envvar="AIRFLOW_EXTRAS", +) +option_answer = click.option( + "-a", + "--answer", + type=AnswerChoice(["y", "n", "q", "yes", "no", "quit"]), + help="Force answer to questions.", + envvar="ANSWER", + expose_value=False, + callback=_set_default_from_parent, +) +option_backend = click.option( + "-b", + "--backend", + type=CacheableChoice(ALLOWED_BACKENDS), + default=CacheableDefault(value=ALLOWED_BACKENDS[0]), + show_default=True, + help="Database backend to use. If 'none' is selected, breeze starts with invalid DB configuration " + "and no database and any attempts to connect to Airflow DB will fail.", + envvar="BACKEND", +) +option_builder = click.option( + "--builder", + help="Buildx builder used to perform `docker buildx build` commands.", + envvar="BUILDER", + show_default=True, + default="autodetect", +) +option_commit_sha = click.option( + "--commit-sha", + show_default=True, + envvar="COMMIT_SHA", + help="Commit SHA that is used to build the images.", +) +option_db_reset = click.option( + "-d", + "--db-reset", + help="Reset DB when entering the container.", + is_flag=True, + envvar="DB_RESET", +) +option_debug_resources = click.option( + "--debug-resources", + is_flag=True, + help="Whether to show resource information while running in parallel.", + envvar="DEBUG_RESOURCES", +) +option_database_isolation = click.option( + "--database-isolation", + help="Run airflow in database isolation mode.", + is_flag=True, + envvar="DATABASE_ISOLATION", +) +option_docker_host = click.option( + "--docker-host", + help="Optional - docker host to use when running docker commands. " + "When set, the `--builder` option is ignored when building images.", + envvar="DOCKER_HOST", +) +option_downgrade_sqlalchemy = click.option( + "--downgrade-sqlalchemy", + help="Downgrade SQLAlchemy to minimum supported version.", + is_flag=True, + envvar="DOWNGRADE_SQLALCHEMY", +) +option_dry_run = click.option( + "-D", + "--dry-run", + is_flag=True, + help="If dry-run is set, commands are only printed, not executed.", + envvar="DRY_RUN", + metavar="BOOLEAN", + expose_value=False, + type=DryRunOption(), + callback=_set_default_from_parent, +) +option_forward_credentials = click.option( + "-f", "--forward-credentials", help="Forward local credentials to container when running.", is_flag=True +) +option_github_token = click.option( + "--github-token", + help="The token used to authenticate to GitHub.", + envvar="GITHUB_TOKEN", +) +option_github_repository = click.option( + "-g", + "--github-repository", + help="GitHub repository used to pull, push run images.", + default=APACHE_AIRFLOW_GITHUB_REPOSITORY, + show_default=True, + envvar="GITHUB_REPOSITORY", + callback=_set_default_from_parent, +) +option_historical_python_version = click.option( + "--python", + type=BetterChoice(ALL_HISTORICAL_PYTHON_VERSIONS), + required=False, + envvar="PYTHON_VERSION", + help="Python version to update sbom from. (defaults to all historical python versions)", +) +option_include_success_outputs = click.option( + "--include-success-outputs", + help="Whether to include outputs of successful parallel runs (skipped by default).", + is_flag=True, + envvar="INCLUDE_SUCCESS_OUTPUTS", +) +option_integration = click.option( + "--integration", + help="Integration(s) to enable when running (can be more than one).", + type=BetterChoice(AUTOCOMPLETE_INTEGRATIONS), + multiple=True, +) +option_image_name = click.option( + "-n", "--image-name", help="Name of the image to verify (overrides --python and --image-tag)." +) +option_image_tag_for_running = click.option( + "--image-tag", + help="Tag of the image which is used to run the image (implies --mount-sources=skip).", + show_default=True, + default="latest", + envvar="IMAGE_TAG", +) +option_max_time = click.option( + "--max-time", + help="Maximum time that the command should take - if it takes longer, the command will fail.", + type=click.IntRange(min=1), + envvar="MAX_TIME", + callback=_set_default_from_parent, +) +option_mount_sources = click.option( + "--mount-sources", + type=BetterChoice(ALLOWED_MOUNT_OPTIONS), + default=ALLOWED_MOUNT_OPTIONS[0], + show_default=True, + envvar="MOUNT_SOURCES", + help="Choose scope of local sources that should be mounted, skipped, or removed (default = selected).", +) +option_mssql_version = click.option( + "-S", + "--mssql-version", + help="Version of MsSQL used.", + type=CacheableChoice(ALLOWED_MSSQL_VERSIONS), + default=CacheableDefault(ALLOWED_MSSQL_VERSIONS[0]), + show_default=True, +) +option_mysql_version = click.option( + "-M", + "--mysql-version", + help="Version of MySQL used.", + type=MySQLBackendVersionType(ALLOWED_MYSQL_VERSIONS), + default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]), + show_default=True, +) +option_installation_package_format = click.option( + "--package-format", + type=BetterChoice(ALLOWED_INSTALLATION_PACKAGE_FORMATS), + help="Format of packages that should be installed from dist.", + default=ALLOWED_INSTALLATION_PACKAGE_FORMATS[0], + show_default=True, + envvar="PACKAGE_FORMAT", +) +option_parallelism = click.option( + "--parallelism", + help="Maximum number of processes to use while running the operation in parallel.", + type=click.IntRange(1, mp.cpu_count() * 3 if not generating_command_images() else 8), + default=mp.cpu_count() if not generating_command_images() else 4, + envvar="PARALLELISM", + show_default=True, +) +option_postgres_version = click.option( + "-P", + "--postgres-version", + type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS), + default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]), + show_default=True, + help="Version of Postgres used.", +) +option_project_name = click.option( + "--project-name", + help="Name of the docker-compose project to bring down. " + "The `docker-compose` is for legacy breeze project name and you can use " + "`breeze down --project-name docker-compose` to stop all containers belonging to it.", + show_default=True, + type=NotVerifiedBetterChoice(ALLOWED_DOCKER_COMPOSE_PROJECTS), + default=ALLOWED_DOCKER_COMPOSE_PROJECTS[0], + envvar="PROJECT_NAME", +) +option_python = click.option( + "-p", + "--python", + type=CacheableChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS), + default=CacheableDefault(value=ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]), + show_default=True, + help="Python major/minor version used in Airflow image for images.", + envvar="PYTHON_MAJOR_MINOR_VERSION", +) +option_python_versions = click.option( + "--python-versions", + help="Space separated list of python versions used for build with multiple versions.", + default=" ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS), + show_default=True, + envvar="PYTHON_VERSIONS", +) +option_run_db_tests_only = click.option( + "--run-db-tests-only", + help="Only runs tests that require a database", + is_flag=True, + envvar="run_db_tests_only", +) +option_run_in_parallel = click.option( + "--run-in-parallel", + help="Run the operation in parallel on all or selected subset of parameters.", + is_flag=True, + envvar="RUN_IN_PARALLEL", +) +option_skip_cleanup = click.option( + "--skip-cleanup", + help="Skip cleanup of temporary files created during parallel run.", + is_flag=True, + envvar="SKIP_CLEANUP", +) +option_skip_db_tests = click.option( + "--skip-db-tests", + help="Skip tests that require a database", + is_flag=True, + envvar="SKIP_DB_TESTS", +) +option_standalone_dag_processor = click.option( + "--standalone-dag-processor", + help="Run standalone dag processor for start-airflow.", + is_flag=True, + envvar="STANDALONE_DAG_PROCESSOR", +) +option_upgrade_boto = click.option( + "--upgrade-boto", + help="Remove aiobotocore and upgrade botocore and boto to the latest version.", + is_flag=True, + envvar="UPGRADE_BOTO", +) +option_use_airflow_version = click.option( + "--use-airflow-version", + help="Use (reinstall at entry) Airflow version from PyPI. It can also be version (to install from PyPI), " + "`none`, `wheel`, or `sdist` to install from `dist` folder, or VCS URL to install from " + "(https://pip.pypa.io/en/stable/topics/vcs-support/). Implies --mount-sources `remove`.", + type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS), + envvar="USE_AIRFLOW_VERSION", +) +option_verbose = click.option( + "-v", + "--verbose", + is_flag=True, + help="Print verbose information about performed steps.", + envvar="VERBOSE", + metavar="BOOLEAN", + expose_value=False, + type=VerboseOption(), + callback=_set_default_from_parent, +) +option_version_suffix_for_pypi = click.option( + "--version-suffix-for-pypi", + help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", + envvar="VERSION_SUFFIX_FOR_PYPI", + default="", +) diff --git a/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py b/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py new file mode 100644 index 0000000000..5a5d428886 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import click + +from airflow_breeze.global_constants import ALLOWED_CONSTRAINTS_MODES_CI, ALLOWED_CONSTRAINTS_MODES_PROD +from airflow_breeze.utils.custom_param_types import BetterChoice + +option_airflow_constraints_reference = click.option( + "--airflow-constraints-reference", + help="Constraint reference to use for airflow installation (used in calculated constraints URL). " + "Can be 'default' in which case the default constraints-reference is used.", + envvar="AIRFLOW_CONSTRAINTS_REFERENCE", +) +option_airflow_constraints_location = click.option( + "--airflow-constraints-location", + type=str, + help="Location of airflow constraints to use (remote URL or local context file).", + envvar="AIRFLOW_CONSTRAINTS_LOCATION", +) +option_airflow_constraints_mode_update = click.option( + "--airflow-constraints-mode", + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), + required=False, + help="Limit constraint update to only selected constraint mode - if selected.", +) +option_airflow_constraints_mode_prod = click.option( + "--airflow-constraints-mode", + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_PROD), + default=ALLOWED_CONSTRAINTS_MODES_PROD[0], + show_default=True, + help="Mode of constraints for Airflow for PROD image building.", +) +option_airflow_constraints_mode_ci = click.option( + "--airflow-constraints-mode", + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), + default=ALLOWED_CONSTRAINTS_MODES_CI[0], + show_default=True, + help="Mode of constraints for Airflow for CI image building.", +) +option_airflow_skip_constraints = click.option( + "--airflow-skip-constraints", + is_flag=True, + help="Do not use constraints when installing airflow.", + envvar="AIRFLOW_SKIP_CONSTRAINTS", +) +option_install_selected_providers = click.option( + "--install-selected-providers", + help="Comma-separated list of providers selected to be installed (implies --use-packages-from-dist).", + envvar="INSTALL_SELECTED_PROVIDERS", + default="", +) +option_providers_constraints_reference = click.option( + "--providers-constraints-reference", + help="Constraint reference to use for providers installation (used in calculated constraints URL). " + "Can be 'default' in which case the default constraints-reference is used.", + envvar="PROVIDERS_CONSTRAINTS_REFERENCE", +) +option_providers_constraints_location = click.option( + "--providers-constraints-location", + type=str, + help="Location of providers constraints to use (remote URL or local context file).", + envvar="PROVIDERS_CONSTRAINTS_LOCATION", +) +option_providers_constraints_mode_prod = click.option( + "--providers-constraints-mode", + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_PROD), + default=ALLOWED_CONSTRAINTS_MODES_PROD[0], + show_default=True, + help="Mode of constraints for Providers for PROD image building.", +) +option_providers_constraints_mode_ci = click.option( + "--providers-constraints-mode", + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), + default=ALLOWED_CONSTRAINTS_MODES_CI[0], + show_default=True, + help="Mode of constraints for Providers for CI image building.", +) +option_providers_skip_constraints = click.option( + "--providers-skip-constraints", + is_flag=True, + help="Do not use constraints when installing providers.", + envvar="PROVIDERS_SKIP_CONSTRAINTS", +) +option_use_packages_from_dist = click.option( + "--use-packages-from-dist", + is_flag=True, + help="Install all found packages (--package-format determines type) from 'dist' folder " + "when entering breeze.", + envvar="USE_PACKAGES_FROM_DIST", +) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 050bc02271..2e347f73c0 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -30,69 +30,66 @@ import click from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed -from airflow_breeze.commands.main_command import main -from airflow_breeze.global_constants import ( - ALLOWED_TTY, - DEFAULT_PYTHON_MAJOR_MINOR_VERSION, -) -from airflow_breeze.params.build_ci_params import BuildCiParams -from airflow_breeze.params.doc_build_params import DocBuildParams -from airflow_breeze.params.shell_params import ShellParams -from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST -from airflow_breeze.utils.coertions import one_or_none_set -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_options import ( argument_doc_packages, - option_airflow_constraints_location, - option_airflow_constraints_mode_ci, - option_airflow_constraints_reference, option_airflow_extras, - option_airflow_skip_constraints, option_answer, option_backend, option_builder, - option_celery_broker, - option_celery_flower, option_database_isolation, option_db_reset, option_docker_host, option_downgrade_sqlalchemy, option_dry_run, - option_executor_shell, - option_executor_start_airflow, - option_force_build, option_forward_credentials, option_github_repository, option_image_tag_for_running, - option_include_mypy_volume, - option_install_selected_providers, option_installation_package_format, option_integration, - option_load_default_connection, - option_load_example_dags, option_max_time, option_mount_sources, option_mssql_version, option_mysql_version, - option_platform_single, option_postgres_version, option_project_name, - option_providers_constraints_location, - option_providers_constraints_mode_ci, - option_providers_constraints_reference, - option_providers_skip_constraints, option_python, - option_restart, option_run_db_tests_only, option_skip_db_tests, - option_skip_environment_initialization, - option_skip_image_upgrade_check, option_standalone_dag_processor, option_upgrade_boto, option_use_airflow_version, - option_use_packages_from_dist, option_verbose, - option_warn_image_upgrade_needed, ) +from airflow_breeze.commands.common_package_installation_options import ( + option_airflow_constraints_location, + option_airflow_constraints_mode_ci, + option_airflow_constraints_reference, + option_airflow_skip_constraints, + option_install_selected_providers, + option_providers_constraints_location, + option_providers_constraints_mode_ci, + option_providers_constraints_reference, + option_providers_skip_constraints, + option_use_packages_from_dist, +) +from airflow_breeze.commands.main_command import main +from airflow_breeze.global_constants import ( + ALLOWED_CELERY_BROKERS, + ALLOWED_EXECUTORS, + ALLOWED_TTY, + DEFAULT_ALLOWED_EXECUTOR, + DEFAULT_CELERY_BROKER, + DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + DOCKER_DEFAULT_PLATFORM, + SINGLE_PLATFORMS, + START_AIRFLOW_ALLOWED_EXECUTORS, + START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR, +) +from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.doc_build_params import DocBuildParams +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST +from airflow_breeze.utils.coertions import one_or_none_set from airflow_breeze.utils.console import get_console from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( @@ -108,6 +105,7 @@ from airflow_breeze.utils.path_utils import ( AIRFLOW_SOURCES_ROOT, cleanup_python_generated_files, ) +from airflow_breeze.utils.recording import generating_command_images from airflow_breeze.utils.run_utils import ( assert_pre_commit_installed, run_command, @@ -158,6 +156,63 @@ class TimerThread(threading.Thread): # Args it should have the same parameters. # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +option_celery_broker = click.option( + "--celery-broker", + type=click.Choice(ALLOWED_CELERY_BROKERS, case_sensitive=False), + help="Specify the celery message broker", + default=DEFAULT_CELERY_BROKER, + show_default=True, +) +option_celery_flower = click.option("--celery-flower", help="Start celery flower", is_flag=True) +option_executor_shell = click.option( + "--executor", + type=click.Choice(ALLOWED_EXECUTORS, case_sensitive=False), + help="Specify the executor to use with shell command.", + default=DEFAULT_ALLOWED_EXECUTOR, + show_default=True, +) +option_force_build = click.option( + "--force-build", help="Force image build no matter if it is determined as needed.", is_flag=True +) +option_include_mypy_volume = click.option( + "--include-mypy-volume", + help="Whether to include mounting of the mypy volume (useful for debugging mypy).", + is_flag=True, + envvar="INCLUDE_MYPY_VOLUME", +) +option_platform_single = click.option( + "--platform", + help="Platform for Airflow image.", + default=DOCKER_DEFAULT_PLATFORM if not generating_command_images() else SINGLE_PLATFORMS[0], + envvar="PLATFORM", + type=BetterChoice(SINGLE_PLATFORMS), +) +option_restart = click.option( + "--restart", + "--remove-orphans", + help="Restart all containers before entering shell (also removes orphan containers).", + is_flag=True, + envvar="RESTART", +) +option_skip_environment_initialization = click.option( + "--skip-environment-initialization", + help="Skip running breeze entrypoint initialization - no user output, no db checks.", + is_flag=True, + envvar="SKIP_ENVIRONMENT_INITIALIZATION", +) +option_skip_image_upgrade_check = click.option( + "--skip-image-upgrade-check", + help="Skip checking if the CI image is up to date.", + is_flag=True, + envvar="SKIP_IMAGE_UPGRADE_CHECK", +) +option_warn_image_upgrade_needed = click.option( + "--warn-image-upgrade-needed", + help="Warn when image upgrade is needed even if --skip-upgrade-check is used.", + is_flag=True, + envvar="WARN_IMAGE_UPGRADE_NEEDED", +) + @main.command() @click.argument("extra-args", nargs=-1, type=click.UNPROCESSED) @@ -339,6 +394,31 @@ def shell( sys.exit(result.returncode) +option_load_example_dags = click.option( + "-e", + "--load-example-dags", + help="Enable configuration to load example DAGs when starting Airflow.", + is_flag=True, + envvar="LOAD_EXAMPLES", +) + +option_load_default_connection = click.option( + "-c", + "--load-default-connections", + help="Enable configuration to load default connections when starting Airflow.", + is_flag=True, + envvar="LOAD_DEFAULT_CONNECTIONS", +) + +option_executor_start_airflow = click.option( + "--executor", + type=click.Choice(START_AIRFLOW_ALLOWED_EXECUTORS, case_sensitive=False), + help="Specify the executor to use with start-airflow command.", + default=START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR, + show_default=True, +) + + @main.command(name="start-airflow") @click.option( "--skip-asset-compilation", diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index d90a856d25..ce8924c9b0 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -28,12 +28,7 @@ from shlex import quote import click -from airflow_breeze.commands.production_image_commands import run_build_production_image -from airflow_breeze.global_constants import ALLOWED_EXECUTORS, ALLOWED_KUBERNETES_VERSIONS -from airflow_breeze.params.build_prod_params import BuildProdParams -from airflow_breeze.utils.ci_group import ci_group -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_options import ( option_debug_resources, option_dry_run, option_include_success_outputs, @@ -44,6 +39,11 @@ from airflow_breeze.utils.common_options import ( option_skip_cleanup, option_verbose, ) +from airflow_breeze.commands.production_image_commands import run_build_production_image +from airflow_breeze.global_constants import ALLOWED_EXECUTORS, ALLOWED_KUBERNETES_VERSIONS +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.utils.ci_group import ci_group +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.custom_param_types import CacheableChoice, CacheableDefault from airflow_breeze.utils.kubernetes_utils import ( @@ -111,92 +111,80 @@ option_executor = click.option( default=CacheableDefault(ALLOWED_EXECUTORS[0]), envvar="EXECUTOR", ) - -option_kubernetes_version = click.option( - "--kubernetes-version", - help="Kubernetes version used to create the KinD cluster of.", - type=CacheableChoice(ALLOWED_KUBERNETES_VERSIONS), - show_default=True, - default=CacheableDefault(ALLOWED_KUBERNETES_VERSIONS[0]), - envvar="KUBERNETES_VERSION", -) - -option_image_tag = click.option( - "-t", - "--image-tag", - help="Image tag used to build K8S image from.", - default="latest", - show_default=True, - envvar="IMAGE_TAG", -) - -option_wait_time_in_seconds = click.option( - "--wait-time-in-seconds", - help="Wait for Airflow webserver for specified number of seconds.", - type=click.IntRange(0), - default=120, - envvar="WAIT_TIME_IN_SECONDS", -) - -option_wait_time_in_seconds_0_default = click.option( - "--wait-time-in-seconds", - help="Wait for Airflow webserver for specified number of seconds.", - type=click.IntRange(0), - default=0, - envvar="WAIT_TIME_IN_SECONDS", -) - option_force_recreate_cluster = click.option( "--force-recreate-cluster", help="Force recreation of the cluster even if it is already created.", is_flag=True, envvar="FORCE_RECREATE_CLUSTER", ) - option_force_venv_setup = click.option( "--force-venv-setup", help="Force recreation of the virtualenv.", is_flag=True, envvar="FORCE_VENV_SETUP", ) - -option_use_standard_naming = click.option( - "--use-standard-naming", - help="Use standard naming.", - is_flag=True, - envvar="USE_STANDARD_NAMING", +option_image_tag = click.option( + "-t", + "--image-tag", + help="Image tag used to build K8S image from.", + default="latest", + show_default=True, + envvar="IMAGE_TAG", +) +option_kubernetes_version = click.option( + "--kubernetes-version", + help="Kubernetes version used to create the KinD cluster of.", + type=CacheableChoice(ALLOWED_KUBERNETES_VERSIONS), + show_default=True, + default=CacheableDefault(ALLOWED_KUBERNETES_VERSIONS[0]), + envvar="KUBERNETES_VERSION", +) +option_kubernetes_versions = click.option( + "--kubernetes-versions", + help="Kubernetes versions used to run in parallel (space separated).", + type=str, + show_default=True, + default=" ".join(ALLOWED_KUBERNETES_VERSIONS), + envvar="KUBERNETES_VERSIONS", ) - option_multi_namespace_mode = click.option( "--multi-namespace-mode", help="Use multi namespace mode.", is_flag=True, envvar="MULTI_NAMESPACE_MODE", ) - option_rebuild_base_image = click.option( "--rebuild-base-image", help="Rebuilds base Airflow image before building K8S image.", is_flag=True, envvar="REBUILD_BASE_IMAGE", ) - -option_kubernetes_versions = click.option( - "--kubernetes-versions", - help="Kubernetes versions used to run in parallel (space separated).", - type=str, - show_default=True, - default=" ".join(ALLOWED_KUBERNETES_VERSIONS), - envvar="KUBERNETES_VERSIONS", -) - option_upgrade = click.option( "--upgrade", help="Upgrade Helm Chart rather than installing it.", is_flag=True, envvar="UPGRADE", ) - +option_use_standard_naming = click.option( + "--use-standard-naming", + help="Use standard naming.", + is_flag=True, + envvar="USE_STANDARD_NAMING", +) +option_wait_time_in_seconds = click.option( + "--wait-time-in-seconds", + help="Wait for Airflow webserver for specified number of seconds.", + type=click.IntRange(0), + default=120, + envvar="WAIT_TIME_IN_SECONDS", +) +option_wait_time_in_seconds_0_default = click.option( + "--wait-time-in-seconds", + help="Wait for Airflow webserver for specified number of seconds.", + type=click.IntRange(0), + default=0, + envvar="WAIT_TIME_IN_SECONDS", +) option_parallelism_cluster = click.option( "--parallelism", help="Maximum number of processes to use while running the operation in parallel for cluster operations.", diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index b931ed36dc..9e09056948 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -23,11 +23,7 @@ import sys from typing import TYPE_CHECKING, Any from airflow_breeze.commands.ci_image_commands import ci_image -from airflow_breeze.commands.production_image_commands import prod_image -from airflow_breeze.commands.testing_commands import group_for_testing -from airflow_breeze.configure_rich_click import click -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_options import ( option_answer, option_backend, option_builder, @@ -47,6 +43,10 @@ from airflow_breeze.utils.common_options import ( option_standalone_dag_processor, option_verbose, ) +from airflow_breeze.commands.production_image_commands import prod_image +from airflow_breeze.commands.testing_commands import group_for_testing +from airflow_breeze.configure_rich_click import click +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import get_console from airflow_breeze.utils.docker_command_utils import remove_docker_networks diff --git a/dev/breeze/src/airflow_breeze/commands/minor_release_command.py b/dev/breeze/src/airflow_breeze/commands/minor_release_command.py index 2fcbd621ac..fe43ecc7e6 100644 --- a/dev/breeze/src/airflow_breeze/commands/minor_release_command.py +++ b/dev/breeze/src/airflow_breeze/commands/minor_release_command.py @@ -20,8 +20,8 @@ import os import click +from airflow_breeze.commands.common_options import option_answer from airflow_breeze.commands.release_management_group import release_management -from airflow_breeze.utils.common_options import option_answer from airflow_breeze.utils.confirm import confirm_action from airflow_breeze.utils.console import console_print from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 45e6bb8c75..b9bb0f8de8 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -21,11 +21,7 @@ from copy import deepcopy import click -from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS -from airflow_breeze.params.build_prod_params import BuildProdParams -from airflow_breeze.utils.ci_group import ci_group -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_image_options import ( option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, @@ -35,46 +31,54 @@ from airflow_breeze.utils.common_options import ( option_additional_runtime_apt_command, option_additional_runtime_apt_deps, option_additional_runtime_apt_env, - option_airflow_constraints_location, - option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, - option_answer, option_build_progress, - option_builder, - option_commit_sha, option_debian_version, - option_debug_resources, option_dev_apt_command, option_dev_apt_deps, option_docker_cache, - option_docker_host, - option_dry_run, - option_github_repository, - option_github_token, - option_image_name, option_image_tag_for_building, option_image_tag_for_pulling, option_image_tag_for_verifying, - option_include_success_outputs, option_install_providers_from_sources, - option_parallelism, option_platform_multiple, option_prepare_buildx_cache, option_pull, option_push, - option_python, option_python_image, - option_python_versions, - option_run_in_parallel, option_runtime_apt_command, option_runtime_apt_deps, - option_skip_cleanup, option_tag_as_latest, - option_verbose, option_verify, - option_version_suffix_for_pypi, option_wait_for_image, ) +from airflow_breeze.commands.common_options import ( + option_answer, + option_builder, + option_commit_sha, + option_debug_resources, + option_docker_host, + option_dry_run, + option_github_repository, + option_github_token, + option_image_name, + option_include_success_outputs, + option_parallelism, + option_python, + option_python_versions, + option_run_in_parallel, + option_skip_cleanup, + option_verbose, + option_version_suffix_for_pypi, +) +from airflow_breeze.commands.common_package_installation_options import ( + option_airflow_constraints_location, + option_airflow_constraints_mode_prod, +) +from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.utils.ci_group import ci_group +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index 5fba66b5fd..f88388a22a 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -20,8 +20,8 @@ import os import click +from airflow_breeze.commands.common_options import option_answer from airflow_breeze.commands.release_management_group import release_management -from airflow_breeze.utils.common_options import option_answer from airflow_breeze.utils.confirm import confirm_action from airflow_breeze.utils.console import console_print from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT diff --git a/dev/breeze/src/airflow_breeze/commands/release_command.py b/dev/breeze/src/airflow_breeze/commands/release_command.py index a7ccfa1d6c..4e2531332e 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_command.py @@ -20,8 +20,8 @@ import os import click +from airflow_breeze.commands.common_options import option_answer from airflow_breeze.commands.release_management_group import release_management -from airflow_breeze.utils.common_options import option_answer from airflow_breeze.utils.confirm import confirm_action from airflow_breeze.utils.console import console_print from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index ff3f9bf9c7..8eb7038694 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -36,9 +36,45 @@ from rich.progress import Progress from rich.syntax import Syntax from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed +from airflow_breeze.commands.common_options import ( + argument_doc_packages, + option_airflow_extras, + option_answer, + option_commit_sha, + option_debug_resources, + option_dry_run, + option_github_repository, + option_historical_python_version, + option_image_tag_for_running, + option_include_success_outputs, + option_installation_package_format, + option_mount_sources, + option_parallelism, + option_python, + option_python_versions, + option_run_in_parallel, + option_skip_cleanup, + option_use_airflow_version, + option_verbose, + option_version_suffix_for_pypi, +) +from airflow_breeze.commands.common_package_installation_options import ( + option_airflow_constraints_location, + option_airflow_constraints_mode_ci, + option_airflow_constraints_mode_update, + option_airflow_constraints_reference, + option_airflow_skip_constraints, + option_install_selected_providers, + option_providers_constraints_location, + option_providers_constraints_mode_ci, + option_providers_constraints_reference, + option_providers_skip_constraints, + option_use_packages_from_dist, +) from airflow_breeze.commands.release_management_group import release_management from airflow_breeze.global_constants import ( ALLOWED_DEBIAN_VERSIONS, + ALLOWED_PACKAGE_FORMATS, ALLOWED_PLATFORMS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, APACHE_AIRFLOW_GITHUB_REPOSITORY, @@ -63,47 +99,9 @@ from airflow_breeze.utils.add_back_references import ( start_generating_back_references, ) from airflow_breeze.utils.ci_group import ci_group -from airflow_breeze.utils.common_options import ( - argument_doc_packages, - argument_provider_packages, - option_airflow_constraints_location, - option_airflow_constraints_mode_ci, - option_airflow_constraints_mode_update, - option_airflow_constraints_reference, - option_airflow_extras, - option_airflow_site_directory, - option_airflow_skip_constraints, - option_answer, - option_chicken_egg_providers, - option_commit_sha, - option_debug_resources, - option_directory, - option_dry_run, - option_github_repository, - option_historical_python_version, - option_image_tag_for_running, - option_include_success_outputs, - option_install_selected_providers, - option_installation_package_format, - option_mount_sources, - option_package_format, - option_parallelism, - option_providers_constraints_location, - option_providers_constraints_mode_ci, - option_providers_constraints_reference, - option_providers_skip_constraints, - option_python, - option_python_versions, - option_run_in_parallel, - option_skip_cleanup, - option_use_airflow_version, - option_use_packages_from_dist, - option_verbose, - option_version_suffix_for_pypi, -) from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import MessageType, Output, get_console -from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( check_remote_ghcr_io_commands, execute_command_in_shell, @@ -150,6 +148,49 @@ from airflow_breeze.utils.run_utils import ( from airflow_breeze.utils.shared_options import get_dry_run, get_verbose from airflow_breeze.utils.versions import is_pre_release +argument_provider_packages = click.argument( + "provider_packages", + nargs=-1, + required=False, + type=NotVerifiedBetterChoice(get_available_packages()), +) +option_airflow_site_directory = click.option( + "-a", + "--airflow-site-directory", + envvar="AIRFLOW_SITE_DIRECTORY", + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + help="Local directory path of cloned airflow-site repo.", + required=True, +) +option_chicken_egg_providers = click.option( + "--chicken-egg-providers", + default="", + help="List of chicken-egg provider packages - " + "those that have airflow_version >= current_version and should " + "be installed in CI from locally built packages with >= current_version.dev0 ", + envvar="CHICKEN_EGG_PROVIDERS", +) +option_debug_release_management = click.option( + "--debug", + is_flag=True, + help="Drop user in shell instead of running the command. Useful for debugging.", + envvar="DEBUG", +) +option_directory = click.option( + "--directory", + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + required=True, + help="Directory to clean the provider artifacts from.", +) +option_package_format = click.option( + "--package-format", + type=BetterChoice(ALLOWED_PACKAGE_FORMATS), + help="Format of packages.", + default=ALLOWED_PACKAGE_FORMATS[0], + show_default=True, + envvar="PACKAGE_FORMAT", +) + if TYPE_CHECKING: from packaging.version import Version diff --git a/dev/breeze/src/airflow_breeze/commands/sbom_commands.py b/dev/breeze/src/airflow_breeze/commands/sbom_commands.py index ef53479659..92c3720039 100644 --- a/dev/breeze/src/airflow_breeze/commands/sbom_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/sbom_commands.py @@ -24,6 +24,17 @@ from pathlib import Path import click +from airflow_breeze.commands.common_options import ( + option_answer, + option_debug_resources, + option_dry_run, + option_historical_python_version, + option_include_success_outputs, + option_parallelism, + option_run_in_parallel, + option_skip_cleanup, + option_verbose, +) from airflow_breeze.global_constants import ( AIRFLOW_PYTHON_COMPATIBILITY_MATRIX, ALL_HISTORICAL_PYTHON_VERSIONS, @@ -41,17 +52,6 @@ from airflow_breeze.utils.cdxgen import ( ) from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( - option_answer, - option_debug_resources, - option_dry_run, - option_historical_python_version, - option_include_success_outputs, - option_parallelism, - option_run_in_parallel, - option_skip_cleanup, - option_verbose, -) from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import get_console from airflow_breeze.utils.custom_param_types import BetterChoice diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands.py b/dev/breeze/src/airflow_breeze/commands/setup_commands.py index d7d1e2db9d..e9dbd670b3 100644 --- a/dev/breeze/src/airflow_breeze/commands/setup_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands.py @@ -31,10 +31,7 @@ from click import Context from rich.console import Console from airflow_breeze import NAME, VERSION -from airflow_breeze.commands.main_command import main -from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_options import ( option_answer, option_backend, option_dry_run, @@ -44,6 +41,9 @@ from airflow_breeze.utils.common_options import ( option_python, option_verbose, ) +from airflow_breeze.commands.main_command import main +from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import get_console, get_stderr_console from airflow_breeze.utils.custom_param_types import BetterChoice diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 3c07d619bf..c7cb5bdccb 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -25,22 +25,12 @@ import click from click import IntRange from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed -from airflow_breeze.global_constants import ( - ALLOWED_HELM_TEST_PACKAGES, -) -from airflow_breeze.params.build_prod_params import BuildProdParams -from airflow_breeze.params.shell_params import ShellParams -from airflow_breeze.utils.ci_group import ci_group -from airflow_breeze.utils.click_utils import BreezeGroup -from airflow_breeze.utils.common_options import ( +from airflow_breeze.commands.common_options import ( option_backend, - option_collect_only, option_db_reset, option_debug_resources, option_downgrade_sqlalchemy, option_dry_run, - option_enable_coverage, - option_excluded_parallel_test_types, option_github_repository, option_image_name, option_image_tag_for_running, @@ -49,26 +39,28 @@ from airflow_breeze.utils.common_options import ( option_mount_sources, option_mssql_version, option_mysql_version, - option_parallel_test_types, option_parallelism, option_postgres_version, option_python, - option_remove_arm_packages, option_run_db_tests_only, option_run_in_parallel, option_skip_cleanup, option_skip_db_tests, - option_skip_docker_compose_down, - option_skip_provider_tests, - option_test_timeout, - option_test_type, option_upgrade_boto, option_use_airflow_version, - option_use_xdist, option_verbose, ) +from airflow_breeze.global_constants import ( + ALLOWED_HELM_TEST_PACKAGES, + ALLOWED_PARALLEL_TEST_TYPE_CHOICES, + ALLOWED_TEST_TYPE_CHOICES, +) +from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.ci_group import ci_group +from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.console import Output, get_console -from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( fix_ownership_using_docker, perform_environment_checks, @@ -87,6 +79,7 @@ from airflow_breeze.utils.run_tests import ( run_docker_compose_tests, ) from airflow_breeze.utils.run_utils import get_filesystem_type, run_command +from airflow_breeze.utils.selective_checks import ALL_CI_SELECTIVE_TEST_TYPES LOW_MEMORY_CONDITION = 8 * 1024 * 1024 * 1024 @@ -371,6 +364,78 @@ def _verify_parallelism_parameters( sys.exit(1) +option_collect_only = click.option( + "--collect-only", + help="Collect tests only, do not run them.", + is_flag=True, + envvar="COLLECT_ONLY", +) +option_enable_coverage = click.option( + "--enable-coverage", + help="Enable coverage capturing for tests in the form of XML files", + is_flag=True, + envvar="ENABLE_COVERAGE", +) +option_excluded_parallel_test_types = click.option( + "--excluded-parallel-test-types", + help="Space separated list of test types that will be excluded from parallel tes runs.", + default="", + show_default=True, + envvar="EXCLUDED_PARALLEL_TEST_TYPES", + type=NotVerifiedBetterChoice(ALLOWED_PARALLEL_TEST_TYPE_CHOICES), +) +option_parallel_test_types = click.option( + "--parallel-test-types", + help="Space separated list of test types used for testing in parallel", + default=ALL_CI_SELECTIVE_TEST_TYPES, + show_default=True, + envvar="PARALLEL_TEST_TYPES", + type=NotVerifiedBetterChoice(ALLOWED_PARALLEL_TEST_TYPE_CHOICES), +) +option_skip_docker_compose_down = click.option( + "--skip-docker-compose-down", + help="Skips running docker-compose down after tests", + is_flag=True, + envvar="SKIP_DOCKER_COMPOSE_DOWN", +) +option_skip_provider_tests = click.option( + "--skip-provider-tests", + help="Skip provider tests", + is_flag=True, + envvar="SKIP_PROVIDER_TESTS", +) +option_test_timeout = click.option( + "--test-timeout", + help="Test timeout in seconds. Set the pytest setup, execution and teardown timeouts to this value", + default=60, + envvar="TEST_TIMEOUT", + type=IntRange(min=0), + show_default=True, +) +option_test_type = click.option( + "--test-type", + help="Type of test to run. With Providers, you can specify tests of which providers " + "should be run: `Providers[airbyte,http]` or " + "excluded from the full test suite: `Providers[-amazon,google]`", + default="Default", + envvar="TEST_TYPE", + show_default=True, + type=NotVerifiedBetterChoice(ALLOWED_TEST_TYPE_CHOICES), +) +option_use_xdist = click.option( + "--use-xdist", + help="Use xdist plugin for pytest", + is_flag=True, + envvar="USE_XDIST", +) +option_remove_arm_packages = click.option( + "--remove-arm-packages", + help="Removes arm packages from the image to test if ARM collection works", + is_flag=True, + envvar="REMOVE_ARM_PACKAGES", +) + + @group_for_testing.command( name="tests", help="Run the specified unit tests. This is a low level testing command that allows you to run " diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py deleted file mode 100644 index bf6288658e..0000000000 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ /dev/null @@ -1,844 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import multiprocessing as mp - -import click -from click import IntRange - -from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH -from airflow_breeze.global_constants import ( - ALL_HISTORICAL_PYTHON_VERSIONS, - ALLOWED_BACKENDS, - ALLOWED_BUILD_CACHE, - ALLOWED_BUILD_PROGRESS, - ALLOWED_CELERY_BROKERS, - ALLOWED_CONSTRAINTS_MODES_CI, - ALLOWED_CONSTRAINTS_MODES_PROD, - ALLOWED_DEBIAN_VERSIONS, - ALLOWED_DOCKER_COMPOSE_PROJECTS, - ALLOWED_EXECUTORS, - ALLOWED_INSTALLATION_PACKAGE_FORMATS, - ALLOWED_MOUNT_OPTIONS, - ALLOWED_MSSQL_VERSIONS, - ALLOWED_MYSQL_VERSIONS, - ALLOWED_PACKAGE_FORMATS, - ALLOWED_PARALLEL_TEST_TYPE_CHOICES, - ALLOWED_PLATFORMS, - ALLOWED_POSTGRES_VERSIONS, - ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, - ALLOWED_TEST_TYPE_CHOICES, - ALLOWED_USE_AIRFLOW_VERSIONS, - APACHE_AIRFLOW_GITHUB_REPOSITORY, - AUTOCOMPLETE_INTEGRATIONS, - DEFAULT_ALLOWED_EXECUTOR, - DEFAULT_CELERY_BROKER, - DOCKER_DEFAULT_PLATFORM, - SINGLE_PLATFORMS, - START_AIRFLOW_ALLOWED_EXECUTORS, - START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR, -) -from airflow_breeze.utils.custom_param_types import ( - AnswerChoice, - BetterChoice, - CacheableChoice, - CacheableDefault, - DryRunOption, - MySQLBackendVersionType, - NotVerifiedBetterChoice, - UseAirflowVersionType, - VerboseOption, -) -from airflow_breeze.utils.packages import get_available_packages -from airflow_breeze.utils.recording import generating_command_images -from airflow_breeze.utils.selective_checks import ALL_CI_SELECTIVE_TEST_TYPES - - -def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, value): - from click.core import ParameterSource - - if ( - ctx.parent - and option.name in ctx.parent.params - and ctx.get_parameter_source(option.name) - in ( - ParameterSource.DEFAULT, - ParameterSource.DEFAULT_MAP, - ) - ): - # Current value is the default, use the parent's value (i.e. for `breeze - # # -v static-checks` respect the "global" option) - value = ctx.parent.params[option.name] - return value - - -option_verbose = click.option( - "-v", - "--verbose", - is_flag=True, - help="Print verbose information about performed steps.", - envvar="VERBOSE", - metavar="BOOLEAN", - expose_value=False, - type=VerboseOption(), - callback=_set_default_from_parent, -) -option_dry_run = click.option( - "-D", - "--dry-run", - is_flag=True, - help="If dry-run is set, commands are only printed, not executed.", - envvar="DRY_RUN", - metavar="BOOLEAN", - expose_value=False, - type=DryRunOption(), - callback=_set_default_from_parent, -) -option_answer = click.option( - "-a", - "--answer", - type=AnswerChoice(["y", "n", "q", "yes", "no", "quit"]), - help="Force answer to questions.", - envvar="ANSWER", - expose_value=False, - callback=_set_default_from_parent, -) -option_github_repository = click.option( - "-g", - "--github-repository", - help="GitHub repository used to pull, push run images.", - default=APACHE_AIRFLOW_GITHUB_REPOSITORY, - show_default=True, - envvar="GITHUB_REPOSITORY", - callback=_set_default_from_parent, -) -option_python = click.option( - "-p", - "--python", - type=CacheableChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS), - default=CacheableDefault(value=ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]), - show_default=True, - help="Python major/minor version used in Airflow image for images.", - envvar="PYTHON_MAJOR_MINOR_VERSION", -) -option_debian_version = click.option( - "--debian-version", - type=BetterChoice(ALLOWED_DEBIAN_VERSIONS), - default=ALLOWED_DEBIAN_VERSIONS[0], - show_default=True, - help="Debian version used in Airflow image as base for building images.", - envvar="DEBIAN_VERSION", -) -option_backend = click.option( - "-b", - "--backend", - type=CacheableChoice(ALLOWED_BACKENDS), - default=CacheableDefault(value=ALLOWED_BACKENDS[0]), - show_default=True, - help="Database backend to use. If 'none' is selected, breeze starts with invalid DB configuration " - "and no database and any attempts to connect to Airflow DB will fail.", - envvar="BACKEND", -) -option_integration = click.option( - "--integration", - help="Integration(s) to enable when running (can be more than one).", - type=BetterChoice(AUTOCOMPLETE_INTEGRATIONS), - multiple=True, -) -option_postgres_version = click.option( - "-P", - "--postgres-version", - type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS), - default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]), - show_default=True, - help="Version of Postgres used.", -) -option_mysql_version = click.option( - "-M", - "--mysql-version", - help="Version of MySQL used.", - type=MySQLBackendVersionType(ALLOWED_MYSQL_VERSIONS), - default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]), - show_default=True, -) -option_mssql_version = click.option( - "-S", - "--mssql-version", - help="Version of MsSQL used.", - type=CacheableChoice(ALLOWED_MSSQL_VERSIONS), - default=CacheableDefault(ALLOWED_MSSQL_VERSIONS[0]), - show_default=True, -) -option_forward_credentials = click.option( - "-f", "--forward-credentials", help="Forward local credentials to container when running.", is_flag=True -) -option_use_airflow_version = click.option( - "--use-airflow-version", - help="Use (reinstall at entry) Airflow version from PyPI. It can also be version (to install from PyPI), " - "`none`, `wheel`, or `sdist` to install from `dist` folder, or VCS URL to install from " - "(https://pip.pypa.io/en/stable/topics/vcs-support/). Implies --mount-sources `remove`.", - type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS), - envvar="USE_AIRFLOW_VERSION", -) -option_airflow_extras = click.option( - "--airflow-extras", - help="Airflow extras to install when --use-airflow-version is used", - default="", - show_default=True, - envvar="AIRFLOW_EXTRAS", -) -option_mount_sources = click.option( - "--mount-sources", - type=BetterChoice(ALLOWED_MOUNT_OPTIONS), - default=ALLOWED_MOUNT_OPTIONS[0], - show_default=True, - envvar="MOUNT_SOURCES", - help="Choose scope of local sources that should be mounted, skipped, or removed (default = selected).", -) -option_force_build = click.option( - "--force-build", help="Force image build no matter if it is determined as needed.", is_flag=True -) -option_db_reset = click.option( - "-d", - "--db-reset", - help="Reset DB when entering the container.", - is_flag=True, - envvar="DB_RESET", -) -option_use_packages_from_dist = click.option( - "--use-packages-from-dist", - is_flag=True, - help="Install all found packages (--package-format determines type) from 'dist' folder " - "when entering breeze.", - envvar="USE_PACKAGES_FROM_DIST", -) -option_docker_cache = click.option( - "-c", - "--docker-cache", - help="Cache option for image used during the build.", - default=ALLOWED_BUILD_CACHE[0], - show_default=True, - type=BetterChoice(ALLOWED_BUILD_CACHE), -) -option_github_token = click.option( - "--github-token", - help="The token used to authenticate to GitHub.", - envvar="GITHUB_TOKEN", -) -option_image_tag_for_pulling = click.option( - "-t", - "--image-tag", - help="Tag of the image which is used to pull the image.", - show_default=True, - default="latest", - envvar="IMAGE_TAG", -) -option_image_tag_for_building = click.option( - "--image-tag", - help="Tag the image after building it.", - show_default=True, - default="latest", - envvar="IMAGE_TAG", -) -option_image_tag_for_running = click.option( - "--image-tag", - help="Tag of the image which is used to run the image (implies --mount-sources=skip).", - show_default=True, - default="latest", - envvar="IMAGE_TAG", -) -option_image_tag_for_verifying = click.option( - "-t", - "--image-tag", - help="Tag of the image when verifying it.", - show_default=True, - default="latest", - envvar="IMAGE_TAG", -) -option_image_name = click.option( - "-n", "--image-name", help="Name of the image to verify (overrides --python and --image-tag)." -) -option_platform_multiple = click.option( - "--platform", - help="Platform for Airflow image.", - envvar="PLATFORM", - default=DOCKER_DEFAULT_PLATFORM if not generating_command_images() else "linux/amd64", - type=BetterChoice(ALLOWED_PLATFORMS), -) -option_platform_single = click.option( - "--platform", - help="Platform for Airflow image.", - envvar="PLATFORM", - default=DOCKER_DEFAULT_PLATFORM if not generating_command_images() else "linux/amd64", - type=BetterChoice(SINGLE_PLATFORMS), -) -option_upgrade_to_newer_dependencies = click.option( - "-u", - "--upgrade-to-newer-dependencies", - is_flag=True, - help="When set, upgrade all PIP packages to latest.", - envvar="UPGRADE_TO_NEWER_DEPENDENCIES", -) -option_upgrade_on_failure = click.option( - "--upgrade-on-failure", - is_flag=True, - help="When set, attempt to run upgrade to newer dependencies when regular build fails.", - envvar="UPGRADE_ON_FAILURE", -) -option_additional_airflow_extras = click.option( - "--additional-airflow-extras", - help="Additional extra package while installing Airflow in the image.", - envvar="ADDITIONAL_AIRFLOW_EXTRAS", -) -option_additional_dev_apt_deps = click.option( - "--additional-dev-apt-deps", - help="Additional apt dev dependencies to use when building the images.", - envvar="ADDITIONAL_DEV_APT_DEPS", -) -option_additional_runtime_apt_deps = click.option( - "--additional-runtime-apt-deps", - help="Additional apt runtime dependencies to use when building the images.", - envvar="ADDITIONAL_RUNTIME_APT_DEPS", -) -option_additional_python_deps = click.option( - "--additional-python-deps", - help="Additional python dependencies to use when building the images.", - envvar="ADDITIONAL_PYTHON_DEPS", -) -option_additional_dev_apt_command = click.option( - "--additional-dev-apt-command", - help="Additional command executed before dev apt deps are installed.", - envvar="ADDITIONAL_DEV_APT_COMMAND", -) -option_additional_runtime_apt_command = click.option( - "--additional-runtime-apt-command", - help="Additional command executed before runtime apt deps are installed.", - envvar="ADDITIONAL_RUNTIME_APT_COMMAND", -) -option_additional_dev_apt_env = click.option( - "--additional-dev-apt-env", - help="Additional environment variables set when adding dev dependencies.", - envvar="ADDITIONAL_DEV_APT_ENV", -) -option_additional_runtime_apt_env = click.option( - "--additional-runtime-apt-env", - help="Additional environment variables set when adding runtime dependencies.", - envvar="ADDITIONAL_RUNTIME_APT_ENV", -) -option_dev_apt_command = click.option( - "--dev-apt-command", - help="Command executed before dev apt deps are installed.", - envvar="DEV_APT_COMMAND", -) -option_dev_apt_deps = click.option( - "--dev-apt-deps", - help="Apt dev dependencies to use when building the images.", - envvar="DEV_APT_DEPS", -) -option_runtime_apt_command = click.option( - "--runtime-apt-command", - help="Command executed before runtime apt deps are installed.", - envvar="RUNTIME_APT_COMMAND", -) -option_runtime_apt_deps = click.option( - "--runtime-apt-deps", - help="Apt runtime dependencies to use when building the images.", - envvar="RUNTIME_APT_DEPS", -) -option_prepare_buildx_cache = click.option( - "--prepare-buildx-cache", - help="Prepares build cache (this is done as separate per-platform steps instead of building the image).", - is_flag=True, - envvar="PREPARE_BUILDX_CACHE", -) -option_push = click.option( - "--push", - help="Push image after building it.", - is_flag=True, - envvar="PUSH", -) -option_wait_for_image = click.option( - "--wait-for-image", - help="Wait until image is available.", - is_flag=True, - envvar="WAIT_FOR_IMAGE", -) -option_tag_as_latest = click.option( - "--tag-as-latest", - help="Tags the image as latest and update checksum of all files after pulling. " - "Useful when you build or pull image with --image-tag.", - is_flag=True, - envvar="TAG_AS_LATEST", -) -option_verify = click.option( - "--verify", - help="Verify image.", - is_flag=True, - envvar="VERIFY", -) -option_additional_pip_install_flags = click.option( - "--additional-pip-install-flags", - help="Additional flags added to `pip install` commands (except reinstalling `pip` itself).", - envvar="ADDITIONAL_PIP_INSTALL_FLAGS", -) - -option_install_providers_from_sources = click.option( - "--install-providers-from-sources", - help="Install providers from sources when installing.", - is_flag=True, - envvar="INSTALL_PROVIDERS_FROM_SOURCES", -) -option_load_example_dags = click.option( - "-e", - "--load-example-dags", - help="Enable configuration to load example DAGs when starting Airflow.", - is_flag=True, - envvar="LOAD_EXAMPLES", -) -option_load_default_connection = click.option( - "-c", - "--load-default-connections", - help="Enable configuration to load default connections when starting Airflow.", - is_flag=True, - envvar="LOAD_DEFAULT_CONNECTIONS", -) -option_version_suffix_for_pypi = click.option( - "--version-suffix-for-pypi", - help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", - envvar="VERSION_SUFFIX_FOR_PYPI", - default="", -) -option_version_suffix_for_pypi_ci = click.option( - "--version-suffix-for-pypi", - help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", - default="dev0", - show_default=True, - envvar="VERSION_SUFFIX_FOR_PYPI", -) -option_package_format = click.option( - "--package-format", - type=BetterChoice(ALLOWED_PACKAGE_FORMATS), - help="Format of packages.", - default=ALLOWED_PACKAGE_FORMATS[0], - show_default=True, - envvar="PACKAGE_FORMAT", -) -option_installation_package_format = click.option( - "--package-format", - type=BetterChoice(ALLOWED_INSTALLATION_PACKAGE_FORMATS), - help="Format of packages that should be installed from dist.", - default=ALLOWED_INSTALLATION_PACKAGE_FORMATS[0], - show_default=True, - envvar="PACKAGE_FORMAT", -) -option_python_versions = click.option( - "--python-versions", - help="Space separated list of python versions used for build with multiple versions.", - default=" ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS), - show_default=True, - envvar="PYTHON_VERSIONS", -) -option_run_in_parallel = click.option( - "--run-in-parallel", - help="Run the operation in parallel on all or selected subset of parameters.", - is_flag=True, - envvar="RUN_IN_PARALLEL", -) -option_parallelism = click.option( - "--parallelism", - help="Maximum number of processes to use while running the operation in parallel.", - type=click.IntRange(1, mp.cpu_count() * 3 if not generating_command_images() else 8), - default=mp.cpu_count() if not generating_command_images() else 4, - envvar="PARALLELISM", - show_default=True, -) -argument_provider_packages = click.argument( - "provider_packages", - nargs=-1, - required=False, - type=NotVerifiedBetterChoice(get_available_packages()), -) -argument_doc_packages = click.argument( - "doc_packages", - nargs=-1, - required=False, - type=NotVerifiedBetterChoice( - get_available_packages(include_non_provider_doc_packages=True, include_all_providers=True) - ), -) -option_airflow_constraints_reference = click.option( - "--airflow-constraints-reference", - help="Constraint reference to use for airflow installation (used in calculated constraints URL). " - "Can be 'default' in which case the default constraints-reference is used.", - envvar="AIRFLOW_CONSTRAINTS_REFERENCE", -) -option_airflow_constraints_location = click.option( - "--airflow-constraints-location", - type=str, - help="Location of airflow constraints to use (remote URL or local context file).", - envvar="AIRFLOW_CONSTRAINTS_LOCATION", -) -option_airflow_constraints_mode_update = click.option( - "--airflow-constraints-mode", - type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), - required=False, - help="Limit constraint update to only selected constraint mode - if selected.", -) -option_airflow_constraints_mode_prod = click.option( - "--airflow-constraints-mode", - type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_PROD), - default=ALLOWED_CONSTRAINTS_MODES_PROD[0], - show_default=True, - help="Mode of constraints for Airflow for PROD image building.", -) -option_airflow_constraints_mode_ci = click.option( - "--airflow-constraints-mode", - type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), - default=ALLOWED_CONSTRAINTS_MODES_CI[0], - show_default=True, - help="Mode of constraints for Airflow for CI image building.", -) -option_providers_constraints_reference = click.option( - "--providers-constraints-reference", - help="Constraint reference to use for providers installation (used in calculated constraints URL). " - "Can be 'default' in which case the default constraints-reference is used.", - envvar="PROVIDERS_CONSTRAINTS_REFERENCE", -) -option_providers_constraints_location = click.option( - "--providers-constraints-location", - type=str, - help="Location of providers constraints to use (remote URL or local context file).", - envvar="PROVIDERS_CONSTRAINTS_LOCATION", -) -option_providers_constraints_mode_prod = click.option( - "--providers-constraints-mode", - type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_PROD), - default=ALLOWED_CONSTRAINTS_MODES_PROD[0], - show_default=True, - help="Mode of constraints for Providers for PROD image building.", -) -option_providers_constraints_mode_ci = click.option( - "--providers-constraints-mode", - type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), - default=ALLOWED_CONSTRAINTS_MODES_CI[0], - show_default=True, - help="Mode of constraints for Providers for CI image building.", -) -option_providers_skip_constraints = click.option( - "--providers-skip-constraints", - is_flag=True, - help="Do not use constraints when installing providers.", - envvar="PROVIDERS_SKIP_CONSTRAINTS", -) -option_airflow_constraints_reference_build = click.option( - "--airflow-constraints-reference", - default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, - help="Constraint reference to use when building the image.", - envvar="AIRFLOW_CONSTRAINTS_REFERENCE", -) -option_pull = click.option( - "--pull", - help="Pull image is missing before attempting to verify it.", - is_flag=True, - envvar="PULL", -) -option_python_image = click.option( - "--python-image", - help="If specified this is the base python image used to build the image. " - "Should be something like: python:VERSION-slim-bookworm.", - envvar="PYTHON_IMAGE", -) -option_docker_host = click.option( - "--docker-host", - help="Optional - docker host to use when running docker commands. " - "When set, the `--builder` option is ignored when building images.", - envvar="DOCKER_HOST", -) -option_builder = click.option( - "--builder", - help="Buildx builder used to perform `docker buildx build` commands.", - envvar="BUILDER", - show_default=True, - default="autodetect", -) -option_build_progress = click.option( - "--build-progress", - help="Build progress.", - type=BetterChoice(ALLOWED_BUILD_PROGRESS), - envvar="BUILD_PROGRESS", - show_default=True, - default=ALLOWED_BUILD_PROGRESS[0], -) -option_include_success_outputs = click.option( - "--include-success-outputs", - help="Whether to include outputs of successful parallel runs (skipped by default).", - is_flag=True, - envvar="INCLUDE_SUCCESS_OUTPUTS", -) -option_skip_cleanup = click.option( - "--skip-cleanup", - help="Skip cleanup of temporary files created during parallel run.", - is_flag=True, - envvar="SKIP_CLEANUP", -) - -option_directory = click.option( - "--directory", - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), - required=True, - help="Directory to clean the provider artifacts from.", -) - -option_include_mypy_volume = click.option( - "--include-mypy-volume", - help="Whether to include mounting of the mypy volume (useful for debugging mypy).", - is_flag=True, - envvar="INCLUDE_MYPY_VOLUME", -) -option_max_time = click.option( - "--max-time", - help="Maximum time that the command should take - if it takes longer, the command will fail.", - type=click.IntRange(min=1), - envvar="MAX_TIME", - callback=_set_default_from_parent, -) -option_debug_resources = click.option( - "--debug-resources", - is_flag=True, - help="Whether to show resource information while running in parallel.", - envvar="DEBUG_RESOURCES", -) -option_executor_start_airflow = click.option( - "--executor", - type=click.Choice(START_AIRFLOW_ALLOWED_EXECUTORS, case_sensitive=False), - help="Specify the executor to use with start-airflow command.", - default=START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR, - show_default=True, -) -option_executor_shell = click.option( - "--executor", - type=click.Choice(ALLOWED_EXECUTORS, case_sensitive=False), - help="Specify the executor to use with shell command.", - default=DEFAULT_ALLOWED_EXECUTOR, - show_default=True, -) -option_celery_broker = click.option( - "--celery-broker", - type=click.Choice(ALLOWED_CELERY_BROKERS, case_sensitive=False), - help="Specify the celery message broker", - default=DEFAULT_CELERY_BROKER, - show_default=True, -) -option_celery_flower = click.option("--celery-flower", help="Start celery flower", is_flag=True) -option_standalone_dag_processor = click.option( - "--standalone-dag-processor", - help="Run standalone dag processor for start-airflow.", - is_flag=True, - envvar="STANDALONE_DAG_PROCESSOR", -) -option_database_isolation = click.option( - "--database-isolation", - help="Run airflow in database isolation mode.", - is_flag=True, - envvar="DATABASE_ISOLATION", -) -option_install_selected_providers = click.option( - "--install-selected-providers", - help="Comma-separated list of providers selected to be installed (implies --use-packages-from-dist).", - envvar="INSTALL_SELECTED_PROVIDERS", - default="", -) -option_airflow_skip_constraints = click.option( - "--airflow-skip-constraints", - is_flag=True, - help="Do not use constraints when installing airflow.", - envvar="AIRFLOW_SKIP_CONSTRAINTS", -) -option_historical_python_version = click.option( - "--python", - type=BetterChoice(ALL_HISTORICAL_PYTHON_VERSIONS), - required=False, - envvar="PYTHON_VERSION", - help="Python version to update sbom from. (defaults to all historical python versions)", -) -option_commit_sha = click.option( - "--commit-sha", - show_default=True, - envvar="COMMIT_SHA", - help="Commit SHA that is used to build the images.", -) -option_build_timeout_minutes = click.option( - "--build-timeout-minutes", - required=False, - type=int, - envvar="BUILD_TIMEOUT_MINUTES", - help="Optional timeout for the build in minutes. Useful to detect `pip` backtracking problems.", -) -option_eager_upgrade_additional_requirements = click.option( - "--eager-upgrade-additional-requirements", - required=False, - type=str, - envvar="EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS", - help="Optional additional requirements to upgrade eagerly to avoid backtracking " - "(see `breeze ci find-backtracking-candidates`).", -) -option_airflow_site_directory = click.option( - "-a", - "--airflow-site-directory", - envvar="AIRFLOW_SITE_DIRECTORY", - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), - help="Local directory path of cloned airflow-site repo.", - required=True, -) -option_upgrade_boto = click.option( - "--upgrade-boto", - help="Remove aiobotocore and upgrade botocore and boto to the latest version.", - is_flag=True, - envvar="UPGRADE_BOTO", -) -option_downgrade_sqlalchemy = click.option( - "--downgrade-sqlalchemy", - help="Downgrade SQLAlchemy to minimum supported version.", - is_flag=True, - envvar="DOWNGRADE_SQLALCHEMY", -) -option_run_db_tests_only = click.option( - "--run-db-tests-only", - help="Only runs tests that require a database", - is_flag=True, - envvar="run_db_tests_only", -) -option_skip_db_tests = click.option( - "--skip-db-tests", - help="Skip tests that require a database", - is_flag=True, - envvar="SKIP_DB_TESTS", -) -option_test_timeout = click.option( - "--test-timeout", - help="Test timeout in seconds. Set the pytest setup, execution and teardown timeouts to this value", - default=60, - envvar="TEST_TIMEOUT", - type=IntRange(min=0), - show_default=True, -) -option_enable_coverage = click.option( - "--enable-coverage", - help="Enable coverage capturing for tests in the form of XML files", - is_flag=True, - envvar="ENABLE_COVERAGE", -) -option_skip_provider_tests = click.option( - "--skip-provider-tests", - help="Skip provider tests", - is_flag=True, - envvar="SKIP_PROVIDER_TESTS", -) -option_use_xdist = click.option( - "--use-xdist", - help="Use xdist plugin for pytest", - is_flag=True, - envvar="USE_XDIST", -) -option_test_type = click.option( - "--test-type", - help="Type of test to run. With Providers, you can specify tests of which providers " - "should be run: `Providers[airbyte,http]` or " - "excluded from the full test suite: `Providers[-amazon,google]`", - default="Default", - envvar="TEST_TYPE", - show_default=True, - type=NotVerifiedBetterChoice(ALLOWED_TEST_TYPE_CHOICES), -) -option_parallel_test_types = click.option( - "--parallel-test-types", - help="Space separated list of test types used for testing in parallel", - default=ALL_CI_SELECTIVE_TEST_TYPES, - show_default=True, - envvar="PARALLEL_TEST_TYPES", - type=NotVerifiedBetterChoice(ALLOWED_PARALLEL_TEST_TYPE_CHOICES), -) -option_excluded_parallel_test_types = click.option( - "--excluded-parallel-test-types", - help="Space separated list of test types that will be excluded from parallel tes runs.", - default="", - show_default=True, - envvar="EXCLUDED_PARALLEL_TEST_TYPES", - type=NotVerifiedBetterChoice(ALLOWED_PARALLEL_TEST_TYPE_CHOICES), -) -option_collect_only = click.option( - "--collect-only", - help="Collect tests only, do not run them.", - is_flag=True, - envvar="COLLECT_ONLY", -) -option_remove_arm_packages = click.option( - "--remove-arm-packages", - help="Removes arm packages from the image to test if ARM collection works", - is_flag=True, - envvar="REMOVE_ARM_PACKAGES", -) -option_skip_docker_compose_down = click.option( - "--skip-docker-compose-down", - help="Skips running docker-compose down after tests", - is_flag=True, - envvar="SKIP_DOCKER_COMPOSE_DOWN", -) -option_skip_image_upgrade_check = click.option( - "--skip-image-upgrade-check", - help="Skip checking if the CI image is up to date.", - is_flag=True, - envvar="SKIP_IMAGE_UPGRADE_CHECK", -) -option_warn_image_upgrade_needed = click.option( - "--warn-image-upgrade-needed", - help="Warn when image upgrade is needed even if --skip-upgrade-check is used.", - is_flag=True, - envvar="WARN_IMAGE_UPGRADE_NEEDED", -) -option_skip_environment_initialization = click.option( - "--skip-environment-initialization", - help="Skip running breeze entrypoint initialization - no user output, no db checks.", - is_flag=True, - envvar="SKIP_ENVIRONMENT_INITIALIZATION", -) -option_project_name = click.option( - "--project-name", - help="Name of the docker-compose project to bring down. " - "The `docker-compose` is for legacy breeze project name and you can use " - "`breeze down --project-name docker-compose` to stop all containers belonging to it.", - show_default=True, - type=NotVerifiedBetterChoice(ALLOWED_DOCKER_COMPOSE_PROJECTS), - default=ALLOWED_DOCKER_COMPOSE_PROJECTS[0], - envvar="PROJECT_NAME", -) -option_restart = click.option( - "--restart", - "--remove-orphans", - help="Restart all containers before entering shell (also removes orphan containers).", - is_flag=True, - envvar="RESTART", -) -option_chicken_egg_providers = click.option( - "--chicken-egg-providers", - default="", - help="List of chicken-egg provider packages - " - "those that have airflow_version >= current_version and should " - "be installed in CI from locally built packages with >= current_version.dev0 ", - envvar="CHICKEN_EGG_PROVIDERS", -)
