This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8986f8f3beba11cf568e0f014883b997e4042f7b Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Dec 18 20:18:30 2023 +0100 Fix using extras when `--use-airflow-version` is used in Breeze (#36280) When we are installing a released version of Airflow in Breeze, we can pass additional extras to install (For example, we need to pass celery extra in order to start airflow with celery executor. The extras could be specified as: ``` breeze start-airflow --use-airflow-version 2.8.0rc4 \ --executor CeleryExecutor --airflow-extras "celery" ``` However recent refactors caused a problem that the extras added were specified after version (which is rejected by newer versions of `pip`). This PR fixes it and also moves the place where CeleryExecutor use triggers adding celery extra when`--use-airflow-version` is used. The warning about this is better visible when moving to Shell Params. (cherry picked from commit 329780649543ab7b9d593a2e2428073fbd4cf274) --- dev/breeze/src/airflow_breeze/params/shell_params.py | 7 +++++++ .../src/airflow_breeze/utils/docker_command_utils.py | 14 -------------- scripts/in_container/install_airflow_and_providers.py | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 58c49ecf9b..e90d348046 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -328,6 +328,13 @@ class ShellParams: if self.executor == "CeleryExecutor": compose_file_list.append(DOCKER_COMPOSE_DIR / "integration-celery.yml") + if self.use_airflow_version: + current_extras = self.airflow_extras + if "celery" not in current_extras.split(","): + get_console().print( + "[warning]Adding `celery` extras as it is implicitly needed by celery executor" + ) + self.airflow_extras = ",".join(current_extras.split(",") + ["celery"]) compose_file_list.append(DOCKER_COMPOSE_DIR / "base.yml") self.add_docker_in_docker(compose_file_list) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 2ad25acb7c..bc54b9bf69 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -751,20 +751,6 @@ def enter_shell(shell_params: ShellParams, output: Output | None = None) -> RunC f"Changing the executor to {SEQUENTIAL_EXECUTOR}.\n" ) shell_params.executor = SEQUENTIAL_EXECUTOR - - if shell_params.executor == "CeleryExecutor" and shell_params.use_airflow_version: - if shell_params.airflow_extras and "celery" not in shell_params.airflow_extras.split(): - get_console().print( - f"\n[warning]CeleryExecutor requires airflow_extras: celery. " - f"Adding celery to extras: '{shell_params.airflow_extras}'.\n" - ) - shell_params.airflow_extras += ",celery" - elif not shell_params.airflow_extras: - get_console().print( - "\n[warning]CeleryExecutor requires airflow_extras: celery. " - "Setting airflow extras to 'celery'.\n" - ) - shell_params.airflow_extras = "celery" if shell_params.restart: bring_compose_project_down(preserve_volumes=False, shell_params=shell_params) if shell_params.include_mypy_volume: diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index 9eadad9789..58f2d86af3 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -253,7 +253,7 @@ def find_installation_spec( ) else: console.print(f"\nInstalling airflow via apache-airflow=={use_airflow_version}") - airflow_package_spec = f"apache-airflow=={use_airflow_version}{airflow_extras}" + airflow_package_spec = f"apache-airflow{airflow_extras}=={use_airflow_version}" airflow_constraints_location = get_airflow_constraints_location( airflow_skip_constraints=airflow_skip_constraints, airflow_constraints_mode=airflow_constraints_mode,
