This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 816d4ccfdecce9c7d8fa64bef9873ec963892d22 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Aug 7 21:28:40 2023 +0200 Fix installing older airflow versions in Breeze with openssl fix (#33171) The dnspython and Flask Application Builder have a weird transitive dependency to openssl. It does not require it - it is an optional dependnency of email validator that uses dnspython and the dnspython uses pyopenssl when installed. This is all fine. But when opendns is used with older FAB < 4.1.4) and NEWER opensl (>=23.0), just creating the FAB applicatio in flask causes an exception than there is a missing parameter in openssl. Since openssl is NOT declared as required dependency in neither FAB nor dnspython, installing airflow with constraints while having a newer version of pyopenssl does not downgrade pyopenssl it is just mentioned in a conflicting message resulting from conflicts with other packages already installed (such as cryptography). This caused a problem when installin older airflow version in breeze with --use-airflow-version switch. Also - during checking this issue, it turned out that we were not - by default - using the constraints for older version when installing it in breeze (you could do that by explicitly specifying the constraints). This was because when you used anoter way to specify versio (commit hash, tag) it is quite complex to figure out which constraints to use. However we can automatically derive constraints when you specify version, or branch, which are the most common scenarios. This PR fixes the `--use-airflow-version` case for Airflow 2.4 and below by: * automatically deriving the right constraints when version or branch are used in `--use-airflow-version` * adding pyopenssl specifically as dependency when --use-airflow-version switch is used. * also a small quality of life improvment - when airflow db migrate fails (because old versio of airflow does not support it) - we inform the user that it happened and that we are attempting to run the legacy airflow db init instead. (cherry picked from commit 5387c0c761fe161f28cee70a46009e1e67286679) --- .../airflow_breeze/commands/developer_commands.py | 43 +++++++++++++++++++--- scripts/in_container/_in_container_utils.sh | 13 ++++++- scripts/in_container/check_environment.sh | 3 ++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 4f0d14a174..f29584bb32 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -17,6 +17,7 @@ from __future__ import annotations import os +import re import shutil import sys import threading @@ -26,6 +27,7 @@ from typing import Iterable import click +from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( @@ -96,12 +98,26 @@ from airflow_breeze.utils.run_utils import ( from airflow_breeze.utils.shared_options import get_dry_run, get_verbose, set_forced_answer from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# Make sure that whatever you add here as an option is also -# Added in the "main" command in breeze.py. The min command above -# Is used for a shorthand of shell and except the extra -# Args it should have the same parameters. -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +def _determine_constraint_branch_used(airflow_constraints_reference: str, use_airflow_version: str | None): + """ + Determine which constraints reference to use. + + When use-airflow-version is branch or version, we derive the constraints branch from it, unless + someone specified the constraints branch explicitly. + + :param airflow_constraints_reference: the constraint reference specified (or default) + :param use_airflow_version: which airflow version we are installing + :return: the actual constraints reference to use + """ + if ( + use_airflow_version + and airflow_constraints_reference == DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH + and re.match(r"[0-9]+\.[0-9]+\.[0-9]+[0-9a-z\.]*|main|v[0-9]_.*", use_airflow_version) + ): + get_console().print(f"[info]Using constraints {use_airflow_version} - matching airflow version used.") + return use_airflow_version + return airflow_constraints_reference class TimerThread(threading.Thread): @@ -116,6 +132,14 @@ class TimerThread(threading.Thread): os.killpg(os.getpgid(0), SIGTERM) +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# Make sure that whatever you add here as an option is also +# Added in the "main" command in breeze.py. The min command above +# Is used for a shorthand of shell and except the extra +# Args it should have the same parameters. +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + @main.command() @option_python @option_platform_single @@ -181,6 +205,9 @@ def shell( if max_time: TimerThread(max_time=max_time).start() set_forced_answer("yes") + airflow_constraints_reference = _determine_constraint_branch_used( + airflow_constraints_reference, use_airflow_version + ) result = enter_shell( python=python, github_repository=github_repository, @@ -292,6 +319,10 @@ def start_airflow( skip_asset_compilation = True if use_airflow_version is None and not skip_asset_compilation: run_compile_www_assets(dev=dev_mode, run_in_background=True) + airflow_constraints_reference = _determine_constraint_branch_used( + airflow_constraints_reference, use_airflow_version + ) + result = enter_shell( python=python, github_repository=github_repository, diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 8d4996a2f5..92d6ccf250 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -262,7 +262,18 @@ function install_released_airflow_version() { if [[ ${constraints_reference} == "none" ]]; then pip install "${airflow_package}${extras}" else - pip install "apache-airflow${BRACKETED_AIRFLOW_EXTRAS}==${version}" \ + local dependency_fix="" + # The pyopenssl is needed to downgrade pyopenssl for older airflow versions when using constraints + # Flask app builder has an optional pyopenssl transitive dependency, that causes import error when + # Pyopenssl is installed in a wrong version for Flask App Builder 4.1 and older. Adding PyOpenSSL + # directly as the dependency, forces downgrading of pyopenssl to the right version. Our constraint + # version has it pinned to the right version, but since it is not directly required, it is not + # downgraded when installing airflow and it is already installed in a newer version + if [[ ${USE_AIRFLOW_VERSION=} != "" ]]; then + dependency_fix="pyopenssl" + fi + + pip install "apache-airflow${BRACKETED_AIRFLOW_EXTRAS}==${version}" ${dependency_fix} \ --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/constraints-${version}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"; fi } diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index 67d4f1814e..9bdb9fcc15 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -123,6 +123,9 @@ function startairflow_if_requested() { airflow connections create-default-connections fi else + echo "${COLOR_YELLOW}Failed to run 'airflow db migrate'.${COLOR_RESET}" + echo "${COLOR_BLUE}This could be because you are installing old airflow version${COLOR_RESET}" + echo "${COLOR_BLUE}Attempting to run deprecated 'airflow db init' instead.${COLOR_RESET}" # For Airflow versions that do not support db migrate, we should run airflow db init and # set the removed AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} airflow db init
