This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0af3dc599b Refactor, speed up, better diagnostics for package tools
installation (#37849)
0af3dc599b is described below
commit 0af3dc599b42f564792206d88770327949003330
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Mar 2 15:07:06 2024 +0100
Refactor, speed up, better diagnostics for package tools installation
(#37849)
When going through various stages of installation of Airlfow in
CI and PROD image, we check several times if the packaging tools
of ours are installed and if they are installed in the right
version. This is in order to make sure that none of the installation
steps upgraded them accidentaly (might happen when --eager upgrade
of pip or --highest resolution of uv are used).
This PR improves the speed of that (rather than installing the
packages again, we just check which version is already installed).
Similarly when constraints are already downloaded, we do not
re-download them again.
Also this change makes a consisten rename of the installation
scripts to reflect we have not only pip but also uv as packaging
tool and installs both `uv` and `pip` regardless which packaging
tool we are using - which gives anyone an opportunity to use
either of the two tools if they wish.
Diagnostics is a bit nicer - rather than complaining with error
when uv is missing, prints information that it's not installed.
We only show traces of installation command (pip and uv) with
`set -x` commands for diagnostics - we do not show details of other
internal commands.
---
Dockerfile | 87 ++++++++++++++--------
Dockerfile.ci | 83 +++++++++++++--------
scripts/docker/common.sh | 49 +++++++-----
scripts/docker/install_additional_dependencies.sh | 4 +-
scripts/docker/install_airflow.sh | 10 ++-
...install_airflow_dependencies_from_branch_tip.sh | 6 +-
.../docker/install_from_docker_context_files.sh | 4 +-
...l_pip_version.sh => install_packaging_tools.sh} | 2 +-
8 files changed, 154 insertions(+), 91 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index bf08405e08..f52ad61a4c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -412,8 +412,8 @@ if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then
fi
EOF
-# The content below is automatically copied from
scripts/docker/install_pip_version.sh
-COPY <<"EOF" /install_pip_version.sh
+# The content below is automatically copied from
scripts/docker/install_packaging_tools.sh
+COPY <<"EOF" /install_packaging_tools.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
@@ -423,7 +423,7 @@ common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_packaging_tool_version_and_location
-common::install_packaging_tool
+common::install_packaging_tools
EOF
# The content below is automatically copied from
scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -454,7 +454,9 @@ function install_airflow_dependencies_from_branch_tip() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
- common::install_packaging_tool
+ set +x
+ common::install_packaging_tools
+ set -x
# Uninstall airflow and providers to keep only the dependencies. In the
future when
# planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
# flag and skip the remove step.
@@ -463,7 +465,9 @@ function install_airflow_dependencies_from_branch_tip() {
echo
echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now
target airflow can be reinstalled using mostly cached
dependencies${COLOR_RESET}"
echo
+ set +x
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow ||
true
+ set -x
}
common::get_colors
@@ -543,6 +547,10 @@ function common::override_pip_version_if_needed() {
}
function common::get_constraints_location() {
+ if [[ -f "${HOME}/constraints.txt" ]]; then
+ # constraints are already downloaded, do not calculate/override again
+ return
+ fi
# auto-detect Airflow-constraint reference and location
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.*
]]; then
@@ -574,40 +582,51 @@ function common::get_constraints_location() {
function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
+ echo "Installed pip: $(pip --version): $(which pip)"
if [[ ${PACKAGING_TOOL} == "pip" ]]; then
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
- echo "pip on path: $(which pip)"
- echo "Using pip: $(pip --version)"
else
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
- echo "uv on path: $(which uv)"
- echo "Using uv: $(uv --version)"
+ echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed
yet"): $(which uv 2>/dev/null)"
fi
}
-function common::install_packaging_tool() {
- echo
- echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
- echo
- if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+function common::install_packaging_tools() {
+ if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "pip
@ ${AIRFLOW_PIP_VERSION}"
else
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ local installed_pip_version
+ installed_pip_version=$(python -c 'from importlib.metadata import
version; print(version("pip"))')
+ if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing pip version:
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ fi
fi
- if [[ ${AIRFLOW_USE_UV} == "true" ]]; then
+ if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
- echo "${COLOR_BLUE}Installing uv version
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
- if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"uv @ ${AIRFLOW_UV_VERSION}"
- else
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check "uv
@ ${AIRFLOW_UV_VERSION}"
+ else
+ local installed_uv_version
+ installed_uv_version=$(python -c 'from importlib.metadata import
version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")
+ if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing uv version:
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
+ # make sure that the venv/user in .local exists
mkdir -p "${HOME}/.local/bin"
}
@@ -756,7 +775,7 @@ function
install_airflow_and_providers_from_docker_context_files(){
${reinstalling_apache_airflow_package}
${reinstalling_apache_airflow_providers_packages}
set +x
fi
- common::install_packaging_tool
+ common::install_packaging_tools
pip check
}
@@ -772,7 +791,7 @@ function
install_all_other_packages_from_docker_context_files() {
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${ADDITIONAL_PIP_INSTALL_FLAGS} \
--force-reinstall --no-deps --no-index
${reinstalling_other_packages}
- common::install_packaging_tool
+ common::install_packaging_tools
set +x
fi
}
@@ -831,14 +850,14 @@ function install_airflow() {
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
set +x
- common::install_packaging_tool
+ common::install_packaging_tools
echo
- echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}"
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with
constraints${COLOR_RESET}"
echo
set -x
# Install all packages with constraints
@@ -846,18 +865,20 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
--constraint "${HOME}/constraints.txt"; then
+ set +x
echo
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
echo
echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
echo
+ set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -895,8 +916,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_HIGHEST_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -909,8 +930,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
"${RESOLUTION_LOWEST_DIRECT_FLAG}" \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -1422,7 +1443,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
# Copy all scripts required for installation - changing any of those should
lead to
# rebuilding from here
-COPY --from=scripts common.sh install_pip_version.sh \
+COPY --from=scripts common.sh install_packaging_tools.sh \
install_airflow_dependencies_from_branch_tip.sh /scripts/docker/
# We can set this value to true in case we want to install .whl/.tar.gz
packages placed in the
@@ -1448,7 +1469,7 @@ ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH}
# the cache is only used when "upgrade to newer dependencies" is not set to
automatically
# account for removed dependencies (we do not install them in the first place)
and in case
# INSTALL_PACKAGES_FROM_CONTEXT is not set (because then caching it from main
makes no sense).
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN bash /scripts/docker/install_packaging_tools.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
@@ -1621,8 +1642,8 @@ RUN sed --in-place=.bak
"s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sud
ARG AIRFLOW_VERSION
-COPY --from=scripts install_pip_version.sh /scripts/docker/
-RUN bash /scripts/docker/install_pip_version.sh
+COPY --from=scripts install_packaging_tools.sh /scripts/docker/
+RUN bash /scripts/docker/install_packaging_tools.sh
# See
https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
# to learn more about the way how signals are handled by the image
diff --git a/Dockerfile.ci b/Dockerfile.ci
index a217ba2938..4372b543cb 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -370,8 +370,8 @@ if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then
fi
EOF
-# The content below is automatically copied from
scripts/docker/install_pip_version.sh
-COPY <<"EOF" /install_pip_version.sh
+# The content below is automatically copied from
scripts/docker/install_packaging_tools.sh
+COPY <<"EOF" /install_packaging_tools.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
@@ -381,7 +381,7 @@ common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_packaging_tool_version_and_location
-common::install_packaging_tool
+common::install_packaging_tools
EOF
# The content below is automatically copied from
scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -412,7 +412,9 @@ function install_airflow_dependencies_from_branch_tip() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
- common::install_packaging_tool
+ set +x
+ common::install_packaging_tools
+ set -x
# Uninstall airflow and providers to keep only the dependencies. In the
future when
# planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
# flag and skip the remove step.
@@ -421,7 +423,9 @@ function install_airflow_dependencies_from_branch_tip() {
echo
echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now
target airflow can be reinstalled using mostly cached
dependencies${COLOR_RESET}"
echo
+ set +x
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow ||
true
+ set -x
}
common::get_colors
@@ -501,6 +505,10 @@ function common::override_pip_version_if_needed() {
}
function common::get_constraints_location() {
+ if [[ -f "${HOME}/constraints.txt" ]]; then
+ # constraints are already downloaded, do not calculate/override again
+ return
+ fi
# auto-detect Airflow-constraint reference and location
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.*
]]; then
@@ -532,40 +540,51 @@ function common::get_constraints_location() {
function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
+ echo "Installed pip: $(pip --version): $(which pip)"
if [[ ${PACKAGING_TOOL} == "pip" ]]; then
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
- echo "pip on path: $(which pip)"
- echo "Using pip: $(pip --version)"
else
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
- echo "uv on path: $(which uv)"
- echo "Using uv: $(uv --version)"
+ echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed
yet"): $(which uv 2>/dev/null)"
fi
}
-function common::install_packaging_tool() {
- echo
- echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
- echo
- if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+function common::install_packaging_tools() {
+ if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "pip
@ ${AIRFLOW_PIP_VERSION}"
else
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ local installed_pip_version
+ installed_pip_version=$(python -c 'from importlib.metadata import
version; print(version("pip"))')
+ if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing pip version:
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ fi
fi
- if [[ ${AIRFLOW_USE_UV} == "true" ]]; then
+ if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
- echo "${COLOR_BLUE}Installing uv version
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
- if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"uv @ ${AIRFLOW_UV_VERSION}"
- else
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check "uv
@ ${AIRFLOW_UV_VERSION}"
+ else
+ local installed_uv_version
+ installed_uv_version=$(python -c 'from importlib.metadata import
version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")
+ if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing uv version:
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
+ # make sure that the venv/user in .local exists
mkdir -p "${HOME}/.local/bin"
}
@@ -671,14 +690,14 @@ function install_airflow() {
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
set +x
- common::install_packaging_tool
+ common::install_packaging_tools
echo
- echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}"
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with
constraints${COLOR_RESET}"
echo
set -x
# Install all packages with constraints
@@ -686,18 +705,20 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
--constraint "${HOME}/constraints.txt"; then
+ set +x
echo
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
echo
echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
echo
+ set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -735,8 +756,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_HIGHEST_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -749,8 +770,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
"${RESOLUTION_LOWEST_DIRECT_FLAG}" \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -1246,7 +1267,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
# Copy all scripts required for installation - changing any of those should
lead to
# rebuilding from here
-COPY --from=scripts install_pip_version.sh
install_airflow_dependencies_from_branch_tip.sh \
+COPY --from=scripts install_packaging_tools.sh
install_airflow_dependencies_from_branch_tip.sh \
common.sh /scripts/docker/
# We are first creating a venv where all python packages and .so binaries
needed by those are
@@ -1257,7 +1278,7 @@ COPY --from=scripts install_pip_version.sh
install_airflow_dependencies_from_bra
# The Airflow and providers are uninstalled, only dependencies remain.
# the cache is only used when "upgrade to newer dependencies" is not set to
automatically
# account for removed dependencies (we do not install them in the first place)
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN bash /scripts/docker/install_packaging_tools.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
@@ -1307,12 +1328,12 @@ COPY --from=scripts entrypoint_ci.sh /entrypoint
COPY --from=scripts entrypoint_exec.sh /entrypoint-exec
RUN chmod a+x /entrypoint /entrypoint-exec
-COPY --from=scripts install_pip_version.sh install_additional_dependencies.sh
/scripts/docker/
+COPY --from=scripts install_packaging_tools.sh
install_additional_dependencies.sh /scripts/docker/
# Additional python deps to install
ARG ADDITIONAL_PYTHON_DEPS=""
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN bash /scripts/docker/install_packaging_tools.sh; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
bash /scripts/docker/install_additional_dependencies.sh; \
fi
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index 08b9f697bb..25fd4ef9a6 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -80,6 +80,10 @@ function common::override_pip_version_if_needed() {
}
function common::get_constraints_location() {
+ if [[ -f "${HOME}/constraints.txt" ]]; then
+ # constraints are already downloaded, do not calculate/override again
+ return
+ fi
# auto-detect Airflow-constraint reference and location
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.*
]]; then
@@ -111,40 +115,51 @@ function common::get_constraints_location() {
function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
+ echo "Installed pip: $(pip --version): $(which pip)"
if [[ ${PACKAGING_TOOL} == "pip" ]]; then
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
- echo "pip on path: $(which pip)"
- echo "Using pip: $(pip --version)"
else
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
- echo "uv on path: $(which uv)"
- echo "Using uv: $(uv --version)"
+ echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed
yet"): $(which uv 2>/dev/null)"
fi
}
-function common::install_packaging_tool() {
- echo
- echo "${COLOR_BLUE}Installing pip version
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
- echo
- if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+function common::install_packaging_tools() {
+ if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "pip
@ ${AIRFLOW_PIP_VERSION}"
else
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ local installed_pip_version
+ installed_pip_version=$(python -c 'from importlib.metadata import
version; print(version("pip"))')
+ if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing pip version:
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
+ fi
fi
- if [[ ${AIRFLOW_USE_UV} == "true" ]]; then
+ if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
- echo "${COLOR_BLUE}Installing uv version
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
- if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then
- # shellcheck disable=SC2086
- pip install --root-user-action ignore --disable-pip-version-check
"uv @ ${AIRFLOW_UV_VERSION}"
- else
+ # shellcheck disable=SC2086
+ pip install --root-user-action ignore --disable-pip-version-check "uv
@ ${AIRFLOW_UV_VERSION}"
+ else
+ local installed_uv_version
+ installed_uv_version=$(python -c 'from importlib.metadata import
version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")
+ if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then
+ echo
+ echo "${COLOR_BLUE}(Re)Installing uv version:
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
+ echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
+ # make sure that the venv/user in .local exists
mkdir -p "${HOME}/.local/bin"
}
diff --git a/scripts/docker/install_additional_dependencies.sh
b/scripts/docker/install_additional_dependencies.sh
index 79e83eeb9e..91281f6233 100644
--- a/scripts/docker/install_additional_dependencies.sh
+++ b/scripts/docker/install_additional_dependencies.sh
@@ -34,8 +34,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_HIGHEST_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
@@ -48,8 +48,8 @@ function install_additional_dependencies() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
"${RESOLUTION_LOWEST_DIRECT_FLAG}" \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
diff --git a/scripts/docker/install_airflow.sh
b/scripts/docker/install_airflow.sh
index 4397ed34ad..1e63c4eafc 100644
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -65,14 +65,14 @@ function install_airflow() {
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
set +x
- common::install_packaging_tool
+ common::install_packaging_tools
echo
- echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}"
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with
constraints${COLOR_RESET}"
echo
set -x
# Install all packages with constraints
@@ -80,18 +80,20 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
--constraint "${HOME}/constraints.txt"; then
+ set +x
echo
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
echo
echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
echo
+ set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
- common::install_packaging_tool
set +x
+ common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
index 202a40c41e..95f38e21e5 100644
--- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
+++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -52,7 +52,9 @@ function install_airflow_dependencies_from_branch_tip() {
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"apache-airflow[${AIRFLOW_EXTRAS}] @
https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz";
- common::install_packaging_tool
+ set +x
+ common::install_packaging_tools
+ set -x
# Uninstall airflow and providers to keep only the dependencies. In the
future when
# planned https://github.com/pypa/pip/issues/11440 is implemented in pip
we might be able to use this
# flag and skip the remove step.
@@ -61,7 +63,9 @@ function install_airflow_dependencies_from_branch_tip() {
echo
echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now
target airflow can be reinstalled using mostly cached
dependencies${COLOR_RESET}"
echo
+ set +x
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow ||
true
+ set -x
}
common::get_colors
diff --git a/scripts/docker/install_from_docker_context_files.sh
b/scripts/docker/install_from_docker_context_files.sh
index 3c327ad312..4523ff469e 100644
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -115,7 +115,7 @@ function
install_airflow_and_providers_from_docker_context_files(){
${reinstalling_apache_airflow_package}
${reinstalling_apache_airflow_providers_packages}
set +x
fi
- common::install_packaging_tool
+ common::install_packaging_tools
pip check
}
@@ -135,7 +135,7 @@ function
install_all_other_packages_from_docker_context_files() {
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${ADDITIONAL_PIP_INSTALL_FLAGS} \
--force-reinstall --no-deps --no-index
${reinstalling_other_packages}
- common::install_packaging_tool
+ common::install_packaging_tools
set +x
fi
}
diff --git a/scripts/docker/install_pip_version.sh
b/scripts/docker/install_packaging_tools.sh
similarity index 97%
rename from scripts/docker/install_pip_version.sh
rename to scripts/docker/install_packaging_tools.sh
index af8d25e06e..788db69221 100644
--- a/scripts/docker/install_pip_version.sh
+++ b/scripts/docker/install_packaging_tools.sh
@@ -25,4 +25,4 @@ common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_packaging_tool_version_and_location
-common::install_packaging_tool
+common::install_packaging_tools
