This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ececdd3207 Speed-up rebuilding of CI image after pip/uv version change
(#43101)
ececdd3207 is described below
commit ececdd32071890ffdb1091d001ab8aac2c473c41
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Oct 17 23:43:48 2024 +0200
Speed-up rebuilding of CI image after pip/uv version change (#43101)
So far `pip` and `uv` version change caused clean reinstallation
of the CI image `pip` dependencies - including the cached layer
from main - which pre-installed airlfow to speed up reinstallation
of "final" airflow package.
However since we started to update `uv` more frequently - those
frequent rebuilds are ... to costly (about 3 minutes extra when
uv version changed.
This change implements optimization of this workflow - the
main cache installation is done using LATEST uv or pip and
only after that UV and PIP get reinstalled to the fixed version
specified in Dockerfile.
Related: #42999
---
Dockerfile | 16 ++++++++++++----
Dockerfile.ci | 39 ++++++++++++++++++++++++++++-----------
scripts/docker/common.sh | 16 ++++++++++++----
3 files changed, 52 insertions(+), 19 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 4e3aa5ac90..59911b3f15 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -515,8 +515,6 @@ function common::get_colors() {
}
function common::get_packaging_tool() {
- : "${AIRFLOW_PIP_VERSION:?Should be set}"
- : "${AIRFLOW_UV_VERSION:?Should be set}"
: "${AIRFLOW_USE_UV:?Should be set}"
## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY
CORRESPONDING FUNCTION IN
@@ -612,7 +610,12 @@ function common::install_packaging_tools() {
echo "${COLOR_BLUE}Checking packaging tools for system Python
installation: $(which python)${COLOR_RESET}"
echo
fi
- if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade pip
+ elif [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
@@ -629,7 +632,12 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade uv
+ elif [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 86268bf920..936423cd96 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -461,8 +461,6 @@ function common::get_colors() {
}
function common::get_packaging_tool() {
- : "${AIRFLOW_PIP_VERSION:?Should be set}"
- : "${AIRFLOW_UV_VERSION:?Should be set}"
: "${AIRFLOW_USE_UV:?Should be set}"
## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY
CORRESPONDING FUNCTION IN
@@ -558,7 +556,12 @@ function common::install_packaging_tools() {
echo "${COLOR_BLUE}Checking packaging tools for system Python
installation: $(which python)${COLOR_RESET}"
echo
fi
- if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade pip
+ elif [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
@@ -575,7 +578,12 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade uv
+ elif [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
@@ -1277,9 +1285,6 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
# It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH
environment variable.
ARG AIRFLOW_CI_BUILD_EPOCH="10"
ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
-ARG AIRFLOW_PIP_VERSION=24.2
-ARG AIRFLOW_UV_VERSION=0.4.23
-ARG AIRFLOW_USE_UV="true"
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
@@ -1301,8 +1306,6 @@ ARG AIRFLOW_VERSION=""
# Additional PIP flags passed to all pip install commands except reinstalling
pip itself
ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
-ARG AIRFLOW_PIP_VERSION=24.2
-ARG AIRFLOW_UV_VERSION=0.4.23
ARG AIRFLOW_USE_UV="true"
ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
@@ -1348,19 +1351,33 @@ COPY --from=scripts install_packaging_tools.sh
install_airflow_dependencies_from
# The Airflow and providers are uninstalled, only dependencies remain.
# the cache is only used when "upgrade to newer dependencies" is not set to
automatically
# account for removed dependencies (we do not install them in the first place)
+#
+# We are installing from branch tip without fixing UV or PIP version - in
order to avoid rebuilding the
+# base cache layer every time the UV or PIP version changes.
RUN bash /scripts/docker/install_packaging_tools.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
+
+# Here we fix the versions so all subsequent commands will use the versions
+# from the sources
+ARG AIRFLOW_PIP_VERSION=24.2
+ARG AIRFLOW_UV_VERSION=0.4.23
+
+ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION}
+
# The PATH is needed for PIPX to find the tools installed
ENV PATH="/root/.local/bin:${PATH}"
COPY --from=scripts install_pipx_tools.sh /scripts/docker/
# Install useful command line tools in their own virtualenv so that they do
not clash with
-# dependencies installed in Airflow
-RUN bash /scripts/docker/install_pipx_tools.sh
+# dependencies installed in Airflow also reinstall PIP and UV to make sure
they are installed
+# in the version specified above
+RUN bash /scripts/docker/install_packaging_tools.sh; \
+ bash /scripts/docker/install_pipx_tools.sh
# Airflow sources change frequently but dependency configuration won't change
that often
# We copy pyproject.toml and other files needed to perform setup of
dependencies
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index a934b6067e..5f50fbc674 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -32,8 +32,6 @@ function common::get_colors() {
}
function common::get_packaging_tool() {
- : "${AIRFLOW_PIP_VERSION:?Should be set}"
- : "${AIRFLOW_UV_VERSION:?Should be set}"
: "${AIRFLOW_USE_UV:?Should be set}"
## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY
CORRESPONDING FUNCTION IN
@@ -129,7 +127,12 @@ function common::install_packaging_tools() {
echo "${COLOR_BLUE}Checking packaging tools for system Python
installation: $(which python)${COLOR_RESET}"
echo
fi
- if [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade pip
+ elif [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing pip version from spec
${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
@@ -146,7 +149,12 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
+ if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore --disable-pip-version-check
--upgrade uv
+ elif [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
echo
echo "${COLOR_BLUE}Installing uv version from spec
${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
