This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9b114c8deee Reuse locked CI dependencies across workspace source
changes (#74466)
9b114c8deee is described below
commit 9b114c8deeeb0ae02d462cf2464350ad09af35fa
Author: Yossi Eliaz <[email protected]>
AuthorDate: Sat Oct 31 21:59:15 2026 +0200
Reuse locked CI dependencies across workspace source changes (#74466)
* Reuse locked CI dependencies across workspace source changes
* Skip the locked dependency pre-install when uv.lock does not fit
A frozen sync fails when uv.lock is missing or lacks a workspace member,
which would fail the whole image build even though the full sync after
COPY already re-resolves in that case.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* Run the locked dependency pre-install through the install script
The pre-install repeated the uv sync flags of install_from_sources, so the
two layers could drift apart and install different builds of the same
packages. INSTALL_LOCKED_THIRD_PARTY_ONLY makes the install script add
--no-install-workspace, skip the re-resolving fallback and skip the steps
that need workspace packages, so both layers run the same uv sync. The
Dockerfile.ci step stays best-effort and is still skipped for upgrade
builds.
* Skip the locked dependency pre-install only when uv.lock does not fit
A frozen dry run now decides whether to skip: it fails when uv.lock is
missing or lacks a workspace member, and only then is the warm-up skipped.
Any other failure, such as a network error or an lxml/xmlsec build error,
would fail the full sync as well, so it now fails the build instead of
leaving an empty layer in the cache. The install script also refuses the
locked mode for upgrade builds itself instead of relying on the caller.
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
Co-authored-by: Shahar Epstein <[email protected]>
---
Dockerfile | 43 ++++++++++---
Dockerfile.ci | 71 +++++++++++++++++++---
.../docker/install_airflow_when_building_images.sh | 48 ++++++++++++---
3 files changed, 140 insertions(+), 22 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index bf64f1a27fa..25a14d98316 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1540,6 +1540,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew;
skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ extra_sync_flags="${extra_sync_flags} --no-install-workspace"
+ fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
@@ -1560,19 +1569,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
- set +x
- echo
- echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
- echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock
without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec
using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
+ # shellcheck disable=SC2206 # extra_sync_flags holds several flags
+ local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec
+ --no-python-downloads --no-managed-python)
+ set +x
+ # Without the workspace sources a lock that does not match the
manifests cannot be re-resolved;
+ # the installation that adds the workspace packages does that later.
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && !
"${frozen_sync[@]}" --dry-run; then
+ echo
+ echo "${COLOR_YELLOW}uv.lock does not match the dependency
manifests; skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ echo
+ echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
+ echo
set -x
- if ! uv sync --all-packages --frozen --group ci-image \
- ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
- --no-python-downloads --no-managed-python; then
+ if ! "${frozen_sync[@]}"; then
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ echo
+ echo "${COLOR_RED}Failing because the locked third-party
installation failed although uv.lock matches the dependency
manifests.${COLOR_RESET}"
+ echo
+ exit 1
+ fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation
failed and fallback is disabled.${COLOR_RESET}"
@@ -1662,6 +1687,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ # Packaging tools and 'pip check' are handled by the installation that
adds the workspace packages
+ return
+ fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
diff --git a/Dockerfile.ci b/Dockerfile.ci
index d164e2acd56..ff756aa62a0 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1233,6 +1233,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew;
skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ extra_sync_flags="${extra_sync_flags} --no-install-workspace"
+ fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
@@ -1253,19 +1262,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
- set +x
- echo
- echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
- echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock
without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec
using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
+ # shellcheck disable=SC2206 # extra_sync_flags holds several flags
+ local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec
+ --no-python-downloads --no-managed-python)
+ set +x
+ # Without the workspace sources a lock that does not match the
manifests cannot be re-resolved;
+ # the installation that adds the workspace packages does that later.
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && !
"${frozen_sync[@]}" --dry-run; then
+ echo
+ echo "${COLOR_YELLOW}uv.lock does not match the dependency
manifests; skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ echo
+ echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
+ echo
set -x
- if ! uv sync --all-packages --frozen --group ci-image \
- ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
- --no-python-downloads --no-managed-python; then
+ if ! "${frozen_sync[@]}"; then
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ echo
+ echo "${COLOR_RED}Failing because the locked third-party
installation failed although uv.lock matches the dependency
manifests.${COLOR_RESET}"
+ echo
+ exit 1
+ fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation
failed and fallback is disabled.${COLOR_RESET}"
@@ -1355,6 +1380,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ # Packaging tools and 'pip check' are handled by the installation that
adds the workspace packages
+ return
+ fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -1992,6 +2021,22 @@ fi
exec /bin/bash "${@}"
EOF
+# Extract manifests separately so COPY below depends on their contents, not
all source files.
+FROM ${BASE_IMAGE} as dependency-manifests
+WORKDIR /source
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o",
"nolog", "-c"]
+RUN --mount=type=bind,target=/source \
+ manifest_list="$(mktemp)" && \
+ trap 'rm -f "${manifest_list}"' EXIT && \
+ find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \
+ mkdir -p /dependency-manifests && \
+ while IFS= read -r -d '' manifest; do \
+ relative_path="${manifest#./}"; \
+ mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit
1; \
+ cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \
+ done < "${manifest_list}" && \
+ if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi
+
FROM ${BASE_IMAGE} as main
# Nolog bash flag is currently ignored - but you can replace it with other
flags (for example
@@ -2233,13 +2278,23 @@ RUN bash /scripts/docker/install_packaging_tools.sh
COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/
+COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/
+ARG UPGRADE_RANDOM_INDICATOR_STRING=""
+# Only preinstall locked third-party dependencies, with the same uv sync as
the full installation.
+# A uv.lock that does not match the manifests only skips this warm-up: the
full sync after COPY still
+# installs workspace packages from the current checkout and retains its
resolution fallback. Any other
+# failure would fail the full sync as well, so it fails the build here.
+RUN
--mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/
\
+ if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \
+ INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash
/scripts/docker/install_airflow_when_building_images.sh; \
+ fi
+
# We can copy everything here. The Context is filtered by dockerignore. This
makes sure we are not
# copying over stuff that is accidentally generated or that we do not need
(such as egg-info)
# if you want to add something that is missing and you expect to see it in the
image you can
# add it with ! in .dockerignore next to the airflow, test etc. directories
there
COPY . ${AIRFLOW_SOURCES}/
-ARG UPGRADE_RANDOM_INDICATOR_STRING=""
ARG VERSION_SUFFIX=""
ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \
diff --git a/scripts/docker/install_airflow_when_building_images.sh
b/scripts/docker/install_airflow_when_building_images.sh
index 772465678a8..45e60d5fdbf 100644
--- a/scripts/docker/install_airflow_when_building_images.sh
+++ b/scripts/docker/install_airflow_when_building_images.sh
@@ -30,6 +30,11 @@
# UPGRADE_RANDOM_INDICATOR_STRING - if set with random value determines that
dependency upgrade should be done
# with highest resolution. It is random in
order to trigger cache
# invalidation when building images
+# INSTALL_LOCKED_THIRD_PARTY_ONLY - if "true", only the third-party
dependencies locked in uv.lock are installed
+# from sources (--no-install-workspace), so
only the pyproject.toml files
+# and uv.lock are needed. When a frozen dry
run shows that uv.lock does not
+# match them, or for upgrade builds, nothing
is installed; any other
+# failure fails. The steps that need the
workspace packages are skipped.
#
# shellcheck shell=bash disable=SC2086
# shellcheck source=scripts/docker/common.sh
@@ -41,6 +46,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew;
skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ extra_sync_flags="${extra_sync_flags} --no-install-workspace"
+ fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
@@ -61,19 +75,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
--no-python-downloads --no-managed-python
else
- set +x
- echo
- echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
- echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock
without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec
using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses
system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
+ # shellcheck disable=SC2206 # extra_sync_flags holds several flags
+ local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
+ ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec
+ --no-python-downloads --no-managed-python)
+ set +x
+ # Without the workspace sources a lock that does not match the
manifests cannot be re-resolved;
+ # the installation that adds the workspace packages does that later.
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && !
"${frozen_sync[@]}" --dry-run; then
+ echo
+ echo "${COLOR_YELLOW}uv.lock does not match the dependency
manifests; skipping the locked third-party installation.${COLOR_RESET}"
+ echo
+ return
+ fi
+ echo
+ echo "${COLOR_BLUE}Installing all packages from uv.lock
(frozen).${COLOR_RESET}"
+ echo
set -x
- if ! uv sync --all-packages --frozen --group ci-image \
- ${extra_sync_flags} --no-binary-package lxml --no-binary-package
xmlsec \
- --no-python-downloads --no-managed-python; then
+ if ! "${frozen_sync[@]}"; then
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ echo
+ echo "${COLOR_RED}Failing because the locked third-party
installation failed although uv.lock matches the dependency
manifests.${COLOR_RESET}"
+ echo
+ exit 1
+ fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true"
]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation
failed and fallback is disabled.${COLOR_RESET}"
@@ -163,6 +193,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
+ if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
+ # Packaging tools and 'pip check' are handled by the installation that
adds the workspace packages
+ return
+ fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
