This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch mirror-hardened-images-more-often
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to
refs/heads/mirror-hardened-images-more-often by this push:
new 00d7e2bdf62 Pin the hardened base images to digests the CI has passed
on
00d7e2bdf62 is described below
commit 00d7e2bdf624815029baa7d1cb918df25bd15aad
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Oct 30 10:14:42 2026 +0100
Pin the hardened base images to digests the CI has passed on
Building on the mirror's moving tags made CI non-reproducible: any
refresh of the mirror changed the image every build started from,
without a change in the repository and without the full CI having run
on it. A CI run could pass or fail depending only on when it ran, and
rebuilding the same commit later gave a different image.
The images now build on a digest, and only the upgrade check moves it,
in a PR that must pass the full CI before the new base is used. The
mirror refreshes every six hours and triggers that upgrade check when it
moves a tag past its pin, so the pins follow Docker's rebuilds - and the
dhi.io apt repository moving with them - within hours rather than drift
behind it. Every pinned and mirrored digest keeps a tag of its own, so a
pinned image never becomes untagged.
The hardened images keep stock Debian's apt sources beside dhi.io, and
apt took whichever offered the higher version, mixing Debian builds into
an image whose dependencies were built by Docker. dhi.io is now
preferred whenever it publishes the package.
Generated-by: Claude Opus 5
---
.github/workflows/mirror-hardened-base-images.yml | 22 +++-
Dockerfile | 20 +++-
Dockerfile.ci | 20 +++-
dev/breeze/doc/ci/02_images.md | 2 +-
.../commands/release_management_commands.py | 89 ++++++++++++----
.../files/hardened_python_image_digests.json | 10 ++
dev/breeze/src/airflow_breeze/global_constants.py | 27 ++++-
dev/breeze/tests/test_build_prod_params.py | 26 ++++-
dev/breeze/tests/test_mirror_base_images.py | 116 +++++++++++++++++++--
docker-stack-docs/build-arg-ref.rst | 5 +-
docker-stack-docs/build.rst | 16 ++-
.../customizing/add-build-essential-custom.sh | 2 +-
.../docker-examples/customizing/custom-sources.sh | 2 +-
.../customizing/pypi-dev-runtime-deps.sh | 2 +-
.../customizing/pypi-extras-and-deps.sh | 2 +-
.../customizing/pypi-selected-version.sh | 2 +-
.../restricted/restricted_environments.sh | 2 +-
.../test_examples_of_prod_image_building.py | 2 +
scripts/ci/prek/upgrade_important_versions.py | 91 +++++++++++++++-
scripts/docker/install_os_dependencies.sh | 14 +++
.../ci/prek/test_upgrade_important_versions.py | 106 +++++++++++++++++++
21 files changed, 518 insertions(+), 60 deletions(-)
diff --git a/.github/workflows/mirror-hardened-base-images.yml
b/.github/workflows/mirror-hardened-base-images.yml
index 55ab27b4d94..90e5b816a4f 100644
--- a/.github/workflows/mirror-hardened-base-images.yml
+++ b/.github/workflows/mirror-hardened-base-images.yml
@@ -27,13 +27,15 @@ name: Mirror hardened base images
#
# `docker buildx imagetools create` copies the multi-platform manifest between
registries without
# pulling the layers into the runner, so a refresh of all Python versions is
cheap.
+#
+# The builds do not use these moving tags: they are pinned to digests that
only "breeze ci upgrade"
+# moves, in a PR that has to pass the full CI. Docker rebuilds the images in
place and moves the
+# dhi.io apt repository on together with them, so a pin left behind soon stops
matching what apt
+# installs. A run that moves a tag past its pin therefore triggers that
upgrade check, and the pins
+# follow Docker's rebuilds within hours.
on: # yamllint disable-line rule:truthy
workflow_dispatch:
schedule:
- # Docker rebuilds the hardened images as CVEs are fixed, keeping the same
tags, and moves the
- # dhi.io apt repository on at the same time. A stale mirror then cannot
install packages whose
- # dependencies were rebuilt (docker-hardened-images/catalog#689), so it is
refreshed several
- # times a day rather than weekly.
- cron: '27 */6 * * *'
permissions:
contents: read
@@ -54,6 +56,7 @@ jobs:
)
)
permissions:
+ actions: write
contents: read
packages: write
steps:
@@ -79,4 +82,15 @@ jobs:
ACTOR: ${{ github.actor }}
run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}"
--password-stdin
- name: "Mirror base images"
+ id: mirror
run: breeze release-management mirror-base-images
+ # Release branches pin their own digests and move them on their own
upgrade schedule. Also after
+ # a failed mirror step: a tag it moved before failing stays moved, so a
re-run would not see it
+ # move again.
+ - name: "Trigger the upgrade check to move the pinned base images"
+ if: >-
+ !cancelled() && steps.mirror.outputs.moved-past-pins == 'true' &&
+ github.ref == 'refs/heads/main'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: gh workflow run scheduled-upgrade-check-main.yml --repo
"${GITHUB_REPOSITORY}" --ref main
diff --git a/Dockerfile b/Dockerfile
index 55f5878e3ce..73f17a9db83 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,7 +54,9 @@ ARG AIRFLOW_VERSION="3.3.2"
# image are based on it, so Python is never compiled during the build.
#
# The default is Airflow's public mirror of the upstream "dhi.io/python"
image, so building needs no
-# registry credentials. The upstream image itself requires a "docker login
dhi.io".
+# registry credentials. The upstream image itself requires a "docker login
dhi.io". It is pinned to a
+# digest, because the mirror's tags move several times a day: "breeze ci
upgrade" moves the pin once
+# the full CI has passed on the new image.
#
# The FIPS-validated variants require a paid Docker subscription and cannot be
mirrored or made the
# default, but you can build a FIPS-compliant image by pointing the build at
one:
@@ -64,7 +66,7 @@ ARG AIRFLOW_VERSION="3.3.2"
# BASE_IMAGE is then a plain Debian image (for example "debian:bookworm-slim")
and Python
# AIRFLOW_PYTHON_VERSION is downloaded from python.org, verified and compiled
from sources.
ARG AIRFLOW_IMAGE_FLAVOR="hardened"
-ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev"
+ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev@sha256:2b8a45bc49c1398c8ef8e9092b9a2d028a878e7229f5ff86913f551a089372b3"
ARG AIRFLOW_PYTHON_VERSION="3.13.16"
# You can swap comments between those two args to test pip from the main
version
@@ -307,6 +309,18 @@ function keep_image_conffiles() {
printf '%s\n' force-confdef force-confold >
/etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles
}
+function prefer_hardened_apt_repository() {
+ # The hardened base images take their packages from dhi.io, but stock
Debian's sources stay
+ # configured beside it and apt installs whichever repository offers the
higher version. Debian's
+ # build of a package then replaces Docker's, and its dependencies no
longer match the packages the
+ # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above
the default priority of
+ # 500 makes apt take Docker's build whenever dhi.io publishes the package,
and fall back to Debian
+ # only when it does not. Staying below 1000 means apt never downgrades an
installed package for it.
+ # The preference covers every apt run - this build's and those of images
extending it.
+ mkdir -p /etc/apt/preferences.d
+ printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 900\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -366,6 +380,7 @@ function align_perl_base_with_perl() {
function install_debian_dev_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
@@ -630,6 +645,7 @@ function install_python() {
function install_debian_runtime_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
diff --git a/Dockerfile.ci b/Dockerfile.ci
index fd9c2012ca9..0c17a867945 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -22,12 +22,14 @@
# image are based on it, so Python is never compiled during the build.
#
# The default is Airflow's public mirror of the upstream "dhi.io/python"
image, so building needs no
-# registry credentials. The upstream image itself requires a "docker login
dhi.io".
+# registry credentials. The upstream image itself requires a "docker login
dhi.io". It is pinned to a
+# digest, because the mirror's tags move several times a day: "breeze ci
upgrade" moves the pin once
+# the full CI has passed on the new image.
#
# The FIPS-validated variants require a paid Docker subscription and cannot be
mirrored or made the
# default, but you can build a FIPS-compliant image by pointing the build at
one:
# docker build . --build-arg
BASE_IMAGE="dhi.io/python:3.13.16-debian13-fips-dev"
-ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev"
+ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev@sha256:2b8a45bc49c1398c8ef8e9092b9a2d028a878e7229f5ff86913f551a089372b3"
##############################################################################################
# This is the script image where we keep all inlined bash scripts needed in
other segments
@@ -243,6 +245,18 @@ function keep_image_conffiles() {
printf '%s\n' force-confdef force-confold >
/etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles
}
+function prefer_hardened_apt_repository() {
+ # The hardened base images take their packages from dhi.io, but stock
Debian's sources stay
+ # configured beside it and apt installs whichever repository offers the
higher version. Debian's
+ # build of a package then replaces Docker's, and its dependencies no
longer match the packages the
+ # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above
the default priority of
+ # 500 makes apt take Docker's build whenever dhi.io publishes the package,
and fall back to Debian
+ # only when it does not. Staying below 1000 means apt never downgrades an
installed package for it.
+ # The preference covers every apt run - this build's and those of images
extending it.
+ mkdir -p /etc/apt/preferences.d
+ printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 900\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -302,6 +316,7 @@ function align_perl_base_with_perl() {
function install_debian_dev_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
@@ -566,6 +581,7 @@ function install_python() {
function install_debian_runtime_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
index 4e14542c15d..a5ecd8e9b3e 100644
--- a/dev/breeze/doc/ci/02_images.md
+++ b/dev/breeze/doc/ci/02_images.md
@@ -429,7 +429,7 @@ can be used for CI images:
| Build argument | Default value |
Description
|
|-----------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------|
-| `BASE_IMAGE` |
`ghcr.io/apache/airflow/base/python:<AIRFLOW_PYTHON_VERSION>-debian13-dev` |
Mirror of the hardened base image Python comes from
|
+| `BASE_IMAGE` |
`ghcr.io/apache/airflow/base/python:<AIRFLOW_PYTHON_VERSION>-debian13-dev@<digest>`
| Mirror of the hardened base image Python comes from, pinned to a digest by
`breeze ci upgrade` |
| `PYTHON_MAJOR_MINOR_VERSION` | `3.11` |
major/minor version of Python (should match base image)
|
| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing
this number will reinstall all apt dependencies
|
| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional
`pip` flags passed to the installation commands (except when reinstalling `pip`
itself) |
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index eea8d9ce160..aa49c6e535e 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import ast
import glob
import itertools
+import json
import operator
import os
import random
@@ -102,6 +103,7 @@ from airflow_breeze.global_constants import (
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES,
DESTINATION_LOCATIONS,
+ HARDENED_PYTHON_IMAGE_DIGESTS,
HARDENED_PYTHON_IMAGE_MIRROR,
HARDENED_PYTHON_IMAGE_SOURCE,
MULTI_PLATFORM,
@@ -111,6 +113,7 @@ from airflow_breeze.global_constants import (
get_airflow_version,
get_airflowctl_version,
get_hardened_python_base_image,
+ get_hardened_python_image_retention_tag,
get_hardened_python_image_tag,
get_task_sdk_version,
get_ts_sdk_version,
@@ -5235,6 +5238,28 @@ def get_latest_hardened_python_patchlevel(
return max(versions, key=lambda version: tuple(int(part) for part in
version.split(".")))
+def get_image_digest(image: str) -> str | None:
+ """Return the digest of an image's (multi-platform) manifest in its
registry, or None if absent."""
+ result = run_command(
+ ["docker", "buildx", "imagetools", "inspect", image, "--format",
"{{json .Manifest}}"],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ if result.returncode != 0 or not result.stdout:
+ return None
+ return json.loads(result.stdout)["digest"]
+
+
+def copy_image(source: str, targets: list[str]) -> bool:
+ console_print(f"[info]Mirroring {source} -> {', '.join(targets)}[/]")
+ # imagetools copies the multi-platform manifest registry-to-registry, so
the layers never travel
+ # through the machine running this.
+ tag_flags = [flag for target in targets for flag in ("--tag", target)]
+ result = run_command(["docker", "buildx", "imagetools", "create",
*tag_flags, source], check=False)
+ return result.returncode == 0
+
+
@release_management_group.command(
name="mirror-base-images",
help="Mirror the hardened Python base images Airflow builds on to
Airflow's public registry.",
@@ -5245,39 +5270,61 @@ def get_latest_hardened_python_patchlevel(
def mirror_base_images(python: str | None):
python_versions = CURRENT_PYTHON_MAJOR_MINOR_VERSIONS if python is None
else [python]
failed: list[str] = []
+ # Tags this run moved to an image other than the one the builds are pinned
to. Only a move counts:
+ # a pin that was already behind has had its upgrade triggered by the run
that moved the tag.
+ moved: list[str] = []
# Every Debian version is mirrored: the CI image and the default PROD
image use the first one, the
# others are still available to build PROD images on.
for python_version, debian_version in itertools.product(python_versions,
ALLOWED_DEBIAN_VERSIONS):
tag = get_hardened_python_image_tag(python_version, debian_version)
- source = f"{HARDENED_PYTHON_IMAGE_SOURCE}:{tag}"
# The floating major/minor tag is what documentation and ad-hoc builds
refer to, so that they
# do not have to be edited on every Python patch release.
floating_tag = tag.replace(
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python_version,
python_version), python_version, 1
)
- targets = [f"{HARDENED_PYTHON_IMAGE_MIRROR}:{tag}",
f"{HARDENED_PYTHON_IMAGE_MIRROR}:{floating_tag}"]
- copies = [(source, targets)]
+ copies = [(tag, [floating_tag])]
pinned = ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python_version,
python_version)
latest = get_latest_hardened_python_patchlevel(python_version,
debian_version)
if latest and latest != pinned:
- latest_tag = tag.replace(pinned, latest, 1)
- copies.append(
- (
- f"{HARDENED_PYTHON_IMAGE_SOURCE}:{latest_tag}",
- [f"{HARDENED_PYTHON_IMAGE_MIRROR}:{latest_tag}"],
- )
- )
- for copy_source, copy_targets in copies:
- console_print(f"[info]Mirroring {copy_source} -> {',
'.join(copy_targets)}[/]")
- # imagetools copies the multi-platform manifest
registry-to-registry, so the layers never
- # travel through the machine running this.
- tag_flags = [flag for target in copy_targets for flag in ("--tag",
target)]
- result = run_command(
- ["docker", "buildx", "imagetools", "create", *tag_flags,
copy_source],
- check=False,
- )
- if result.returncode != 0:
- failed.append(copy_source)
+ copies.append((tag.replace(pinned, latest, 1), []))
+ for copy_tag, extra_tags in copies:
+ mirrored = f"{HARDENED_PYTHON_IMAGE_MIRROR}:{copy_tag}"
+ targets = [mirrored, *(f"{HARDENED_PYTHON_IMAGE_MIRROR}:{extra}"
for extra in extra_tags)]
+ previous_digest = get_image_digest(mirrored)
+ if not copy_image(f"{HARDENED_PYTHON_IMAGE_SOURCE}:{copy_tag}",
targets):
+ failed.append(copy_tag)
+ continue
+ digest = get_image_digest(mirrored)
+ if digest is None:
+ continue
+ if digest != previous_digest and digest !=
HARDENED_PYTHON_IMAGE_DIGESTS.get(copy_tag):
+ moved.append(copy_tag)
+ # The moving tag is replaced by the next refresh, which would
leave the digest an upgrade
+ # PR is testing untagged.
+ if not copy_image(
+ f"{mirrored}@{digest}",
+ [
+
f"{HARDENED_PYTHON_IMAGE_MIRROR}:{get_hardened_python_image_retention_tag(copy_tag,
digest)}"
+ ],
+ ):
+ failed.append(f"{copy_tag}@{digest}")
+ # The builds keep using the pinned digest after the moving tag has
moved on, so it stays tagged
+ # too - also when it was pinned before this command started tagging
digests.
+ pinned_digest = HARDENED_PYTHON_IMAGE_DIGESTS.get(tag)
+ if pinned_digest and not copy_image(
+ f"{HARDENED_PYTHON_IMAGE_MIRROR}:{tag}@{pinned_digest}",
+
[f"{HARDENED_PYTHON_IMAGE_MIRROR}:{get_hardened_python_image_retention_tag(tag,
pinned_digest)}"],
+ ):
+ failed.append(f"{tag}@{pinned_digest}")
+ if moved:
+ console_print(
+ f"[warning]The mirror moved past the pinned base images for: {',
'.join(moved)}. "
+ "Run 'breeze ci upgrade' to move the pins.[/]"
+ )
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with open(github_output, "a") as output_file:
+ output_file.write(f"moved-past-pins={'true' if moved else
'false'}\n")
if failed:
console_print(f"[error]Failed to mirror: {', '.join(failed)}[/]")
sys.exit(1)
diff --git
a/dev/breeze/src/airflow_breeze/files/hardened_python_image_digests.json
b/dev/breeze/src/airflow_breeze/files/hardened_python_image_digests.json
new file mode 100644
index 00000000000..e524e0d33c0
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/files/hardened_python_image_digests.json
@@ -0,0 +1,10 @@
+{
+ "3.11.17-debian13-dev":
"sha256:a2897780be27c9a241ac00328e2cbc3d9e830713a2588895c7de1ddc0ebfe13c",
+ "3.11.17-debian12-dev":
"sha256:283766595a4ccaafd6c428e5e71a5a105eb80c60f7aadb2c07a6ddcce2ab83ed",
+ "3.12.15-debian13-dev":
"sha256:1f6b0b79911a523f04af1051c6063714ae3f10807d4b8a40c6c24be13302454d",
+ "3.12.15-debian12-dev":
"sha256:a03b1cfea23b915c6e22a28de497776873097819998aa749270cb6ca9a84181e",
+ "3.13.16-debian13-dev":
"sha256:2b8a45bc49c1398c8ef8e9092b9a2d028a878e7229f5ff86913f551a089372b3",
+ "3.13.16-debian12-dev":
"sha256:8562353d35d625c6030d9417ffd4a753bf17e04975dbf9492ea14210f6681b58",
+ "3.14.8-debian13-dev":
"sha256:ce0302a81b4f7e78b6ac935098f1aa8af8921fa2375fe2d1e9179df09b4bc90b",
+ "3.14.8-debian12-dev":
"sha256:4bd030dab376b001202b15dfe8e4ce493e53ea2842f0f9335c4df2df24c4183a"
+}
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 70146552743..9f652dbd19c 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -824,6 +824,7 @@ DEVEL_DEPS_PATH = AIRFLOW_ROOT_PATH / "generated" /
"devel_deps.txt"
# Initialize files for rebuild check
FILES_FOR_REBUILD_CHECK = [
"Dockerfile.ci",
+ "dev/breeze/src/airflow_breeze/files/hardened_python_image_digests.json",
".dockerignore",
"scripts/docker/common.sh",
"scripts/docker/install_additional_dependencies.sh",
@@ -954,8 +955,32 @@ def get_hardened_python_image_tag(python: str,
debian_version: str = ALLOWED_DEB
return f"{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python,
python)}-{distro}-dev"
+# The mirror refreshes its tags several times a day, so images are built on
the digest each tag had
+# when "breeze ci upgrade" last moved it. A new base image therefore reaches
the builds only through
+# the upgrade PR, after the full CI has passed on it. The digests live in a
file of their own, which
+# scripts/ci/prek/upgrade_important_versions.py rewrites and the CI image
rebuild check watches.
+HARDENED_PYTHON_IMAGE_DIGESTS_PATH = (
+ AIRFLOW_ROOT_PATH
+ / "dev"
+ / "breeze"
+ / "src"
+ / "airflow_breeze"
+ / "files"
+ / "hardened_python_image_digests.json"
+)
+HARDENED_PYTHON_IMAGE_DIGESTS: dict[str, str] =
json.loads(HARDENED_PYTHON_IMAGE_DIGESTS_PATH.read_text())
+
+
def get_hardened_python_base_image(python: str, debian_version: str =
ALLOWED_DEBIAN_VERSIONS[0]) -> str:
- return
f"{HARDENED_PYTHON_IMAGE_MIRROR}:{get_hardened_python_image_tag(python,
debian_version)}"
+ tag = get_hardened_python_image_tag(python, debian_version)
+ image = f"{HARDENED_PYTHON_IMAGE_MIRROR}:{tag}"
+ digest = HARDENED_PYTHON_IMAGE_DIGESTS.get(tag)
+ return f"{image}@{digest}" if digest else image
+
+
+def get_hardened_python_image_retention_tag(tag: str, digest: str) -> str:
+ """Return the mirror tag that keeps a pinned digest tagged after its
moving tag has moved on."""
+ return f"{tag}-{digest.removeprefix('sha256:')[:12]}"
def get_legacy_base_image(debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) ->
str:
diff --git a/dev/breeze/tests/test_build_prod_params.py
b/dev/breeze/tests/test_build_prod_params.py
index 659733cada1..eaa1a2e77ad 100644
--- a/dev/breeze/tests/test_build_prod_params.py
+++ b/dev/breeze/tests/test_build_prod_params.py
@@ -18,10 +18,16 @@ from __future__ import annotations
import pytest
-from airflow_breeze.global_constants import
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION
+from airflow_breeze.global_constants import (
+ ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION,
+ HARDENED_PYTHON_IMAGE_DIGESTS,
+)
from airflow_breeze.params.build_ci_params import BuildCiParams
from airflow_breeze.params.build_prod_params import BuildProdParams
+PYTHON_3_12_TRIXIE_TAG =
f"{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.12']}-debian13-dev"
+PYTHON_3_12_BOOKWORM_TAG =
f"{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.12']}-debian12-dev"
+
def _build_args(params) -> dict[str, str]:
values = params.prepare_arguments_for_docker_build_command()
@@ -34,13 +40,15 @@ def _build_args(params) -> dict[str, str]:
pytest.param(
"hardened",
"trixie",
-
f"ghcr.io/apache/airflow/base/python:{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.12']}-debian13-dev",
+ f"ghcr.io/apache/airflow/base/python:{PYTHON_3_12_TRIXIE_TAG}"
+ f"@{HARDENED_PYTHON_IMAGE_DIGESTS[PYTHON_3_12_TRIXIE_TAG]}",
id="hardened-trixie",
),
pytest.param(
"hardened",
"bookworm",
-
f"ghcr.io/apache/airflow/base/python:{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.12']}-debian12-dev",
+ f"ghcr.io/apache/airflow/base/python:{PYTHON_3_12_BOOKWORM_TAG}"
+ f"@{HARDENED_PYTHON_IMAGE_DIGESTS[PYTHON_3_12_BOOKWORM_TAG]}",
id="hardened-bookworm",
),
pytest.param("legacy", "trixie", "debian:trixie-slim",
id="legacy-trixie"),
@@ -70,13 +78,21 @@ def test_prod_image_defaults_to_hardened_trixie():
build_args = _build_args(BuildProdParams(python="3.12"))
assert build_args["AIRFLOW_IMAGE_FLAVOR"] == "hardened"
- assert build_args["BASE_IMAGE"].endswith("-debian13-dev")
+ assert build_args["BASE_IMAGE"].split("@")[0].endswith("-debian13-dev")
def test_ci_image_is_hardened_trixie():
build_args = _build_args(BuildCiParams(python="3.12"))
assert build_args["BASE_IMAGE"] == (
-
f"ghcr.io/apache/airflow/base/python:{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.12']}-debian13-dev"
+ f"ghcr.io/apache/airflow/base/python:{PYTHON_3_12_TRIXIE_TAG}"
+ f"@{HARDENED_PYTHON_IMAGE_DIGESTS[PYTHON_3_12_TRIXIE_TAG]}"
)
assert "AIRFLOW_IMAGE_FLAVOR" not in build_args
+
+
+def test_base_image_without_a_pinned_digest_uses_the_tag():
+ # A Python version Airflow does not pin - for example one passed in to try
a new release.
+ build_args = _build_args(BuildCiParams(python="3.99"))
+
+ assert build_args["BASE_IMAGE"] ==
"ghcr.io/apache/airflow/base/python:3.99-debian13-dev"
diff --git a/dev/breeze/tests/test_mirror_base_images.py
b/dev/breeze/tests/test_mirror_base_images.py
index 7b04f6e6c87..33d004f7e9f 100644
--- a/dev/breeze/tests/test_mirror_base_images.py
+++ b/dev/breeze/tests/test_mirror_base_images.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+import json
from subprocess import CompletedProcess
from unittest import mock
@@ -27,7 +28,41 @@ from airflow_breeze.commands.release_management_commands
import (
get_latest_hardened_python_patchlevel,
mirror_base_images,
)
-from airflow_breeze.global_constants import
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION
+from airflow_breeze.global_constants import (
+ ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION,
+ HARDENED_PYTHON_IMAGE_DIGESTS,
+)
+
+MIRROR = "ghcr.io/apache/airflow/base/python"
+PINNED_TAG = f"{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION['3.14']}-debian13-dev"
+PINNED_DIGEST = HARDENED_PYTHON_IMAGE_DIGESTS[PINNED_TAG]
+NEW_DIGEST = "sha256:" + "ab" * 32
+
+
+class FakeRegistry:
+ """Answers run_command for imagetools, moving each mirrored tag to the
digest given for it."""
+
+ def __init__(self, digests_after_copy: dict[str, str]):
+ self.digests: dict[str, str] = {}
+ self.digests_after_copy = digests_after_copy
+ self.copies: list[tuple[str, list[str]]] = []
+
+ def __call__(self, cmd, **kwargs):
+ if cmd[3] == "inspect":
+ digest = self.digests.get(cmd[4])
+ stdout = json.dumps({"digest": digest}) if digest else ""
+ return CompletedProcess(args=cmd, returncode=0 if digest else 1,
stdout=stdout)
+ targets = [cmd[i + 1] for i, arg in enumerate(cmd) if arg == "--tag"]
+ self.copies.append((cmd[-1], targets))
+ for target in targets:
+ tag = target.split(":", 1)[1]
+ if tag in self.digests_after_copy:
+ self.digests[target] = self.digests_after_copy[tag]
+ return CompletedProcess(args=cmd, returncode=0)
+
+ @property
+ def sources(self) -> list[str]:
+ return [source for source, _ in self.copies if
source.startswith("dhi.io/")]
def _catalog_response(text: str) -> mock.MagicMock:
@@ -70,6 +105,7 @@ def
test_latest_hardened_python_patchlevel_is_none_when_the_catalog_is_unreachab
def test_mirror_also_copies_a_newer_published_patchlevel(newer_available):
pinned = ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION["3.14"]
latest = "3.14.99" if newer_available else pinned
+ registry = FakeRegistry({})
with (
mock.patch(
"airflow_breeze.commands.release_management_commands.get_latest_hardened_python_patchlevel",
@@ -79,20 +115,84 @@ def
test_mirror_also_copies_a_newer_published_patchlevel(newer_available):
mock.patch(
"airflow_breeze.commands.release_management_commands.run_command",
autospec=True,
- return_value=CompletedProcess(args=[], returncode=0),
- ) as run,
+ side_effect=registry,
+ ),
):
- result = CliRunner().invoke(mirror_base_images, ["--python", "3.14"],
catch_exceptions=False)
+ result = CliRunner().invoke(
+ mirror_base_images, ["--python", "3.14"], env={"GITHUB_OUTPUT":
None}, catch_exceptions=False
+ )
assert result.exit_code == 0
- sources = [call.args[0][-1] for call in run.call_args_list]
if newer_available:
- assert sources == [
+ assert registry.sources == [
f"dhi.io/python:{pinned}-debian13-dev",
"dhi.io/python:3.14.99-debian13-dev",
f"dhi.io/python:{pinned}-debian12-dev",
"dhi.io/python:3.14.99-debian12-dev",
]
- assert "ghcr.io/apache/airflow/base/python:3.14.99-debian13-dev" in
run.call_args_list[1].args[0]
+ assert (
+ "dhi.io/python:3.14.99-debian13-dev",
+ [f"{MIRROR}:3.14.99-debian13-dev"],
+ ) in registry.copies
else:
- assert sources == [f"dhi.io/python:{pinned}-debian13-dev",
f"dhi.io/python:{pinned}-debian12-dev"]
+ assert registry.sources == [
+ f"dhi.io/python:{pinned}-debian13-dev",
+ f"dhi.io/python:{pinned}-debian12-dev",
+ ]
+
+
[email protected](
+
"airflow_breeze.commands.release_management_commands.get_latest_hardened_python_patchlevel",
+ autospec=True,
+ return_value=None,
+)
+def test_mirror_keeps_the_pinned_and_the_mirrored_digests_tagged(mock_latest):
+ registry = FakeRegistry({PINNED_TAG: NEW_DIGEST})
+ with mock.patch(
+ "airflow_breeze.commands.release_management_commands.run_command",
+ autospec=True,
+ side_effect=registry,
+ ):
+ result = CliRunner().invoke(
+ mirror_base_images, ["--python", "3.14"], env={"GITHUB_OUTPUT":
None}, catch_exceptions=False
+ )
+
+ assert result.exit_code == 0
+ assert (f"{MIRROR}:{PINNED_TAG}@{NEW_DIGEST}",
[f"{MIRROR}:{PINNED_TAG}-{'ab' * 6}"]) in registry.copies
+ assert (
+ f"{MIRROR}:{PINNED_TAG}@{PINNED_DIGEST}",
+
[f"{MIRROR}:{PINNED_TAG}-{PINNED_DIGEST.removeprefix('sha256:')[:12]}"],
+ ) in registry.copies
+
+
[email protected](
+ ("previous_digest", "new_digest", "expected"),
+ [
+ pytest.param(PINNED_DIGEST, NEW_DIGEST, "true",
id="moved-past-the-pin"),
+ pytest.param(NEW_DIGEST, NEW_DIGEST, "false",
id="already-behind-not-moved"),
+ pytest.param(NEW_DIGEST, PINNED_DIGEST, "false",
id="moved-back-to-the-pin"),
+ ],
+)
[email protected](
+
"airflow_breeze.commands.release_management_commands.get_latest_hardened_python_patchlevel",
+ autospec=True,
+ return_value=None,
+)
+def test_mirror_reports_moving_past_the_pins(mock_latest, previous_digest,
new_digest, expected, tmp_path):
+ registry = FakeRegistry({PINNED_TAG: new_digest})
+ registry.digests[f"{MIRROR}:{PINNED_TAG}"] = previous_digest
+ github_output = tmp_path / "github_output"
+ with mock.patch(
+ "airflow_breeze.commands.release_management_commands.run_command",
+ autospec=True,
+ side_effect=registry,
+ ):
+ result = CliRunner().invoke(
+ mirror_base_images,
+ ["--python", "3.14"],
+ env={"GITHUB_OUTPUT": str(github_output)},
+ catch_exceptions=False,
+ )
+
+ assert result.exit_code == 0
+ assert github_output.read_text() == f"moved-past-pins={expected}\n"
diff --git a/docker-stack-docs/build-arg-ref.rst
b/docker-stack-docs/build-arg-ref.rst
index 0164efba611..39712a8b910 100644
--- a/docker-stack-docs/build-arg-ref.rst
+++ b/docker-stack-docs/build-arg-ref.rst
@@ -98,8 +98,9 @@ Those are the most common arguments that you use when you
want to build a custom
The default ``BASE_IMAGE`` is Airflow's public mirror of the `Docker Hardened
Image <https://dhi.io>`_ for
Python, tagged with the same Python version as ``AIRFLOW_PYTHON_VERSION`` -
for Airflow |airflow-version| that
-is ``ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev``. Pulling the
mirror needs no credentials;
-pulling ``dhi.io`` directly requires a ``docker login dhi.io``.
+is ``ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev``, pinned to the
digest the Airflow CI last
+passed on (see :ref:`the hardened base images <image-hardened-base>`). Pulling
the mirror needs no
+credentials; pulling ``dhi.io`` directly requires a ``docker login dhi.io``.
For the 3.4.x line Airflow also publishes ``legacy`` images, built the way all
images were built before
3.4.0: Python is downloaded from python.org, its signature verified and
compiled on a ``debian:bookworm-slim``
diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst
index b2b113c0b9d..538d95946d5 100644
--- a/docker-stack-docs/build.rst
+++ b/docker-stack-docs/build.rst
@@ -963,11 +963,17 @@ previous images that can matter when you extend or run
the image are:
``dpkg`` keeps the image's copy of a configuration file without asking -
also in images that extend
it. A package upgrade therefore does not replace a configuration file that
differs from the
package's version.
-* **The tags are rebuilt in place.** Docker republishes the same tags as CVEs
are fixed, and Airflow
- refreshes its mirror every six hours, so rebuilding an image from the same
``BASE_IMAGE`` tag can pick up
- a newer base. The mirror also copies the newest Python patch release Docker
publishes, and Airflow's
- regular dependency upgrade moves the pinned patch level to it, so the images
follow Docker's releases
- rather than python.org's.
+* **The tags are rebuilt in place, so the default is pinned to a digest.**
Docker republishes the same tags
+ as CVEs are fixed, and Airflow refreshes its mirror every six hours, so
building from a tag alone can pick
+ up a newer base at any time. The default ``BASE_IMAGE`` therefore names the
digest as well, and Airflow's
+ regular dependency upgrade moves it - in a pull request that has to pass the
full CI before the new base
+ is used. Pass a tag without a digest to build on the newest mirrored base
instead. The mirror also copies
+ the newest Python patch release Docker publishes, and the same upgrade moves
the pinned patch level to it,
+ so the images follow Docker's releases rather than python.org's.
+* **Packages come from Docker's apt repository first.** The image keeps stock
Debian's apt sources beside
+ ``dhi.io``, and ``/etc/apt/preferences.d/airflow-prefer-dhi`` ranks
``dhi.io`` higher, so apt installs
+ Docker's build of a package whenever ``dhi.io`` publishes it and falls back
to Debian's otherwise - also
+ in images that extend it.
.. _image-legacy-flavor:
diff --git
a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
index e211cd5e2dc..849abc57e95 100755
---
a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
+++
b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
@@ -31,7 +31,7 @@ export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg ADDITIONAL_PYTHON_DEPS="mpi4py==4.1.1" \
diff --git a/docker-stack-docs/docker-examples/customizing/custom-sources.sh
b/docker-stack-docs/docker-examples/customizing/custom-sources.sh
index cae38a28fa9..55ec8cf7ccf 100755
--- a/docker-stack-docs/docker-examples/customizing/custom-sources.sh
+++ b/docker-stack-docs/docker-examples/customizing/custom-sources.sh
@@ -32,7 +32,7 @@ export DOCKER_BUILDKIT=1
docker build . -f Dockerfile \
--pull \
--platform 'linux/amd64' \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack,odbc" \
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
index 0687a3deb94..996f3b6f57e 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
@@ -32,7 +32,7 @@ export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
index c5ce80d6141..442868c7636 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
@@ -31,7 +31,7 @@ export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" \
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
index 0e7e256b3a8..af5f2ddf12d 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
@@ -30,7 +30,7 @@ export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--tag "my-pypi-selected-version:0.0.1"
diff --git
a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
index be15b9e2096..ee9c8d79a4c 100755
--- a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
+++ b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
@@ -47,7 +47,7 @@ export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" \
+ --build-arg
BASE_IMAGE="${BASE_IMAGE:-ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev}"
\
--build-arg AIRFLOW_PYTHON_VERSION="3.13.16" \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
diff --git
a/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
b/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
index 438e349eb35..73ba6df1980 100644
--- a/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
+++ b/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
@@ -53,6 +53,8 @@ def get_latest_airflow_image():
)
@pytest.mark.parametrize("script_file",
glob.glob(f"{DOCKER_EXAMPLES_DIR}/**/*.sh", recursive=True))
def test_shell_script_example(script_file):
+ # The examples build on the floating base image tag unless BASE_IMAGE is
set - set it to the pinned
+ # image (`breeze` builds use it) to build them on the base the CI passed
on.
run_command(["bash", script_file])
diff --git a/scripts/ci/prek/upgrade_important_versions.py
b/scripts/ci/prek/upgrade_important_versions.py
index 3fc4d800aca..0a98b7b6349 100755
--- a/scripts/ci/prek/upgrade_important_versions.py
+++ b/scripts/ci/prek/upgrade_important_versions.py
@@ -33,6 +33,7 @@
from __future__ import annotations
+import json
import os
import re
import subprocess
@@ -442,6 +443,77 @@ def get_latest_python_version(python_major_minor: str) ->
str:
return str(latest_version)
+HARDENED_PYTHON_IMAGE_MIRROR_REPOSITORY = "apache/airflow/base/python"
+MANIFEST_LIST_MEDIA_TYPES = (
+ "application/vnd.oci.image.index.v1+json,
application/vnd.docker.distribution.manifest.list.v2+json"
+)
+# The digests the images built on the mirror are pinned to - breeze reads them
from this file.
+PYTHON_IMAGE_DIGESTS_PATH = (
+ AIRFLOW_ROOT_PATH
+ / "dev"
+ / "breeze"
+ / "src"
+ / "airflow_breeze"
+ / "files"
+ / "hardened_python_image_digests.json"
+)
+# Only references that already carry a digest are re-pinned - documentation
that shows the tag alone
+# stays readable.
+PINNED_BASE_IMAGE_PATTERN = re.compile(
+
r"(ghcr\.io/apache/airflow/base/python:)([0-9]+\.[0-9]+\.[0-9]+-debian[0-9]+-dev)@sha256:[0-9a-f]{64}"
+)
+
+
+def get_mirrored_python_image_digest(tag: str) -> str:
+ """Return the digest the mirror's moving tag currently points at - the
image the pin moves to."""
+ token = requests.get(
+ "https://ghcr.io/token";,
+ params={"scope":
f"repository:{HARDENED_PYTHON_IMAGE_MIRROR_REPOSITORY}:pull"},
+ timeout=30,
+ ).json()["token"]
+ response = requests.head(
+
f"https://ghcr.io/v2/{HARDENED_PYTHON_IMAGE_MIRROR_REPOSITORY}/manifests/{tag}";,
+ headers={"Authorization": f"Bearer {token}", "Accept":
MANIFEST_LIST_MEDIA_TYPES},
+ timeout=30,
+ )
+ if response.status_code == 404:
+ console.print(
+
f"[bright_red]ghcr.io/{HARDENED_PYTHON_IMAGE_MIRROR_REPOSITORY}:{tag} is not
mirrored yet. "
+ "Run the 'Mirror hardened base images' workflow and try again."
+ )
+ sys.exit(1)
+ response.raise_for_status()
+ return response.headers["Docker-Content-Digest"]
+
+
+def fetch_python_image_digests(latest_python_versions: dict[str, str]) ->
dict[str, str]:
+ """Fetch the digest to pin for every hardened base image tag the images
are built on."""
+ return {
+ tag: get_mirrored_python_image_digest(tag)
+ for python_version in latest_python_versions.values()
+ for debian in HARDENED_PYTHON_DEBIAN_RELEASES
+ for tag in [f"{python_version}-debian{debian}-dev"]
+ }
+
+
+def write_python_image_digests(python_image_digests: dict[str, str]) -> bool:
+ """Write the digests to pin to, returning whether they changed."""
+ new_content = json.dumps(python_image_digests, indent=2) + "\n"
+ if PYTHON_IMAGE_DIGESTS_PATH.read_text() == new_content:
+ return False
+ PYTHON_IMAGE_DIGESTS_PATH.write_text(new_content)
+ console.print(f"[bright_blue]Updated {PYTHON_IMAGE_DIGESTS_PATH}")
+ return True
+
+
+def update_python_image_digests(content: str, python_image_digests: dict[str,
str]) -> str:
+ def repin(match: re.Match[str]) -> str:
+ digest = python_image_digests.get(match.group(2))
+ return f"{match.group(1)}{match.group(2)}@{digest}" if digest else
match.group(0)
+
+ return PINNED_BASE_IMAGE_PATTERN.sub(repin, content)
+
+
def get_latest_golang_version() -> str:
if not UPGRADE_GOLANG:
return ""
@@ -979,6 +1051,7 @@ def update_file_with_versions(
keep_length: bool,
versions: dict[str, str],
latest_python_versions: dict[str, str],
+ python_image_digests: dict[str, str] | None = None,
) -> str:
"""Update file content with all version replacements."""
new_content = file_content
@@ -1005,6 +1078,9 @@ def update_file_with_versions(
new_content = apply_pattern_replacements(
new_content, latest_python_version,
AIRFLOW_IMAGE_PYTHON_PATTERNS, keep_length
)
+ # After the patchlevel replacements above, so the references name the
tags the digests are for.
+ if python_image_digests:
+ new_content = update_python_image_digests(new_content,
python_image_digests)
return _apply_simple_regexp_replacements(new_content, versions)
@@ -1019,7 +1095,11 @@ def _apply_simple_regexp_replacements(new_content: str,
versions: dict[str, str]
return new_content
-def process_all_files(versions: dict[str, str], latest_python_versions:
dict[str, str]) -> bool:
+def process_all_files(
+ versions: dict[str, str],
+ latest_python_versions: dict[str, str],
+ python_image_digests: dict[str, str] | None = None,
+) -> bool:
"""
Process all files and apply version updates.
@@ -1030,7 +1110,9 @@ def process_all_files(versions: dict[str, str],
latest_python_versions: dict[str
for file_to_update, keep_length in FILES_TO_UPDATE:
console.print(f"[bright_blue]Updating {file_to_update}")
file_content = file_to_update.read_text()
- new_content = update_file_with_versions(file_content, keep_length,
versions, latest_python_versions)
+ new_content = update_file_with_versions(
+ file_content, keep_length, versions, latest_python_versions,
python_image_digests
+ )
if new_content != file_content:
file_to_update.write_text(new_content)
@@ -1254,8 +1336,11 @@ def main() -> None:
versions = fetch_all_package_versions()
log_special_versions(versions)
latest_python_versions = fetch_python_versions()
+ python_image_digests = fetch_python_image_digests(latest_python_versions)
- changed = process_all_files(versions, latest_python_versions)
+ changed = process_all_files(versions, latest_python_versions,
python_image_digests)
+ if python_image_digests:
+ changed = write_python_image_digests(python_image_digests) or changed
# Resolve hatchling transitive dependencies and update all pyproject.toml
build-system sections
if UPGRADE_HATCHLING or UPGRADE_FLIT_CORE:
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index 681f6950ce8..cf080942f79 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -218,6 +218,18 @@ function keep_image_conffiles() {
printf '%s\n' force-confdef force-confold >
/etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles
}
+function prefer_hardened_apt_repository() {
+ # The hardened base images take their packages from dhi.io, but stock
Debian's sources stay
+ # configured beside it and apt installs whichever repository offers the
higher version. Debian's
+ # build of a package then replaces Docker's, and its dependencies no
longer match the packages the
+ # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above
the default priority of
+ # 500 makes apt take Docker's build whenever dhi.io publishes the package,
and fall back to Debian
+ # only when it does not. Staying below 1000 means apt never downgrades an
installed package for it.
+ # The preference covers every apt run - this build's and those of images
extending it.
+ mkdir -p /etc/apt/preferences.d
+ printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 900\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -277,6 +289,7 @@ function align_perl_base_with_perl() {
function install_debian_dev_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
@@ -541,6 +554,7 @@ function install_python() {
function install_debian_runtime_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
+ prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
diff --git a/scripts/tests/ci/prek/test_upgrade_important_versions.py
b/scripts/tests/ci/prek/test_upgrade_important_versions.py
index 6a9f9a1a088..1696b1106c8 100644
--- a/scripts/tests/ci/prek/test_upgrade_important_versions.py
+++ b/scripts/tests/ci/prek/test_upgrade_important_versions.py
@@ -19,6 +19,7 @@
from __future__ import annotations
+import json
from datetime import UTC
from unittest import mock
@@ -349,3 +350,108 @@ def test_get_latest_pypi_version_latest_package_managers(
monkeypatch.setattr(uiv.requests, "get",
mock.MagicMock(return_value=response))
assert uiv.get_latest_pypi_version(package_name, should_upgrade=True) ==
expected_version
+
+
+OLD_DIGEST = "sha256:" + "0" * 64
+NEW_DIGEST = "sha256:" + "1" * 64
+
+
+def test_python_image_digests_repin_only_the_pinned_references():
+ from ci.prek import upgrade_important_versions as uiv
+
+ content = (
+ f'ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.17-debian13-dev@{OLD_DIGEST}"\n'
+ "See ghcr.io/apache/airflow/base/python:3.13.17-debian13-dev for the
tag.\n"
+ )
+
+ new_content = uiv.update_python_image_digests(content,
{"3.13.17-debian13-dev": NEW_DIGEST})
+
+ assert new_content == (
+ f'ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.17-debian13-dev@{NEW_DIGEST}"\n'
+ "See ghcr.io/apache/airflow/base/python:3.13.17-debian13-dev for the
tag.\n"
+ )
+
+
[email protected](
+ ("existing", "expected_changed"),
+ [
+ pytest.param({"3.13.16-debian13-dev": OLD_DIGEST}, True, id="moved"),
+ pytest.param({"3.13.17-debian13-dev": NEW_DIGEST}, False,
id="unchanged"),
+ ],
+)
+def test_python_image_digests_are_written_to_the_pins_file(monkeypatch,
tmp_path, existing, expected_changed):
+ from ci.prek import upgrade_important_versions as uiv
+
+ pins_file = tmp_path / "hardened_python_image_digests.json"
+ pins_file.write_text(json.dumps(existing, indent=2) + "\n")
+ monkeypatch.setattr(uiv, "PYTHON_IMAGE_DIGESTS_PATH", pins_file)
+
+ assert uiv.write_python_image_digests({"3.13.17-debian13-dev":
NEW_DIGEST}) is expected_changed
+ assert json.loads(pins_file.read_text()) == {"3.13.17-debian13-dev":
NEW_DIGEST}
+
+
+def test_pins_file_is_the_one_breeze_reads():
+ from ci.prek import upgrade_important_versions as uiv
+
+ assert uiv.PYTHON_IMAGE_DIGESTS_PATH.is_file()
+
+
+def test_python_patchlevel_upgrade_also_repins_the_base_image(monkeypatch):
+ from ci.prek import upgrade_important_versions as uiv
+
+ monkeypatch.setattr(uiv, "UPGRADE_PYTHON", True)
+ monkeypatch.setattr(uiv, "ALL_PYTHON_MAJOR_MINOR_VERSIONS", ["3.13"])
+ monkeypatch.setattr(uiv, "DEFAULT_PROD_IMAGE_PYTHON_VERSION", "3.13")
+ content = f'ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev@{OLD_DIGEST}"\n'
+
+ new_content = uiv.update_file_with_versions(
+ content,
+ False,
+ {},
+ latest_python_versions={"3.13": "3.13.17"},
+ python_image_digests={"3.13.17-debian13-dev": NEW_DIGEST},
+ )
+
+ assert (
+ new_content
+ == f'ARG
BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.17-debian13-dev@{NEW_DIGEST}"\n'
+ )
+
+
+def _ghcr_responses(uiv, status_code: int):
+ token_response = mock.MagicMock(spec=uiv.requests.Response)
+ token_response.json.return_value = {"token": "anonymous"}
+ manifest_response = mock.MagicMock(spec=uiv.requests.Response)
+ manifest_response.status_code = status_code
+ manifest_response.headers = {"Docker-Content-Digest": NEW_DIGEST}
+ get = mock.create_autospec(uiv.requests.get, return_value=token_response)
+ head = mock.create_autospec(uiv.requests.head,
return_value=manifest_response)
+ return get, head
+
+
+def test_python_image_digests_are_read_from_the_mirror(monkeypatch):
+ from ci.prek import upgrade_important_versions as uiv
+
+ get, head = _ghcr_responses(uiv, 200)
+ monkeypatch.setattr(uiv.requests, "get", get)
+ monkeypatch.setattr(uiv.requests, "head", head)
+
+ assert uiv.fetch_python_image_digests({"3.13": "3.13.17"}) == {
+ "3.13.17-debian13-dev": NEW_DIGEST,
+ "3.13.17-debian12-dev": NEW_DIGEST,
+ }
+ assert [call.args[0] for call in head.call_args_list] == [
+
"https://ghcr.io/v2/apache/airflow/base/python/manifests/3.13.17-debian13-dev";,
+
"https://ghcr.io/v2/apache/airflow/base/python/manifests/3.13.17-debian12-dev";,
+ ]
+
+
+def test_python_image_digest_exits_when_the_tag_is_not_mirrored(monkeypatch):
+ from ci.prek import upgrade_important_versions as uiv
+
+ get, head = _ghcr_responses(uiv, 404)
+ monkeypatch.setattr(uiv.requests, "get", get)
+ monkeypatch.setattr(uiv.requests, "head", head)
+
+ with pytest.raises(SystemExit):
+ uiv.get_mirrored_python_image_digest("3.13.17-debian13-dev")
