This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 2c4e0c5c678 [v3-4-test] Install OS dependencies on hardened images
after dhi.io renumbering (#76200) (#76202)
2c4e0c5c678 is described below
commit 2c4e0c5c67875845673ee7e31ada43069ba230c5
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Nov 1 10:01:46 2026 +0100
[v3-4-test] Install OS dependencies on hardened images after dhi.io
renumbering (#76200) (#76202)
The dhi.io apt repository is live, so it keeps moving under a base image
pinned to a digest. Docker renumbered its builds again ("14.2.0-19+dhi5" became
"14.2.0-19+deb13u0+dhi5", which sorts lower) and dropped the old numbers, while
the pinned images still ship them. apt will not downgrade the image's copies at
the dhi.io pin priority, so gcc, libgomp1 and libgmp-dev become uninstallable
and the CI image build fails on main (docker-hardened-images/catalog#689).
Pinning the base image [...]
(cherry picked from commit 72d752a00c14261c1d80fb71258e058689363d3b)
Generated-by: Claude Opus 5
Co-authored-by: Jarek Potiuk <[email protected]>
---
Dockerfile | 31 ++++++++++++++++++++++++++++---
Dockerfile.ci | 31 ++++++++++++++++++++++++++++---
scripts/docker/install_os_dependencies.sh | 31 ++++++++++++++++++++++++++++---
3 files changed, 84 insertions(+), 9 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index bb5b1b35e2a..1bcb2b1f857 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -331,14 +331,37 @@ function prefer_hardened_apt_repository() {
# The priority stays below 1000 so the preference never forces a
downgrade: apt still favors
# dhi.io's build of a package it publishes, but may take a newer version
from another repository
# when a dependency requires it - for example PGDG's libpq5, which
postgresql-client-18 needs at
- # ">= 18.6" and dhi.io does not carry. Pinning the base image to a digest
CI has passed on keeps
- # dhi.io and the image consistent, so the renumbering that once needed a
forced "downgrade" of the
- # image's copy (docker-hardened-images/catalog#689) no longer arises
within a build.
+ # ">= 18.6" and dhi.io does not carry. The renumbered builds that sort
below the image's own copy
+ # are handled by align_image_packages_with_hardened_repository instead.
# The preference covers every apt run - this build's and those of images
extending it.
mkdir -p /etc/apt/preferences.d
printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
}
+function align_image_packages_with_hardened_repository() {
+ # The dhi.io apt repository is live, so it moves on even under a base
image pinned to a digest.
+ # Docker renumbered its builds ("14.2.0-19+dhi5" became
"14.2.0-19+deb13u0+dhi5", which sorts
+ # lower) and the repository dropped the old numbers, while the image still
ships them. Packages
+ # depending on the exact new version (gcc-14 on gcc-14-base, libgmp-dev on
libgmp10) are then
+ # uninstallable, because apt will not downgrade the image's copy at a
priority below 1000
+ # (docker-hardened-images/catalog#689). Only installed versions no
repository offers any more
+ # ("local" ones) are moved to dhi.io's build, so newer packages from other
repositories - such as
+ # PGDG's libpq5 - are left alone.
+ local package
+ local dhi_version
+ local -a downgrades=()
+ while read -r package; do
+ dhi_version=$(apt-cache madison "${package}" | awk -F' [|] ' '$3 ~
/dhi\.io/ {print $2; exit}')
+ if [[ -n "${dhi_version}" ]]; then
+ downgrades+=("${package}=${dhi_version}")
+ fi
+ done < <(apt list --installed 2>/dev/null | awk -F/ '/\[installed,local\]/
{print $1}')
+ if (( ${#downgrades[@]} )); then
+ echo "Aligning image packages with the dhi.io repository:
${downgrades[*]}"
+ apt-get install -y --no-install-recommends --allow-downgrades
"${downgrades[@]}"
+ fi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -404,6 +427,7 @@ function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
@@ -700,6 +724,7 @@ function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 3a0d8758d56..7d0771bccc3 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -262,14 +262,37 @@ function prefer_hardened_apt_repository() {
# The priority stays below 1000 so the preference never forces a
downgrade: apt still favors
# dhi.io's build of a package it publishes, but may take a newer version
from another repository
# when a dependency requires it - for example PGDG's libpq5, which
postgresql-client-18 needs at
- # ">= 18.6" and dhi.io does not carry. Pinning the base image to a digest
CI has passed on keeps
- # dhi.io and the image consistent, so the renumbering that once needed a
forced "downgrade" of the
- # image's copy (docker-hardened-images/catalog#689) no longer arises
within a build.
+ # ">= 18.6" and dhi.io does not carry. The renumbered builds that sort
below the image's own copy
+ # are handled by align_image_packages_with_hardened_repository instead.
# The preference covers every apt run - this build's and those of images
extending it.
mkdir -p /etc/apt/preferences.d
printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
}
+function align_image_packages_with_hardened_repository() {
+ # The dhi.io apt repository is live, so it moves on even under a base
image pinned to a digest.
+ # Docker renumbered its builds ("14.2.0-19+dhi5" became
"14.2.0-19+deb13u0+dhi5", which sorts
+ # lower) and the repository dropped the old numbers, while the image still
ships them. Packages
+ # depending on the exact new version (gcc-14 on gcc-14-base, libgmp-dev on
libgmp10) are then
+ # uninstallable, because apt will not downgrade the image's copy at a
priority below 1000
+ # (docker-hardened-images/catalog#689). Only installed versions no
repository offers any more
+ # ("local" ones) are moved to dhi.io's build, so newer packages from other
repositories - such as
+ # PGDG's libpq5 - are left alone.
+ local package
+ local dhi_version
+ local -a downgrades=()
+ while read -r package; do
+ dhi_version=$(apt-cache madison "${package}" | awk -F' [|] ' '$3 ~
/dhi\.io/ {print $2; exit}')
+ if [[ -n "${dhi_version}" ]]; then
+ downgrades+=("${package}=${dhi_version}")
+ fi
+ done < <(apt list --installed 2>/dev/null | awk -F/ '/\[installed,local\]/
{print $1}')
+ if (( ${#downgrades[@]} )); then
+ echo "Aligning image packages with the dhi.io repository:
${downgrades[*]}"
+ apt-get install -y --no-install-recommends --allow-downgrades
"${downgrades[@]}"
+ fi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -335,6 +358,7 @@ function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
@@ -631,6 +655,7 @@ function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index 655f282a3fd..834ba721f57 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -237,14 +237,37 @@ function prefer_hardened_apt_repository() {
# The priority stays below 1000 so the preference never forces a
downgrade: apt still favors
# dhi.io's build of a package it publishes, but may take a newer version
from another repository
# when a dependency requires it - for example PGDG's libpq5, which
postgresql-client-18 needs at
- # ">= 18.6" and dhi.io does not carry. Pinning the base image to a digest
CI has passed on keeps
- # dhi.io and the image consistent, so the renumbering that once needed a
forced "downgrade" of the
- # image's copy (docker-hardened-images/catalog#689) no longer arises
within a build.
+ # ">= 18.6" and dhi.io does not carry. The renumbered builds that sort
below the image's own copy
+ # are handled by align_image_packages_with_hardened_repository instead.
# The preference covers every apt run - this build's and those of images
extending it.
mkdir -p /etc/apt/preferences.d
printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' >
/etc/apt/preferences.d/airflow-prefer-dhi
}
+function align_image_packages_with_hardened_repository() {
+ # The dhi.io apt repository is live, so it moves on even under a base
image pinned to a digest.
+ # Docker renumbered its builds ("14.2.0-19+dhi5" became
"14.2.0-19+deb13u0+dhi5", which sorts
+ # lower) and the repository dropped the old numbers, while the image still
ships them. Packages
+ # depending on the exact new version (gcc-14 on gcc-14-base, libgmp-dev on
libgmp10) are then
+ # uninstallable, because apt will not downgrade the image's copy at a
priority below 1000
+ # (docker-hardened-images/catalog#689). Only installed versions no
repository offers any more
+ # ("local" ones) are moved to dhi.io's build, so newer packages from other
repositories - such as
+ # PGDG's libpq5 - are left alone.
+ local package
+ local dhi_version
+ local -a downgrades=()
+ while read -r package; do
+ dhi_version=$(apt-cache madison "${package}" | awk -F' [|] ' '$3 ~
/dhi\.io/ {print $2; exit}')
+ if [[ -n "${dhi_version}" ]]; then
+ downgrades+=("${package}=${dhi_version}")
+ fi
+ done < <(apt list --installed 2>/dev/null | awk -F/ '/\[installed,local\]/
{print $1}')
+ if (( ${#downgrades[@]} )); then
+ echo "Aligning image packages with the dhi.io repository:
${downgrades[*]}"
+ apt-get install -y --no-install-recommends --allow-downgrades
"${downgrades[@]}"
+ fi
+}
+
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so
installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() -
which reads that cache -
@@ -310,6 +333,7 @@ function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
@@ -606,6 +630,7 @@ function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
+ align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
