This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch hardened-python-base-images in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8dfdb58131a7f110e3ce1d72195628ca372a521a Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Sep 12 19:26:15 2026 +0200 Base CI and PROD images on hardened Python images Airflow compiled Python from source in every image build, which meant downloading a tarball, verifying it with sigstore or GPG, and paying for a full CPython build on each cache miss. Docker's hardened Python images are Apache-2.0 licensed and already provide a maintained, patched Python, so the build can consume one instead of reproducing that work. Pulling from dhi.io requires a Docker Hub login. Requiring one to build an Airflow image would force credentials on every contributor and CI job, so the tags are mirrored to a public registry and BASE_IMAGE points there. PYTHON_LTO existed only to disable Link-Time Optimization while compiling Python in FIPS mode. With no compilation left it has nothing to configure, and the hardened images offer FIPS variants that serve that need directly. Closes: #59625 Claude-Session: https://claude.ai/code/session_01DcRW9x8n7Jor7ftsKFxQHD --- .github/workflows/mirror-hardened-base-images.yml | 73 ++++++ Dockerfile | 276 +++++++-------------- Dockerfile.ci | 270 +++++++------------- .../hardened-base-images.significant.rst | 17 ++ dev/breeze/doc/ci/02_images.md | 18 +- .../commands/release_management_commands.py | 41 ++- dev/breeze/src/airflow_breeze/global_constants.py | 21 +- .../airflow_breeze/params/common_build_params.py | 3 +- docker-stack-docs/build-arg-ref.rst | 14 +- docker-stack-docs/build.rst | 15 +- docker-stack-docs/changelog.rst | 26 ++ .../customizing/add-build-essential-custom.sh | 2 +- .../docker-examples/customizing/custom-sources.sh | 2 +- .../customizing/pypi-dev-runtime-deps.sh | 2 +- .../customizing/pypi-extras-and-deps.sh | 2 +- .../customizing/pypi-selected-version.sh | 2 +- .../restricted/restricted_environments.sh | 2 +- scripts/ci/prek/update_docker_gpg_keys.py | 2 - scripts/ci/prek/upgrade_important_versions.py | 11 +- scripts/docker/common.sh | 10 + scripts/docker/install_mssql.sh | 2 +- scripts/docker/install_mysql.sh | 2 +- scripts/docker/install_os_dependencies.sh | 228 +++++------------ scripts/docker/install_postgres.sh | 2 +- scripts/docker/keys/python-3.10.asc | 109 -------- 25 files changed, 466 insertions(+), 686 deletions(-) diff --git a/.github/workflows/mirror-hardened-base-images.yml b/.github/workflows/mirror-hardened-base-images.yml new file mode 100644 index 00000000000..8cd43953671 --- /dev/null +++ b/.github/workflows/mirror-hardened-base-images.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Mirror hardened base images +# Copies the Docker Hardened Image (https://dhi.io) tags that Airflow images are built on into a +# public ghcr.io repository owned by the project. +# +# Pulling from dhi.io requires a Docker Hub login. Requiring one to build an Airflow image would +# mean every contributor and every CI job needs Docker Hub credentials, so the tags are mirrored +# instead and `BASE_IMAGE` defaults to the mirror. The images are Apache-2.0 licensed, so +# redistributing them is allowed. +# +# `docker buildx imagetools create` copies the multi-platform manifest between registries without +# pulling the layers into the runner, so a refresh of all Python versions is cheap. +on: # yamllint disable-line rule:truthy + workflow_dispatch: + schedule: + # Docker rebuilds the hardened images as CVEs are fixed, keeping the same tags. Re-running + # weekly is what picks those rebuilds up. + - cron: '27 4 * * 1' +permissions: + contents: read +concurrency: + group: mirror-hardened-base-images + cancel-in-progress: false +jobs: + mirror: + name: "Mirror hardened base images" + runs-on: ["ubuntu-22.04"] + # Never runs on forks - they have neither the credentials nor the package to push to. + if: github.repository == 'apache/airflow' + permissions: + contents: read + packages: write + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + # dhi.io accepts any Docker Hub account - the free tier covers the images we mirror - so this + # reuses the credentials the DockerHub release workflows already use. + - name: "Login to dhi.io" + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: echo "${DOCKERHUB_TOKEN}" | docker login dhi.io -u "${DOCKERHUB_USER}" --password-stdin + - name: "Login to ghcr.io" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}" --password-stdin + - name: "Mirror base images" + run: breeze release-management mirror-base-images diff --git a/Dockerfile b/Dockerfile index 9f845276cfe..749212b4e67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,23 +50,17 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here ARG AIRFLOW_VERSION="3.3.1" -ARG BASE_IMAGE="debian:bookworm-slim" -ARG AIRFLOW_PYTHON_VERSION="3.13.15" - -# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO). -# -# Link-Time Optimization uses MD5 checksums during the compilation process to verify -# object files and intermediate representations. In FIPS-compliant environments, MD5 -# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3). -# This can cause Python builds with LTO to fail when FIPS mode is enabled. +# BASE_IMAGE: the Docker Hardened Image that Python comes from. Both the build stage and the final +# image are based on it, so Python is never compiled during the build. # -# When building FIPS-compliant images, set this to "false" to disable LTO: -# docker build --build-arg PYTHON_LTO="false" ... +# The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no +# registry credentials. The upstream image itself requires a "docker login dhi.io". # -# Default: "true" (LTO enabled for better performance) -# -# Related: https://github.com/apache/airflow/issues/58337 -ARG PYTHON_LTO="true" +# The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the +# default, but you can build a FIPS-compliant image by pointing the build at one: +# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.15-debian12-fips-dev" +ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" +ARG AIRFLOW_PYTHON_VERSION="3.13.15" # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds @@ -119,15 +113,13 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} -PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} NODEJS_VERSION=${NODEJS_VERSION:-22.23.1} PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} -COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} +PYTHON_HOME=${PYTHON_HOME:-/opt/python} if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" @@ -154,38 +146,25 @@ freetds-dev \ git \ graphviz \ graphviz-dev \ +gzip \ krb5-user \ -lcov \ ldap-utils \ -libbluetooth-dev \ -libbz2-dev \ libc6-dev \ -libdb-dev \ libev-dev \ libev4 \ libffi-dev \ -libgdbm-compat-dev \ -libgdbm-dev \ libgeos-dev \ libkrb5-dev \ libldap2-dev \ libleveldb-dev \ libleveldb1d \ -liblzma-dev \ -libncurses5-dev \ -libreadline6-dev \ libsasl2-2 \ libsasl2-dev \ libsasl2-modules \ -libsqlite3-dev \ libssl-dev \ libxmlsec1 \ libxmlsec1-dev \ -libzstd-dev \ locales \ -lsb-release \ -lzma \ -lzma-dev \ openssh-client \ openssl \ pkg-config \ @@ -194,10 +173,8 @@ sasl2-bin \ sqlite3 \ sudo \ tdsodbc \ -tk-dev \ unixodbc \ unixodbc-dev \ -uuid-dev \ wget \ xz-utils \ zlib1g-dev \ @@ -224,6 +201,9 @@ netcat-openbsd\ echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo + # libxmlsec1-openssl was added because libxmlsec1 ships no crypto engine of its own - the engines + # are separate packages - so the "xmlsec" module (pulled in by python3-saml) imported with + # "libxmlsec1-openssl.so.1: cannot open shared object file" without it. if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then RUNTIME_APT_DEPS="\ ${debian_version_apt_deps} \ @@ -242,8 +222,8 @@ libgeos-dev \ libsasl2-2 \ libsasl2-modules \ libxmlsec1 \ +libxmlsec1-openssl \ locales \ -lsb-release \ openssh-client \ rsync \ sasl2-bin \ @@ -272,10 +252,25 @@ function install_docker_cli() { apt-get install -y --no-install-recommends docker-ce-cli } +function restore_debian_base_files() { + # The hardened base images ship a minimal /etc, but Debian maintainer scripts assume the files + # a stock Debian has: sasl2-bin chowns its run directory to the "sasl" group from base-passwd, + # and tmux registers its shell with add-shell, which reads /etc/shells. The base-passwd package + # only ships the master account files - update-passwd is what merges them into /etc. + # libpam-runtime generates the /etc/pam.d/common-* files that the PAM configs already in the + # image "@include" - without them "adduser --gecos" aborts with a PAM error from chfn. + apt-get install -y --no-install-recommends base-passwd libpam-runtime + update-passwd + if [[ ! -e /etc/shells ]]; then + printf '%s\n' "# /etc/shells: valid login shells" /bin/sh /bin/bash > /etc/shells + fi +} + function install_debian_dev_dependencies() { apt-get update apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_DEV_APT_ENV?} if [[ ${DEV_APT_COMMAND} != "" ]]; then @@ -305,14 +300,25 @@ function install_additional_dev_dependencies() { } function link_python() { + # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and + # user customizations refer to that path - while the hardened base images ship it in /opt/python, + # so keep the historical location working as a symlink. + if [[ ! -e /usr/python ]]; then + ln -sv "${PYTHON_HOME}" /usr/python + fi + # The hardened base images have no /usr/local tree at all + mkdir -p /usr/local/bin /usr/local/lib # link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix # Links in /usr/local/bin are needed for tools that expect python to be there # Links in /usr/python/bin are needed for tools that are detecting home of python installation including # lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part. for dst in pip3 python3 python3-config; do src="$(echo "${dst}" | tr -d 3)" + if [[ ! -e "/usr/python/bin/${dst}" ]]; then + continue + fi echo "Linking ${dst} in /usr/local/bin and /usr/python/bin" - ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" + ln -sfv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" for dir in /usr/local/bin /usr/python/bin; do if [[ ! -e "${dir}/${src}" ]]; then echo "Creating ${src} - > ${dst} link in ${dir}" @@ -332,10 +338,33 @@ function link_python() { ldconfig } +function check_no_system_python() { + # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # image. A system Python pulled in as a dependency of an apt package shares its shared libraries + # with ours and leads to errors such as: + # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add + if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + echo + echo "ERROR! System python is installed by one of the previous steps" + echo + echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" + echo + apt-get install -yqq aptitude >/dev/null + aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + echo + exit 1 + else + echo + echo "GOOD! System python is not installed - OK" + echo + fi +} + function install_debian_runtime_dependencies() { apt-get update apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_RUNTIME_APT_ENV?} if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then @@ -349,152 +378,11 @@ function install_debian_runtime_dependencies() { apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} apt-get autoremove -yqq --purge apt-get clean + check_no_system_python link_python rm -rf /var/lib/apt/lists/* /var/log/* } -function install_cosign() { - local arch - arch="$(dpkg --print-architecture)" - declare -A cosign_sha256s=( - # https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt - [amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2" - [arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab" - ) - local cosign_sha256="${cosign_sha256s[${arch}]}" - if [[ -z "${cosign_sha256}" ]]; then - echo "Unsupported architecture for cosign: ${arch}" - exit 1 - fi - curl -fsSL --retry 3 --retry-delay 5 \ - "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"; \ - -o /tmp/cosign - echo "${cosign_sha256} /tmp/cosign" | sha256sum --check - chmod +x /tmp/cosign -} - -function install_python() { - # If system python (3.11 in bookworm) is installed (via automatic installation of some dependencies for example), we need - # to fail and make sure that it is not there, because there can be strange interactions if we install - # newer version and system libraries are installed, because - # when you create a virtualenv part of the shared libraries of Python can be taken from the system - # Installation leading to weird errors when you want to install some modules - for example when you install ssl: - # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then - echo - echo "ERROR! System python is installed by one of the previous steps" - echo - echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython3.11 is installed:" - echo - apt-get install -yqq aptitude >/dev/null - aptitude why libpython3.11 - echo - exit 1 - else - echo - echo "GOOD! System python is not installed - OK" - echo - fi - wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"; - local major_minor_version - major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" - echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="[email protected]" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="[email protected]" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="[email protected]" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="[email protected]" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com"; - [3.12]="https://accounts.google.com"; - [3.13]="https://accounts.google.com"; - [3.14]="https://github.com/login/oauth"; - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"; - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado <[email protected]>" imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc"; - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi - mkdir -p /usr/src/python - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz - rm python.tar.xz - cd /usr/src/python - arch="$(dpkg --print-architecture)"; arch="${arch##*-}" - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" - EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" - EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"; - LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" - LDFLAGS="${LDFLAGS:--Wl},--strip-all" - # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during - # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds - # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds. - # See: https://github.com/apache/airflow/issues/58337 - local lto_option="" - if [[ "${PYTHON_LTO:-true}" == "true" ]]; then - lto_option="--with-lto" - fi - local build_log - build_log=$(mktemp) - echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..." - if ! ( - ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \ - --enable-loadable-sqlite-extensions --enable-option-checking=fatal \ - --enable-shared ${lto_option} && \ - make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \ - make -s -j "$(nproc)" install - ) > "${build_log}" 2>&1; then - echo - echo "ERROR! Python build failed. Build output:" - echo - cat "${build_log}" - rm -f "${build_log}" - exit 1 - fi - rm -f "${build_log}" - cd / - rm -rf /usr/src/python - find /usr/python -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + - link_python -} - function install_golang() { curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz @@ -588,7 +476,8 @@ if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then else get_dev_apt_deps install_debian_dev_dependencies - install_python + check_no_system_python + link_python install_additional_dev_dependencies install_rustup if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then @@ -692,7 +581,7 @@ install_mariadb_client() { echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mariadb.list # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb @@ -736,7 +625,7 @@ function install_mssql_client() { echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(common::debian_release)/prod $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mssql-release.list && mkdir -p /opt/microsoft/msodbcsql18 && touch /opt/microsoft/msodbcsql18/ACCEPT_EULA && @@ -780,7 +669,7 @@ install_postgres_client() { common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres" - echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \ + echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" @@ -1008,6 +897,16 @@ function common::install_packaging_tools() { fi } +function common::debian_codename() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_CODENAME}") +} + +function common::debian_release() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_ID}") +} + function common::import_trusted_gpg() { common::get_colors @@ -1910,15 +1809,12 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV} \ AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION} -ARG PYTHON_LTO - ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/home/airflow/.cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" COPY --from=scripts install_os_dependencies.sh /scripts/docker/ -COPY scripts/docker/keys/ /scripts/docker/keys/ -RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh dev +RUN bash /scripts/docker/install_os_dependencies.sh dev # In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python # libraries will be accidentally used before the library installed from sources (which is newer and @@ -1940,6 +1836,7 @@ COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other # scripts which are needed much later will not invalidate the docker layer here COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ +COPY scripts/docker/keys/ /scripts/docker/keys/ RUN bash /scripts/docker/install_mysql.sh dev && \ bash /scripts/docker/install_mssql.sh dev && \ @@ -2170,9 +2067,6 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} -ARG PYTHON_LTO - -COPY --from=airflow-build-image "/usr/python/" "/usr/python/" COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh runtime diff --git a/Dockerfile.ci b/Dockerfile.ci index ddb9ae98e84..c266f080df6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -18,21 +18,16 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO). +# BASE_IMAGE: the Docker Hardened Image that Python comes from. Both the build stage and the final +# image are based on it, so Python is never compiled during the build. # -# Link-Time Optimization uses MD5 checksums during the compilation process to verify -# object files and intermediate representations. In FIPS-compliant environments, MD5 -# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3). -# This can cause Python builds with LTO to fail when FIPS mode is enabled. +# The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no +# registry credentials. The upstream image itself requires a "docker login dhi.io". # -# When building FIPS-compliant images, set this to "false" to disable LTO: -# docker build --build-arg PYTHON_LTO="false" ... -# -# Default: "true" (LTO enabled for better performance) -# -# Related: https://github.com/apache/airflow/issues/58337 -ARG PYTHON_LTO="true" -ARG BASE_IMAGE="debian:bookworm-slim" +# The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the +# default, but you can build a FIPS-compliant image by pointing the build at one: +# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.15-debian12-fips-dev" +ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments @@ -59,15 +54,13 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} -PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} NODEJS_VERSION=${NODEJS_VERSION:-22.23.1} PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} -COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} +PYTHON_HOME=${PYTHON_HOME:-/opt/python} if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" @@ -94,38 +87,25 @@ freetds-dev \ git \ graphviz \ graphviz-dev \ +gzip \ krb5-user \ -lcov \ ldap-utils \ -libbluetooth-dev \ -libbz2-dev \ libc6-dev \ -libdb-dev \ libev-dev \ libev4 \ libffi-dev \ -libgdbm-compat-dev \ -libgdbm-dev \ libgeos-dev \ libkrb5-dev \ libldap2-dev \ libleveldb-dev \ libleveldb1d \ -liblzma-dev \ -libncurses5-dev \ -libreadline6-dev \ libsasl2-2 \ libsasl2-dev \ libsasl2-modules \ -libsqlite3-dev \ libssl-dev \ libxmlsec1 \ libxmlsec1-dev \ -libzstd-dev \ locales \ -lsb-release \ -lzma \ -lzma-dev \ openssh-client \ openssl \ pkg-config \ @@ -134,10 +114,8 @@ sasl2-bin \ sqlite3 \ sudo \ tdsodbc \ -tk-dev \ unixodbc \ unixodbc-dev \ -uuid-dev \ wget \ xz-utils \ zlib1g-dev \ @@ -164,6 +142,9 @@ netcat-openbsd\ echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo + # libxmlsec1-openssl was added because libxmlsec1 ships no crypto engine of its own - the engines + # are separate packages - so the "xmlsec" module (pulled in by python3-saml) imported with + # "libxmlsec1-openssl.so.1: cannot open shared object file" without it. if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then RUNTIME_APT_DEPS="\ ${debian_version_apt_deps} \ @@ -182,8 +163,8 @@ libgeos-dev \ libsasl2-2 \ libsasl2-modules \ libxmlsec1 \ +libxmlsec1-openssl \ locales \ -lsb-release \ openssh-client \ rsync \ sasl2-bin \ @@ -212,10 +193,25 @@ function install_docker_cli() { apt-get install -y --no-install-recommends docker-ce-cli } +function restore_debian_base_files() { + # The hardened base images ship a minimal /etc, but Debian maintainer scripts assume the files + # a stock Debian has: sasl2-bin chowns its run directory to the "sasl" group from base-passwd, + # and tmux registers its shell with add-shell, which reads /etc/shells. The base-passwd package + # only ships the master account files - update-passwd is what merges them into /etc. + # libpam-runtime generates the /etc/pam.d/common-* files that the PAM configs already in the + # image "@include" - without them "adduser --gecos" aborts with a PAM error from chfn. + apt-get install -y --no-install-recommends base-passwd libpam-runtime + update-passwd + if [[ ! -e /etc/shells ]]; then + printf '%s\n' "# /etc/shells: valid login shells" /bin/sh /bin/bash > /etc/shells + fi +} + function install_debian_dev_dependencies() { apt-get update apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_DEV_APT_ENV?} if [[ ${DEV_APT_COMMAND} != "" ]]; then @@ -245,14 +241,25 @@ function install_additional_dev_dependencies() { } function link_python() { + # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and + # user customizations refer to that path - while the hardened base images ship it in /opt/python, + # so keep the historical location working as a symlink. + if [[ ! -e /usr/python ]]; then + ln -sv "${PYTHON_HOME}" /usr/python + fi + # The hardened base images have no /usr/local tree at all + mkdir -p /usr/local/bin /usr/local/lib # link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix # Links in /usr/local/bin are needed for tools that expect python to be there # Links in /usr/python/bin are needed for tools that are detecting home of python installation including # lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part. for dst in pip3 python3 python3-config; do src="$(echo "${dst}" | tr -d 3)" + if [[ ! -e "/usr/python/bin/${dst}" ]]; then + continue + fi echo "Linking ${dst} in /usr/local/bin and /usr/python/bin" - ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" + ln -sfv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" for dir in /usr/local/bin /usr/python/bin; do if [[ ! -e "${dir}/${src}" ]]; then echo "Creating ${src} - > ${dst} link in ${dir}" @@ -272,10 +279,33 @@ function link_python() { ldconfig } +function check_no_system_python() { + # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # image. A system Python pulled in as a dependency of an apt package shares its shared libraries + # with ours and leads to errors such as: + # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add + if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + echo + echo "ERROR! System python is installed by one of the previous steps" + echo + echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" + echo + apt-get install -yqq aptitude >/dev/null + aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + echo + exit 1 + else + echo + echo "GOOD! System python is not installed - OK" + echo + fi +} + function install_debian_runtime_dependencies() { apt-get update apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_RUNTIME_APT_ENV?} if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then @@ -289,152 +319,11 @@ function install_debian_runtime_dependencies() { apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} apt-get autoremove -yqq --purge apt-get clean + check_no_system_python link_python rm -rf /var/lib/apt/lists/* /var/log/* } -function install_cosign() { - local arch - arch="$(dpkg --print-architecture)" - declare -A cosign_sha256s=( - # https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt - [amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2" - [arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab" - ) - local cosign_sha256="${cosign_sha256s[${arch}]}" - if [[ -z "${cosign_sha256}" ]]; then - echo "Unsupported architecture for cosign: ${arch}" - exit 1 - fi - curl -fsSL --retry 3 --retry-delay 5 \ - "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"; \ - -o /tmp/cosign - echo "${cosign_sha256} /tmp/cosign" | sha256sum --check - chmod +x /tmp/cosign -} - -function install_python() { - # If system python (3.11 in bookworm) is installed (via automatic installation of some dependencies for example), we need - # to fail and make sure that it is not there, because there can be strange interactions if we install - # newer version and system libraries are installed, because - # when you create a virtualenv part of the shared libraries of Python can be taken from the system - # Installation leading to weird errors when you want to install some modules - for example when you install ssl: - # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then - echo - echo "ERROR! System python is installed by one of the previous steps" - echo - echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython3.11 is installed:" - echo - apt-get install -yqq aptitude >/dev/null - aptitude why libpython3.11 - echo - exit 1 - else - echo - echo "GOOD! System python is not installed - OK" - echo - fi - wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"; - local major_minor_version - major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" - echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="[email protected]" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="[email protected]" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="[email protected]" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="[email protected]" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com"; - [3.12]="https://accounts.google.com"; - [3.13]="https://accounts.google.com"; - [3.14]="https://github.com/login/oauth"; - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"; - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado <[email protected]>" imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc"; - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi - mkdir -p /usr/src/python - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz - rm python.tar.xz - cd /usr/src/python - arch="$(dpkg --print-architecture)"; arch="${arch##*-}" - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" - EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" - EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"; - LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" - LDFLAGS="${LDFLAGS:--Wl},--strip-all" - # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during - # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds - # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds. - # See: https://github.com/apache/airflow/issues/58337 - local lto_option="" - if [[ "${PYTHON_LTO:-true}" == "true" ]]; then - lto_option="--with-lto" - fi - local build_log - build_log=$(mktemp) - echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..." - if ! ( - ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \ - --enable-loadable-sqlite-extensions --enable-option-checking=fatal \ - --enable-shared ${lto_option} && \ - make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \ - make -s -j "$(nproc)" install - ) > "${build_log}" 2>&1; then - echo - echo "ERROR! Python build failed. Build output:" - echo - cat "${build_log}" - rm -f "${build_log}" - exit 1 - fi - rm -f "${build_log}" - cd / - rm -rf /usr/src/python - find /usr/python -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + - link_python -} - function install_golang() { curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz @@ -528,7 +417,8 @@ if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then else get_dev_apt_deps install_debian_dev_dependencies - install_python + check_no_system_python + link_python install_additional_dev_dependencies install_rustup if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then @@ -632,7 +522,7 @@ install_mariadb_client() { echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mariadb.list # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb @@ -676,7 +566,7 @@ function install_mssql_client() { echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(common::debian_release)/prod $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mssql-release.list && mkdir -p /opt/microsoft/msodbcsql18 && touch /opt/microsoft/msodbcsql18/ACCEPT_EULA && @@ -720,7 +610,7 @@ install_postgres_client() { common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres" - echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \ + echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" @@ -948,6 +838,16 @@ function common::install_packaging_tools() { fi } +function common::debian_codename() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_CODENAME}") +} + +function common::debian_release() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_ID}") +} + function common::import_trusted_gpg() { common::get_colors @@ -1785,12 +1685,9 @@ ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/usr/local/cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" -ARG PYTHON_LTO - COPY --from=scripts install_os_dependencies.sh /scripts/docker/ -COPY scripts/docker/keys/ /scripts/docker/keys/ -RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh ci +RUN bash /scripts/docker/install_os_dependencies.sh ci # In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python # libraries will be accidentally used before the library installed from sources (which is newer and @@ -1802,6 +1699,7 @@ COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other # scripts which are needed much later will not invalidate the docker layer here. COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ +COPY scripts/docker/keys/ /scripts/docker/keys/ ARG HOME=/root ARG AIRFLOW_IMAGE_TYPE="ci" diff --git a/airflow-core/newsfragments/hardened-base-images.significant.rst b/airflow-core/newsfragments/hardened-base-images.significant.rst new file mode 100644 index 00000000000..8a4672b1786 --- /dev/null +++ b/airflow-core/newsfragments/hardened-base-images.significant.rst @@ -0,0 +1,17 @@ +Airflow's CI and production images are now based on `Docker Hardened Images <https://dhi.io>`_ for Python + +Python is no longer compiled from source during the image build - it comes from the hardened base image +instead, which removes the source download, its signature verification and the compilation itself from the +build (the OS dependency layer of the production image went from roughly 210s to 105s in a local +cache-disabled build). Python still lives in ``/usr/python``, which is now a symlink to the ``/opt/python`` +location the base image uses, so paths inside the image are unchanged. + +The ``BASE_IMAGE`` build argument now defaults to ``ghcr.io/apache/airflow/base/python:<version>-debian12-dev``, +Airflow's public mirror of the upstream ``dhi.io/python`` image - building an image needs no registry +credentials. Anyone overriding ``BASE_IMAGE`` with a plain ``debian:bookworm-slim`` image has to switch to +an image that already provides Python. + +The ``PYTHON_LTO`` build argument is gone. It existed to disable Link-Time Optimization when compiling +Python in FIPS mode, and there is no Python compilation left to configure. To build a FIPS-compliant image, +point ``BASE_IMAGE`` at a FIPS variant of the hardened image instead, for example +``dhi.io/python:3.13.15-debian12-fips-dev``. diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index f3959be9218..79c524c6f37 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -379,7 +379,7 @@ plugin installed. ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \ --build-arg ADDITIONAL_PYTHON_DEPS="pandas" \ --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" \ @@ -404,7 +404,7 @@ comment](https://github.com/apache/airflow/issues/8605#issuecomment-690065621): ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack" \ --build-arg ADDITIONAL_PYTHON_DEPS="apache-airflow-providers-odbc \ @@ -429,7 +429,7 @@ can be used for CI images: | Build argument | Default value | Description | |-----------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------| -| `PYTHON_BASE_IMAGE` | `python:3.10-slim-bookworm` | Base Python image | +| `BASE_IMAGE` | `ghcr.io/apache/airflow/base/python:<AIRFLOW_PYTHON_VERSION>-debian12-dev` | Mirror of the hardened base image Python comes from | | `PYTHON_MAJOR_MINOR_VERSION` | `3.10` | major/minor version of Python (should match base image) | | `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | | `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | @@ -465,7 +465,7 @@ This builds the CI image in version 3.10 with default extras ("all"). ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" --tag my-image:0.0.1 + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" --tag my-image:0.0.1 ``` This builds the CI image in version 3.10 with "gcp" extra only. @@ -473,7 +473,7 @@ This builds the CI image in version 3.10 with "gcp" extra only. ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg AIRFLOW_EXTRAS=gcp --tag my-image:0.0.1 ``` @@ -482,7 +482,7 @@ This builds the CI image in version 3.10 with "apache-beam" extra added. ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="apache-beam" --tag my-image:0.0.1 ``` @@ -492,7 +492,7 @@ added. ``` bash DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg ADDITIONAL_PYTHON_DEPS="mssql" --tag my-image:0.0.1 ``` @@ -502,7 +502,7 @@ apt dev dependencies added. ``` DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" --tag my-image:0.0.1 ``` @@ -512,7 +512,7 @@ This builds the CI image in version 3.10 with "jdbc" extra and ``` DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.10-debian12-dev" \ --build-arg AIRFLOW_EXTRAS=jdbc \ --tag my-image:0.0.1 ``` diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index def232cddc6..41cbe2d4b7d 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -99,12 +99,16 @@ from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES, DESTINATION_LOCATIONS, + HARDENED_PYTHON_IMAGE_MIRROR, + HARDENED_PYTHON_IMAGE_SOURCE, MULTI_PLATFORM, SCHEMA_DESTINATION_LOCATIONS, UV_VERSION, get_airflow_mypy_version, get_airflow_version, get_airflowctl_version, + get_hardened_python_base_image, + get_hardened_python_image_tag, get_task_sdk_version, get_ts_sdk_version, ) @@ -2471,7 +2475,7 @@ def release_prod_images( for python in python_versions: build_args = { "AIRFLOW_CONSTRAINTS": "constraints-no-providers", - "BASE_IMAGE": "debian:bookworm-slim", + "BASE_IMAGE": get_hardened_python_base_image(python), "AIRFLOW_PYTHON_VERSION": ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python, python), "AIRFLOW_VERSION": airflow_version, "INCLUDE_PRE_RELEASE": "true" if include_pre_release else "false", @@ -5175,3 +5179,38 @@ def check_release_files( else: console.print("\n[success]All expected files are present![/]") sys.exit(0) + + +@release_management_group.command( + name="mirror-base-images", + help="Mirror the hardened Python base images Airflow builds on to Airflow's public registry.", +) +@option_python_no_default +@option_verbose +@option_dry_run +def mirror_base_images(python: str | None): + python_versions = CURRENT_PYTHON_MAJOR_MINOR_VERSIONS if python is None else [python] + failed: list[str] = [] + for python_version in python_versions: + tag = get_hardened_python_image_tag(python_version) + source = f"{HARDENED_PYTHON_IMAGE_SOURCE}:{tag}" + # The floating major/minor tag is what documentation and ad-hoc builds refer to, so that they + # do not have to be edited on every Python patch release. + floating_tag = tag.replace( + ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python_version, python_version), python_version, 1 + ) + targets = [f"{HARDENED_PYTHON_IMAGE_MIRROR}:{tag}", f"{HARDENED_PYTHON_IMAGE_MIRROR}:{floating_tag}"] + console_print(f"[info]Mirroring {source} -> {', '.join(targets)}[/]") + # imagetools copies the multi-platform manifest registry-to-registry, so the layers never + # travel through the machine running this. + tag_flags = [flag for target in targets for flag in ("--tag", target)] + result = run_command( + ["docker", "buildx", "imagetools", "create", *tag_flags, source], + check=False, + ) + if result.returncode != 0: + failed.append(source) + if failed: + console_print(f"[error]Failed to mirror: {', '.join(failed)}[/]") + sys.exit(1) + console_print("[success]All base images mirrored[/]") diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index d4b0fd9186c..c57020f00be 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -905,9 +905,28 @@ ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION: dict[str, str] = { "3.11": "3.11.16", "3.12": "3.12.14", "3.13": "3.13.15", - "3.14": "3.14.3", + "3.14": "3.14.7", } +# Airflow images are based on Docker Hardened Images (https://dhi.io). The "-dev" variant carries +# apt, a shell and runs as root; the tags encode the Debian release rather than its codename. +# Pulling from dhi.io requires a Docker Hub login, so the tags Airflow builds against are mirrored +# to a public ghcr.io repository by the "Mirror hardened base images" workflow - building Airflow +# images needs no registry credentials at all, which is what the default below points at. +HARDENED_PYTHON_IMAGE_SOURCE = "dhi.io/python" +HARDENED_PYTHON_IMAGE_MIRROR = "ghcr.io/apache/airflow/base/python" +DEBIAN_VERSION_TO_HARDENED_IMAGE_DISTRO = {"bookworm": "debian12"} + + +def get_hardened_python_image_tag(python: str, debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) -> str: + distro = DEBIAN_VERSION_TO_HARDENED_IMAGE_DISTRO[debian_version] + return f"{ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python, python)}-{distro}-dev" + + +def get_hardened_python_base_image(python: str, debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) -> str: + return f"{HARDENED_PYTHON_IMAGE_MIRROR}:{get_hardened_python_image_tag(python, debian_version)}" + + # Number of slices for low dep tests NUMBER_OF_LOW_DEP_SLICES = 5 diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 01fc0d54061..6c84b30d92f 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -29,6 +29,7 @@ from airflow_breeze.global_constants import ( APACHE_AIRFLOW_GITHUB_REPOSITORY, DOCKER_DEFAULT_PLATFORM, get_airflow_version, + get_hardened_python_base_image, ) from airflow_breeze.utils.console import console_print from airflow_breeze.utils.platforms import get_normalized_platform @@ -124,7 +125,7 @@ class CommonBuildParams: """Construct Python Base Image""" if self.python_image is not None: return self.python_image - return f"debian:{self.debian_version}-slim" + return get_hardened_python_base_image(self.python, self.debian_version) @property def airflow_image_repository(self): diff --git a/docker-stack-docs/build-arg-ref.rst b/docker-stack-docs/build-arg-ref.rst index 3de99867fa7..4f189f524a9 100644 --- a/docker-stack-docs/build-arg-ref.rst +++ b/docker-stack-docs/build-arg-ref.rst @@ -32,7 +32,14 @@ Those are the most common arguments that you use when you want to build a custom +==========================================+===========================================+=============================================+ | ``AIRFLOW_VERSION`` | :subst-code:`|airflow-version|` | Version of Airflow. | +------------------------------------------+-------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PYTHON_VERSION`` | ``3.13.15`` | Version of Python. | +| ``AIRFLOW_PYTHON_VERSION`` | ``3.13.15`` | Python version recorded in the image | +| | | labels. The Python itself comes from | +| | | ``BASE_IMAGE``, so change both together. | ++------------------------------------------+-------------------------------------------+---------------------------------------------+ +| ``BASE_IMAGE`` | (see below the table) | Airflow's public mirror of the Docker | +| | | Hardened Image that Python comes from. | +| | | Point it at a ``dhi.io`` ``-fips-`` variant | +| | | to build a FIPS-compliant image. | +------------------------------------------+-------------------------------------------+---------------------------------------------+ | ``AIRFLOW_EXTRAS`` | (see below the table) | Default extras with which Airflow is | | | | installed. | @@ -81,6 +88,11 @@ Those are the most common arguments that you use when you want to build a custom Before Airflow 2.2, the image also had ``AIRFLOW_GID`` parameter, but it did not provide any additional functionality - only added confusion - so it has been removed. +The default ``BASE_IMAGE`` is Airflow's public mirror of the `Docker Hardened Image <https://dhi.io>`_ for +Python, tagged with the same Python version as ``AIRFLOW_PYTHON_VERSION`` - for Airflow |airflow-version| that +is ``ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev``. Pulling the mirror needs no credentials; +pulling ``dhi.io`` directly requires a ``docker login dhi.io``. + List of default extras in the production Dockerfile: .. BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst index 79ffe2bd7b2..b527a43296a 100644 --- a/docker-stack-docs/build.rst +++ b/docker-stack-docs/build.rst @@ -889,21 +889,18 @@ you want to follow that route. Build images in FIPS-compliant environments ........................................... -If you are building images in a FIPS-compliant environment, you might encounter issues with the default -build process. For example, the default build process uses ``--with-lto`` (Link Time Optimization) when -building Python, which might fail in FIPS mode because LTO uses MD5 checksums to verify object files -during compilation, and MD5 is blocked in FIPS mode. - -In order to build the image in FIPS-compliant environment, you can use ``PYTHON_LTO`` build argument -and set it to ``false``. +Airflow images are based on the `Docker Hardened Images <https://dhi.io>`_ for Python, and Docker +publishes FIPS-validated variants of those images. Those variants are only available with a paid +Docker subscription, so they cannot be the default, but you can point the build at one with the +``BASE_IMAGE`` build argument. .. code-block:: bash - docker build . --build-arg PYTHON_LTO="false" --tag my-image:my-tag + docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.15-debian12-fips-dev" --tag my-image:my-tag .. note:: - While disabling LTO is necessary for FIPS compliance during the build process, it is not sufficient + While building on a FIPS-validated base image is necessary for FIPS compliance, it is not sufficient to make the image fully FIPS compliant. There might be other reasons for FIPS incompatibility (for example usage of non-FIPS compliant algorithms in the software installed in the image). You should verify the compliance of the image yourself. diff --git a/docker-stack-docs/changelog.rst b/docker-stack-docs/changelog.rst index 7eb66dee3e7..e043c1ed7b0 100644 --- a/docker-stack-docs/changelog.rst +++ b/docker-stack-docs/changelog.rst @@ -40,6 +40,32 @@ Airflow 3.4.0 * The ``tdsodbc`` package was added to the image so that the FreeTDS ODBC driver (``libtdsodbc.so``) is available for connecting to Sybase/TDS databases via ODBC. + * The ``libxmlsec1-openssl`` package was added to the image. ``libxmlsec1`` contains no crypto + engine of its own, so ``import xmlsec`` (used by ``python3-saml``) failed with + ``libxmlsec1-openssl.so.1: cannot open shared object file``. + +In Airflow 3.4.0 the base image changed again - from ``debian:bookworm-slim`` with a Python compiled in +the image, to the `Docker Hardened Image <https://dhi.io>`_ for Python, which already carries a Python +built by Docker. This removes the Python compilation (and the download and signature verification that +went with it) from the build entirely. + +* The ``BASE_IMAGE`` arg now defaults to ``ghcr.io/apache/airflow/base/python:<version>-debian12-dev``, + which is Airflow's public mirror of the upstream ``dhi.io/python`` image. Pulling from ``dhi.io`` + directly requires a ``docker login dhi.io``; pulling the mirror requires nothing. If you pass your own + ``BASE_IMAGE``, it now has to be an image that already provides Python - a bare ``debian:bookworm-slim`` + no longer works. +* The ``PYTHON_LTO`` arg was removed. It only controlled Link-Time Optimization while compiling Python, + and there is no compilation left. To build a FIPS-compliant image, point ``BASE_IMAGE`` at a FIPS + variant of the hardened image, for example ``dhi.io/python:3.13.15-debian12-fips-dev`` (those variants + require a paid Docker subscription). +* Python is installed in ``/opt/python/`` by the base image. ``/usr/python`` is a symlink to it, and the + ``/usr/local/bin`` symlinks are unchanged, so paths that worked before keep working. + +As with any base image change, some ``apt`` packages that used to be present as a side effect are not +there any more. The hardened images are deliberately minimal - they ship no compiler, no ``curl``, +``wget``, ``git`` or ``gzip``, and a stripped ``/etc`` - so a custom image that relied on something being +present may need to install it explicitly. See :doc:`Building the image <build>`. + Airflow 3.1.4 ~~~~~~~~~~~~~ diff --git a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh index 8429e14a36f..b17fcbe905c 100755 --- a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh +++ b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh @@ -31,7 +31,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg ADDITIONAL_PYTHON_DEPS="mpi4py==4.1.1" \ diff --git a/docker-stack-docs/docker-examples/customizing/custom-sources.sh b/docker-stack-docs/docker-examples/customizing/custom-sources.sh index b0a9e828ab4..04984280f28 100755 --- a/docker-stack-docs/docker-examples/customizing/custom-sources.sh +++ b/docker-stack-docs/docker-examples/customizing/custom-sources.sh @@ -32,7 +32,7 @@ export DOCKER_BUILDKIT=1 docker build . -f Dockerfile \ --pull \ --platform 'linux/amd64' \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack,odbc" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh index 348171baebb..10cb7a2e600 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh @@ -32,7 +32,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh index ee431e7db3f..0cf49c728d7 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh @@ -31,7 +31,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh index 12f27f821d2..4455b7f98b7 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh @@ -30,7 +30,7 @@ export AIRFLOW_VERSION=3.0.3 export DOCKER_BUILDKIT=1 docker build . \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --tag "my-pypi-selected-version:0.0.1" diff --git a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh index 05b1a9b2da2..fcdebcf5144 100755 --- a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh +++ b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh @@ -47,7 +47,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ - --build-arg BASE_IMAGE="debian:bookworm-slim" \ + --build-arg BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.15-debian12-dev" \ --build-arg AIRFLOW_PYTHON_VERSION="3.13.15" \ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ diff --git a/scripts/ci/prek/update_docker_gpg_keys.py b/scripts/ci/prek/update_docker_gpg_keys.py index 26916d81d64..37eee4e1e32 100755 --- a/scripts/ci/prek/update_docker_gpg_keys.py +++ b/scripts/ci/prek/update_docker_gpg_keys.py @@ -41,8 +41,6 @@ KEYS: dict[str, str] = { "postgres": "7FCC7D46ACCC4CF8", # Microsoft APT repository signing key (MSSQL ODBC) "microsoft": "EB3E94ADBE1229CF", - # Python 3.10 release manager (Pablo Galindo Salgado) - "python-3.10": "A035C8C19219BA821ECEA86B64E628F8D684696D", } diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 1a135830584..22db953219d 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -46,6 +46,7 @@ from common_prek_utils import ( AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH, console, + read_allowed_python_major_minor_versions, read_default_python_major_minor_version_for_images, retrieve_gh_token, ) @@ -666,6 +667,11 @@ GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ AIRFLOW_IMAGE_PYTHON_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ (re.compile(r"(AIRFLOW_PYTHON_VERSION=)(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + # Base image tags pin the same Python patchlevel as AIRFLOW_PYTHON_VERSION and have to move with it + ( + re.compile(r"((?:dhi\.io|ghcr\.io/apache/airflow/base)/python:)([0-9]+\.[0-9]+\.[0-9.abrc]+)"), + Quoting.UNQUOTED, + ), ( re.compile(r"(\| ``AIRFLOW_PYTHON_VERSION`` *\| )(``[0-9.abrc]+``)( *\|)"), Quoting.REVERSE_DOUBLE_QUOTED, @@ -769,7 +775,10 @@ UPGRADE_PROTOC: bool = get_env_bool("UPGRADE_PROTOC") UPGRADE_OPENAPI_GENERATOR: bool = get_env_bool("UPGRADE_OPENAPI_GENERATOR") UPGRADE_SPHINX_AIRFLOW_THEME: bool = get_env_bool("UPGRADE_SPHINX_AIRFLOW_THEME") -ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.10", "3.11", "3.12", "3.13"] +# Read from global_constants.py rather than repeating the list here. A hard-coded copy silently +# stops upgrading the versions it does not know about: 3.14 was added to Airflow in March 2026 and +# its pinned patchlevel was never bumped afterwards, because this list still ended at 3.13. +ALL_PYTHON_MAJOR_MINOR_VERSIONS = read_allowed_python_major_minor_versions() DEFAULT_PROD_IMAGE_PYTHON_VERSION = read_default_python_major_minor_version_for_images() diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index d8ebb2e261f..9185cf4c2fb 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -217,6 +217,16 @@ function common::install_packaging_tools() { fi } +function common::debian_codename() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_CODENAME}") +} + +function common::debian_release() { + # shellcheck disable=SC1091 + (. /etc/os-release; printf '%s\n' "${VERSION_ID}") +} + function common::import_trusted_gpg() { common::get_colors diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh index f50f213d851..94abe6c7655 100644 --- a/scripts/docker/install_mssql.sh +++ b/scripts/docker/install_mssql.sh @@ -43,7 +43,7 @@ function install_mssql_client() { echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(common::debian_release)/prod $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mssql-release.list && mkdir -p /opt/microsoft/msodbcsql18 && touch /opt/microsoft/msodbcsql18/ACCEPT_EULA && diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh index 012ef9f14ac..68c5b768fe6 100644 --- a/scripts/docker/install_mysql.sh +++ b/scripts/docker/install_mysql.sh @@ -106,7 +106,7 @@ install_mariadb_client() { echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo - echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mariadb.list # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh index d802fe8e0ed..57349515f61 100644 --- a/scripts/docker/install_os_dependencies.sh +++ b/scripts/docker/install_os_dependencies.sh @@ -25,8 +25,6 @@ if [[ "$#" != 1 ]]; then exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18} -PYTHON_LTO=${PYTHON_LTO:-true} GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} NODEJS_VERSION=${NODEJS_VERSION:-22.23.1} @@ -35,7 +33,8 @@ NODEJS_VERSION=${NODEJS_VERSION:-22.23.1} PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} -COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} +# The hardened Python base images ship Python under /opt/python. +PYTHON_HOME=${PYTHON_HOME:-/opt/python} if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" @@ -62,38 +61,25 @@ freetds-dev \ git \ graphviz \ graphviz-dev \ +gzip \ krb5-user \ -lcov \ ldap-utils \ -libbluetooth-dev \ -libbz2-dev \ libc6-dev \ -libdb-dev \ libev-dev \ libev4 \ libffi-dev \ -libgdbm-compat-dev \ -libgdbm-dev \ libgeos-dev \ libkrb5-dev \ libldap2-dev \ libleveldb-dev \ libleveldb1d \ -liblzma-dev \ -libncurses5-dev \ -libreadline6-dev \ libsasl2-2 \ libsasl2-dev \ libsasl2-modules \ -libsqlite3-dev \ libssl-dev \ libxmlsec1 \ libxmlsec1-dev \ -libzstd-dev \ locales \ -lsb-release \ -lzma \ -lzma-dev \ openssh-client \ openssl \ pkg-config \ @@ -102,10 +88,8 @@ sasl2-bin \ sqlite3 \ sudo \ tdsodbc \ -tk-dev \ unixodbc \ unixodbc-dev \ -uuid-dev \ wget \ xz-utils \ zlib1g-dev \ @@ -132,6 +116,9 @@ netcat-openbsd\ echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo + # libxmlsec1-openssl was added because libxmlsec1 ships no crypto engine of its own - the engines + # are separate packages - so the "xmlsec" module (pulled in by python3-saml) imported with + # "libxmlsec1-openssl.so.1: cannot open shared object file" without it. if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then RUNTIME_APT_DEPS="\ ${debian_version_apt_deps} \ @@ -150,8 +137,8 @@ libgeos-dev \ libsasl2-2 \ libsasl2-modules \ libxmlsec1 \ +libxmlsec1-openssl \ locales \ -lsb-release \ openssh-client \ rsync \ sasl2-bin \ @@ -180,10 +167,25 @@ function install_docker_cli() { apt-get install -y --no-install-recommends docker-ce-cli } +function restore_debian_base_files() { + # The hardened base images ship a minimal /etc, but Debian maintainer scripts assume the files + # a stock Debian has: sasl2-bin chowns its run directory to the "sasl" group from base-passwd, + # and tmux registers its shell with add-shell, which reads /etc/shells. The base-passwd package + # only ships the master account files - update-passwd is what merges them into /etc. + # libpam-runtime generates the /etc/pam.d/common-* files that the PAM configs already in the + # image "@include" - without them "adduser --gecos" aborts with a PAM error from chfn. + apt-get install -y --no-install-recommends base-passwd libpam-runtime + update-passwd + if [[ ! -e /etc/shells ]]; then + printf '%s\n' "# /etc/shells: valid login shells" /bin/sh /bin/bash > /etc/shells + fi +} + function install_debian_dev_dependencies() { apt-get update apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_DEV_APT_ENV?} if [[ ${DEV_APT_COMMAND} != "" ]]; then @@ -213,14 +215,25 @@ function install_additional_dev_dependencies() { } function link_python() { + # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and + # user customizations refer to that path - while the hardened base images ship it in /opt/python, + # so keep the historical location working as a symlink. + if [[ ! -e /usr/python ]]; then + ln -sv "${PYTHON_HOME}" /usr/python + fi + # The hardened base images have no /usr/local tree at all + mkdir -p /usr/local/bin /usr/local/lib # link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix # Links in /usr/local/bin are needed for tools that expect python to be there # Links in /usr/python/bin are needed for tools that are detecting home of python installation including # lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part. for dst in pip3 python3 python3-config; do src="$(echo "${dst}" | tr -d 3)" + if [[ ! -e "/usr/python/bin/${dst}" ]]; then + continue + fi echo "Linking ${dst} in /usr/local/bin and /usr/python/bin" - ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" + ln -sfv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" for dir in /usr/local/bin /usr/python/bin; do if [[ ! -e "${dir}/${src}" ]]; then echo "Creating ${src} - > ${dst} link in ${dir}" @@ -240,10 +253,33 @@ function link_python() { ldconfig } +function check_no_system_python() { + # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # image. A system Python pulled in as a dependency of an apt package shares its shared libraries + # with ours and leads to errors such as: + # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add + if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + echo + echo "ERROR! System python is installed by one of the previous steps" + echo + echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" + echo + apt-get install -yqq aptitude >/dev/null + aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + echo + exit 1 + else + echo + echo "GOOD! System python is not installed - OK" + echo + fi +} + function install_debian_runtime_dependencies() { apt-get update apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates + restore_debian_base_files + apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_RUNTIME_APT_ENV?} if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then @@ -257,152 +293,11 @@ function install_debian_runtime_dependencies() { apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} apt-get autoremove -yqq --purge apt-get clean + check_no_system_python link_python rm -rf /var/lib/apt/lists/* /var/log/* } -function install_cosign() { - local arch - arch="$(dpkg --print-architecture)" - declare -A cosign_sha256s=( - # https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt - [amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2" - [arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab" - ) - local cosign_sha256="${cosign_sha256s[${arch}]}" - if [[ -z "${cosign_sha256}" ]]; then - echo "Unsupported architecture for cosign: ${arch}" - exit 1 - fi - curl -fsSL --retry 3 --retry-delay 5 \ - "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"; \ - -o /tmp/cosign - echo "${cosign_sha256} /tmp/cosign" | sha256sum --check - chmod +x /tmp/cosign -} - -function install_python() { - # If system python (3.11 in bookworm) is installed (via automatic installation of some dependencies for example), we need - # to fail and make sure that it is not there, because there can be strange interactions if we install - # newer version and system libraries are installed, because - # when you create a virtualenv part of the shared libraries of Python can be taken from the system - # Installation leading to weird errors when you want to install some modules - for example when you install ssl: - # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then - echo - echo "ERROR! System python is installed by one of the previous steps" - echo - echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython3.11 is installed:" - echo - apt-get install -yqq aptitude >/dev/null - aptitude why libpython3.11 - echo - exit 1 - else - echo - echo "GOOD! System python is not installed - OK" - echo - fi - wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"; - local major_minor_version - major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" - local major minor - major="${major_minor_version%.*}" - minor="${major_minor_version#*.}" - echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" - if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then - # Sigstore verification for Python >= 3.11 (PEP 761) - declare -A sigstore_identities=( - # https://peps.python.org/pep-0664/#release-manager-and-crew - [3.11]="[email protected]" - # https://peps.python.org/pep-0693/#release-manager-and-crew - [3.12]="[email protected]" - # https://peps.python.org/pep-0719/#release-manager-and-crew - [3.13]="[email protected]" - # https://peps.python.org/pep-0745/#release-manager-and-crew - [3.14]="[email protected]" - ) - declare -A sigstore_issuers=( - [3.11]="https://accounts.google.com"; - [3.12]="https://accounts.google.com"; - [3.13]="https://accounts.google.com"; - [3.14]="https://github.com/login/oauth"; - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"; - install_cosign - local identity="${sigstore_identities[${major_minor_version}]}" - local issuer="${sigstore_issuers[${major_minor_version}]}" - /tmp/cosign verify-blob \ - --bundle python.tar.xz.sigstore \ - --certificate-identity "${identity}" \ - --certificate-oidc-issuer "${issuer}" \ - python.tar.xz - rm -f python.tar.xz.sigstore /tmp/cosign - else - # PGP verification for Python 3.10 - declare -A keys=( - # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado <[email protected]>" imported - # https://peps.python.org/pep-0619/#release-manager-and-crew - [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D" - ) - wget --tries=3 --waitretry=5 -O python.tar.xz.asc \ - "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc"; - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME - local gpg_key="${keys[${major_minor_version}]}" - echo "Using GPG key ${gpg_key}" - gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc" - gpg --batch --verify python.tar.xz.asc python.tar.xz - gpgconf --kill all - rm -rf "${GNUPGHOME}" python.tar.xz.asc - fi - mkdir -p /usr/src/python - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz - rm python.tar.xz - cd /usr/src/python - arch="$(dpkg --print-architecture)"; arch="${arch##*-}" - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" - EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" - EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"; - LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" - LDFLAGS="${LDFLAGS:--Wl},--strip-all" - # Link-Time Optimization (LTO) uses MD5 checksums for object file verification during - # compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds - # to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds. - # See: https://github.com/apache/airflow/issues/58337 - local lto_option="" - if [[ "${PYTHON_LTO:-true}" == "true" ]]; then - lto_option="--with-lto" - fi - local build_log - build_log=$(mktemp) - echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..." - if ! ( - ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \ - --enable-loadable-sqlite-extensions --enable-option-checking=fatal \ - --enable-shared ${lto_option} && \ - make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \ - make -s -j "$(nproc)" install - ) > "${build_log}" 2>&1; then - echo - echo "ERROR! Python build failed. Build output:" - echo - cat "${build_log}" - rm -f "${build_log}" - exit 1 - fi - rm -f "${build_log}" - cd / - rm -rf /usr/src/python - find /usr/python -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + - link_python -} - function install_golang() { curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz @@ -496,7 +391,8 @@ if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then else get_dev_apt_deps install_debian_dev_dependencies - install_python + check_no_system_python + link_python install_additional_dev_dependencies install_rustup if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then diff --git a/scripts/docker/install_postgres.sh b/scripts/docker/install_postgres.sh index 4af446a7a68..9a6c625bdee 100644 --- a/scripts/docker/install_postgres.sh +++ b/scripts/docker/install_postgres.sh @@ -43,7 +43,7 @@ install_postgres_client() { common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres" - echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \ + echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" diff --git a/scripts/docker/keys/python-3.10.asc b/scripts/docker/keys/python-3.10.asc deleted file mode 100644 index 0cd2e1412fd..00000000000 --- a/scripts/docker/keys/python-3.10.asc +++ /dev/null @@ -1,109 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBFq+ToQBEADRYvIVtbK6owynD3j3nxwpW2KEk/p+aDvtXmc2SR2dBcZ8sFW2 -R5vEsG8d3/D3wgv5pcL3KfNNXQYUnXVbobrFUUWQYc79qIsE3MgiPf5NVOtwKPUR -i5g9YJgKvpBxkQfqp3LYGm9ZBtwo3DVLA3yn7KsazCmAgTNFJYw7ku1XxgmIzY6K -5J30DfbJiqDqj4f9GslCCCCH3qiPnuLG/HUyVLHMpbWlaiy9NI0GcaLxjJewHj9w -W2D2lydkxe5JGo7egUkV3ILcuLVSVKA35SKY27dYqfuyqp9tAzaRbjDYjsYdHA6G -BqrNrKBn/GwlFDPrVdcvN3ZSY2wMLTxWE3Axc/FweuHxFnou/80FwX7F3JD+oEQ6 -rofmcxOBCC7J98I7HZAhP9jBn88XIS2hztbLq8d6rZJZRtcz0k61VR0ddO+TrFmf -9rMYCPgCckRtVxeFIVIabrN1IzKynLFeo040h8hSGswd6YKDOVwjJY6Oa6EmVefZ -a8QSt4+M65RSzH6SEPY008F3nJUAK6MEkzTak+tFltZNrVWu8p2xd1j9nmxAwEhZ -/lgbxLqzYgaUWmfyHeZ8yVA0MhHzdiAL8nVUEdG3KecIq0RWCJLGLWWIjd6KAJl1 -yAmhRYKK/sjPDsL3elHsFACfZbyx3o5GGQNlas1FYoPLWbaNGaJtgFTF2QARAQAB -tCtQYWJsbyBHYWxpbmRvIFNhbGdhZG8gPHBhYmxvZ3NhbEBnbWFpbC5jb20+iQJO -BBMBCgA4FiEEoDXIwZIZuoIezqhrZOYo+NaEaW0FAlq+ToQCGwMFCwkIBwMFFQoJ -CAsFFgIDAQACHgECF4AACgkQZOYo+NaEaW2bmA/+PXIap2udLoUVOHxnsIBdqYwp -sv1Aj5lfIJmNhmxPbHShwp1Jg+w4urxe+2Dj5ofKVlIo1i83bQkvnKJMDXDVuc/K -P6zqhBJ3rT4Q3qx2mzX8bIfQoJ2JHuH4lkP+I7doDcHHRyeNASyk72VdQmU4twNw -Ibn8nSNV6ThKHdoPYzVnO2rZUFcGIqH5HNsvR+B7cc1MBCHsgURYwSVhSePIFGlZ -iasdBD6QQkDSe4QWi7AcJFWFElw4kbOKJWxAWsrEk+tMXJVGRjnmL289EmPCx/vx -BqKy7Mse0yWCSRR3vB+O6TB1S5SgEyEgqlYsfGNv1qf/rfRD4KkyCbNU3LhY1Aim -vJP4pDW+KFxTk2Ks8vrx8gOSd2aFqPeO/pFDrpsF7PD62XwsfoXu4xc5V0Giw7r1 -Nai0nax7kOrldNF8TbbtRjW0jmoC7wLIDujAkwDIOroZ0CXA3N4HVHdSbrHm/urX -nyxJXupXAQNwGx64JCBcbF2fp3Kvu1VAXBEFnd01KaopthHcbG5pA50Kl2Vhe+98 -OdezUX42fHkQpQkB7HgtXfm6W1bw6YRBamrNvs1OoHBYmUjlECpe566IIu25Hc8s -x3qA+6eca7iqizyLG+WyMT8ZIYTWGAS59jxwR4esqGczbbZPSAPHFwLbGv7Wr0Rd -TPu5B0FcKpDkTd4IxQW5Ag0EWr5PIAEQALfh9vPD2B+miHDTMADI8aRZ7g9tnzyn -ZYkk3+2sCiiusetsQQ+HIPJ/ASEJB7Onane9dyT/LTRhrK9qaxgVMimk2COXB/xy -h7Mnw7nJgFU0aRSbtX0vbvQz2suSzrQ69mPKzan28JGoClqB0bw1vwf3VjjxHV2d -gD57CmqFPv7kAC/2a56dE+etzXattZAL+2JWTpmfQ0ePRRadtBm0VahQhnU8x0+j -vAVrEawqpVW83ozYFyW/0WInM2J7jHgQ16OosY4lj5L/DxpVxaArhRFoRfWPXfC3 -7iE8Mou/I95isvPQIhp1wTo4jG0KM02BoIVbp/QRNBQ6WtpOzvJs1gqQiJJTfqbK -JXQ3NDEY9crpVS83HJ+Zv99PNsyNkFjGQpU84U3ZhsI4ygjdY45mpZueqI1RVcRQ -du8Hgvoo/78Q/Sir6gMGop3mVdVo2guIkFcJrXh0Xk3ech4aVqrmKx/mPXGwOAQU -0DAul4RW3fKg1QxQE7Tlw3+95Ee/+q5jHARL0uDbCJpRO8Sl8NDEuL32n/2Ot6kQ -eCSHrU7KJRYAkTxkKvr8zNow7hFhHFPESnHvTnskI6noh0VY6NwMhmLvhm0wKkRx -ZPzUNc3sgLvbK1NymIZ9aKCZamzhZrmGvnblEz/OSLwGUua465H3hM1vvBQiartj -7+6ZqWIkSmBPABEBAAGJAjYEGAEKACAWIQSgNcjBkhm6gh7OqGtk5ij41oRpbQUC -Wr5PIAIbIAAKCRBk5ij41oRpbWmeEACG+axtDC8UoNp9ORiYwEWLzZWDuugE+ah7 -DYYGD4Vs633FXVZW3SgM/bFtJ/0Lg8CF74jI4LMHyIjDzEjcoItwnhBLix+kUoJT -vrY58GPydwekLuw1p4KXLqtRs4fsZbNQYTknl4jYtRWoxO98x7tun7Gq2gqmJkIB -2uj630fKz5cBk6p6oDFKjzyrHe+V7BiK3okQPaD4x7hq8OnTy7lOy92ZZAqztS4t -NEb4DkYW1MpuwsJ7hbBZitc1siI+FVVbGjVVGZz6ssXoW67Tz8+VxdWJxNLXlv27 -eMcj4sme5S0th/YYNA5fRRv6zuzqZAruYNGLpYYU7JLvZJ+3lCwa5j5ycOGBF0Gv -sGs6gj6h+CHkjR/BgzAgWC+GgUgslt6qaH04rWtV6rVz+Y91LcrX5P6OM4anmXD3 -Gp3kl35AypXb4KyASF19+11RUziD4Z7qwQEWfbwOltNyZv2lD8s2jPr7P02axWRQ -UbZAEhxRmvOQev/FZPyCF6gqUo/HxRbQy3bzmnipyHSv1DlXNfCFCHvN8kGyZnRW -ARqIKRg+j9ediJgOUqlLhg6KmrTVxd5v3Dfv52PW2UODDTM20s3cQGuX/UswzMRw -PI/+P44iCMwEKdm7duM/5oisZT9Vhy7gP15MreFZLcZvUVgjqgy0u57cstyGK1Bo -9e2sFcK2fLkCDQRavk76ARAAvDstYOqT1gW6wjISKu1VmNn48pSdX8IOQT15DilS -tXbBmDuEA+6FvS9GkqnBUjRRhfQpdEYuExV2Qt4ZxuB3kl1FgKbinQS59tR9422t -+gMtaBnBn3ZtL2YIbeHOgZTFQaRgsNsCJCW1DRMuiJzDPjvAgm22h94iZCeNeNT+ -6Tgx4oJ8eQJmSHKJzLouGDoQviF0F8p1RL9SbpO3qwdTYdUXEb7AOJW+OB1sePZS -MHvOXozpqGdDt+n/TSUXq+m1dPqx1poOCFeUPYfxp+CIRA2cQNA2ZdUUCO3TIGjG -yBW1xquyoM7LcxFJzqMtvYV6HcVZsCLDKwIkFGLcCfYVvhErMOO/ltojx1YL4hlp -e6/0oHIDGJn+puZIkwbcFOwUD0vRt88MD9kZtwAP+oN/UPrUCozsItSPPhmbGNjp -+OGZcuwa3gTYOeICYHQBnDDmFPXra9OdHJR91KU2HABLV/vmobTioSYCyDzXCAED -ZZ4n6r1NFWKYMEeGKWe4YIPgFQ8wu9zyDNJRSNF4g7qgoW8RPitY5i11nuYiRdHc -4FU8D/j5rInzWVoN/mMUrEvoQzD0NsT4kDePYE+CSQx3A7soR8ghmIMjIlSQrJLv -Sq5Lm231UYr8jWRdmX6N3/p260cbH7c9MowKvG8gr29FdFzqbeKSf3XZboPcrUcQ -CzcAEQEAAYkCNgQYAQoAIBYhBKA1yMGSGbqCHs6oa2TmKPjWhGltBQJavk76AhsM -AAoJEGTmKPjWhGlt7qwP/1EP3CkOx71o3C6Yc0DmXPupuWC9MMb2ZEo3plO5qo2X -aWnLXiL0QKHq7BKlx3NtPJYiAbTX0c6K3l3XFzc4nuucV5K0Z1qB1R/301suJTpx -69gjFy3h41IIPzOj2IvPrAEZtOWBQfJ7GIZYcdyHS4XH3lVWyXnUCjtMi9witaJE -99A1FauS6ul/ygwugMbiUdw+lDEqE9Fi2qF/Km4QKfPKi8pyJ3wVS9fHfe4fdelj -y0z3j9Hh8IcDQ22LSgwjA7ikPV1+fkLDR9EZm8ZV9u+Cx61DDL5CysXHwgbz/orq -qwXiLqHlc12CWY+2SbRB+/W1uOHnnJfwmP0kxWIEkNlxk6dgSNqutMAJrSpTYcMW -OXp2tBYMhZek5uTgR1vz8KPp3UCRn26wslpE4EWsvXPHq94JdxkGCAR+l9JAohu4 -fxKY9G+Qhzj1OYQsNJBAfDAeKaEyI3omPkNNeO9v5UOvAi9w19efIUeXd12j0iMQ -DfqhbHwdaN9eqco8YXytTA/yH6zoI5YIDz53Pypi5As6YHcCnGcCwnWavYWAbxdW -bN+B22lTGG1kyrsS6MLVxlON72rQl16pjzr5eP4OLWQ+q8rCQ4Dqx0JxB80/jWqZ -9O7NAUQtCG2zZxcp1K5K+SjErkc+x4O8TAKRDTq29J6nR0lCfdeGabjCEukYSYBr -uQINBFq+TtoBEADIy3uArW0n6H5ia8+cHzZDFbGfNRm6njKGHd3bZCZ0GBEVI/H5 -cwY97nGsRifgrn+8m4d52j6KSrxCNatL5HIAg6qxIFyv/4yiWz9vLj0jBoThtvKu -D22phueYMuRbTN3s8wMT1linvagaVeuRjJhcu3gUMdoibMogyDjkEOJK2kvnE6jR -YphFwBiwjkETS3MCGc8AAB6FuxxKTVH1cGgQ/ZDJW6NTQyOtCCnDazXJnJkcYSan -8Y4/GATMHChzuhgRNwFaWhP/lDvc8q+Ex/tGLkFIMSpT1g5IOV5ZA6NEtRsi2OXv -XJCAyJZE3Q2tqTpKBNsgeTEAhIGt2Ih6ylFU01msZ/V1SeseWY1Bem4ElhpKxuXV -KkmKDtYgzLjVY7Yrozcn/6734+XDcUu/5YMXIYUCmR2Vy1lasZZ3nlSerND+PkTl -RvYGtkr3u2lCesFyHsGcLpQXFdrBP1vw4MR4CwYudwlELbxMxcyr6OPBpw6i+VWI -wO5dBzGB3eBpLAi4vBPFOgoI/opl4jPL6C+6R0WX7PJmPLnUflE+HtUnJdjGD9qg -Sff9ewf1+dgd7NRGXaiP6/oXzsqq4T7PnWwDdJrTD5+vPp672Nfp7HL60mBD+mQZ -XPR6dpoCXW+iHE7yDVwRHi0Y4HN6Y1Yr2uON6ZIhafQMZnnDnJDCj97kpQARAQAB -iQRsBBgBCgAgFiEEoDXIwZIZuoIezqhrZOYo+NaEaW0FAlq+TtoCGwICQAkQZOYo -+NaEaW3BdCAEGQEKAB0WIQTP3KJFsQQ88qX5eGX/6HQEFovYRwUCWr5O2gAKCRD/ -6HQEFovYR6+2D/0Qi63tQpjqFmKQLaGtclQnmDA1enkICwePPPwRGR0BNc262Mte -l5ITkaG9yRf5FDI5iVu12hRfBa1r0XHY/8WlBmzjhLpSoL/vM1n7MGEEYmePyZcB -EeL0S4un0xPPTFREehZI9gEg+A9nIn2PhM7Q+H5Vd7Kt7LyKIBhWjfSL3a3XqbTl -Rl7NUZgMqE7IktpikOsbBofAfWQJueVvHwThoksy19tV5Dn3cOdwaxUMZ93LvyYC -CqXXmcYLpHLxACOtgel/mC+jWuI9HqhBi87DbVNfsXiZaFrVb5kcJ0RxksRen8yG -d5pP6f2hYlwBKv8SEwk2OrQ7+8e6RRP0aXZSWcjSfkeWOEkIrg7MpcTQmv+BuRAe -qvu4lWLWxEPTJ2YNqYaMBBONrZY9K8/P67wytyx98Ym6LShyKHQlipVgcsR/fbtj -7h1ENHNaAlQVBNSpK7jKbyNt+g6VP3b3auCS0oP31q8rP1SI6i2YLCBOYPyjvV6p -4Fh5C3AS4eAjQJhyfCImvrPv634LHo0FgebiW+nmINiYweFj9RCNK1CzOn3awjQf -jCd+sBNi52+G309KYTjII4x+fyou2Qm5MZDrAbOEkY4kvsHgrJqQLTKqHVmPwp8P -aVkgP6i0kQ+f5inm/Ujujmfzj3wGrcJxh7L2drALnguq0dOFuQX12Vq5yVF0D/4n -fxChNSrKeXyqMUgm5CkwooPkn+QH8Ht+shTfp4mQas5+WwYC/xyCmmTPe6Ax3q7D -zwDXhIEviVZQX/eYiiGJ0G2T+NxS1W7zrmiDYQSFbRTueNmOxGMJeNZD+yCwxJVc -t+igJnUMgQH6452YkQO9MUoTsQKS7oNS9JcYUd2H5i8DYkSdFeip2Ayn9KX3YTvV -RchRVlKI19brJ9kpkrFzYs7Dbb0ah18z519K2yc2ffjeDvZNZ8coGBRv9kf4xqJ1 -Zk40c3AB6wJDSREzFlzAO//META8Jbs/I/nApXXsvcfFZCcICxp2Um0LVaXBQJ5u -a/9+hrfvcHNVVjMq1K1a+meu/WqgGxuzzGnkeYecoDI9ZFakU2AFD0pNX/G2SkbL -EJEMBG6fXVSI5vUrEfFU+/7QflLgfOXQAcYB3796bCUU/rS/IPVe5EmakH3CzRpr -xFb69mZfTbccF7bBTNjgaa8Qnp8G/nvT6ZtS43yRvzRO9pgJzhEXw2oi7eG6jaN+ -hIubBgieBcetaYczQV85HUTTOO4vCeVGUShZBPdXPfQVTBylGmOtTUStuH0dMz/f -H7Q1PNmBcNFURSaU3gazuYrzdTJUMVD81FsSsTMTvttMGlDl66BMNdU2+EkSxjpW -taQpepQlDyUcvSM2nTw4E6/zkzJKG/7J7f1y3Z1efw== -=XrQs ------END PGP PUBLIC KEY BLOCK-----
