This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch hardened-python-base-images in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a0babc024c8e51278dcadbaf35fa9ee0f7b27e07 Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Oct 10 12:14:24 2026 +0100 Support the trixie hardened images' Debian-packaged Python and test legacy images like regular ones The Debian 13 hardened images install Python as Debian packages under /usr instead of /opt/python, so the image has to find the base Python's prefix and the system-Python guard must not mistake those packages for Debian's own Python. The legacy canary now runs the regular PROD image tests, not just a smoke check, so the legacy images are covered as well as the default ones. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014np1fVAxHmGaBFt3qebJ8x --- .../scheduled-legacy-prod-image-canary.yml | 250 ++++++++++++++++----- Dockerfile | 39 ++-- Dockerfile.ci | 39 ++-- docker-stack-docs/build.rst | 6 +- docker-stack-docs/changelog.rst | 2 +- scripts/docker/install_os_dependencies.sh | 39 ++-- 6 files changed, 270 insertions(+), 105 deletions(-) diff --git a/.github/workflows/scheduled-legacy-prod-image-canary.yml b/.github/workflows/scheduled-legacy-prod-image-canary.yml index 64821143b9c..19cdc30b89e 100644 --- a/.github/workflows/scheduled-legacy-prod-image-canary.yml +++ b/.github/workflows/scheduled-legacy-prod-image-canary.yml @@ -19,11 +19,11 @@ name: "Scheduled legacy PROD image canary" # The default PROD images are hardened and based on Debian trixie, and that is all that PR and canary # builds test. The "legacy" flavor - Python compiled from sources on Debian bookworm - is still -# published for the 3.4.x line, so this checks once a week that it can still be built and passes the -# PROD image checks, instead of finding out only when a release is being prepared. +# published for the 3.4.x line, so once a week this builds legacy bookworm PROD images and runs the +# regular PROD image tests against them, instead of finding out only when a release is being prepared. # -# The image is built the way release images are: with the Airflow version released to PyPI and its -# constraints, on top of the Dockerfile from the checked-out branch. +# It reuses the PROD image workflows of the main CI build. The constraints those workflows normally get +# from the CI image are taken from the constraints branch instead, so no CI image has to be built. on: # yamllint disable-line rule:truthy schedule: - cron: '43 2 * * 0' @@ -34,53 +34,149 @@ concurrency: group: scheduled-legacy-prod-image-canary-${{ github.ref }} cancel-in-progress: true jobs: - python-versions: - name: "Python versions to test" - runs-on: ["ubuntu-slim"] + build-info: + name: "Build info" + runs-on: ["ubuntu-22.04"] if: github.repository == 'apache/airflow' || github.event_name == 'workflow_dispatch' - timeout-minutes: 10 + timeout-minutes: 15 outputs: - python-versions: ${{ steps.versions.outputs.python-versions }} - airflow-version: ${{ steps.versions.outputs.airflow-version }} + canary-run: ${{ steps.source-run-info.outputs.canary-run }} + debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} + java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }} + kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} + providers-compatibility-tests-matrix: > + ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + run-airflow-ctl-integration-tests: >- + ${{ steps.selective-checks.outputs.run-airflow-ctl-integration-tests }} + run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }} + run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} + run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }} + run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }} + run-remote-logging-elasticsearch-e2e-tests: >- + ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }} + run-remote-logging-opensearch-e2e-tests: >- + ${{ steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }} + run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }} + run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} + run-ts-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-e2e-tests }} + run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 2 persist-credentials: false - - name: "Install uv" - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - version-file: uv.lock - - name: "Get Python versions and the latest released Airflow version" - id: versions + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Save github context to file" + # Same as in ci-amd.yml: a scheduled or dispatched run has no attacker-controlled context, and + # the single-quoted heredoc disables every shell expansion of the JSON anyway. + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: "Get information about the Workflow" + id: source-run-info + run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} + env: + SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + PLATFORM: "linux/amd64" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + + constraints: + name: "Constraints ${{ matrix.python-version }}" + needs: [build-info] + runs-on: ["ubuntu-slim"] + timeout-minutes: 10 + strategy: + matrix: + python-version: ${{ fromJSON(needs.build-info.outputs.python-versions) }} + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} + steps: + - name: "Download constraints from ${{ env.CONSTRAINTS_BRANCH }}" run: | - python_versions=$(uv run --project dev/breeze python -c " - import json - from airflow_breeze.global_constants import CURRENT_PYTHON_MAJOR_MINOR_VERSIONS - print(json.dumps(CURRENT_PYTHON_MAJOR_MINOR_VERSIONS)) - ") - airflow_version=$(curl -fsSL --retry 3 https://pypi.org/pypi/apache-airflow/json \ - | jq -r .info.version) - echo "python-versions=${python_versions}" >> "${GITHUB_OUTPUT}" - echo "airflow-version=${airflow_version}" >> "${GITHUB_OUTPUT}" - echo "Python versions: ${python_versions}, Airflow version: ${airflow_version}" + base_url="https://raw.githubusercontent.com/apache/airflow/${CONSTRAINTS_BRANCH}"; + target_dir="files/constraints-${PYTHON_MAJOR_MINOR_VERSION}" + mkdir -p "${target_dir}" + for prefix in constraints constraints-no-providers constraints-source-providers; do + file="${prefix}-${PYTHON_MAJOR_MINOR_VERSION}.txt" + curl -fsSL --retry 5 "${base_url}/${file}" -o "${target_dir}/${file}" + done + ls -l "files/constraints-${PYTHON_MAJOR_MINOR_VERSION}" + # Same artifact name and layout as generate-constraints.yml uploads, which is what the PROD image + # build downloads. + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: constraints-${{ matrix.python-version }} + path: ./files/constraints-${{ matrix.python-version }}/constraints-*.txt + retention-days: 7 + if-no-files-found: error - legacy-prod-image: - name: "Legacy bookworm PROD image ${{ matrix.python-version }}" - needs: [python-versions] + build-prod-images: + name: "Build legacy bookworm PROD images" + needs: [build-info, constraints] + uses: ./.github/workflows/prod-image-build.yml + permissions: + contents: read + packages: read + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + build-type: "Legacy" + debian-version: "bookworm" + image-flavor: "legacy" + push-image: "false" + upload-image-artifact: "true" + upload-package-artifact: "true" + python-versions: ${{ needs.build-info.outputs.python-versions }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: "false" + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + # The cache in the registry is the one of the hardened images - it would not match anyway. + docker-cache: "disabled" + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + prod-image-build: "true" + + verify-legacy-flavor: + name: "Verify legacy bookworm flavor ${{ matrix.python-version }}" + needs: [build-info, build-prod-images] runs-on: ["ubuntu-22.04"] - timeout-minutes: 90 + timeout-minutes: 20 strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(needs.python-versions.outputs.python-versions) }} + python-version: ${{ fromJSON(needs.build-info.outputs.python-versions) }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ needs.python-versions.outputs.airflow-version }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} - VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash @@ -89,32 +185,78 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: "Make /mnt writeable" - run: ./scripts/ci/make_mnt_writeable.sh - - name: "Move docker to /mnt" - run: ./scripts/ci/move_docker_to_mnt.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Build legacy bookworm PROD image ${{ matrix.python-version }}" - run: > - breeze prod-image build - --python "${PYTHON_MAJOR_MINOR_VERSION}" - --image-flavor legacy - --debian-version bookworm - --installation-method apache-airflow - --install-airflow-version "${AIRFLOW_VERSION}" - --airflow-constraints-reference "constraints-${AIRFLOW_VERSION}" - --docker-cache disabled + - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: "linux/amd64" + image-type: "prod" + python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + make-mnt-writeable-and-cleanup: true + # Guards against the canary silently testing the default flavor if the flavor or Debian version + # stop being passed through to the build. - name: "Verify the image is the legacy bookworm flavor" + env: + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} run: | - image="ghcr.io/${GITHUB_REPOSITORY,,}/main/prod/python${PYTHON_MAJOR_MINOR_VERSION}" + image="ghcr.io/${GITHUB_REPOSITORY,,}/${DEFAULT_BRANCH}/prod/python${PYTHON_MAJOR_MINOR_VERSION}" flavor=$(docker inspect "${image}" \ --format '{{ index .Config.Labels "org.apache.airflow.image.flavor" }}') codename=$(docker run --rm --entrypoint /bin/bash "${image}" \ -c '. /etc/os-release && echo "${VERSION_CODENAME}"') echo "Image flavor: ${flavor}, Debian: ${codename}" [[ "${flavor}" == "legacy" && "${codename}" == "bookworm" ]] - - name: "Verify PROD image ${{ matrix.python-version }}" - run: breeze prod-image verify --python "${PYTHON_MAJOR_MINOR_VERSION}" - - name: "Test docker-compose quick start ${{ matrix.python-version }}" - run: breeze testing docker-compose-tests --python "${PYTHON_MAJOR_MINOR_VERSION}" + + additional-prod-image-tests: + name: "PROD image tests" + needs: [build-info, build-prod-images, constraints] + uses: ./.github/workflows/additional-prod-image-tests.yml + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + default-branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + upgrade-to-newer-dependencies: "false" + docker-cache: "disabled" + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + run-task-sdk-integration-tests: ${{ needs.build-info.outputs.run-task-sdk-integration-tests }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + run-remote-logging-elasticsearch-e2e-tests: >- + ${{ needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }} + run-remote-logging-opensearch-e2e-tests: >- + ${{ needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }} + run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }} + run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} + run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} + run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} + run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }} + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + run-ts-sdk-e2e-tests: ${{ needs.build-info.outputs.run-ts-sdk-e2e-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} + run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} + + tests-kubernetes: + name: "Kubernetes tests" + needs: [build-info, build-prod-images] + uses: ./.github/workflows/k8s-tests.yml + permissions: + contents: read + packages: read + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} + # yamllint disable rule:line-length + lang-sdk-kubernetes-combo: >- + ${{ needs.build-info.outputs.default-python-version }}-${{ needs.build-info.outputs.default-kubernetes-version }} + # yamllint enable rule:line-length diff --git a/Dockerfile b/Dockerfile index af763424a36..b42423b85c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -128,8 +128,9 @@ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then - # The hardened Python base images ship Python under /opt/python. - PYTHON_HOME=${PYTHON_HOME:-/opt/python} + # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as + # Debian packages under /usr - so ask the base image's Python where it lives. + PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else @@ -351,8 +352,8 @@ function install_additional_dev_dependencies() { function link_python() { # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and - # user customizations refer to that path - while the hardened base images ship it in /opt/python, - # so keep the historical location working as a symlink. + # user customizations refer to that path - while the hardened base images ship it in /opt/python + # (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink. if [[ ! -e /usr/python ]]; then ln -sv "${PYTHON_HOME}" /usr/python fi @@ -376,15 +377,19 @@ function link_python() { fi done done - for dst in /usr/python/lib/* - do - src="/usr/local/lib/$(basename "${dst}")" - if [[ -e "${src}" ]]; then - rm -rf "${src}" - fi - echo "Linking ${dst} to ${src}" - ln -sv "${dst}" "${src}" - done + # A Python installed under /usr already has its libraries where the dynamic linker looks, and + # linking all of /usr/lib into /usr/local/lib would only shadow the system libraries. + if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then + for dst in /usr/python/lib/* + do + src="/usr/local/lib/$(basename "${dst}")" + if [[ -e "${src}" ]]; then + rm -rf "${src}" + fi + echo "Linking ${dst} to ${src}" + ln -sv "${dst}" "${src}" + done + fi ldconfig } @@ -417,18 +422,20 @@ function compile_python_stdlib() { } function check_no_system_python() { - # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # Python - from the hardened base image or compiled from sources - must stay the only Python in the # image. A system Python pulled in as a dependency of an apt package shares its shared libraries # with ours and leads to errors such as: # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + # Debian names its Python libraries "libpython3.13". The trixie hardened images register their own + # Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match. + if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then echo echo "ERROR! System python is installed by one of the previous steps" echo echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" echo apt-get install -yqq aptitude >/dev/null - aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')" echo exit 1 else diff --git a/Dockerfile.ci b/Dockerfile.ci index 84498e6dfd9..95e3d61521d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -64,8 +64,9 @@ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then - # The hardened Python base images ship Python under /opt/python. - PYTHON_HOME=${PYTHON_HOME:-/opt/python} + # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as + # Debian packages under /usr - so ask the base image's Python where it lives. + PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else @@ -287,8 +288,8 @@ function install_additional_dev_dependencies() { function link_python() { # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and - # user customizations refer to that path - while the hardened base images ship it in /opt/python, - # so keep the historical location working as a symlink. + # user customizations refer to that path - while the hardened base images ship it in /opt/python + # (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink. if [[ ! -e /usr/python ]]; then ln -sv "${PYTHON_HOME}" /usr/python fi @@ -312,15 +313,19 @@ function link_python() { fi done done - for dst in /usr/python/lib/* - do - src="/usr/local/lib/$(basename "${dst}")" - if [[ -e "${src}" ]]; then - rm -rf "${src}" - fi - echo "Linking ${dst} to ${src}" - ln -sv "${dst}" "${src}" - done + # A Python installed under /usr already has its libraries where the dynamic linker looks, and + # linking all of /usr/lib into /usr/local/lib would only shadow the system libraries. + if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then + for dst in /usr/python/lib/* + do + src="/usr/local/lib/$(basename "${dst}")" + if [[ -e "${src}" ]]; then + rm -rf "${src}" + fi + echo "Linking ${dst} to ${src}" + ln -sv "${dst}" "${src}" + done + fi ldconfig } @@ -353,18 +358,20 @@ function compile_python_stdlib() { } function check_no_system_python() { - # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # Python - from the hardened base image or compiled from sources - must stay the only Python in the # image. A system Python pulled in as a dependency of an apt package shares its shared libraries # with ours and leads to errors such as: # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + # Debian names its Python libraries "libpython3.13". The trixie hardened images register their own + # Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match. + if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then echo echo "ERROR! System python is installed by one of the previous steps" echo echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" echo apt-get install -yqq aptitude >/dev/null - aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')" echo exit 1 else diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst index b44e89d52da..ca4210cb76a 100644 --- a/docker-stack-docs/build.rst +++ b/docker-stack-docs/build.rst @@ -913,8 +913,10 @@ previous images that can matter when you extend or run the image are: * **Python is built without profile-guided optimization.** The previous images were built with ``--enable-optimizations`` and ``--with-lto``; the hardened Python uses only ``--with-lto``, so CPU-bound Python code can run measurably slower. -* **Python lives in** ``/opt/python``. ``/usr/python`` and the ``/usr/local/bin`` links point to it, so - the paths used before keep working. The standard library ships without ``.pyc`` files, so the build +* **Python lives under** ``/usr``. The Debian 13 hardened images install Python as Debian packages + (``python-3.13``, ``libpython-3.13``, ...), so it is ``/usr/bin/python3`` with the standard library in + ``/usr/lib/python3.13`` (the Debian 12 ones used ``/opt/python``). ``/usr/python`` and the + ``/usr/local/bin`` links point to it, so the paths used before keep working. The standard library ships without ``.pyc`` files, so the build compiles it into the image. * **The OS is minimal.** The base ships no compiler, ``curl``, ``wget``, ``git``, ``gzip``, ``which`` or ``ldconfig``, no ``dash`` (``/bin/sh`` is ``bash``) and a stripped ``/etc`` - for example without diff --git a/docker-stack-docs/changelog.rst b/docker-stack-docs/changelog.rst index 1d439d67840..79454aa9a66 100644 --- a/docker-stack-docs/changelog.rst +++ b/docker-stack-docs/changelog.rst @@ -70,7 +70,7 @@ roughly 210s to 105s in a local cache-disabled build. the legacy flavor always compiles with LTO. To build a FIPS-compliant image, point ``BASE_IMAGE`` at a FIPS variant of the hardened image, for example ``dhi.io/python:3.13.16-debian13-fips-dev`` (those variants require a paid Docker subscription). -* Python is installed in ``/opt/python/`` by the base image. ``/usr/python`` is a symlink to it, and the +* Python is installed under ``/usr`` by the base image, as Debian packages. ``/usr/python`` is a symlink to it, and the ``/usr/local/bin`` symlinks are unchanged, so paths that worked before keep working. * The hardened images ship the standard library with no ``.pyc`` files, so the build compiles it into the image. This keeps the behaviour introduced in Airflow 3.1.4 (see below): the standard library is owned diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh index 939ffe8063c..ef74fa7d468 100644 --- a/scripts/docker/install_os_dependencies.sh +++ b/scripts/docker/install_os_dependencies.sh @@ -39,8 +39,9 @@ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then - # The hardened Python base images ship Python under /opt/python. - PYTHON_HOME=${PYTHON_HOME:-/opt/python} + # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as + # Debian packages under /usr - so ask the base image's Python where it lives. + PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else @@ -262,8 +263,8 @@ function install_additional_dev_dependencies() { function link_python() { # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and - # user customizations refer to that path - while the hardened base images ship it in /opt/python, - # so keep the historical location working as a symlink. + # user customizations refer to that path - while the hardened base images ship it in /opt/python + # (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink. if [[ ! -e /usr/python ]]; then ln -sv "${PYTHON_HOME}" /usr/python fi @@ -287,15 +288,19 @@ function link_python() { fi done done - for dst in /usr/python/lib/* - do - src="/usr/local/lib/$(basename "${dst}")" - if [[ -e "${src}" ]]; then - rm -rf "${src}" - fi - echo "Linking ${dst} to ${src}" - ln -sv "${dst}" "${src}" - done + # A Python installed under /usr already has its libraries where the dynamic linker looks, and + # linking all of /usr/lib into /usr/local/lib would only shadow the system libraries. + if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then + for dst in /usr/python/lib/* + do + src="/usr/local/lib/$(basename "${dst}")" + if [[ -e "${src}" ]]; then + rm -rf "${src}" + fi + echo "Linking ${dst} to ${src}" + ln -sv "${dst}" "${src}" + done + fi ldconfig } @@ -328,18 +333,20 @@ function compile_python_stdlib() { } function check_no_system_python() { - # Python comes from the hardened base image (in /opt/python) and must stay the only Python in the + # Python - from the hardened base image or compiled from sources - must stay the only Python in the # image. A system Python pulled in as a dependency of an apt package shares its shared libraries # with ours and leads to errors such as: # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add - if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then + # Debian names its Python libraries "libpython3.13". The trixie hardened images register their own + # Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match. + if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then echo echo "ERROR! System python is installed by one of the previous steps" echo echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" echo apt-get install -yqq aptitude >/dev/null - aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')" + aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')" echo exit 1 else
