This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch pin-os-deps-image-layer in repository https://gitbox.apache.org/repos/asf/airflow.git
commit fe464d365fd9078cd59cfb8e7d62ff2aae7df1da Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 1 11:33:37 2026 +0100 Fix the legacy PROD image and isolate the OS-dependencies builds The legacy flavor's runtime OS installation links the compiled Python, so the Python copied from the build image has to be in place before it runs - copying it afterwards broke the legacy image. A cache-disabled build is asked to install everything from scratch, and the CI image's DEPENDENCIES_EPOCH_NUMBER forces reinstallation, so neither may be satisfied by a pinned OS-dependencies image. The daily OS-dependencies builds no longer hold up mirror-only runs, which the upgrade check wa [...] Generated-by: Claude Opus 5 --- .github/workflows/mirror-hardened-base-images.yml | 22 ++++-- Dockerfile | 21 +++--- Dockerfile.ci | 2 +- dev/breeze/doc/09_release_management_tasks.rst | 5 ++ ...put_release-management_build-os-deps-images.svg | 42 ++++++++---- ...put_release-management_build-os-deps-images.txt | 2 +- ...t_release-management_publish-os-deps-images.svg | 38 ++++++---- ...t_release-management_publish-os-deps-images.txt | 2 +- .../commands/release_management_commands.py | 69 ++++++++++++++++--- .../commands/release_management_commands_config.py | 2 + .../airflow_breeze/params/common_build_params.py | 2 + dev/breeze/tests/test_mirror_base_images.py | 5 +- dev/breeze/tests/test_os_deps_images.py | 80 +++++++++++++++++++++- scripts/docker/install_os_dependencies.sh | 5 +- 14 files changed, 238 insertions(+), 59 deletions(-) diff --git a/.github/workflows/mirror-hardened-base-images.yml b/.github/workflows/mirror-hardened-base-images.yml index e689c741cea..02dd77c4c52 100644 --- a/.github/workflows/mirror-hardened-base-images.yml +++ b/.github/workflows/mirror-hardened-base-images.yml @@ -62,9 +62,10 @@ on: # yamllint disable-line rule:truthy - cron: '47 2 * * *' permissions: contents: read -concurrency: - group: mirror-hardened-base-images - cancel-in-progress: false +# Concurrency is per job, not per run: the mirror jobs of all runs are serialized, the OS-dependencies +# builds have groups of their own, so a mirror-only run - such as the one the upgrade check dispatches +# and waits for - never queues behind a daily build. The builds use the digests their own run's mirror +# job mirrored, so a later mirror moving the tags does not change what they build or publish. jobs: # Picks the runners the way CI does: RunsOn in apache/airflow, public runners when the repository # variable USE_PUBLIC_RUNNERS is "true". @@ -127,11 +128,15 @@ jobs: # branch must not be able to replace it. Main mirrors every Python version an active release # branch still builds on, so the release branches need no runs of their own. if: github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' + concurrency: + group: mirror-hardened-base-images + cancel-in-progress: false permissions: contents: read packages: write outputs: moved-past-pins: ${{ steps.mirror.outputs.moved-past-pins }} + os-deps-base-image-digests: ${{ steps.mirror.outputs.os-deps-base-image-digests }} steps: - name: "Cleanup repo" shell: bash @@ -178,6 +183,9 @@ jobs: ${{ format(fromJSON(matrix.platform == 'linux/arm64' && needs.build-info.outputs.arm-runner-type || needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} timeout-minutes: 120 + concurrency: + group: mirror-hardened-base-images-os-deps-${{ matrix.platform }}-${{ matrix.python }} + cancel-in-progress: false permissions: contents: read packages: write @@ -200,12 +208,13 @@ jobs: env: PLATFORM: ${{ matrix.platform }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python }} + OS_DEPS_BASE_IMAGE_DIGESTS: ${{ needs.mirror.outputs.os-deps-base-image-digests }} run: > breeze release-management build-os-deps-images --platform "${PLATFORM}" --python "${PYTHON_MAJOR_MINOR_VERSION}" publish-os-deps-images: name: "Publish OS-dependencies images" - needs: [build-info, build-os-deps-images] + needs: [build-info, mirror, build-os-deps-images] # Also after a failed build: the images that were built are published, and the job fails for the # rest. if: >- @@ -215,6 +224,9 @@ jobs: (github.event_name == 'workflow_dispatch' && inputs.build-os-deps-images)) runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '2+4', '8+16') }} timeout-minutes: 30 + concurrency: + group: mirror-hardened-base-images-os-deps-publish + cancel-in-progress: false permissions: contents: read packages: write @@ -237,6 +249,8 @@ jobs: run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}" --password-stdin - name: "Publish OS-dependencies images" id: publish + env: + OS_DEPS_BASE_IMAGE_DIGESTS: ${{ needs.mirror.outputs.os-deps-base-image-digests }} run: breeze release-management publish-os-deps-images # Only main's pins: release branches move theirs on their own upgrade schedule. Also after a failed # mirror or publication: a tag moved before the failure stays moved, so a re-run would not see it diff --git a/Dockerfile b/Dockerfile index e460d8a870a..e83df12c028 100644 --- a/Dockerfile +++ b/Dockerfile @@ -143,7 +143,7 @@ OS_DEPENDENCIES_INPUTS_HASH=$( ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1 @@ -2276,18 +2276,24 @@ ENV PATH="${CARGO_HOME}/bin:${PATH}" COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh dev -FROM dev-os-deps as airflow-build-image +FROM dev-os-deps as python-dist +# The final image copies /python-dist over its root before its runtime OS dependencies are installed - +# the legacy flavor's installation links the compiled Python. It is empty for the hardened flavor, whose +# Python comes with the base image, and holds the compiled /usr/python for the legacy flavor - so that +# COPY needs no condition and adds nothing to the hardened image. A stage of its own, so the +# "runtime-os-deps" stage needs the OS dependencies of the build image, but not the build image itself. SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] -# The final image copies /python-dist over its root. It is empty for the hardened flavor, whose Python -# comes with the base image, and holds the compiled /usr/python for the legacy flavor - so that COPY -# needs no condition and adds nothing to the hardened image. RUN mkdir -p /python-dist; \ if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then \ mkdir -p /python-dist/usr && cp -a /usr/python /python-dist/usr/; \ fi +FROM dev-os-deps as airflow-build-image + +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] + # In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python # libraries will be accidentally used before the library installed from sources (which is newer and # python interpreter might break if accidentally the old system libraries are used. @@ -2537,6 +2543,7 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR} +COPY --from=python-dist /python-dist/ / COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh runtime @@ -2552,10 +2559,6 @@ LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.image="airflow" \ org.apache.airflow.uid="${AIRFLOW_UID}" -# Copied after the runtime OS dependencies, so the "runtime-os-deps" stage does not depend on the build -# image. It holds the compiled Python of the legacy flavor, whose runtime installation does not use it. -COPY --from=airflow-build-image /python-dist/ / - # Having the variable in final image allows to disable providers manager warnings when # production image is prepared from sources rather than from package ARG AIRFLOW_IMAGE_REPOSITORY diff --git a/Dockerfile.ci b/Dockerfile.ci index d4cc2e1a429..4d460f96800 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -73,7 +73,7 @@ OS_DEPENDENCIES_INPUTS_HASH=$( ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1 diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 0596878978e..5babcd1ffaa 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -1371,6 +1371,11 @@ Example usage: breeze release-management build-os-deps-images --platform linux/arm64 --python 3.13 +In the workflow, the build and publish jobs get the base image digests the run's own mirror job mirrored +(``--base-image-digests``), so a mirror-only run moving the tags meanwhile does not change what they +build and publish - the OS-dependencies jobs have concurrency groups of their own and do not hold up the +mirror runs. + The script records what it installed with in the image, and a build on it installs nothing when nothing differs. When a PR changes the script or the variables it installs with (such as the Go or Node.js versions), the installation runs again on top of the pinned diff --git a/dev/breeze/doc/images/output_release-management_build-os-deps-images.svg b/dev/breeze/doc/images/output_release-management_build-os-deps-images.svg index d33f5fb8d08..99eb2c0b036 100644 --- a/dev/breeze/doc/images/output_release-management_build-os-deps-images.svg +++ b/dev/breeze/doc/images/output_release-management_build-os-deps-images.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 489.2" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 562.4" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -45,7 +45,7 @@ <defs> <clipPath id="breeze-release-management-build-os-deps-images-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="438.2" /> + <rect x="0" y="0" width="1463.0" height="511.4" /> </clipPath> <clipPath id="breeze-release-management-build-os-deps-images-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -98,9 +98,18 @@ <clipPath id="breeze-release-management-build-os-deps-images-line-16"> <rect x="0" y="391.9" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-build-os-deps-images-line-17"> + <rect x="0" y="416.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-build-os-deps-images-line-18"> + <rect x="0" y="440.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-build-os-deps-images-line-19"> + <rect x="0" y="465.1" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="487.2" rx="8"/><text class="breeze-release-management-build-os-deps-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management build-os-deps-images</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="560.4" rx="8"/><text class="breeze-release-management-build-os-deps-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management build-os-deps-images</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -116,18 +125,21 @@ </text><text class="breeze-release-management-build-os-deps-images-r1" x="12.2" y="93.2" textLength="1183.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-3)">Build and push the OS-dependencies images for one platform, on the mirrored hardened base images.</text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-release-manage [...] </text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-4)"> </text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-5)">╭─</text><text class="breeze-release-management-build-os-deps-images-r5" x="24.4" y="142" textLength="158.6" clip-path="url(#breeze-release-management-build-os-deps-images-line-5)"> Build flags </text><text class="breeze-release-management-build-os-deps-images-r5" x="183" y="142" textLength="1256.6" c [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-6)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="166.4" textLength="146.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-6)">--python    </text><text class="breeze-release-management-build-os-deps-images-r6" x="231.8" y="166.4" textLength=" [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-7)">│</text><text class="breeze-release-management-build-os-deps-images-r1" x="280.6" y="190.8" textLength="244" clip-path="url(#breeze-release-management-build-os-deps-images-line-7)">versions are used). </text><text class="breeze-release-management-build-os-deps-images-r7" x="524.6" y="190.8" textLeng [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-8)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="215.2" textLength="146.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-8)">--image-type</text><text class="breeze-release-management-build-os-deps-images-r1" x="280.6" y="215.2" textLength="1000.4" clip-path="u [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-9)">│</text><text class="breeze-release-management-build-os-deps-images-r7" x="280.6" y="239.6" textLength="97.6" clip-path="url(#breeze-release-management-build-os-deps-images-line-9)">runtime)</text><text class="breeze-release-management-build-os-deps-images-r5" x="1451.8" y="239.6" textLength="12.2" clip-path="url( [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-10)">│</text><text class="breeze-release-management-build-os-deps-images-r8" x="24.4" y="264" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-10)">*</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="264" textLength="146.4" clip-path="url(#breeze-release [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-11)">│</text><text class="breeze-release-management-build-os-deps-images-r9" x="280.6" y="288.4" textLength="122" clip-path="url(#breeze-release-management-build-os-deps-images-line-11)">[required]</text><text class="breeze-release-management-build-os-deps-images-r7" x="414.8" y="288.4" textLength="329.4" clip-path="u [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="312.8" textLength="1464" clip-path="url(#breeze-release-management-build-os-deps-images-line-12)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-12)"> -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="337.2" textLength="24.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-13)">╭─</text><text class="breeze-release-management-build-os-deps-images-r5" x="24.4" y="337.2" textLength="195.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-13)"> Common options </text><text class="breeze-release-management-build-os-deps-images-r5" x="219.6" y="337.2" textLeng [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-14)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="361.6" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-14)">--verbose</text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="361.6" textLength="24.4" clip-path="ur [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-15)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="386" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-15)">--dry-run</text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="386" textLength="24.4" clip-path="url(#bre [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-16)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="410.4" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-16)">--help   </text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="410.4" textLength="24.4 [...] -</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="434.8" textLength="1464" clip-path="url(#breeze-release-management-build-os-deps-images-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-17)"> +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-6)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="166.4" textLength="244" clip-path="url(#breeze-release-management-build-os-deps-images-line-6)">--python            </text><text class="breeze-release-management-build-os-de [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-7)">│</text><text class="breeze-release-management-build-os-deps-images-r1" x="378.2" y="190.8" textLength="329.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-7)">python versions are used). </text><text class="breeze-release-management-build-os-deps-images-r7" x="707.6" y="1 [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-8)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="215.2" textLength="244" clip-path="url(#breeze-release-management-build-os-deps-images-line-8)">--image-type        </text><text class="breeze-release-management-build-os-deps-images-r1" x="378 [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-9)">│</text><text class="breeze-release-management-build-os-deps-images-r7" x="378.2" y="239.6" textLength="170.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-9)">dev | runtime)</text><text class="breeze-release-management-build-os-deps-images-r5" x="1451.8" y="239.6" textLength="12.2 [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-10)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="264" textLength="244" clip-path="url(#breeze-release-management-build-os-deps-images-line-10)">--base-image-digests</text><text class="breeze-release-management-build-os-deps-images-r1" x="378.2" y="264" textLength="1061.4" clip-path= [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-11)">│</text><text class="breeze-release-management-build-os-deps-images-r1" x="378.2" y="288.4" textLength="1061.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-11)">'os-deps-base-image-digests' output of mirror-base-images. Tags missing from it  [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-12)">│</text><text class="breeze-release-management-build-os-deps-images-r1" x="378.2" y="312.8" textLength="268.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-12)">read from the mirror. </text><text class="breeze-release-management-build-os-deps-images-r7" x="646.6" y="312. [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-13)">│</text><text class="breeze-release-management-build-os-deps-images-r8" x="24.4" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-13)">*</text><text class="breeze-release-management-build-os-deps-images-r4" x="61" y="337.2" textLength="244" clip-path="url(#breeze-rel [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-14)">│</text><text class="breeze-release-management-build-os-deps-images-r9" x="378.2" y="361.6" textLength="122" clip-path="url(#breeze-release-management-build-os-deps-images-line-14)">[required]</text><text class="breeze-release-management-build-os-deps-images-r7" x="512.4" y="361.6" textLength="329.4" clip-path="u [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-release-management-build-os-deps-images-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-15)"> +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="410.4" textLength="24.4" clip-path="url(#breeze-release-management-build-os-deps-images-line-16)">╭─</text><text class="breeze-release-management-build-os-deps-images-r5" x="24.4" y="410.4" textLength="195.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-16)"> Common options </text><text class="breeze-release-management-build-os-deps-images-r5" x="219.6" y="410.4" textLeng [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-17)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="434.8" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-17)">--verbose</text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="434.8" textLength="24.4" clip-path="ur [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-18)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="459.2" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-18)">--dry-run</text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="459.2" textLength="24.4" clip-path="ur [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-19)">│</text><text class="breeze-release-management-build-os-deps-images-r4" x="24.4" y="483.6" textLength="109.8" clip-path="url(#breeze-release-management-build-os-deps-images-line-19)">--help   </text><text class="breeze-release-management-build-os-deps-images-r6" x="158.6" y="483.6" textLength="24.4 [...] +</text><text class="breeze-release-management-build-os-deps-images-r5" x="0" y="508" textLength="1464" clip-path="url(#breeze-release-management-build-os-deps-images-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-build-os-deps-images-r1" x="1464" y="508" textLength="12.2" clip-path="url(#breeze-release-management-build-os-deps-images-line-20)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_release-management_build-os-deps-images.txt b/dev/breeze/doc/images/output_release-management_build-os-deps-images.txt index 4a71ed5eb92..cc3b205d2e2 100644 --- a/dev/breeze/doc/images/output_release-management_build-os-deps-images.txt +++ b/dev/breeze/doc/images/output_release-management_build-os-deps-images.txt @@ -1 +1 @@ -f846e207e99d7f9e9b5a706cf27a075a +e71e100f2bc640e668188f04e5068e41 diff --git a/dev/breeze/doc/images/output_release-management_publish-os-deps-images.svg b/dev/breeze/doc/images/output_release-management_publish-os-deps-images.svg index 4db6145b5aa..365836078e2 100644 --- a/dev/breeze/doc/images/output_release-management_publish-os-deps-images.svg +++ b/dev/breeze/doc/images/output_release-management_publish-os-deps-images.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 440.4" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 513.5999999999999" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-publish-os-deps-images-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="389.4" /> + <rect x="0" y="0" width="1463.0" height="462.59999999999997" /> </clipPath> <clipPath id="breeze-release-management-publish-os-deps-images-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -90,9 +90,18 @@ <clipPath id="breeze-release-management-publish-os-deps-images-line-14"> <rect x="0" y="343.1" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-publish-os-deps-images-line-15"> + <rect x="0" y="367.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-publish-os-deps-images-line-16"> + <rect x="0" y="391.9" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-publish-os-deps-images-line-17"> + <rect x="0" y="416.3" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="438.4" rx="8"/><text class="breeze-release-management-publish-os-deps-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management publish-os-deps-images</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="511.6" rx="8"/><text class="breeze-release-management-publish-os-deps-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management publish-os-deps-images</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -108,16 +117,19 @@ </text><text class="breeze-release-management-publish-os-deps-images-r1" x="12.2" y="93.2" textLength="1061.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-3)">Merge the per-platform OS-dependencies images and move the tags the upgrade check pins.</text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-o [...] </text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-4)"> </text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-5)">╭─</text><text class="breeze-release-management-publish-os-deps-images-r5" x="24.4" y="142" textLength="183" clip-path="url(#breeze-release-management-publish-os-deps-images-line-5)"> Publish flags </text><text class="breeze-release-management-publish-os-deps-images-r5" x="207.4" y="142" textLengt [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-6)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="166.4" textLength="146.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-6)">--python    </text><text class="breeze-release-management-publish-os-deps-images-r6" x="195.2" y="166.4" [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-7)">│</text><text class="breeze-release-management-publish-os-deps-images-r1" x="244" y="190.8" textLength="244" clip-path="url(#breeze-release-management-publish-os-deps-images-line-7)">versions are used). </text><text class="breeze-release-management-publish-os-deps-images-r7" x="488" y="190.8" te [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-8)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="215.2" textLength="146.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-8)">--image-type</text><text class="breeze-release-management-publish-os-deps-images-r1" x="244" y="215.2" textLength="1000.4" cl [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-9)">│</text><text class="breeze-release-management-publish-os-deps-images-r7" x="244" y="239.6" textLength="97.6" clip-path="url(#breeze-release-management-publish-os-deps-images-line-9)">runtime)</text><text class="breeze-release-management-publish-os-deps-images-r5" x="1451.8" y="239.6" textLength="12.2" clip-pa [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="264" textLength="1464" clip-path="url(#breeze-release-management-publish-os-deps-images-line-10)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="264" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-10)"> -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="288.4" textLength="24.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-11)">╭─</text><text class="breeze-release-management-publish-os-deps-images-r5" x="24.4" y="288.4" textLength="195.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-11)"> Common options </text><text class="breeze-release-management-publish-os-deps-images-r5" x="219.6" y="288.4 [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-12)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="312.8" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-12)">--verbose</text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="312.8" textLength="24.4" cli [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-13)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="337.2" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-13)">--dry-run</text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="337.2" textLength="24.4" cli [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-14)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="361.6" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-14)">--help   </text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="361.6" textLe [...] -</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-release-management-publish-os-deps-images-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-15)"> +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-6)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="166.4" textLength="244" clip-path="url(#breeze-release-management-publish-os-deps-images-line-6)">--python            </text><text class="breeze-release-management-p [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-7)">│</text><text class="breeze-release-management-publish-os-deps-images-r1" x="341.6" y="190.8" textLength="244" clip-path="url(#breeze-release-management-publish-os-deps-images-line-7)">versions are used). </text><text class="breeze-release-management-publish-os-deps-images-r7" x="585.6" y="190.8 [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-8)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="215.2" textLength="244" clip-path="url(#breeze-release-management-publish-os-deps-images-line-8)">--image-type        </text><text class="breeze-release-management-publish-os-deps-image [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-9)">│</text><text class="breeze-release-management-publish-os-deps-images-r7" x="341.6" y="239.6" textLength="170.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-9)">dev | runtime)</text><text class="breeze-release-management-publish-os-deps-images-r5" x="1451.8" y="239.6" textLe [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-10)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="264" textLength="244" clip-path="url(#breeze-release-management-publish-os-deps-images-line-10)">--base-image-digests</text><text class="breeze-release-management-publish-os-deps-images-r1" x="341.6" y="264" textLength="1098" [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-11)">│</text><text class="breeze-release-management-publish-os-deps-images-r1" x="341.6" y="288.4" textLength="1098" clip-path="url(#breeze-release-management-publish-os-deps-images-line-11)">'os-deps-base-image-digests' output of mirror-base-images. Tags missing from i [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-12)">│</text><text class="breeze-release-management-publish-os-deps-images-r1" x="341.6" y="312.8" textLength="207.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-12)">from the mirror. </text><text class="breeze-release-management-publish-os-deps-images-r7" x="549" y="312.8" [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="337.2" textLength="1464" clip-path="url(#breeze-release-management-publish-os-deps-images-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-13)"> +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="361.6" textLength="24.4" clip-path="url(#breeze-release-management-publish-os-deps-images-line-14)">╭─</text><text class="breeze-release-management-publish-os-deps-images-r5" x="24.4" y="361.6" textLength="195.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-14)"> Common options </text><text class="breeze-release-management-publish-os-deps-images-r5" x="219.6" y="361.6 [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-15)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="386" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-15)">--verbose</text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="386" textLength="24.4" clip-path [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-16)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="410.4" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-16)">--dry-run</text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="410.4" textLength="24.4" cli [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-17)">│</text><text class="breeze-release-management-publish-os-deps-images-r4" x="24.4" y="434.8" textLength="109.8" clip-path="url(#breeze-release-management-publish-os-deps-images-line-17)">--help   </text><text class="breeze-release-management-publish-os-deps-images-r6" x="158.6" y="434.8" textLe [...] +</text><text class="breeze-release-management-publish-os-deps-images-r5" x="0" y="459.2" textLength="1464" clip-path="url(#breeze-release-management-publish-os-deps-images-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-publish-os-deps-images-r1" x="1464" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-publish-os-deps-images-line-18)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_release-management_publish-os-deps-images.txt b/dev/breeze/doc/images/output_release-management_publish-os-deps-images.txt index 742502dfc91..8d0b85eefbc 100644 --- a/dev/breeze/doc/images/output_release-management_publish-os-deps-images.txt +++ b/dev/breeze/doc/images/output_release-management_publish-os-deps-images.txt @@ -1 +1 @@ -e1e5084d1a1ee125beaad65e61b08c0e +7aa3d481ffae3a06027feddbb37d765d diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 595733d74b3..980eb4d91c9 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2505,10 +2505,10 @@ def release_prod_images( "DOCKER_CONTEXT_FILES": "./docker-context-files", "AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION": "false", } - for image_type in OS_DEPS_IMAGE_TYPES: - os_deps_image = get_pinned_os_deps_image(image_type.name, python) - if image_type.dockerfile == "Dockerfile" and os_deps_image: - build_args[image_type.build_arg] = os_deps_image + for os_deps_image_type in OS_DEPS_IMAGE_TYPES: + os_deps_image = get_pinned_os_deps_image(os_deps_image_type.name, python) + if os_deps_image_type.dockerfile == "Dockerfile" and os_deps_image: + build_args[os_deps_image_type.build_arg] = os_deps_image if commit_sha: build_args["COMMIT_SHA"] = commit_sha if slim_images: @@ -5334,6 +5334,9 @@ def mirror_base_images(python: str | None): # Tags this run moved to an image other than the one the builds are pinned to. Only a move counts: # a pin that was already behind has had its upgrade triggered by the run that moved the tag. moved: list[str] = [] + # The digests the OS-dependencies images are built on in the same workflow run - read here rather + # than from the moving tags later, which a mirror-only run may move while the images are building. + os_deps_base_image_digests: dict[str, str] = {} # Every Debian version is mirrored: the CI image and the default PROD image use the first one, the # others are still available to build PROD images on. for python_version, debian_version in itertools.product(python_versions, ALLOWED_DEBIAN_VERSIONS): @@ -5358,6 +5361,8 @@ def mirror_base_images(python: str | None): digest = get_image_digest(mirrored) if digest is None: continue + if copy_tag == tag and debian_version == OS_DEPS_IMAGE_DEBIAN_VERSION: + os_deps_base_image_digests[tag] = digest if digest != previous_digest and digest != HARDENED_PYTHON_IMAGE_DIGESTS.get(copy_tag): moved.append(copy_tag) # The moving tag is replaced by the next refresh, which would leave the digest an upgrade @@ -5386,6 +5391,7 @@ def mirror_base_images(python: str | None): if github_output: with open(github_output, "a") as output_file: output_file.write(f"moved-past-pins={'true' if moved else 'false'}\n") + output_file.write(f"os-deps-base-image-digests={json.dumps(os_deps_base_image_digests)}\n") if failed: console_print(f"[error]Failed to mirror: {', '.join(failed)}[/]") sys.exit(1) @@ -5449,13 +5455,28 @@ class OsDepsImageBuild(NamedTuple): content_tag: str -def get_os_deps_image_builds(python: str | None, image_types: tuple[str, ...]) -> list[OsDepsImageBuild]: - """Describe the OS-dependencies images to build on the base images the mirror currently serves.""" +DRY_RUN_BASE_IMAGE_DIGEST = "sha256:" + "0" * 64 + + +def get_os_deps_image_builds( + python: str | None, image_types: tuple[str, ...], base_image_digests: dict[str, str] | None = None +) -> list[OsDepsImageBuild]: + """ + Describe the OS-dependencies images to build on the mirrored base images. + + :param base_image_digests: the base image digest for each mirrored tag - the ones the mirror job of + the same workflow run mirrored. A tag missing from it is read from the mirror's moving tag. + """ python_versions = CURRENT_PYTHON_MAJOR_MINOR_VERSIONS if python is None else [python] builds = [] for python_version in python_versions: python_tag = get_hardened_python_image_tag(python_version, OS_DEPS_IMAGE_DEBIAN_VERSION) - python_digest = get_image_digest(f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}") + python_digest = (base_image_digests or {}).get(python_tag) or get_image_digest( + f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}" + ) + if python_digest is None and get_dry_run(): + # A dry run does not inspect the registry, so the plan is printed for a placeholder digest. + python_digest = DRY_RUN_BASE_IMAGE_DIGEST if python_digest is None: console_print(f"[error]{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag} is not mirrored[/]") sys.exit(1) @@ -5481,6 +5502,26 @@ def get_platform_tag(tag: str, platform: str) -> str: return f"{tag}-{platform.split('/')[-1]}" +def parse_base_image_digests(ctx: click.Context, param: click.Parameter, value: str | None) -> dict[str, str]: + if not value: + return {} + try: + digests = json.loads(value) + except json.JSONDecodeError as e: + raise click.BadParameter(f"not valid JSON: {e}") from e + if not isinstance(digests, dict): + raise click.BadParameter("must be a JSON object mapping base image tags to digests") + return digests + + +option_base_image_digests = click.option( + "--base-image-digests", + callback=parse_base_image_digests, + envvar="OS_DEPS_BASE_IMAGE_DIGESTS", + help="JSON object with the digest of each mirrored base image tag to build on - the " + "'os-deps-base-image-digests' output of mirror-base-images. Tags missing from it are read from the mirror.", +) + option_os_deps_image_type = click.option( "--image-type", "image_types", @@ -5496,6 +5537,7 @@ option_os_deps_image_type = click.option( ) @option_python_no_default @option_os_deps_image_type +@option_base_image_digests @click.option( "--platform", type=BetterChoice(OS_DEPS_IMAGE_PLATFORMS), @@ -5504,9 +5546,11 @@ option_os_deps_image_type = click.option( ) @option_verbose @option_dry_run -def build_os_deps_images(python: str | None, image_types: tuple[str, ...], platform: str): +def build_os_deps_images( + python: str | None, image_types: tuple[str, ...], base_image_digests: dict[str, str], platform: str +): failed: list[str] = [] - for build in get_os_deps_image_builds(python, image_types): + for build in get_os_deps_image_builds(python, image_types, base_image_digests): image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.content_tag}" platform_image = ( f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{get_platform_tag(build.content_tag, platform)}" @@ -5554,12 +5598,15 @@ def build_os_deps_images(python: str | None, image_types: tuple[str, ...], platf ) @option_python_no_default @option_os_deps_image_type +@option_base_image_digests @option_verbose @option_dry_run -def publish_os_deps_images(python: str | None, image_types: tuple[str, ...]): +def publish_os_deps_images( + python: str | None, image_types: tuple[str, ...], base_image_digests: dict[str, str] +): failed: list[str] = [] moved: list[str] = [] - for build in get_os_deps_image_builds(python, image_types): + for build in get_os_deps_image_builds(python, image_types, base_image_digests): image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.content_tag}" if get_image_digest(image) is None: platform_images = [ diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 2d7b8c26306..c6fdb60edf1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -121,6 +121,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "options": [ "--python", "--image-type", + "--base-image-digests", "--platform", ], } @@ -131,6 +132,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "options": [ "--python", "--image-type", + "--base-image-digests", ], } ], diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 3aea6759917..2fb576b19c4 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -148,9 +148,11 @@ class CommonBuildParams: Those images are built on the pinned base image of the default Debian version with the default apt dependencies, so any other base image or apt customization installs from the repositories. + A build with the image cache disabled is asked to install everything from scratch, so it does too. """ return ( self.use_pinned_os_deps_images + and self.docker_cache != "disabled" and self.python_image is None and self.debian_version == OS_DEPS_IMAGE_DEBIAN_VERSION ) diff --git a/dev/breeze/tests/test_mirror_base_images.py b/dev/breeze/tests/test_mirror_base_images.py index 63105d6371b..5c398f788d2 100644 --- a/dev/breeze/tests/test_mirror_base_images.py +++ b/dev/breeze/tests/test_mirror_base_images.py @@ -195,7 +195,10 @@ def test_mirror_reports_moving_past_the_pins(mock_latest, previous_digest, new_d ) assert result.exit_code == 0 - assert github_output.read_text() == f"moved-past-pins={expected}\n" + # Only the default Debian version is in the digests the OS-dependencies images are built on. + assert github_output.read_text() == ( + f"moved-past-pins={expected}\nos-deps-base-image-digests={json.dumps({PINNED_TAG: new_digest})}\n" + ) @mock.patch( diff --git a/dev/breeze/tests/test_os_deps_images.py b/dev/breeze/tests/test_os_deps_images.py index 49a6a9baaf4..55d4f5265a7 100644 --- a/dev/breeze/tests/test_os_deps_images.py +++ b/dev/breeze/tests/test_os_deps_images.py @@ -361,8 +361,9 @@ def test_publish_os_deps_images_fails_for_a_platform_not_built(registry): @mock.patch(f"{PATCH_PREFIX}.check_remote_ghcr_io_commands", autospec=True) @mock.patch(f"{PATCH_PREFIX}.check_airflow_cache_builder_configured", autospec=True) @mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True) [email protected](f"{PATCH_PREFIX}.console_print", autospec=True) def test_release_prod_images_start_from_the_pinned_os_deps_images( - mock_run_command, mock_builder, mock_ghcr, mock_environment, pins, expected, tmp_path + mock_console_print, mock_run_command, mock_builder, mock_ghcr, mock_environment, pins, expected, tmp_path ): with mock.patch.dict(HARDENED_OS_DEPS_IMAGE_DIGESTS, pins, clear=True): result = CliRunner().invoke( @@ -385,3 +386,80 @@ def test_release_prod_images_start_from_the_pinned_os_deps_images( command = mock_run_command.call_args.args[0] build_args = dict(command[i + 1].split("=", 1) for i, arg in enumerate(command) if arg == "--build-arg") assert {name: build_args[name] for name in build_args if name.endswith("_OS_DEPS_IMAGE")} == expected + mock_console_print.assert_any_call("[info]Building regular 3.2.0 image for Python 3.12[/]") + + [email protected](HARDENED_OS_DEPS_IMAGE_DIGESTS, PINS, clear=True) +def test_cache_disabled_ci_image_installs_os_dependencies_itself(): + assert "CI_OS_DEPS_IMAGE" not in _build_args(BuildCiParams(python="3.12", docker_cache="disabled")) + + [email protected](HARDENED_OS_DEPS_IMAGE_DIGESTS, PINS, clear=True) +def test_cache_disabled_prod_image_installs_os_dependencies_itself(): + build_args = _build_args(BuildProdParams(python="3.12", docker_cache="disabled")) + + assert not [name for name in build_args if name.endswith("_OS_DEPS_IMAGE")] + + +def test_ci_dependencies_epoch_is_a_marker_input(): + script = (AIRFLOW_ROOT_PATH / "scripts" / "docker" / "install_os_dependencies.sh").read_text() + marker_variables = script.split("for variable in", 1)[1].split("; do", 1)[0].split() + stage = extract_dockerfile_stage((AIRFLOW_ROOT_PATH / "Dockerfile.ci").read_text(), "ci-os-deps") + + assert "DEPENDENCIES_EPOCH_NUMBER" in marker_variables + assert "DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER}" in stage + + +def test_legacy_python_is_in_place_before_the_runtime_os_dependencies_install(): + dockerfile = (AIRFLOW_ROOT_PATH / "Dockerfile").read_text() + runtime_stage = extract_dockerfile_stage(dockerfile, "runtime-os-deps") + python_dist_stage = extract_dockerfile_stage(dockerfile, "python-dist") + main_stage = extract_dockerfile_stage(dockerfile, "main") + + assert python_dist_stage.startswith("FROM dev-os-deps as python-dist\n") + assert runtime_stage.index("COPY --from=python-dist /python-dist/ /") < runtime_stage.index( + "RUN bash /scripts/docker/install_os_dependencies.sh runtime" + ) + assert "/python-dist" not in main_stage + + +def test_os_deps_image_builds_use_the_given_base_image_digests(): + # No registry: the digest the mirror job of the same run mirrored is used as given. + with mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True, side_effect=FakeRegistry()): + (build,) = get_os_deps_image_builds("3.12", ("ci",), {PYTHON_TAG: PYTHON_DIGEST}) + + assert build.build_args["BASE_IMAGE"] == f"{MIRROR}:{PYTHON_TAG}@{PYTHON_DIGEST}" + + [email protected](f"{PATCH_PREFIX}.get_dry_run", autospec=True, return_value=True) +def test_os_deps_image_builds_plan_with_a_placeholder_digest_in_dry_run(mock_dry_run): + with mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True, side_effect=FakeRegistry()): + (build,) = get_os_deps_image_builds("3.12", ("ci",)) + + assert build.build_args["BASE_IMAGE"] == f"{MIRROR}:{PYTHON_TAG}@sha256:{'0' * 64}" + + +def test_build_os_deps_images_reads_the_base_image_digests_from_the_environment(): + registry = FakeRegistry() + with mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True, side_effect=registry): + result = CliRunner().invoke( + build_os_deps_images, + ["--python", "3.12", "--image-type", "ci", "--platform", "linux/amd64"], + env={"OS_DEPS_BASE_IMAGE_DIGESTS": json.dumps({PYTHON_TAG: PYTHON_DIGEST})}, + catch_exceptions=False, + ) + + assert result.exit_code == 0 + (command,) = registry.builds + assert f"BASE_IMAGE={MIRROR}:{PYTHON_TAG}@{PYTHON_DIGEST}" in command + + [email protected]("value", ["not-json", "[]"]) +def test_build_os_deps_images_rejects_invalid_base_image_digests(value): + result = CliRunner().invoke( + build_os_deps_images, + ["--platform", "linux/amd64", "--base-image-digests", value], + ) + + assert result.exit_code == 2 + assert "--base-image-digests" in result.output diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh index 47907b559d4..73ad398efb4 100644 --- a/scripts/docker/install_os_dependencies.sh +++ b/scripts/docker/install_os_dependencies.sh @@ -28,7 +28,8 @@ fi # The images built on a pinned OS-dependencies image (see "breeze release-management # build-os-deps-images") start from a layer this script already ran in. The marker records what it # ran with: this script, the installation type and every environment variable that changes what -# gets installed - read before the defaults below are applied, as those are part of the script. +# gets installed - read before the defaults below are applied, as those are part of the script. The CI +# image's DEPENDENCIES_EPOCH_NUMBER is among them, so bumping it reinstalls on a pinned image too. # When nothing differs there is nothing left to install. When anything does - a PR changing this # script or the apt dependencies - the installation runs again on top, so the change is exercised. OS_DEPENDENCIES_MARKER=/usr/share/airflow/os-dependencies.sha256 @@ -41,7 +42,7 @@ OS_DEPENDENCIES_INPUTS_HASH=$( ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1
