This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch pin-os-deps-image-layer in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 82d2aa8dc55e1365fe96d26ca7556684aa68220d Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 1 13:33:06 2026 +0100 Install the database clients with the pinned OS dependencies The MySQL, MSSQL and PostgreSQL clients were installed after the pinned OS-dependencies layer, so every CI and PROD build still ran apt against the live repositories for them - without the alignment with the dhi.io repository, which the skipped OS installation no longer ran first. Installing them with the OS dependencies puts them in the pinned images, and the marker now covers their scripts and selection, so a different choice reinstalls them. The runtime OS-dependencies build also s [...] Generated-by: Claude Opus 5 --- Dockerfile | 84 +++++++++------ Dockerfile.ci | 64 ++++++----- .../commands/release_management_commands.py | 52 ++++++++- .../src/airflow_breeze/params/build_prod_params.py | 9 ++ .../airflow_breeze/params/common_build_params.py | 8 +- dev/breeze/tests/test_os_deps_images.py | 118 ++++++++++++++++++++- docker-stack-docs/build-arg-ref.rst | 4 +- docker-stack-docs/build.rst | 11 +- scripts/docker/install_os_dependencies.sh | 41 +++++-- 9 files changed, 310 insertions(+), 81 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd50894b59b..ebc7202ff86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -133,17 +133,23 @@ if [[ "$#" != 1 ]]; then exit 1 fi +OS_DEPENDENCIES_SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")" OS_DEPENDENCIES_MARKER=/usr/share/airflow/os-dependencies.sha256 OS_DEPENDENCIES_INPUTS_HASH=$( { echo "${1}" - sha256sum "${BASH_SOURCE[0]}" | cut -d ' ' -f 1 + for file in "${BASH_SOURCE[0]}" "${OS_DEPENDENCIES_SCRIPTS_DIR}/common.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}"/keys/*; do + sha256sum "${file}" | cut -d ' ' -f 1 + done for variable in AIRFLOW_IMAGE_FLAVOR AIRFLOW_PYTHON_VERSION PYTHON_HOME \ DEV_APT_DEPS ADDITIONAL_DEV_APT_DEPS DEV_APT_COMMAND ADDITIONAL_DEV_APT_COMMAND \ ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER \ + INSTALL_MYSQL_CLIENT INSTALL_MYSQL_CLIENT_TYPE INSTALL_MSSQL_CLIENT INSTALL_POSTGRES_CLIENT; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1 @@ -864,6 +870,25 @@ function install_rustup() { rm -f /tmp/rustup-init } +function install_database_clients() { + # Part of the OS dependencies, so the pinned OS-dependencies images carry the clients too and an + # image built on one does not install them from the live apt repositories. They run in the same + # order as before they moved here: after the rest of the OS dependencies, each cleaning up after apt. + if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" prod + return + fi + # The CI image runs Airflow from the sources as well, so it also gets the MySQL runtime client. + if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + fi + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" dev +} + function apt_clean() { apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false rm -rf /var/lib/apt/lists/* /var/log/* @@ -896,6 +921,7 @@ else install_docker_cli apt_clean fi +install_database_clients mkdir -p "$(dirname "${OS_DEPENDENCIES_MARKER}")" echo "${OS_DEPENDENCIES_INPUTS_HASH}" > "${OS_DEPENDENCIES_MARKER}" @@ -2283,7 +2309,21 @@ ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/home/airflow/.cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" -COPY --from=scripts install_os_dependencies.sh /scripts/docker/ +ARG INSTALL_MYSQL_CLIENT="true" +ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" +ARG INSTALL_MSSQL_CLIENT="true" +ARG INSTALL_POSTGRES_CLIENT="true" + +ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ + INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ + INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \ + INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT} + +# The database clients are part of the OS dependencies - install_os_dependencies.sh runs their +# scripts, and its marker covers them, so the pinned OS-dependencies images carry them too. +COPY --from=scripts install_os_dependencies.sh common.sh install_mysql.sh install_mssql.sh \ + install_postgres.sh /scripts/docker/ +COPY scripts/docker/keys/ /scripts/docker/keys/ RUN bash /scripts/docker/install_os_dependencies.sh dev FROM dev-os-deps as python-dist @@ -2309,26 +2349,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "n # python interpreter might break if accidentally the old system libraries are used. ENV LD_LIBRARY_PATH="/usr/python/lib" -ARG INSTALL_MYSQL_CLIENT="true" -ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" -ARG INSTALL_MSSQL_CLIENT="true" -ARG INSTALL_POSTGRES_CLIENT="true" - -ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ - INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ - INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \ - INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT} - COPY --from=scripts common.sh /scripts/docker/ -# Only copy mysql/mssql installation scripts for now - so that changing the other -# scripts which are needed much later will not invalidate the docker layer here -COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ -COPY scripts/docker/keys/ /scripts/docker/keys/ - -RUN bash /scripts/docker/install_mysql.sh dev && \ - bash /scripts/docker/install_mssql.sh dev && \ - bash /scripts/docker/install_postgres.sh dev ENV PATH=${PATH}:/opt/mssql-tools/bin # By default we do not install from docker context files but if we decide to install from docker context @@ -2560,7 +2582,11 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR} COPY --from=python-dist /python-dist/ / -COPY --from=scripts install_os_dependencies.sh /scripts/docker/ +# The database clients are part of the OS dependencies - install_os_dependencies.sh runs their +# scripts, and its marker covers them, so the pinned OS-dependencies images carry them too. +COPY --from=scripts install_os_dependencies.sh common.sh install_mysql.sh install_mssql.sh \ + install_postgres.sh /scripts/docker/ +COPY scripts/docker/keys/ /scripts/docker/keys/ RUN bash /scripts/docker/install_os_dependencies.sh runtime FROM runtime-os-deps as main @@ -2593,19 +2619,7 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:/usr/python/bin:${PATH}" \ COPY --from=scripts common.sh /scripts/docker/ -COPY scripts/docker/keys/ /scripts/docker/keys/ - -# Only copy mysql/mssql installation scripts for now - so that changing the other -# scripts which are needed much later will not invalidate the docker layer here. -COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ -# We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an -# unexpected result - the cache for Dockerfiles might get invalidated in case the host system -# had different umask set and group x bit was not set. In Azure the bit might be not set at all. -# That also protects against AUFS Docker backend problem where changing the executable bit required sync -RUN bash /scripts/docker/install_mysql.sh prod \ - && bash /scripts/docker/install_mssql.sh prod \ - && bash /scripts/docker/install_postgres.sh prod \ - && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ +RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" \ # Make Airflow files belong to the root group and are accessible. This is to accommodate the guidelines from # OpenShift https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html diff --git a/Dockerfile.ci b/Dockerfile.ci index c1d7fa61d0e..75779430953 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -63,17 +63,23 @@ if [[ "$#" != 1 ]]; then exit 1 fi +OS_DEPENDENCIES_SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")" OS_DEPENDENCIES_MARKER=/usr/share/airflow/os-dependencies.sha256 OS_DEPENDENCIES_INPUTS_HASH=$( { echo "${1}" - sha256sum "${BASH_SOURCE[0]}" | cut -d ' ' -f 1 + for file in "${BASH_SOURCE[0]}" "${OS_DEPENDENCIES_SCRIPTS_DIR}/common.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}"/keys/*; do + sha256sum "${file}" | cut -d ' ' -f 1 + done for variable in AIRFLOW_IMAGE_FLAVOR AIRFLOW_PYTHON_VERSION PYTHON_HOME \ DEV_APT_DEPS ADDITIONAL_DEV_APT_DEPS DEV_APT_COMMAND ADDITIONAL_DEV_APT_COMMAND \ ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER \ + INSTALL_MYSQL_CLIENT INSTALL_MYSQL_CLIENT_TYPE INSTALL_MSSQL_CLIENT INSTALL_POSTGRES_CLIENT; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1 @@ -794,6 +800,25 @@ function install_rustup() { rm -f /tmp/rustup-init } +function install_database_clients() { + # Part of the OS dependencies, so the pinned OS-dependencies images carry the clients too and an + # image built on one does not install them from the live apt repositories. They run in the same + # order as before they moved here: after the rest of the OS dependencies, each cleaning up after apt. + if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" prod + return + fi + # The CI image runs Airflow from the sources as well, so it also gets the MySQL runtime client. + if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + fi + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" dev +} + function apt_clean() { apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false rm -rf /var/lib/apt/lists/* /var/log/* @@ -826,6 +851,7 @@ else install_docker_cli apt_clean fi +install_database_clients mkdir -p "$(dirname "${OS_DEPENDENCIES_MARKER}")" echo "${OS_DEPENDENCIES_INPUTS_HASH}" > "${OS_DEPENDENCIES_MARKER}" @@ -2168,7 +2194,14 @@ ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/usr/local/cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" -COPY --from=scripts install_os_dependencies.sh /scripts/docker/ +ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" +ENV INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} + +# The database clients are part of the OS dependencies - install_os_dependencies.sh runs their +# scripts, and its marker covers them, so the pinned OS-dependencies images carry them too. +COPY --from=scripts install_os_dependencies.sh common.sh install_mysql.sh install_mssql.sh \ + install_postgres.sh /scripts/docker/ +COPY scripts/docker/keys/ /scripts/docker/keys/ RUN bash /scripts/docker/install_os_dependencies.sh ci @@ -2183,35 +2216,18 @@ ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"; # python interpreter might break if accidentally the old system libraries are used. ENV LD_LIBRARY_PATH="/usr/python/lib" -COPY --from=scripts common.sh /scripts/docker/ - -# Only copy mysql/mssql installation scripts for now - so that changing the other -# scripts which are needed much later will not invalidate the docker layer here. -COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ -COPY scripts/docker/keys/ /scripts/docker/keys/ - ARG HOME=/root ARG AIRFLOW_IMAGE_TYPE="ci" ARG AIRFLOW_HOME=/root/airflow ARG AIRFLOW_SOURCES=/opt/airflow -ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ENV HOME=${HOME} \ AIRFLOW_IMAGE_TYPE=${AIRFLOW_IMAGE_TYPE} \ AIRFLOW_HOME=${AIRFLOW_HOME} \ - AIRFLOW_SOURCES=${AIRFLOW_SOURCES} \ - INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} - -# We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an -# unexpected result - the cache for Dockerfiles might get invalidated in case the host system -# had different umask set and group x bit was not set. In Azure the bit might be not set at all. -# That also protects against AUFS Docker backend problem where changing the executable bit required sync -RUN bash /scripts/docker/install_mysql.sh prod \ - && bash /scripts/docker/install_mysql.sh dev \ - && bash /scripts/docker/install_mssql.sh dev \ - && bash /scripts/docker/install_postgres.sh dev \ - # The user is added to allow ssh debugging (you can connect with airflow/airflow over ssh) - && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ + AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + +# The user is added to allow ssh debugging (you can connect with airflow/airflow over ssh) +RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ --quiet "airflow" --home "/home/airflow" \ && echo -e "airflow\nairflow" | passwd airflow 2>&1 \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 12b0171a9d3..8213d049c7f 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -127,6 +127,7 @@ from airflow_breeze.global_constants import ( get_ts_sdk_version, ) from airflow_breeze.params.build_ci_params import BuildCiParams +from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.prepare_providers.provider_distributions import ( PrepareReleasePackageErrorBuildingPackageException, @@ -5420,14 +5421,26 @@ def get_os_deps_image_build_args( They must give the installation the same environment the CI and PROD builds give it, or the marker install_os_dependencies.sh writes would not match there and the installation would run again. - Those builds pass the Python patchlevel they are pinned to - the one in the base image tag. + Those builds pass the Python patchlevel they are pinned to - the one in the base image tag - and + the database clients of their default parameters, which the images install too. """ build_args = { "BASE_IMAGE": f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}@{python_digest}", "AIRFLOW_PYTHON_VERSION": python_tag.split("-", 1)[0], } if image_type.dockerfile == "Dockerfile": - build_args["AIRFLOW_IMAGE_FLAVOR"] = "hardened" + prod_params = BuildProdParams() + build_args.update( + { + "AIRFLOW_IMAGE_FLAVOR": "hardened", + "INSTALL_MYSQL_CLIENT": prod_params.install_mysql_client, + "INSTALL_MYSQL_CLIENT_TYPE": prod_params.install_mysql_client_type, + "INSTALL_MSSQL_CLIENT": prod_params.install_mssql_client, + "INSTALL_POSTGRES_CLIENT": prod_params.install_postgres_client, + } + ) + else: + build_args["INSTALL_MYSQL_CLIENT_TYPE"] = BuildCiParams().install_mysql_client_type return build_args @@ -5533,6 +5546,21 @@ option_base_image_digests = click.option( "'os-deps-base-image-digests' output of mirror-base-images. Tags missing from it are read from the mirror.", ) + +def find_built_os_deps_image(build: OsDepsImageBuild, platform: str) -> str | None: + """ + Return an already built OS-dependencies image of a build, for a build of another type to start from. + + The merged image if it is published, the image of the platform otherwise - the one this job has just + built, or an earlier run that failed before publishing it. + """ + for tag in (build.content_tag, get_platform_tag(build.content_tag, platform)): + image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{tag}" + if get_image_digest(image): + return image + return None + + option_os_deps_image_type = click.option( "--image-type", "image_types", @@ -5561,6 +5589,11 @@ def build_os_deps_images( python: str | None, image_types: tuple[str, ...], base_image_digests: dict[str, str], platform: str ): failed: list[str] = [] + # The dev images built - now or before - for each base image, which the runtime builds start from. + dev_builds = { + build.build_args["BASE_IMAGE"]: build + for build in get_os_deps_image_builds(python, ("dev",), base_image_digests) + } for build in get_os_deps_image_builds(python, image_types, base_image_digests): image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.content_tag}" platform_image = ( @@ -5570,6 +5603,14 @@ def build_os_deps_images( console_print(f"[info]{platform_image} is already built[/]") continue console_print(f"[info]Building {platform_image}[/]") + # Not part of the content tag: the runtime stage only copies the legacy flavor's Python from the + # dev stage, which is empty for the hardened flavor, so the runtime image does not change. + extra_build_args = {} + dev_build = dev_builds.get(build.build_args["BASE_IMAGE"]) + if build.image_type.name == "runtime" and dev_build: + dev_image = find_built_os_deps_image(dev_build, platform) + if dev_image: + extra_build_args["DEV_OS_DEPS_IMAGE"] = dev_image result = run_command( [ "docker", @@ -5583,14 +5624,15 @@ def build_os_deps_images( platform, # The point of the image is a fresh installation from the apt repositories - but only of # the target stage: the runtime stage copies from one built on the dev OS dependencies, - # which the dev image built just before in the same job leaves in the cache, so this - # does not install them a second time. The installation types are built in this order. + # which starts from the dev image built before, so it does not install them a second time. "--no-cache-filter", build.image_type.target, "--pull", # A plain image manifest per platform, for publish-os-deps-images to merge. "--provenance=false", - *convert_build_args_dict_to_array_of_args(build_args=build.build_args), + *convert_build_args_dict_to_array_of_args( + build_args={**build.build_args, **extra_build_args} + ), "--tag", platform_image, "--push", diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 54ca594f7a1..7bde13fbfc8 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -69,6 +69,15 @@ class BuildProdParams(CommonBuildParams): return get_legacy_base_image(self.debian_version) return super().python_base_image + def _has_custom_database_clients(self) -> bool: + return super()._has_custom_database_clients() or any( + [ + self.disable_mssql_client_installation, + self.disable_mysql_client_installation, + self.disable_postgres_client_installation, + ] + ) + @property def _has_custom_runtime_apt_dependencies(self) -> bool: return any( diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index ac34ca387df..d12905c783f 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -143,17 +143,23 @@ class CommonBuildParams: ] ) + def _has_custom_database_clients(self) -> bool: + """Whether the build installs other database clients than the OS-dependencies images carry.""" + return self.install_mysql_client_type != ALLOWED_INSTALL_MYSQL_CLIENT_TYPES[0] + def _can_use_pinned_os_deps_images(self) -> bool: """Whether the build may start from the pinned OS-dependencies images instead of BASE_IMAGE. Those images are built on the pinned base image of the default Debian version with the default - apt dependencies, so any other base image or apt customization installs from the repositories. + apt dependencies and database clients, so any other base image, apt customization or choice + of database clients installs from the repositories. A disabled image cache does not turn them off - the CI build retries with it after any failure. """ return ( self.use_pinned_os_deps_images and self.python_image is None and self.debian_version == OS_DEPS_IMAGE_DEBIAN_VERSION + and not self._has_custom_database_clients() ) def _set_os_deps_image_arg(self, image_type: str, build_arg: str): diff --git a/dev/breeze/tests/test_os_deps_images.py b/dev/breeze/tests/test_os_deps_images.py index 9d5cd728fca..95cb8d5daea 100644 --- a/dev/breeze/tests/test_os_deps_images.py +++ b/dev/breeze/tests/test_os_deps_images.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import re from subprocess import CompletedProcess from unittest import mock @@ -187,12 +188,20 @@ def test_extract_dockerfile_stage_fails_for_a_missing_stage(): extract_dockerfile_stage("FROM scratch as main\n", "no-such-stage") +PROD_CLIENT_ARGS = { + "INSTALL_MYSQL_CLIENT": "true", + "INSTALL_MYSQL_CLIENT_TYPE": "mariadb", + "INSTALL_MSSQL_CLIENT": "true", + "INSTALL_POSTGRES_CLIENT": "true", +} + + @pytest.mark.parametrize( ("image_type", "expected"), [ - pytest.param("ci", {}, id="ci"), - pytest.param("dev", {"AIRFLOW_IMAGE_FLAVOR": "hardened"}, id="dev"), - pytest.param("runtime", {"AIRFLOW_IMAGE_FLAVOR": "hardened"}, id="runtime"), + pytest.param("ci", {"INSTALL_MYSQL_CLIENT_TYPE": "mariadb"}, id="ci"), + pytest.param("dev", {"AIRFLOW_IMAGE_FLAVOR": "hardened", **PROD_CLIENT_ARGS}, id="dev"), + pytest.param("runtime", {"AIRFLOW_IMAGE_FLAVOR": "hardened", **PROD_CLIENT_ARGS}, id="runtime"), ], ) def test_os_deps_image_build_args_match_the_image_builds(image_type, expected): @@ -205,6 +214,25 @@ def test_os_deps_image_build_args_match_the_image_builds(image_type, expected): } [email protected]( + ("image_type", "params"), + [ + pytest.param("ci", BuildCiParams(python="3.12"), id="ci"), + pytest.param("dev", BuildProdParams(python="3.12"), id="dev"), + pytest.param("runtime", BuildProdParams(python="3.12"), id="runtime"), + ], +) +def test_os_deps_images_install_what_the_default_image_builds_pass(image_type, params): + # Every arg the marker depends on must have the value the CI and PROD builds pass by default, or + # every build on the pinned image would install all of its OS dependencies again. + os_deps_build_args = get_os_deps_image_build_args(IMAGE_TYPES[image_type], PYTHON_TAG, PYTHON_DIGEST) + image_build_args = _build_args(params) + + assert {name: image_build_args[name] for name in os_deps_build_args if name != "BASE_IMAGE"} == { + name: value for name, value in os_deps_build_args.items() if name != "BASE_IMAGE" + } + + @pytest.mark.parametrize( "change", [ @@ -537,6 +565,7 @@ def test_os_deps_image_builds_include_a_newer_mirrored_patchlevel(): { "BASE_IMAGE": f"{MIRROR}:{PYTHON_TAG}@{PYTHON_DIGEST}", "AIRFLOW_PYTHON_VERSION": ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION["3.12"], + "INSTALL_MYSQL_CLIENT_TYPE": "mariadb", }, ), ( @@ -544,6 +573,7 @@ def test_os_deps_image_builds_include_a_newer_mirrored_patchlevel(): { "BASE_IMAGE": f"{MIRROR}:3.12.99-debian13-dev@{newer_digest}", "AIRFLOW_PYTHON_VERSION": "3.12.99", + "INSTALL_MYSQL_CLIENT_TYPE": "mariadb", }, ), ] @@ -582,3 +612,85 @@ def test_os_deps_stages_expose_the_image_their_os_dependencies_come_from(): (AIRFLOW_ROOT_PATH / image_type.dockerfile).read_text(), image_type.target ) assert f"ENV AIRFLOW_OS_DEPS_IMAGE=${{{image_type.build_arg}}}" in stage + + +CLIENT_SCRIPTS = ("install_mysql.sh", "install_mssql.sh", "install_postgres.sh") +CLIENT_VARIABLES = ( + "INSTALL_MYSQL_CLIENT", + "INSTALL_MYSQL_CLIENT_TYPE", + "INSTALL_MSSQL_CLIENT", + "INSTALL_POSTGRES_CLIENT", +) + + [email protected]("image_type", OS_DEPS_IMAGE_TYPES, ids=lambda image_type: image_type.name) +def test_database_clients_are_installed_in_the_os_deps_stages(image_type): + dockerfile = (AIRFLOW_ROOT_PATH / image_type.dockerfile).read_text() + stage = extract_dockerfile_stage(dockerfile, image_type.target) + main_stage = extract_dockerfile_stage(dockerfile, "main") + + copied = stage.split("COPY --from=scripts install_os_dependencies.sh", 1)[1].split("\n\n", 1)[0] + assert all(script in copied for script in ("common.sh", *CLIENT_SCRIPTS)) + assert "COPY scripts/docker/keys/ /scripts/docker/keys/" in stage + # Set in the stage, before its installation runs - from an arg, or fixed for the CI image. + before_installation = stage.split("RUN bash /scripts/docker/install_os_dependencies.sh", 1)[0] + assert all(f"{variable}=" in before_installation for variable in CLIENT_VARIABLES) + assert not [script for script in CLIENT_SCRIPTS if f"bash /scripts/docker/{script}" in main_stage] + + +def test_marker_covers_the_database_clients(): + script = (AIRFLOW_ROOT_PATH / "scripts" / "docker" / "install_os_dependencies.sh").read_text() + marker_block = script.split("OS_DEPENDENCIES_INPUTS_HASH=$(", 1)[1].split("\n)\n", 1)[0] + + assert all(f"/{name}" in marker_block for name in ("common.sh", *CLIENT_SCRIPTS)) + assert "/keys/*" in marker_block + assert all(re.search(rf"\b{variable}\b", marker_block) for variable in CLIENT_VARIABLES) + + [email protected]( + "params", + [ + pytest.param({"disable_mysql_client_installation": True}, id="no-mysql"), + pytest.param({"disable_mssql_client_installation": True}, id="no-mssql"), + pytest.param({"disable_postgres_client_installation": True}, id="no-postgres"), + pytest.param({"install_mysql_client_type": "mysql"}, id="other-mysql-client"), + ], +) [email protected](HARDENED_OS_DEPS_IMAGE_DIGESTS, PINS, clear=True) +def test_prod_image_with_other_database_clients_installs_them_itself(params): + build_args = _build_args(BuildProdParams(python="3.12", **params)) + + assert not [name for name in build_args if name.endswith("_OS_DEPS_IMAGE")] + + [email protected](HARDENED_OS_DEPS_IMAGE_DIGESTS, PINS, clear=True) +def test_ci_image_with_another_mysql_client_installs_it_itself(): + assert "CI_OS_DEPS_IMAGE" not in _build_args( + BuildCiParams(python="3.12", install_mysql_client_type="mysql") + ) + + [email protected]("existing", ["merged", "platform", "none"]) +def test_runtime_os_deps_image_build_starts_from_the_built_dev_image(registry, existing): + (dev_build,) = get_os_deps_image_builds("3.12", ("dev",)) + (runtime_build,) = get_os_deps_image_builds("3.12", ("runtime",)) + dev_image = { + "merged": f"{REPOSITORY}:{dev_build.content_tag}", + "platform": f"{REPOSITORY}:{dev_build.content_tag}-arm64", + "none": None, + }[existing] + if dev_image: + registry.digests[dev_image] = OS_DEPS_DIGEST + + result = CliRunner().invoke( + build_os_deps_images, + ["--python", "3.12", "--image-type", "runtime", "--platform", "linux/arm64"], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + (command,) = registry.builds + build_args = dict(command[i + 1].split("=", 1) for i, arg in enumerate(command) if arg == "--build-arg") + assert build_args.get("DEV_OS_DEPS_IMAGE") == dev_image + # The dev image the runtime build starts from is not one of its inputs. + assert command[command.index("--tag") + 1] == f"{REPOSITORY}:{runtime_build.content_tag}-arm64" diff --git a/docker-stack-docs/build-arg-ref.rst b/docker-stack-docs/build-arg-ref.rst index 747fca5d97c..e56cefed5d8 100644 --- a/docker-stack-docs/build-arg-ref.rst +++ b/docker-stack-docs/build-arg-ref.rst @@ -59,7 +59,9 @@ Those are the most common arguments that you use when you want to build a custom | | | builds pass a pinned image that already has | | | | them, built from ``BASE_IMAGE`` by the | | | | ``dev-os-deps`` stage. Any other image runs | -| | | the installation. | +| | | the installation. The OS dependencies | +| | | include the database clients the | +| | | ``INSTALL_*_CLIENT`` args select. | +------------------------------------------+-------------------------------------------+---------------------------------------------+ | ``RUNTIME_OS_DEPS_IMAGE`` | ``BASE_IMAGE`` | The image the final image installs its OS | | | | dependencies on - like | diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst index ef6a401bc93..5dabf354b5d 100644 --- a/docker-stack-docs/build.rst +++ b/docker-stack-docs/build.rst @@ -973,11 +973,12 @@ previous images that can matter when you extend or run the image are: * **The OS dependencies come from pinned images too.** A digest pins the base image, but not ``dhi.io``'s apt repository: it moves on - and drops the package versions it replaced - whenever Docker rebuilds its images, so installing the OS dependencies from it can fail for a commit that built fine before. Airflow's - CI and release builds therefore start their OS dependencies from images in which - ``install_os_dependencies.sh`` already ran, built daily on the mirrored base images and pinned to a digest - by the same upgrade, passed as ``DEV_OS_DEPS_IMAGE`` and ``RUNTIME_OS_DEPS_IMAGE``. The script records what it - installed with and does nothing when nothing differs, so a build with other apt dependencies still - installs them. A plain ``docker build`` leaves both at ``BASE_IMAGE`` and installs everything as before. + CI and release builds therefore start their OS dependencies - the database clients included - from + images in which ``install_os_dependencies.sh`` already ran, built daily on the mirrored base images and + pinned to a digest by the same upgrade, passed as ``DEV_OS_DEPS_IMAGE`` and ``RUNTIME_OS_DEPS_IMAGE``. The + script records what it installed with and does nothing when nothing differs, so a build with other apt + dependencies or database clients still installs them. A plain ``docker build`` leaves both at + ``BASE_IMAGE`` and installs everything as before. The ``AIRFLOW_OS_DEPS_IMAGE`` environment variable of an image names the image its OS dependencies came from, and ``/usr/share/airflow/os-dependencies.base-image`` the base image they were installed on - which can be an older digest than ``BASE_IMAGE``. diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh index 8128142eee6..1ce527f1c9f 100644 --- a/scripts/docker/install_os_dependencies.sh +++ b/scripts/docker/install_os_dependencies.sh @@ -27,22 +27,29 @@ fi # The images built on a pinned OS-dependencies image (see "breeze release-management # build-os-deps-images") start from a layer this script already ran in. The marker records what it -# ran with: this script, the installation type and every environment variable that changes what -# gets installed - read before the defaults below are applied, as those are part of the script. The CI -# image's DEPENDENCIES_EPOCH_NUMBER is among them, so bumping it reinstalls on a pinned image too. -# When nothing differs there is nothing left to install. When anything does - a PR changing this -# script or the apt dependencies - the installation runs again on top, so the change is exercised. +# ran with: this script, the database client scripts it runs with common.sh and the keys they use, the +# installation type and every environment variable that changes what gets installed - read before the +# defaults below are applied, as those are part of the script. The CI image's DEPENDENCIES_EPOCH_NUMBER +# is among them, so bumping it reinstalls on a pinned image too. When nothing differs there is nothing +# left to install. When anything does - a PR changing one of the scripts, the apt dependencies or the +# database clients - all of the installation runs again on top, so the change is exercised. +OS_DEPENDENCIES_SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")" OS_DEPENDENCIES_MARKER=/usr/share/airflow/os-dependencies.sha256 OS_DEPENDENCIES_INPUTS_HASH=$( { echo "${1}" - sha256sum "${BASH_SOURCE[0]}" | cut -d ' ' -f 1 + for file in "${BASH_SOURCE[0]}" "${OS_DEPENDENCIES_SCRIPTS_DIR}/common.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" \ + "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" "${OS_DEPENDENCIES_SCRIPTS_DIR}"/keys/*; do + sha256sum "${file}" | cut -d ' ' -f 1 + done for variable in AIRFLOW_IMAGE_FLAVOR AIRFLOW_PYTHON_VERSION PYTHON_HOME \ DEV_APT_DEPS ADDITIONAL_DEV_APT_DEPS DEV_APT_COMMAND ADDITIONAL_DEV_APT_COMMAND \ ADDITIONAL_DEV_APT_ENV RUNTIME_APT_DEPS ADDITIONAL_RUNTIME_APT_DEPS RUNTIME_APT_COMMAND \ ADDITIONAL_RUNTIME_APT_COMMAND ADDITIONAL_RUNTIME_APT_ENV GOLANG_MAJOR_MINOR_VERSION \ TEMURIN_VERSION NODEJS_VERSION PNPM_VERSION RUSTUP_DEFAULT_TOOLCHAIN RUSTUP_VERSION \ - RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER; do + RUSTUP_HOME CARGO_HOME COSIGN_VERSION LDFLAGS EXTRA_CFLAGS DEPENDENCIES_EPOCH_NUMBER \ + INSTALL_MYSQL_CLIENT INSTALL_MYSQL_CLIENT_TYPE INSTALL_MSSQL_CLIENT INSTALL_POSTGRES_CLIENT; do printf '%s=%s\n' "${variable}" "${!variable-}" done } | sha256sum | cut -d ' ' -f 1 @@ -769,6 +776,25 @@ function install_rustup() { rm -f /tmp/rustup-init } +function install_database_clients() { + # Part of the OS dependencies, so the pinned OS-dependencies images carry the clients too and an + # image built on one does not install them from the live apt repositories. They run in the same + # order as before they moved here: after the rest of the OS dependencies, each cleaning up after apt. + if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" prod + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" prod + return + fi + # The CI image runs Airflow from the sources as well, so it also gets the MySQL runtime client. + if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" prod + fi + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mysql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_mssql.sh" dev + bash "${OS_DEPENDENCIES_SCRIPTS_DIR}/install_postgres.sh" dev +} + function apt_clean() { apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false rm -rf /var/lib/apt/lists/* /var/log/* @@ -801,6 +827,7 @@ else install_docker_cli apt_clean fi +install_database_clients mkdir -p "$(dirname "${OS_DEPENDENCIES_MARKER}")" echo "${OS_DEPENDENCIES_INPUTS_HASH}" > "${OS_DEPENDENCIES_MARKER}"
