This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch pin-os-deps-image-layer in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ce2bbb0a7f09b93d65553a5d4c57dbf07c85814c Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 1 12:01:45 2026 +0100 Publish OS-dependencies images ahead of patch releases and pin them on main only The upgrade check moves the base image pin to a newer Python patch release as soon as the mirror has it, and without OS-dependencies images for that patch release the upgrade PR and main installed from the live apt repository again - the failure the pinned images are there to avoid. Building them for every patch release the mirror copied publishes them before the pin moves. The images are built from main's install script and Dockerfiles, so a release branch pinning them would never ma [...] Generated-by: Claude Opus 5 --- dev/breeze/doc/09_release_management_tasks.rst | 13 ++-- .../commands/release_management_commands.py | 75 +++++++++++++--------- dev/breeze/src/airflow_breeze/global_constants.py | 10 +-- dev/breeze/tests/test_mirror_base_images.py | 31 +++++++++ dev/breeze/tests/test_os_deps_images.py | 54 +++++++++++++++- scripts/ci/prek/upgrade_important_versions.py | 31 ++++++++- .../ci/prek/test_upgrade_important_versions.py | 44 +++++++++++++ 7 files changed, 215 insertions(+), 43 deletions(-) diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 5babcd1ffaa..91f49e894d6 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -1351,12 +1351,17 @@ by CI and the release workflows therefore start from images in which ``install_o already ran, one per installation type (``ci``, ``dev`` and ``runtime``). They are built from the ``ci-os-deps``, ``dev-os-deps`` and ``runtime-os-deps`` stages of the Dockerfiles, on the base images the mirror serves, and pinned in ``dev/breeze/src/airflow_breeze/files/hardened_os_deps_image_digests.json`` -by the same upgrade that moves the base image pins. +by the same upgrade that moves the base image pins. Only ``main`` carries pins: the images are built from +``main``'s script and Dockerfiles, so on a release branch they would not match its own installation - the +upgrade leaves the file empty on any other branch until there are images per branch. The "Mirror hardened base images" workflow runs this command daily, natively on an AMD and an ARM runner, -after it mirrored the base images - its six-hourly runs only mirror, and the run the upgrade check -dispatches (``breeze workflow-run mirror-base-images --no-build-os-deps-images``) builds nothing either. Every input of an image - the base image digest, the script, its Dockerfile -stage and its build args - is part of its tag, so an image is built only when one of them changed. +after it mirrored the base images - also for a newer Python patch release the mirror already copied, so +its images are published by the time the upgrade moves the base image pin to it. Its six-hourly runs only +mirror, and the run the upgrade check dispatches +(``breeze workflow-run mirror-base-images --no-build-os-deps-images``) builds nothing either. Every input +of an image - the base image digest, the script, its Dockerfile stage and its build args - is part of its +tag, so an image is built only when one of them changed. These are all available flags of ``release-management build-os-deps-images`` command: diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 980eb4d91c9..1266c27f5af 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -122,7 +122,6 @@ from airflow_breeze.global_constants import ( get_hardened_python_image_retention_tag, get_hardened_python_image_tag, get_os_deps_image_python_tag, - get_os_deps_image_tag, get_pinned_os_deps_image, get_task_sdk_version, get_ts_sdk_version, @@ -5361,8 +5360,9 @@ def mirror_base_images(python: str | None): digest = get_image_digest(mirrored) if digest is None: continue - if copy_tag == tag and debian_version == OS_DEPS_IMAGE_DEBIAN_VERSION: - os_deps_base_image_digests[tag] = digest + # The newer patchlevel too, so its OS-dependencies images are published before its pin. + if debian_version == OS_DEPS_IMAGE_DEBIAN_VERSION: + os_deps_base_image_digests[copy_tag] = digest if digest != previous_digest and digest != HARDENED_PYTHON_IMAGE_DIGESTS.get(copy_tag): moved.append(copy_tag) # The moving tag is replaced by the next refresh, which would leave the digest an upgrade @@ -5413,18 +5413,18 @@ def extract_dockerfile_stage(dockerfile_content: str, target: str) -> str: def get_os_deps_image_build_args( - image_type: OsDepsImageType, python: str, python_digest: str + image_type: OsDepsImageType, python_tag: str, python_digest: str ) -> dict[str, str]: """ - Return the build args of an OS-dependencies image. + Return the build args of an OS-dependencies image built on a mirrored base image tag. They must give the installation the same environment the CI and PROD builds give it, or the marker install_os_dependencies.sh writes would not match there and the installation would run again. + Those builds pass the Python patchlevel they are pinned to - the one in the base image tag. """ - python_tag = get_hardened_python_image_tag(python, OS_DEPS_IMAGE_DEBIAN_VERSION) build_args = { "BASE_IMAGE": f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}@{python_digest}", - "AIRFLOW_PYTHON_VERSION": ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python, python), + "AIRFLOW_PYTHON_VERSION": python_tag.split("-", 1)[0], } if image_type.dockerfile == "Dockerfile": build_args["AIRFLOW_IMAGE_FLAVOR"] = "hardened" @@ -5455,7 +5455,7 @@ class OsDepsImageBuild(NamedTuple): content_tag: str -DRY_RUN_BASE_IMAGE_DIGEST = "sha256:" + "0" * 64 +DRY_RUN_IMAGE_DIGEST = "sha256:" + "0" * 64 def get_os_deps_image_builds( @@ -5465,36 +5465,47 @@ def get_os_deps_image_builds( Describe the OS-dependencies images to build on the mirrored base images. :param base_image_digests: the base image digest for each mirrored tag - the ones the mirror job of - the same workflow run mirrored. A tag missing from it is read from the mirror's moving tag. + the same workflow run mirrored. The pinned patchlevel missing from it is read from the mirror's + moving tag. A newer patchlevel the mirror already copied is built too, so its images are published + by the time the upgrade check moves the base image pin to it. """ python_versions = CURRENT_PYTHON_MAJOR_MINOR_VERSIONS if python is None else [python] + base_image_digests = base_image_digests or {} + distro = DEBIAN_VERSION_TO_HARDENED_IMAGE_DISTRO[OS_DEPS_IMAGE_DEBIAN_VERSION] builds = [] for python_version in python_versions: - python_tag = get_hardened_python_image_tag(python_version, OS_DEPS_IMAGE_DEBIAN_VERSION) - python_digest = (base_image_digests or {}).get(python_tag) or get_image_digest( - f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}" + pinned_tag = get_hardened_python_image_tag(python_version, OS_DEPS_IMAGE_DEBIAN_VERSION) + python_digest = base_image_digests.get(pinned_tag) or get_image_digest( + f"{HARDENED_PYTHON_IMAGE_MIRROR}:{pinned_tag}" ) if python_digest is None and get_dry_run(): # A dry run does not inspect the registry, so the plan is printed for a placeholder digest. - python_digest = DRY_RUN_BASE_IMAGE_DIGEST + python_digest = DRY_RUN_IMAGE_DIGEST if python_digest is None: - console_print(f"[error]{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag} is not mirrored[/]") + console_print(f"[error]{HARDENED_PYTHON_IMAGE_MIRROR}:{pinned_tag} is not mirrored[/]") sys.exit(1) - for image_type in OS_DEPS_IMAGE_TYPES: - if image_types and image_type.name not in image_types: - continue - build_args = get_os_deps_image_build_args(image_type, python_version, python_digest) - os_deps_python_tag = get_os_deps_image_python_tag(image_type.name, python_version, python_digest) - inputs_hash = compute_os_deps_image_inputs_hash(image_type, build_args) - builds.append( - OsDepsImageBuild( - image_type=image_type, - python=python_version, - build_args=build_args, - latest_tag=get_os_deps_image_tag(image_type.name, python_version), - content_tag=f"{os_deps_python_tag}-{inputs_hash[:12]}", + python_tag_digests = {pinned_tag: python_digest} + for tag, digest in sorted(base_image_digests.items()): + if tag != pinned_tag and tag.startswith(f"{python_version}.") and tag.endswith(f"-{distro}-dev"): + python_tag_digests[tag] = digest + for python_tag, tag_digest in python_tag_digests.items(): + for image_type in OS_DEPS_IMAGE_TYPES: + if image_types and image_type.name not in image_types: + continue + build_args = get_os_deps_image_build_args(image_type, python_tag, tag_digest) + inputs_hash = compute_os_deps_image_inputs_hash(image_type, build_args) + builds.append( + OsDepsImageBuild( + image_type=image_type, + python=python_version, + build_args=build_args, + latest_tag=f"{image_type.name}-{python_tag}", + content_tag=( + f"{get_os_deps_image_python_tag(image_type.name, python_tag, tag_digest)}" + f"-{inputs_hash[:12]}" + ), + ) ) - ) return builds @@ -5606,6 +5617,8 @@ def publish_os_deps_images( ): failed: list[str] = [] moved: list[str] = [] + # A dry run does not inspect the registry, so it takes every image as built to print the plan. + dry_run = get_dry_run() for build in get_os_deps_image_builds(python, image_types, base_image_digests): image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.content_tag}" if get_image_digest(image) is None: @@ -5614,7 +5627,9 @@ def publish_os_deps_images( for platform in OS_DEPS_IMAGE_PLATFORMS ] missing = [ - platform_image for platform_image in platform_images if not get_image_digest(platform_image) + platform_image + for platform_image in platform_images + if not dry_run and not get_image_digest(platform_image) ] if missing: console_print(f"[error]Not built: {', '.join(missing)}[/]") @@ -5627,7 +5642,7 @@ def publish_os_deps_images( if result.returncode != 0: failed.append(image) continue - digest = get_image_digest(image) + digest = get_image_digest(image) or (DRY_RUN_IMAGE_DIGEST if dry_run else None) if digest is None: failed.append(image) continue diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 95d9e722939..c2fea1fdeea 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -1019,7 +1019,9 @@ OS_DEPS_IMAGE_TYPES = ( # ones install their OS dependencies from the apt repositories. OS_DEPS_IMAGE_DEBIAN_VERSION = ALLOWED_DEBIAN_VERSIONS[0] # Written by scripts/ci/prek/upgrade_important_versions.py together with the Python base image digests, -# so the pins only move in the upgrade PR, after the full CI has passed on them. +# so the pins only move in the upgrade PR, after the full CI has passed on them. Only main carries pins: +# the images are built from main's install_os_dependencies.sh and Dockerfiles, so the upgrade leaves the +# file empty on other branches until there are images per branch. HARDENED_OS_DEPS_IMAGE_DIGESTS_PATH = ( AIRFLOW_ROOT_PATH / "dev" @@ -1037,9 +1039,9 @@ def get_os_deps_image_tag(image_type: str, python: str) -> str: return f"{image_type}-{get_hardened_python_image_tag(python, OS_DEPS_IMAGE_DEBIAN_VERSION)}" -def get_os_deps_image_python_tag(image_type: str, python: str, python_digest: str) -> str: - """Return the prefix of the tags of the OS-dependencies images built on one Python base image digest.""" - return f"{get_os_deps_image_tag(image_type, python)}-{python_digest.removeprefix('sha256:')[:12]}" +def get_os_deps_image_python_tag(image_type: str, python_tag: str, python_digest: str) -> str: + """Return the prefix of the tags of the OS-dependencies images built on one base image digest.""" + return f"{image_type}-{python_tag}-{python_digest.removeprefix('sha256:')[:12]}" def get_pinned_os_deps_image(image_type: str, python: str) -> str | None: diff --git a/dev/breeze/tests/test_mirror_base_images.py b/dev/breeze/tests/test_mirror_base_images.py index 5c398f788d2..b6860cc657d 100644 --- a/dev/breeze/tests/test_mirror_base_images.py +++ b/dev/breeze/tests/test_mirror_base_images.py @@ -225,3 +225,34 @@ def test_mirror_covers_the_python_versions_release_branches_still_use(mock_lates assert result.exit_code == 0 assert {source.split(":")[1].rsplit(".", 1)[0] for source in registry.sources} == {"3.13", "3.14"} + + [email protected]( + "airflow_breeze.commands.release_management_commands.get_latest_hardened_python_patchlevel", + autospec=True, + return_value="3.14.99", +) +def test_mirror_outputs_the_os_deps_base_digests_of_a_newer_patchlevel_too(mock_latest, tmp_path): + newer_digest = "sha256:" + "cd" * 32 + registry = FakeRegistry( + {PINNED_TAG: NEW_DIGEST, "3.14.99-debian13-dev": newer_digest, "3.14.99-debian12-dev": NEW_DIGEST} + ) + github_output = tmp_path / "github_output" + with mock.patch( + "airflow_breeze.commands.release_management_commands.run_command", + autospec=True, + side_effect=registry, + ): + result = CliRunner().invoke( + mirror_base_images, + ["--python", "3.14"], + env={"GITHUB_OUTPUT": str(github_output)}, + catch_exceptions=False, + ) + + assert result.exit_code == 0 + digests_line = github_output.read_text().splitlines()[1] + assert json.loads(digests_line.split("=", 1)[1]) == { + PINNED_TAG: NEW_DIGEST, + "3.14.99-debian13-dev": newer_digest, + } diff --git a/dev/breeze/tests/test_os_deps_images.py b/dev/breeze/tests/test_os_deps_images.py index 55d4f5265a7..9bc5be09918 100644 --- a/dev/breeze/tests/test_os_deps_images.py +++ b/dev/breeze/tests/test_os_deps_images.py @@ -103,7 +103,7 @@ def registry(): def test_os_deps_image_tags(): assert get_os_deps_image_tag("ci", "3.12") == f"ci-{PYTHON_TAG}" - assert get_os_deps_image_python_tag("ci", "3.12", PYTHON_DIGEST) == f"ci-{PYTHON_TAG}-{'12' * 6}" + assert get_os_deps_image_python_tag("ci", PYTHON_TAG, PYTHON_DIGEST) == f"ci-{PYTHON_TAG}-{'12' * 6}" @mock.patch.dict(HARDENED_OS_DEPS_IMAGE_DIGESTS, PINS, clear=True) @@ -193,7 +193,7 @@ def test_extract_dockerfile_stage_fails_for_a_missing_stage(): ], ) def test_os_deps_image_build_args_match_the_image_builds(image_type, expected): - build_args = get_os_deps_image_build_args(IMAGE_TYPES[image_type], "3.12", PYTHON_DIGEST) + build_args = get_os_deps_image_build_args(IMAGE_TYPES[image_type], PYTHON_TAG, PYTHON_DIGEST) assert build_args == { "BASE_IMAGE": f"{MIRROR}:{PYTHON_TAG}@{PYTHON_DIGEST}", @@ -463,3 +463,53 @@ def test_build_os_deps_images_rejects_invalid_base_image_digests(value): assert result.exit_code == 2 assert "--base-image-digests" in result.output + + +def test_os_deps_image_builds_include_a_newer_mirrored_patchlevel(): + newer_digest = "sha256:" + "34" * 32 + base_image_digests = { + PYTHON_TAG: PYTHON_DIGEST, + "3.12.99-debian13-dev": newer_digest, + "3.12.99-debian12-dev": "sha256:" + "56" * 32, + "3.13.99-debian13-dev": "sha256:" + "78" * 32, + } + with mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True, side_effect=FakeRegistry()): + builds = get_os_deps_image_builds("3.12", ("ci",), base_image_digests) + + assert [(build.latest_tag, build.build_args) for build in builds] == [ + ( + f"ci-{PYTHON_TAG}", + { + "BASE_IMAGE": f"{MIRROR}:{PYTHON_TAG}@{PYTHON_DIGEST}", + "AIRFLOW_PYTHON_VERSION": ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION["3.12"], + }, + ), + ( + "ci-3.12.99-debian13-dev", + { + "BASE_IMAGE": f"{MIRROR}:3.12.99-debian13-dev@{newer_digest}", + "AIRFLOW_PYTHON_VERSION": "3.12.99", + }, + ), + ] + assert builds[1].content_tag.startswith(f"ci-3.12.99-debian13-dev-{'34' * 6}-") + + [email protected](f"{PATCH_PREFIX}.get_dry_run", autospec=True, return_value=True) +def test_publish_os_deps_images_prints_the_plan_in_dry_run(mock_dry_run): + # In a dry run every registry lookup comes back empty, as nothing is inspected. + registry = FakeRegistry() + with mock.patch(f"{PATCH_PREFIX}.run_command", autospec=True, side_effect=registry): + (build,) = get_os_deps_image_builds("3.12", ("ci",)) + result = CliRunner().invoke( + publish_os_deps_images, + ["--python", "3.12", "--image-type", "ci"], + env={"GITHUB_OUTPUT": None}, + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert [targets for _, targets in registry.copies] == [ + [f"{REPOSITORY}:{build.content_tag}"], + [f"{REPOSITORY}:ci-{PYTHON_TAG}"], + ] diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 00e84f787fd..3c2a98e72a7 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -510,7 +510,10 @@ HARDENED_OS_DEPS_IMAGE_REPOSITORY = "apache/airflow/base/os-deps" OS_DEPS_IMAGE_TYPES = ["ci", "dev", "runtime"] OS_DEPS_IMAGE_DEBIAN_RELEASE = HARDENED_PYTHON_DEBIAN_RELEASES[0] # The digests the CI and release builds start their OS dependencies from - breeze reads them from this -# file. A branch without it does not build on OS-dependencies images. +# file. A branch without it does not build on OS-dependencies images. Only main carries pins: the images +# are built from main's install_os_dependencies.sh and Dockerfiles, so on another branch they would never +# match its marker and would install on top of main's dependencies - until there are images per branch, +# the upgrade empties the file anywhere else. OS_DEPS_IMAGE_DIGESTS_PATH = ( AIRFLOW_ROOT_PATH / "dev" @@ -520,6 +523,7 @@ OS_DEPS_IMAGE_DIGESTS_PATH = ( / "files" / "hardened_os_deps_image_digests.json" ) +BRANCH_DEFAULTS_PATH = AIRFLOW_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "branch_defaults.py" def fetch_os_deps_image_digests( @@ -557,6 +561,27 @@ def fetch_os_deps_image_digests( return pins +def get_airflow_branch() -> str: + """Return the branch this checkout belongs to - every branch sets AIRFLOW_BRANCH in its breeze.""" + match = re.search(r'^AIRFLOW_BRANCH = "([^"]+)"$', BRANCH_DEFAULTS_PATH.read_text(), re.MULTILINE) + if not match: + console.print(f"[bright_red]No AIRFLOW_BRANCH in {BRANCH_DEFAULTS_PATH}") + sys.exit(1) + return match.group(1) + + +def compute_os_deps_image_pins( + python_image_digests: dict[str, str], current_pins: dict[str, str], branch: str +) -> dict[str, str]: + """Return the OS-dependencies image pins of a branch - none anywhere but on main.""" + if branch != "main": + console.print( + f"[yellow]The OS-dependencies images are built from main only - {branch} does not pin them." + ) + return {} + return fetch_os_deps_image_digests(python_image_digests, current_pins) + + def write_pins_file(path: Path, pins: dict[str, str]) -> bool: """Write one of the JSON files breeze reads its pins from, returning whether they changed.""" new_content = json.dumps(pins, indent=2) + "\n" @@ -1439,8 +1464,8 @@ def main() -> None: if python_image_digests: changed = write_pins_file(PYTHON_IMAGE_DIGESTS_PATH, python_image_digests) or changed if OS_DEPS_IMAGE_DIGESTS_PATH.exists(): - os_deps_image_digests = fetch_os_deps_image_digests( - python_image_digests, json.loads(OS_DEPS_IMAGE_DIGESTS_PATH.read_text()) + os_deps_image_digests = compute_os_deps_image_pins( + python_image_digests, json.loads(OS_DEPS_IMAGE_DIGESTS_PATH.read_text()), get_airflow_branch() ) changed = write_pins_file(OS_DEPS_IMAGE_DIGESTS_PATH, os_deps_image_digests) or changed if legacy_python_versions: diff --git a/scripts/tests/ci/prek/test_upgrade_important_versions.py b/scripts/tests/ci/prek/test_upgrade_important_versions.py index f41762feba4..16f9c0fe04a 100644 --- a/scripts/tests/ci/prek/test_upgrade_important_versions.py +++ b/scripts/tests/ci/prek/test_upgrade_important_versions.py @@ -551,6 +551,50 @@ def test_os_deps_image_digests_pin_the_latest_published_images(monkeypatch, buil ] [email protected]( + ("branch", "expected"), + [ + pytest.param("main", {"ci-3.13.17-debian13-dev": OS_DEPS_DIGEST}, id="main-pins"), + pytest.param("v3-4-test", {}, id="release-branch-pins-nothing"), + ], +) +def test_only_main_pins_os_deps_images(monkeypatch, branch, expected): + from ci.prek import upgrade_important_versions as uiv + + fetch = mock.create_autospec(uiv.fetch_os_deps_image_digests, return_value=expected) + monkeypatch.setattr(uiv, "fetch_os_deps_image_digests", fetch) + + pins = uiv.compute_os_deps_image_pins( + {"3.13.17-debian13-dev": NEW_DIGEST}, {"ci-3.13.17-debian13-dev": OLD_DIGEST}, branch + ) + + assert pins == expected + assert fetch.called is (branch == "main") + + [email protected]( + ("content", "expected"), + [ + pytest.param(' AIRFLOW_BRANCH = "v2-2-test"\nAIRFLOW_BRANCH = "main"\n', "main", id="main"), + pytest.param('AIRFLOW_BRANCH = "v3-4-test"\n', "v3-4-test", id="release-branch"), + ], +) +def test_airflow_branch_is_read_from_branch_defaults(monkeypatch, tmp_path, content, expected): + from ci.prek import upgrade_important_versions as uiv + + branch_defaults = tmp_path / "branch_defaults.py" + branch_defaults.write_text(content) + monkeypatch.setattr(uiv, "BRANCH_DEFAULTS_PATH", branch_defaults) + + assert uiv.get_airflow_branch() == expected + + +def test_airflow_branch_of_this_checkout_is_found(): + from ci.prek import upgrade_important_versions as uiv + + assert uiv.get_airflow_branch() + + def test_ghcr_image_digest_is_none_for_a_missing_tag(monkeypatch): from ci.prek import upgrade_important_versions as uiv
