This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch ci-image-keyed-cache
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 0f0a97162db9d5745b7f6338c63d597daecbf3bc
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 15 16:04:11 2026 +0900

    Push keyed cache for every Python version and sweep stale keyed tags
    
    The keyed cache a pull request pushes followed the test matrix, which a
    dependency change usually narrows to the default Python version, so the
    others still built dependencies from scratch after the merge.
    
    A keyed cache version that kept the shared tag past the keyed cleanup's
    scanned window escaped that cleanup for good; the slow cleanup already
    walks every old version, so it takes those as well.
    
    Generated-by: Claude Opus 5
    Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq
---
 .github/workflows/ci-amd.yml                       |  4 +-
 .github/workflows/ci-arm.yml                       |  4 +-
 dev/breeze/doc/ci/04_selective_checks.md           |  1 +
 .../src/airflow_breeze/utils/selective_checks.py   |  6 +++
 dev/breeze/tests/test_ci_image_commands.py         |  6 +++
 dev/breeze/tests/test_selective_checks.py          | 13 ++++++
 scripts/ci/cleanup_keyed_image_cache.py            |  3 +-
 scripts/ci/cleanup_old_image_versions.py           | 18 ++++++--
 .../tests/ci/test_cleanup_old_image_versions.py    | 52 ++++++++++++++++++++++
 9 files changed, 99 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 2584151dec4..4b11afb210d 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -277,7 +277,7 @@ jobs:
       arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
       basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only 
}}
       canary-run: ${{ steps.source-run-info.outputs.canary-run }}
-      all-python-versions: ${{ 
steps.selective-checks.outputs.all-python-versions }}
+      ci-image-cache-python-versions: ${{ 
steps.selective-checks.outputs.ci-image-cache-python-versions }}
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       ci-image-dependencies-changed: ${{ 
steps.selective-checks.outputs.ci-image-dependencies-changed }}
       core-test-types-list-as-strings-in-json: >-
@@ -579,7 +579,7 @@ jobs:
       include-prod-images: "false"
       push-latest-images: "false"
       keyed-cache-only: "true"
-      python-versions: ${{ needs.build-info.outputs.all-python-versions }}
+      python-versions: ${{ 
needs.build-info.outputs.ci-image-cache-python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ 
needs.build-info.outputs.default-constraints-branch }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index ccb40c5c6e4..ff16edd40f8 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -153,7 +153,7 @@ jobs:
       arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
       basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only 
}}
       canary-run: ${{ steps.source-run-info.outputs.canary-run }}
-      all-python-versions: ${{ 
steps.selective-checks.outputs.all-python-versions }}
+      ci-image-cache-python-versions: ${{ 
steps.selective-checks.outputs.ci-image-cache-python-versions }}
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       ci-image-dependencies-changed: ${{ 
steps.selective-checks.outputs.ci-image-dependencies-changed }}
       core-test-types-list-as-strings-in-json: >-
@@ -455,7 +455,7 @@ jobs:
       include-prod-images: "false"
       push-latest-images: "false"
       keyed-cache-only: "true"
-      python-versions: ${{ needs.build-info.outputs.all-python-versions }}
+      python-versions: ${{ 
needs.build-info.outputs.ci-image-cache-python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ 
needs.build-info.outputs.default-constraints-branch }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md 
b/dev/breeze/doc/ci/04_selective_checks.md
index 40932879727..b1e21f16438 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -754,6 +754,7 @@ GitHub Actions to pass the list of parameters to a command 
to execute
 | all-versions                                            | If set to true, 
then all python, k8s, DB versions are used for tests.                           
        | false                                    |      |
 | basic-checks-only                                       | Whether to run all 
static checks ("false") or only basic set of static checks ("true")             
     | false                                    |      |
 | ci-image-build                                          | Whether CI image 
build is needed                                                                 
       | true                                     |      |
+| ci-image-cache-python-versions                          | Every supported 
Python version, for the keyed image cache an apache/airflow PR pushes           
        | \['3.11', '3.12'\]                       |      |
 | ci-image-dependencies-changed                           | Whether dependency 
inputs of the CI image changed (an apache/airflow PR then pushes its keyed 
cache)    | false                                    |      |
 | core-test-types-list-as-strings-in-json                 | Which test types 
should be run for unit tests for core (2 slices; 1 on RunsOn)                   
       | API Always Providers                     | *    |
 | debug-resources                                         | Whether resources 
usage should be printed during parallel job execution ("true"/ "false")         
      | false                                    |      |
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py 
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 33faee53d50..d6291e2acd3 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -2281,6 +2281,12 @@ class SelectiveChecks:
             return True
         return False
 
+    @cached_property
+    def ci_image_cache_python_versions(self) -> list[str]:
+        # Every supported version, whatever the test matrix: a keyed cache 
missing for one version
+        # means a full dependency build for everyone using it.
+        return CURRENT_PYTHON_MAJOR_MINOR_VERSIONS
+
     @cached_property
     def ci_image_dependencies_changed(self) -> bool:
         return 
len(self._matching_files(FileGroupForCi.CI_IMAGE_DEPENDENCY_FILES, 
CI_FILE_GROUP_MATCHES)) > 0
diff --git a/dev/breeze/tests/test_ci_image_commands.py 
b/dev/breeze/tests/test_ci_image_commands.py
index af1ec3500e5..431ffb243db 100644
--- a/dev/breeze/tests/test_ci_image_commands.py
+++ b/dev/breeze/tests/test_ci_image_commands.py
@@ -181,6 +181,7 @@ def 
test_build_ci_image_if_needed_reuses_image_built_in_another_checkout(
     mock_run_build_ci_image.assert_not_called()
 
 
[email protected]("airflow_breeze.commands.ci_image_commands.check_buildx_version")
 
@mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed")
 @mock.patch("airflow_breeze.commands.ci_image_commands.run_build_ci_image")
 
@mock.patch("airflow_breeze.commands.ci_image_commands.is_ci_image_built_from_current_sources")
@@ -188,6 +189,7 @@ def 
test_build_ci_image_if_needed_forces_build_when_image_does_not_match_sources
     mock_is_ci_image_built_from_current_sources,
     mock_run_build_ci_image,
     mock_check_if_image_building_is_needed,
+    mock_check_buildx_version,
     tmp_path,
     monkeypatch,
 ):
@@ -198,8 +200,10 @@ def 
test_build_ci_image_if_needed_forces_build_when_image_does_not_match_sources
     build_ci_image_if_needed(command_params=BuildCiParams())
     assert 
mock_check_if_image_building_is_needed.call_args.kwargs["ci_image_params"].force_build
 is True
     mock_run_build_ci_image.assert_called_once()
+    mock_check_buildx_version.assert_called_once_with(quiet=True)
 
 
[email protected]("airflow_breeze.commands.ci_image_commands.check_buildx_version")
 
@mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed")
 @mock.patch("airflow_breeze.commands.ci_image_commands.run_build_ci_image")
 
@mock.patch("airflow_breeze.commands.ci_image_commands.is_ci_image_built_from_current_sources")
@@ -207,6 +211,7 @@ def 
test_build_ci_image_if_needed_does_not_reuse_image_when_force_build_requeste
     mock_is_ci_image_built_from_current_sources,
     mock_run_build_ci_image,
     mock_check_if_image_building_is_needed,
+    mock_check_buildx_version,
     tmp_path,
     monkeypatch,
 ):
@@ -216,6 +221,7 @@ def 
test_build_ci_image_if_needed_does_not_reuse_image_when_force_build_requeste
     build_ci_image_if_needed(command_params=BuildCiParams(force_build=True))
     mock_is_ci_image_built_from_current_sources.assert_not_called()
     mock_run_build_ci_image.assert_called_once()
+    mock_check_buildx_version.assert_called_once_with(quiet=True)
 
 
 
@mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed")
diff --git a/dev/breeze/tests/test_selective_checks.py 
b/dev/breeze/tests/test_selective_checks.py
index 75a61091e42..f7c684a2d25 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -3802,6 +3802,19 @@ def test_ci_image_dependencies_changed(files: tuple[str, 
...], expected: str):
     assert_outputs_are_printed({"ci-image-dependencies-changed": expected}, 
str(stderr))
 
 
+def test_ci_image_cache_python_versions_are_all_current_versions():
+    stderr = SelectiveChecks(
+        files=("providers/amazon/pyproject.toml",),
+        commit_ref=NEUTRAL_COMMIT,
+        github_event=GithubEvents.PULL_REQUEST,
+        default_branch="main",
+    )
+    assert_outputs_are_printed(
+        {"ci-image-cache-python-versions": CURRENT_PYTHON_VERSIONS_AS_LIST},
+        str(stderr),
+    )
+
+
 @pytest.mark.parametrize(
     ("files", "pr_labels", "github_event", "expected_constraints", 
"expected_pypi"),
     [
diff --git a/scripts/ci/cleanup_keyed_image_cache.py 
b/scripts/ci/cleanup_keyed_image_cache.py
index 8520661309a..78b540296e6 100644
--- a/scripts/ci/cleanup_keyed_image_cache.py
+++ b/scripts/ci/cleanup_keyed_image_cache.py
@@ -27,7 +27,8 @@ and anything else are never touched.
 The age is counted from creation, not from the last update: a push of 
identical cache content
 reuses the version and only moves its ``updated_at``, and versions are listed 
by creation, so a
 version updated after creation could otherwise fall out of the scanned window 
and stay forever.
-A key still in use is pushed again within hours of being deleted.
+A key still in use is pushed again within hours of being deleted. A version 
that kept another
+tag past the scanned window is left to cleanup_old_image_versions.py, which 
walks every version.
 
 Usage: cleanup_keyed_image_cache.py [--days N] [--dry-run]
 Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with 
``packages: write``.
diff --git a/scripts/ci/cleanup_old_image_versions.py 
b/scripts/ci/cleanup_old_image_versions.py
index 3d321ea10f4..48201cfbfae 100644
--- a/scripts/ci/cleanup_old_image_versions.py
+++ b/scripts/ci/cleanup_old_image_versions.py
@@ -61,6 +61,8 @@ MANIFEST_MEDIA_TYPES = ", ".join(
 )
 LINK_LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"')
 PUSH_WINDOW = timedelta(minutes=5)
+# The keyed CI image cache tags, see cleanup_keyed_image_cache.py.
+KEYED_CACHE_TAG = re.compile(r"^cache-linux-(amd64|arm64)-[0-9a-f]{16}$")
 
 
 def get_image_package_pattern(repo: str) -> re.Pattern[str]:
@@ -79,9 +81,17 @@ def _get_tags(version: dict) -> list[str]:
     return version.get("metadata", {}).get("container", {}).get("tags") or []
 
 
+def _is_disposable(version: dict) -> bool:
+    """Untagged, or tagged only as keyed cache, which nothing pulls by 
reference."""
+    return all(KEYED_CACHE_TAG.match(tag) for tag in _get_tags(version))
+
+
 def find_deletable_versions(versions: list[dict], keep: set[str], cutoff: 
datetime) -> list[dict]:
     """
-    Return the untagged versions of one page that are safe to delete.
+    Return the versions of one page that are safe to delete.
+
+    Untagged versions, and the keyed cache tags that the daily keyed-cache 
cleanup misses when a
+    version kept another tag past its scan window, are deleted once nothing 
references them.
 
     ``keep`` holds every manifest a tag references. As a second guard, an 
untagged version pushed
     within PUSH_WINDOW of a tagged one, or of the page edges where a tagged 
one may sit on the
@@ -89,12 +99,12 @@ def find_deletable_versions(versions: list[dict], keep: 
set[str], cutoff: dateti
     """
     if not versions:
         return []
-    anchors = [_parse_time(v["created_at"]) for v in versions if _get_tags(v)]
+    anchors = [_parse_time(v["created_at"]) for v in versions if not 
_is_disposable(v)]
     anchors += [_parse_time(versions[0]["created_at"]), 
_parse_time(versions[-1]["created_at"])]
     return [
         version
         for version in versions
-        if not _get_tags(version)
+        if _is_disposable(version)
         and version["name"] not in keep
         and (created := _parse_time(version["created_at"])) < cutoff
         and all(abs(created - anchor) > PUSH_WINDOW for anchor in anchors)
@@ -163,6 +173,8 @@ def collect_keep_set(image: str) -> set[str]:
         next_page = re.search(r"<([^>]+)>;\s*rel=\"next\"", link)
         path = next_page.group(1) if next_page else None
 
+    tags = [tag for tag in tags if not KEYED_CACHE_TAG.match(tag)]
+
     def read_tag(tag: str) -> set[str]:
         manifest, digest, _ = registry.get(f"/v2/{image}/manifests/{tag}", 
MANIFEST_MEDIA_TYPES)
         return get_referenced_digests(manifest) | ({digest} if digest else 
set())
diff --git a/scripts/tests/ci/test_cleanup_old_image_versions.py 
b/scripts/tests/ci/test_cleanup_old_image_versions.py
index c1542963867..73e4c0ee89d 100644
--- a/scripts/tests/ci/test_cleanup_old_image_versions.py
+++ b/scripts/tests/ci/test_cleanup_old_image_versions.py
@@ -145,6 +145,58 @@ def 
test_select_versions_to_delete_does_not_load_tags_without_candidates(cleanup
     assert cleanup.select_versions_to_delete(pages.get, 1, load_keep, CUTOFF, 
budget=10) == []
 
 
+def test_find_deletable_versions_takes_keyed_cache_only_versions(cleanup):
+    keyed = _version("sha256:keyed", OLD, 
["cache-linux-amd64-0123456789abcdef"])
+    shared_too = _version(
+        "sha256:shared", "2026-08-20T00:00:00Z", 
["cache-linux-amd64-0123456789abcdef", "cache-linux-amd64"]
+    )
+
+    deletable = cleanup.find_deletable_versions(_page(keyed, shared_too), 
set(), CUTOFF)
+
+    assert [v["name"] for v in deletable] == ["sha256:keyed"]
+
+
+class _FakeRegistry:
+    """Two pages of tags; the commit-tagged index, on the second page, lists 
an old manifest."""
+
+    def __init__(self, image: str):
+        self.image = image
+
+    def get(self, path: str, accept: str = "application/json"):
+        pages: dict[str, tuple[dict, str]] = {
+            f"/v2/{self.image}/tags/list?n=1000": (
+                {"tags": ["latest", "cache-linux-amd64-0123456789abcdef"]},
+                f'</v2/{self.image}/tags/list?n=1000&last=latest>; rel="next"',
+            ),
+            f"/v2/{self.image}/tags/list?n=1000&last=latest": ({"tags": 
["0123abcd4567ef89"]}, ""),
+        }
+        if path in pages:
+            listing, link = pages[path]
+            return listing, None, link
+        manifests: dict[str, tuple[dict, str]] = {
+            "latest": ({}, "sha256:latest"),
+            "0123abcd4567ef89": (
+                {"manifests": [{"digest": "sha256:old-platform-manifest"}]},
+                "sha256:commit-index",
+            ),
+        }
+        tag = path.rsplit("/", 1)[1]
+        if tag not in manifests:
+            raise AssertionError(f"unexpected manifest read: {tag}")
+        body, digest = manifests[tag]
+        return body, digest, ""
+
+
+def test_collect_keep_set_protects_manifests_listed_by_commit_tags(cleanup, 
monkeypatch):
+    monkeypatch.setattr(cleanup, "_Registry", _FakeRegistry)
+
+    keep = cleanup.collect_keep_set("apache/airflow/main/ci/python3.11")
+
+    assert keep == {"sha256:latest", "sha256:commit-index", 
"sha256:old-platform-manifest"}
+    old_manifest = _version("sha256:old-platform-manifest", 
"2026-08-20T00:00:00Z")
+    assert cleanup.find_deletable_versions(_page(old_manifest), keep, CUTOFF) 
== []
+
+
 def test_get_referenced_digests_lists_index_children(cleanup):
     index = {
         "mediaType": "application/vnd.oci.image.index.v1+json",

Reply via email to