This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch ci-image-keyed-cache
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 59e44d9eb33072822693601505b40158697e56d0
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 15 15:52:33 2026 +0900

    Prove image versions unreferenced before the cleanup deletes them
    
    A version close in time to a tagged image is not proof it is not part
    of it: a rebuilt identical manifest keeps its old digest and creation
    time while a new image lists it. Reading what every tag references
    costs thousands of registry reads per package, so a run takes one
    package, in turn.
    
    An oldest page with nothing to delete stopped the whole package, and
    keyed cache re-pushed with identical content could fall out of the
    scanned window for good, since the scan follows creation time.
    
    Generated-by: Claude Opus 5
    Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq
---
 .github/workflows/cleanup-old-image-versions.yml   |   6 +-
 scripts/ci/cleanup_keyed_image_cache.py            |  11 +-
 scripts/ci/cleanup_old_image_versions.py           | 179 +++++++++++++--------
 scripts/tests/ci/test_cleanup_keyed_image_cache.py |  15 +-
 .../tests/ci/test_cleanup_old_image_versions.py    |  52 ++++++
 5 files changed, 189 insertions(+), 74 deletions(-)

diff --git a/.github/workflows/cleanup-old-image-versions.yml 
b/.github/workflows/cleanup-old-image-versions.yml
index 74a2de5bcfa..945b84308f2 100644
--- a/.github/workflows/cleanup-old-image-versions.yml
+++ b/.github/workflows/cleanup-old-image-versions.yml
@@ -17,9 +17,9 @@
 #
 ---
 name: Clean up old image versions
-# Deletes, a few hundred per run, the untagged CI and PROD image versions 
older than 30 days that
-# no tagged image references. Small, spaced-out runs keep it far below the API 
rate limit of the
-# workflow token. See scripts/ci/cleanup_old_image_versions.py.
+# Deletes, a few hundred per run from one package in turn, the untagged CI and 
PROD image versions
+# older than 30 days that no tag references. Small, spaced-out runs keep it 
far below the API rate
+# limit of the workflow token. See scripts/ci/cleanup_old_image_versions.py.
 on:  # yamllint disable-line rule:truthy
   schedule:
     - cron: '41 */3 * * *'
diff --git a/scripts/ci/cleanup_keyed_image_cache.py 
b/scripts/ci/cleanup_keyed_image_cache.py
index 2eaddc6f836..8520661309a 100644
--- a/scripts/ci/cleanup_keyed_image_cache.py
+++ b/scripts/ci/cleanup_keyed_image_cache.py
@@ -16,14 +16,19 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-Delete keyed CI image cache entries that have not been pushed for a while.
+Delete keyed CI image cache entries a while after they were first pushed.
 
 ``breeze ci-image build --prepare-buildx-cache`` pushes, next to the shared 
``cache-<platform>`` tag,
 a ``cache-<platform>-<dependency key>`` tag for every set of dependency inputs 
it builds. Those pile
 up with every dependency change, so the versions of the CI image packages 
whose only tags are keyed
-cache tags, and which were last updated before the cutoff, are deleted. Shared 
cache tags, ``latest``
+cache tags, and which were created before the cutoff, are deleted. Shared 
cache tags, ``latest``
 and anything else are never touched.
 
+The age is counted from creation, not from the last update: a push of 
identical cache content
+reuses the version and only moves its ``updated_at``, and versions are listed 
by creation, so a
+version updated after creation could otherwise fall out of the scanned window 
and stay forever.
+A key still in use is pushed again within hours of being deleted.
+
 Usage: cleanup_keyed_image_cache.py [--days N] [--dry-run]
 Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with 
``packages: write``.
 """
@@ -52,7 +57,7 @@ def is_expired_keyed_cache(version: dict, cutoff: datetime) 
-> bool:
     tags = version.get("metadata", {}).get("container", {}).get("tags", [])
     if not tags or not all(KEYED_CACHE_TAG.match(tag) for tag in tags):
         return False
-    return _parse_time(version["updated_at"]) < cutoff
+    return _parse_time(version["created_at"]) < cutoff
 
 
 def find_expired_versions(versions: Iterable[dict], cutoff: datetime) -> 
list[dict]:
diff --git a/scripts/ci/cleanup_old_image_versions.py 
b/scripts/ci/cleanup_old_image_versions.py
index f507ba5b373..3d321ea10f4 100644
--- a/scripts/ci/cleanup_old_image_versions.py
+++ b/scripts/ci/cleanup_old_image_versions.py
@@ -19,29 +19,35 @@
 Slowly delete old untagged versions of the CI and PROD image packages.
 
 Every image or cache push leaves the manifest it replaced behind as an 
untagged package version,
-so a package such as ``airflow/main/ci/python3.11`` holds tens of thousands of 
them. This deletes
-the untagged versions created before the cutoff, oldest first, up to a budget 
per run and with a
-pause between deletions, so that a run stays far below the API rate limit of 
the workflow token
-and the next run continues where this one stopped.
+so a package such as ``airflow/main/ci/python3.11`` holds tens of thousands of 
them. Each run
+takes one package, in turn by run number, and deletes its untagged versions 
created before the
+cutoff, oldest first, up to a budget and with a pause between deletions, so 
that a run stays far
+below the API rate limit of the workflow token and a later run continues where 
this one stopped.
 
-A version is only deleted when no tagged manifest of its package references 
it: the platform
-manifests and attestations of a multi-platform image are untagged versions of 
their own and must
-stay while the image tag that lists them exists.
+A version is only deleted when no tag of its package references it: the 
platform manifests and
+attestations of a multi-platform image are untagged versions of their own and 
must stay while the
+tag that lists them exists. The references are read from the registry for 
every tag, and an
+untagged version pushed within PUSH_WINDOW of a tagged one is kept as well.
 
 Usage: cleanup_old_image_versions.py [--days N] [--max-deletions N] [--pause 
SECONDS] [--dry-run]
-Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with 
``packages: write``.
+Environment: GITHUB_REPOSITORY (``owner/repo``), GITHUB_RUN_NUMBER (picks the 
package), and a
+token for ``gh`` with ``packages: write``.
 """
 
 from __future__ import annotations
 
 import argparse
+import functools
 import json
 import os
 import re
 import subprocess
 import sys
 import time
+import urllib.error
 import urllib.request
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime, timedelta
 from urllib.parse import quote
 
@@ -55,7 +61,6 @@ MANIFEST_MEDIA_TYPES = ", ".join(
 )
 LINK_LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"')
 PUSH_WINDOW = timedelta(minutes=5)
-COMMIT_TAG = re.compile(r"^[0-9a-f]{7,40}$")
 
 
 def get_image_package_pattern(repo: str) -> re.Pattern[str]:
@@ -78,11 +83,9 @@ def find_deletable_versions(versions: list[dict], keep: 
set[str], cutoff: dateti
     """
     Return the untagged versions of one page that are safe to delete.
 
-    The few long-lived tags have their listed manifests in ``keep``. The 
packages also hold
-    thousands of old per-commit tags, too many to read the manifests of, so an 
untagged version
-    pushed within PUSH_WINDOW of a tagged one, or of the page edges where the 
tagged one may sit
-    on the next page, is taken for part of that image and kept: the manifests 
of a
-    multi-platform image are pushed together with it.
+    ``keep`` holds every manifest a tag references. As a second guard, an 
untagged version pushed
+    within PUSH_WINDOW of a tagged one, or of the page edges where a tagged 
one may sit on the
+    next page, is kept: the manifests of a multi-platform image are pushed 
together with it.
     """
     if not versions:
         return []
@@ -112,45 +115,96 @@ def _get_page(versions_path: str, page: int) -> 
list[dict]:
     return json.loads(_gh_api([f"{versions_path}?per_page=100&page={page}"]))
 
 
-def _registry_get(url: str, token: str, accept: str = "application/json") -> 
tuple[dict, str | None, str]:
-    request = urllib.request.Request(url, headers={"Authorization": f"Bearer 
{token}", "Accept": accept})
-    with urllib.request.urlopen(request, timeout=60) as response:
-        return (
-            json.load(response),
-            response.headers.get("Docker-Content-Digest"),
-            response.headers.get("Link", ""),
-        )
+class _Registry:
+    """Anonymous, read-only access to the public images on ghcr.io."""
+
+    def __init__(self, image: str):
+        self.image = image
+        self.token = self._get_token()
+
+    def _get_token(self) -> str:
+        url = f"https://ghcr.io/token?scope=repository:{self.image}:pull";
+        with urllib.request.urlopen(url, timeout=60) as response:
+            return json.load(response)["token"]
+
+    def get(self, path: str, accept: str = "application/json") -> tuple[dict, 
str | None, str]:
+        for attempt in range(2):
+            request = urllib.request.Request(
+                f"https://ghcr.io{path}";, headers={"Authorization": f"Bearer 
{self.token}", "Accept": accept}
+            )
+            try:
+                with urllib.request.urlopen(request, timeout=60) as response:
+                    return (
+                        json.load(response),
+                        response.headers.get("Docker-Content-Digest"),
+                        response.headers.get("Link", ""),
+                    )
+            except urllib.error.HTTPError as error:
+                if error.code != 401 or attempt:
+                    raise
+                self.token = self._get_token()
+        raise AssertionError("unreachable")
 
 
 def collect_keep_set(image: str) -> set[str]:
     """
-    Return the digests of the long-lived (non-commit) tags of ``image`` and of 
the manifests they list.
+    Return the digests of every tagged manifest of ``image`` and of the 
manifests they list.
 
     Read from the registry (the images are public) rather than from the 
packages API, which can
-    only find the few tagged versions by listing every one of the tens of 
thousands of versions.
+    only find the tagged versions by listing every one of the tens of 
thousands of versions, and
+    whose rate limit the deletions need.
     """
-    with urllib.request.urlopen(
-        f"https://ghcr.io/token?scope=repository:{image}:pull";, timeout=60
-    ) as response:
-        token = json.load(response)["token"]
+    registry = _Registry(image)
     tags: list[str] = []
-    url: str | None = f"https://ghcr.io/v2/{image}/tags/list?n=1000";
-    while url:
-        listing, _, link = _registry_get(url, token)
+    path: str | None = f"/v2/{image}/tags/list?n=1000"
+    while path:
+        listing, _, link = registry.get(path)
         tags.extend(listing.get("tags") or [])
         next_page = re.search(r"<([^>]+)>;\s*rel=\"next\"", link)
-        url = f"https://ghcr.io{next_page.group(1)}" if next_page else None
+        path = next_page.group(1) if next_page else None
+
+    def read_tag(tag: str) -> set[str]:
+        manifest, digest, _ = registry.get(f"/v2/{image}/manifests/{tag}", 
MANIFEST_MEDIA_TYPES)
+        return get_referenced_digests(manifest) | ({digest} if digest else 
set())
+
     keep: set[str] = set()
-    for tag in (tag for tag in tags if not COMMIT_TAG.match(tag)):
-        manifest, digest, _ = _registry_get(
-            f"https://ghcr.io/v2/{image}/manifests/{tag}";, token, 
MANIFEST_MEDIA_TYPES
-        )
-        if digest:
-            keep.add(digest)
-        keep |= get_referenced_digests(manifest)
+    with ThreadPoolExecutor(max_workers=8) as executor:
+        for digests in executor.map(read_tag, tags):
+            keep |= digests
+    print(f"{image}: {len(tags)} tag(s) reference {len(keep)} manifest(s)")
     return keep
 
 
+def select_versions_to_delete(
+    fetch_page: Callable[[int], list[dict]],
+    last_page: int,
+    load_keep: Callable[[], set[str]],
+    cutoff: datetime,
+    budget: int,
+) -> list[dict]:
+    """
+    Walk the pages from the last, oldest one, and return up to ``budget`` 
versions to delete.
+
+    Versions are listed newest first, so deleting from a page never moves the 
versions of the
+    pages before it, and the selection can be made before deleting. The tag 
references are only
+    loaded once a page has a candidate, and a page with none does not stop the 
walk.
+    """
+    selected: list[dict] = []
+    keep: set[str] | None = None
+    for page in range(last_page, 0, -1):
+        if len(selected) >= budget:
+            break
+        batch = fetch_page(page)
+        if not batch or _parse_time(batch[-1]["created_at"]) >= cutoff:
+            break
+        if keep is None:
+            if not find_deletable_versions(batch, set(), cutoff):
+                continue
+            keep = load_keep()
+        selected.extend(find_deletable_versions(batch, keep, cutoff)[: budget 
- len(selected)])
+    return selected
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__, 
formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument("--days", type=int, default=30, help="Keep versions 
created within this many days")
@@ -166,34 +220,27 @@ def main() -> int:
         ["--paginate", 
f"/orgs/{owner}/packages?package_type=container&per_page=100", "--jq", 
".[].name"]
     )
     packages = sorted(name for name in package_listing.split() if 
package_pattern.match(name))
-    budget = args.max_deletions
-    for package in packages:
-        if budget <= 0:
-            break
-        versions_path = f"/orgs/{owner}/packages/container/{quote(package, 
safe='')}/versions"
-        page = _get_last_page(versions_path)
-        if not find_deletable_versions(_get_page(versions_path, page), set(), 
cutoff):
-            continue
-        keep = collect_keep_set(f"{owner}/{package}")
-        # Versions are listed newest first, so walking from the last page goes 
oldest first, and
-        # deleting from a page never moves the versions of the pages before it.
-        while page >= 1 and budget > 0:
-            batch = _get_page(versions_path, page)
-            if not batch or _parse_time(batch[-1]["created_at"]) >= cutoff:
-                break
-            for version in find_deletable_versions(batch, keep, 
cutoff)[:budget]:
-                print(
-                    f"{'Would delete' if args.dry_run else 'Deleting'} "
-                    f"{package}@{version['name']} ({version['created_at']})"
-                )
-                if not args.dry_run:
-                    _gh_api(["-X", "DELETE", 
f"{versions_path}/{version['id']}"])
-                    time.sleep(args.pause)
-                budget -= 1
-            page -= 1
-    print(
-        f"{args.max_deletions - budget} old untagged version(s) {'would be ' 
if args.dry_run else ''}deleted."
+    if not packages:
+        print("No image packages found.")
+        return 0
+    package = packages[int(os.environ.get("GITHUB_RUN_NUMBER", "0")) % 
len(packages)]
+    versions_path = f"/orgs/{owner}/packages/container/{quote(package, 
safe='')}/versions"
+    selected = select_versions_to_delete(
+        functools.partial(_get_page, versions_path),
+        _get_last_page(versions_path),
+        functools.partial(collect_keep_set, f"{owner}/{package}"),
+        cutoff,
+        args.max_deletions,
     )
+    for version in selected:
+        print(
+            f"{'Would delete' if args.dry_run else 'Deleting'} "
+            f"{package}@{version['name']} ({version['created_at']})"
+        )
+        if not args.dry_run:
+            _gh_api(["-X", "DELETE", f"{versions_path}/{version['id']}"])
+            time.sleep(args.pause)
+    print(f"{package}: {len(selected)} old untagged version(s) {'would be ' if 
args.dry_run else ''}deleted.")
     return 0
 
 
diff --git a/scripts/tests/ci/test_cleanup_keyed_image_cache.py 
b/scripts/tests/ci/test_cleanup_keyed_image_cache.py
index d6b5ac3e0ef..fe14a59ab38 100644
--- a/scripts/tests/ci/test_cleanup_keyed_image_cache.py
+++ b/scripts/tests/ci/test_cleanup_keyed_image_cache.py
@@ -42,8 +42,13 @@ def cleanup():
     return module
 
 
-def _version(tags: list[str], updated_at: str) -> dict:
-    return {"id": 1, "updated_at": updated_at, "metadata": {"container": 
{"tags": tags}}}
+def _version(tags: list[str], created_at: str, updated_at: str | None = None) 
-> dict:
+    return {
+        "id": 1,
+        "created_at": created_at,
+        "updated_at": updated_at or created_at,
+        "metadata": {"container": {"tags": tags}},
+    }
 
 
 @pytest.mark.parametrize(
@@ -93,6 +98,12 @@ def 
test_get_versions_created_since_stops_on_empty_page(cleanup):
     assert fetched == [1, 2]
 
 
+def test_keyed_cache_expires_by_creation_even_when_pushed_again(cleanup):
+    version = _version(["cache-linux-amd64-0123456789abcdef"], created_at=OLD, 
updated_at=RECENT)
+
+    assert cleanup.is_expired_keyed_cache(version, CUTOFF) is True
+
+
 @pytest.mark.parametrize(
     ("package", "matches"),
     [
diff --git a/scripts/tests/ci/test_cleanup_old_image_versions.py 
b/scripts/tests/ci/test_cleanup_old_image_versions.py
index eae80564278..c1542963867 100644
--- a/scripts/tests/ci/test_cleanup_old_image_versions.py
+++ b/scripts/tests/ci/test_cleanup_old_image_versions.py
@@ -93,6 +93,58 @@ def 
test_find_deletable_versions_keeps_versions_near_the_page_edges(cleanup):
     assert [v["name"] for v in cleanup.find_deletable_versions(page, set(), 
CUTOFF)] == ["sha256:middle"]
 
 
+def _pages() -> dict[int, list[dict]]:
+    return {
+        1: [_version("sha256:new-1", RECENT), _version("sha256:new-2", 
RECENT)],
+        2: [
+            _version("sha256:p2-edge", "2026-10-01T00:00:00Z"),
+            _version("sha256:p2-a", "2026-09-20T00:00:00Z"),
+            _version("sha256:p2-kept", "2026-09-15T00:00:00Z"),
+            _version("sha256:p2-b", "2026-09-10T00:00:00Z"),
+            _version("sha256:p2-last", "2026-09-05T00:00:00Z"),
+        ],
+        # Only an edge version, which is never deleted: the walk must go on to 
page 2.
+        3: [_version("sha256:p3-only", "2026-08-01T00:00:00Z")],
+    }
+
+
+def 
test_select_versions_to_delete_walks_past_pages_without_candidates(cleanup):
+    pages = _pages()
+    fetched: list[int] = []
+    keep_loads: list[int] = []
+
+    def fetch_page(page: int) -> list[dict]:
+        fetched.append(page)
+        return pages.get(page, [])
+
+    def load_keep() -> set[str]:
+        keep_loads.append(1)
+        return {"sha256:p2-kept"}
+
+    selected = cleanup.select_versions_to_delete(fetch_page, 3, load_keep, 
CUTOFF, budget=10)
+
+    assert [v["name"] for v in selected] == ["sha256:p2-a", "sha256:p2-b"]
+    assert fetched == [3, 2, 1]
+    assert keep_loads == [1]
+
+
+def test_select_versions_to_delete_stops_at_budget(cleanup):
+    pages = _pages()
+
+    selected = cleanup.select_versions_to_delete(pages.get, 3, set, CUTOFF, 
budget=1)
+
+    assert [v["name"] for v in selected] == ["sha256:p2-a"]
+
+
+def 
test_select_versions_to_delete_does_not_load_tags_without_candidates(cleanup):
+    pages = {1: [_version("sha256:new", RECENT)]}
+
+    def load_keep() -> set[str]:
+        raise AssertionError("tags loaded without a candidate")
+
+    assert cleanup.select_versions_to_delete(pages.get, 1, load_keep, CUTOFF, 
budget=10) == []
+
+
 def test_get_referenced_digests_lists_index_children(cleanup):
     index = {
         "mediaType": "application/vnd.oci.image.index.v1+json",

Reply via email to