This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch ci-image-keyed-cache in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 59e44d9eb33072822693601505b40158697e56d0 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 15 15:52:33 2026 +0900 Prove image versions unreferenced before the cleanup deletes them A version close in time to a tagged image is not proof it is not part of it: a rebuilt identical manifest keeps its old digest and creation time while a new image lists it. Reading what every tag references costs thousands of registry reads per package, so a run takes one package, in turn. An oldest page with nothing to delete stopped the whole package, and keyed cache re-pushed with identical content could fall out of the scanned window for good, since the scan follows creation time. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq --- .github/workflows/cleanup-old-image-versions.yml | 6 +- scripts/ci/cleanup_keyed_image_cache.py | 11 +- scripts/ci/cleanup_old_image_versions.py | 179 +++++++++++++-------- scripts/tests/ci/test_cleanup_keyed_image_cache.py | 15 +- .../tests/ci/test_cleanup_old_image_versions.py | 52 ++++++ 5 files changed, 189 insertions(+), 74 deletions(-) diff --git a/.github/workflows/cleanup-old-image-versions.yml b/.github/workflows/cleanup-old-image-versions.yml index 74a2de5bcfa..945b84308f2 100644 --- a/.github/workflows/cleanup-old-image-versions.yml +++ b/.github/workflows/cleanup-old-image-versions.yml @@ -17,9 +17,9 @@ # --- name: Clean up old image versions -# Deletes, a few hundred per run, the untagged CI and PROD image versions older than 30 days that -# no tagged image references. Small, spaced-out runs keep it far below the API rate limit of the -# workflow token. See scripts/ci/cleanup_old_image_versions.py. +# Deletes, a few hundred per run from one package in turn, the untagged CI and PROD image versions +# older than 30 days that no tag references. Small, spaced-out runs keep it far below the API rate +# limit of the workflow token. See scripts/ci/cleanup_old_image_versions.py. on: # yamllint disable-line rule:truthy schedule: - cron: '41 */3 * * *' diff --git a/scripts/ci/cleanup_keyed_image_cache.py b/scripts/ci/cleanup_keyed_image_cache.py index 2eaddc6f836..8520661309a 100644 --- a/scripts/ci/cleanup_keyed_image_cache.py +++ b/scripts/ci/cleanup_keyed_image_cache.py @@ -16,14 +16,19 @@ # specific language governing permissions and limitations # under the License. """ -Delete keyed CI image cache entries that have not been pushed for a while. +Delete keyed CI image cache entries a while after they were first pushed. ``breeze ci-image build --prepare-buildx-cache`` pushes, next to the shared ``cache-<platform>`` tag, a ``cache-<platform>-<dependency key>`` tag for every set of dependency inputs it builds. Those pile up with every dependency change, so the versions of the CI image packages whose only tags are keyed -cache tags, and which were last updated before the cutoff, are deleted. Shared cache tags, ``latest`` +cache tags, and which were created before the cutoff, are deleted. Shared cache tags, ``latest`` and anything else are never touched. +The age is counted from creation, not from the last update: a push of identical cache content +reuses the version and only moves its ``updated_at``, and versions are listed by creation, so a +version updated after creation could otherwise fall out of the scanned window and stay forever. +A key still in use is pushed again within hours of being deleted. + Usage: cleanup_keyed_image_cache.py [--days N] [--dry-run] Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with ``packages: write``. """ @@ -52,7 +57,7 @@ def is_expired_keyed_cache(version: dict, cutoff: datetime) -> bool: tags = version.get("metadata", {}).get("container", {}).get("tags", []) if not tags or not all(KEYED_CACHE_TAG.match(tag) for tag in tags): return False - return _parse_time(version["updated_at"]) < cutoff + return _parse_time(version["created_at"]) < cutoff def find_expired_versions(versions: Iterable[dict], cutoff: datetime) -> list[dict]: diff --git a/scripts/ci/cleanup_old_image_versions.py b/scripts/ci/cleanup_old_image_versions.py index f507ba5b373..3d321ea10f4 100644 --- a/scripts/ci/cleanup_old_image_versions.py +++ b/scripts/ci/cleanup_old_image_versions.py @@ -19,29 +19,35 @@ Slowly delete old untagged versions of the CI and PROD image packages. Every image or cache push leaves the manifest it replaced behind as an untagged package version, -so a package such as ``airflow/main/ci/python3.11`` holds tens of thousands of them. This deletes -the untagged versions created before the cutoff, oldest first, up to a budget per run and with a -pause between deletions, so that a run stays far below the API rate limit of the workflow token -and the next run continues where this one stopped. +so a package such as ``airflow/main/ci/python3.11`` holds tens of thousands of them. Each run +takes one package, in turn by run number, and deletes its untagged versions created before the +cutoff, oldest first, up to a budget and with a pause between deletions, so that a run stays far +below the API rate limit of the workflow token and a later run continues where this one stopped. -A version is only deleted when no tagged manifest of its package references it: the platform -manifests and attestations of a multi-platform image are untagged versions of their own and must -stay while the image tag that lists them exists. +A version is only deleted when no tag of its package references it: the platform manifests and +attestations of a multi-platform image are untagged versions of their own and must stay while the +tag that lists them exists. The references are read from the registry for every tag, and an +untagged version pushed within PUSH_WINDOW of a tagged one is kept as well. Usage: cleanup_old_image_versions.py [--days N] [--max-deletions N] [--pause SECONDS] [--dry-run] -Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with ``packages: write``. +Environment: GITHUB_REPOSITORY (``owner/repo``), GITHUB_RUN_NUMBER (picks the package), and a +token for ``gh`` with ``packages: write``. """ from __future__ import annotations import argparse +import functools import json import os import re import subprocess import sys import time +import urllib.error import urllib.request +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from urllib.parse import quote @@ -55,7 +61,6 @@ MANIFEST_MEDIA_TYPES = ", ".join( ) LINK_LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"') PUSH_WINDOW = timedelta(minutes=5) -COMMIT_TAG = re.compile(r"^[0-9a-f]{7,40}$") def get_image_package_pattern(repo: str) -> re.Pattern[str]: @@ -78,11 +83,9 @@ def find_deletable_versions(versions: list[dict], keep: set[str], cutoff: dateti """ Return the untagged versions of one page that are safe to delete. - The few long-lived tags have their listed manifests in ``keep``. The packages also hold - thousands of old per-commit tags, too many to read the manifests of, so an untagged version - pushed within PUSH_WINDOW of a tagged one, or of the page edges where the tagged one may sit - on the next page, is taken for part of that image and kept: the manifests of a - multi-platform image are pushed together with it. + ``keep`` holds every manifest a tag references. As a second guard, an untagged version pushed + within PUSH_WINDOW of a tagged one, or of the page edges where a tagged one may sit on the + next page, is kept: the manifests of a multi-platform image are pushed together with it. """ if not versions: return [] @@ -112,45 +115,96 @@ def _get_page(versions_path: str, page: int) -> list[dict]: return json.loads(_gh_api([f"{versions_path}?per_page=100&page={page}"])) -def _registry_get(url: str, token: str, accept: str = "application/json") -> tuple[dict, str | None, str]: - request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}", "Accept": accept}) - with urllib.request.urlopen(request, timeout=60) as response: - return ( - json.load(response), - response.headers.get("Docker-Content-Digest"), - response.headers.get("Link", ""), - ) +class _Registry: + """Anonymous, read-only access to the public images on ghcr.io.""" + + def __init__(self, image: str): + self.image = image + self.token = self._get_token() + + def _get_token(self) -> str: + url = f"https://ghcr.io/token?scope=repository:{self.image}:pull" + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response)["token"] + + def get(self, path: str, accept: str = "application/json") -> tuple[dict, str | None, str]: + for attempt in range(2): + request = urllib.request.Request( + f"https://ghcr.io{path}", headers={"Authorization": f"Bearer {self.token}", "Accept": accept} + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return ( + json.load(response), + response.headers.get("Docker-Content-Digest"), + response.headers.get("Link", ""), + ) + except urllib.error.HTTPError as error: + if error.code != 401 or attempt: + raise + self.token = self._get_token() + raise AssertionError("unreachable") def collect_keep_set(image: str) -> set[str]: """ - Return the digests of the long-lived (non-commit) tags of ``image`` and of the manifests they list. + Return the digests of every tagged manifest of ``image`` and of the manifests they list. Read from the registry (the images are public) rather than from the packages API, which can - only find the few tagged versions by listing every one of the tens of thousands of versions. + only find the tagged versions by listing every one of the tens of thousands of versions, and + whose rate limit the deletions need. """ - with urllib.request.urlopen( - f"https://ghcr.io/token?scope=repository:{image}:pull", timeout=60 - ) as response: - token = json.load(response)["token"] + registry = _Registry(image) tags: list[str] = [] - url: str | None = f"https://ghcr.io/v2/{image}/tags/list?n=1000" - while url: - listing, _, link = _registry_get(url, token) + path: str | None = f"/v2/{image}/tags/list?n=1000" + while path: + listing, _, link = registry.get(path) tags.extend(listing.get("tags") or []) next_page = re.search(r"<([^>]+)>;\s*rel=\"next\"", link) - url = f"https://ghcr.io{next_page.group(1)}" if next_page else None + path = next_page.group(1) if next_page else None + + def read_tag(tag: str) -> set[str]: + manifest, digest, _ = registry.get(f"/v2/{image}/manifests/{tag}", MANIFEST_MEDIA_TYPES) + return get_referenced_digests(manifest) | ({digest} if digest else set()) + keep: set[str] = set() - for tag in (tag for tag in tags if not COMMIT_TAG.match(tag)): - manifest, digest, _ = _registry_get( - f"https://ghcr.io/v2/{image}/manifests/{tag}", token, MANIFEST_MEDIA_TYPES - ) - if digest: - keep.add(digest) - keep |= get_referenced_digests(manifest) + with ThreadPoolExecutor(max_workers=8) as executor: + for digests in executor.map(read_tag, tags): + keep |= digests + print(f"{image}: {len(tags)} tag(s) reference {len(keep)} manifest(s)") return keep +def select_versions_to_delete( + fetch_page: Callable[[int], list[dict]], + last_page: int, + load_keep: Callable[[], set[str]], + cutoff: datetime, + budget: int, +) -> list[dict]: + """ + Walk the pages from the last, oldest one, and return up to ``budget`` versions to delete. + + Versions are listed newest first, so deleting from a page never moves the versions of the + pages before it, and the selection can be made before deleting. The tag references are only + loaded once a page has a candidate, and a page with none does not stop the walk. + """ + selected: list[dict] = [] + keep: set[str] | None = None + for page in range(last_page, 0, -1): + if len(selected) >= budget: + break + batch = fetch_page(page) + if not batch or _parse_time(batch[-1]["created_at"]) >= cutoff: + break + if keep is None: + if not find_deletable_versions(batch, set(), cutoff): + continue + keep = load_keep() + selected.extend(find_deletable_versions(batch, keep, cutoff)[: budget - len(selected)]) + return selected + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--days", type=int, default=30, help="Keep versions created within this many days") @@ -166,34 +220,27 @@ def main() -> int: ["--paginate", f"/orgs/{owner}/packages?package_type=container&per_page=100", "--jq", ".[].name"] ) packages = sorted(name for name in package_listing.split() if package_pattern.match(name)) - budget = args.max_deletions - for package in packages: - if budget <= 0: - break - versions_path = f"/orgs/{owner}/packages/container/{quote(package, safe='')}/versions" - page = _get_last_page(versions_path) - if not find_deletable_versions(_get_page(versions_path, page), set(), cutoff): - continue - keep = collect_keep_set(f"{owner}/{package}") - # Versions are listed newest first, so walking from the last page goes oldest first, and - # deleting from a page never moves the versions of the pages before it. - while page >= 1 and budget > 0: - batch = _get_page(versions_path, page) - if not batch or _parse_time(batch[-1]["created_at"]) >= cutoff: - break - for version in find_deletable_versions(batch, keep, cutoff)[:budget]: - print( - f"{'Would delete' if args.dry_run else 'Deleting'} " - f"{package}@{version['name']} ({version['created_at']})" - ) - if not args.dry_run: - _gh_api(["-X", "DELETE", f"{versions_path}/{version['id']}"]) - time.sleep(args.pause) - budget -= 1 - page -= 1 - print( - f"{args.max_deletions - budget} old untagged version(s) {'would be ' if args.dry_run else ''}deleted." + if not packages: + print("No image packages found.") + return 0 + package = packages[int(os.environ.get("GITHUB_RUN_NUMBER", "0")) % len(packages)] + versions_path = f"/orgs/{owner}/packages/container/{quote(package, safe='')}/versions" + selected = select_versions_to_delete( + functools.partial(_get_page, versions_path), + _get_last_page(versions_path), + functools.partial(collect_keep_set, f"{owner}/{package}"), + cutoff, + args.max_deletions, ) + for version in selected: + print( + f"{'Would delete' if args.dry_run else 'Deleting'} " + f"{package}@{version['name']} ({version['created_at']})" + ) + if not args.dry_run: + _gh_api(["-X", "DELETE", f"{versions_path}/{version['id']}"]) + time.sleep(args.pause) + print(f"{package}: {len(selected)} old untagged version(s) {'would be ' if args.dry_run else ''}deleted.") return 0 diff --git a/scripts/tests/ci/test_cleanup_keyed_image_cache.py b/scripts/tests/ci/test_cleanup_keyed_image_cache.py index d6b5ac3e0ef..fe14a59ab38 100644 --- a/scripts/tests/ci/test_cleanup_keyed_image_cache.py +++ b/scripts/tests/ci/test_cleanup_keyed_image_cache.py @@ -42,8 +42,13 @@ def cleanup(): return module -def _version(tags: list[str], updated_at: str) -> dict: - return {"id": 1, "updated_at": updated_at, "metadata": {"container": {"tags": tags}}} +def _version(tags: list[str], created_at: str, updated_at: str | None = None) -> dict: + return { + "id": 1, + "created_at": created_at, + "updated_at": updated_at or created_at, + "metadata": {"container": {"tags": tags}}, + } @pytest.mark.parametrize( @@ -93,6 +98,12 @@ def test_get_versions_created_since_stops_on_empty_page(cleanup): assert fetched == [1, 2] +def test_keyed_cache_expires_by_creation_even_when_pushed_again(cleanup): + version = _version(["cache-linux-amd64-0123456789abcdef"], created_at=OLD, updated_at=RECENT) + + assert cleanup.is_expired_keyed_cache(version, CUTOFF) is True + + @pytest.mark.parametrize( ("package", "matches"), [ diff --git a/scripts/tests/ci/test_cleanup_old_image_versions.py b/scripts/tests/ci/test_cleanup_old_image_versions.py index eae80564278..c1542963867 100644 --- a/scripts/tests/ci/test_cleanup_old_image_versions.py +++ b/scripts/tests/ci/test_cleanup_old_image_versions.py @@ -93,6 +93,58 @@ def test_find_deletable_versions_keeps_versions_near_the_page_edges(cleanup): assert [v["name"] for v in cleanup.find_deletable_versions(page, set(), CUTOFF)] == ["sha256:middle"] +def _pages() -> dict[int, list[dict]]: + return { + 1: [_version("sha256:new-1", RECENT), _version("sha256:new-2", RECENT)], + 2: [ + _version("sha256:p2-edge", "2026-10-01T00:00:00Z"), + _version("sha256:p2-a", "2026-09-20T00:00:00Z"), + _version("sha256:p2-kept", "2026-09-15T00:00:00Z"), + _version("sha256:p2-b", "2026-09-10T00:00:00Z"), + _version("sha256:p2-last", "2026-09-05T00:00:00Z"), + ], + # Only an edge version, which is never deleted: the walk must go on to page 2. + 3: [_version("sha256:p3-only", "2026-08-01T00:00:00Z")], + } + + +def test_select_versions_to_delete_walks_past_pages_without_candidates(cleanup): + pages = _pages() + fetched: list[int] = [] + keep_loads: list[int] = [] + + def fetch_page(page: int) -> list[dict]: + fetched.append(page) + return pages.get(page, []) + + def load_keep() -> set[str]: + keep_loads.append(1) + return {"sha256:p2-kept"} + + selected = cleanup.select_versions_to_delete(fetch_page, 3, load_keep, CUTOFF, budget=10) + + assert [v["name"] for v in selected] == ["sha256:p2-a", "sha256:p2-b"] + assert fetched == [3, 2, 1] + assert keep_loads == [1] + + +def test_select_versions_to_delete_stops_at_budget(cleanup): + pages = _pages() + + selected = cleanup.select_versions_to_delete(pages.get, 3, set, CUTOFF, budget=1) + + assert [v["name"] for v in selected] == ["sha256:p2-a"] + + +def test_select_versions_to_delete_does_not_load_tags_without_candidates(cleanup): + pages = {1: [_version("sha256:new", RECENT)]} + + def load_keep() -> set[str]: + raise AssertionError("tags loaded without a candidate") + + assert cleanup.select_versions_to_delete(pages.get, 1, load_keep, CUTOFF, budget=10) == [] + + def test_get_referenced_digests_lists_index_children(cleanup): index = { "mediaType": "application/vnd.oci.image.index.v1+json",
