This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch replace-dependabot-with-update-workflow in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
commit bfc203e397ae1a3763b6bcefe4bb706bb178efdf Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Aug 9 16:13:26 2026 +0800 Replace Dependabot with a sparse-checkout update workflow Dependabot clones the whole repository before its file fetcher runs. At 826k files and 47 GB that never finishes inside a Dependabot job, so the runs time out and the pinned actions and pre-commit hooks fall behind - only one Dependabot PR has landed here since April. There is no knob in dependabot.yml that avoids the clone, so this drops the config and does the same job from a workflow that only ever needs the configuration files: * .github/workflows/update-pinned-deps.yml runs weekly (and on demand), once per target branch, on a sparse blobless checkout of .github. * scripts/update_pinned_deps.py resolves each SHA-pinned action and pre-commit hook to the newest release outside a 7-day cooldown window - the same cooldown the Dependabot config used - and rewrites the pins in place, preserving the `# vX.Y.Z` / `# frozen: vX.Y.Z` style. Branch pins such as `# main` move along their branch instead. Pull requests opened with GITHUB_TOKEN do not trigger other workflows, so the workflow runs pre-commit on the changed files itself before opening the PR. Dependabot security alerts are unaffected - they do not come from dependabot.yml. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> --- .github/dependabot.yml | 63 ------- .github/workflows/update-pinned-deps.yml | 131 ++++++++++++++ scripts/update_pinned_deps.py | 300 +++++++++++++++++++++++++++++++ 3 files changed, 431 insertions(+), 63 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index e2da1f1f0fc..00000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,63 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -version: 2 -updates: - - package-ecosystem: github-actions - cooldown: - default-days: 7 - directory: / - schedule: - interval: daily - groups: - github-actions: - patterns: - - ".github/*" - - package-ecosystem: github-actions - cooldown: - default-days: 7 - directory: / - target-branch: staging - schedule: - interval: daily - groups: - github-actions: - patterns: - - ".github/*" - - package-ecosystem: pre-commit - cooldown: - default-days: 7 - directory: / - schedule: - interval: weekly - groups: - pre-commit-hooks: - patterns: - - ".pre-commit-config.yaml" - - package-ecosystem: pre-commit - cooldown: - default-days: 7 - directory: / - target-branch: staging - schedule: - interval: weekly - groups: - pre-commit-hooks: - patterns: - - ".pre-commit-config.yaml" diff --git a/.github/workflows/update-pinned-deps.yml b/.github/workflows/update-pinned-deps.yml new file mode 100644 index 00000000000..4351eb2a7ec --- /dev/null +++ b/.github/workflows/update-pinned-deps.yml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +name: Update pinned dependencies +# Replaces Dependabot for GitHub Actions and pre-commit hooks. Dependabot clones +# the whole repository before its file fetcher runs, and at 800k+ files / 47+ GB +# that never finishes inside a Dependabot job - so updates simply stopped +# landing. This workflow only needs the configuration files, which a sparse +# blobless checkout gets in seconds. +on: # yamllint disable-line rule:truthy + schedule: + - cron: '0 6 * * 1' # Mondays, 06:00 UTC + workflow_dispatch: + inputs: + cooldown-days: + description: "Ignore releases published more recently than this many days" + default: "7" + type: string +permissions: + contents: read +concurrency: + group: update-pinned-deps + cancel-in-progress: false +jobs: + update: + name: "Update pins on ${{ matrix.branch }}" + runs-on: ubuntu-latest + if: github.repository == 'apache/airflow-site-archive' + permissions: + contents: write # push the update branch + pull-requests: write # open the update pull request + strategy: + fail-fast: false + matrix: + branch: [main, staging] + env: + TARGET_BRANCH: ${{ matrix.branch }} + UPDATE_BRANCH: update-pinned-deps-${{ matrix.branch }} + COOLDOWN_DAYS: ${{ inputs.cooldown-days || '7' }} + steps: + - name: "Checkout ${{ matrix.branch }} configuration" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Only the configuration is needed - never check out docs-archive here. + with: + ref: ${{ matrix.branch }} + fetch-depth: 1 + filter: blob:none + sparse-checkout: | + .github + persist-credentials: false + + - name: "Checkout ${{ github.sha }} for scripts" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The updater always comes from the ref this workflow runs from, so it + # does not have to exist on the branch being updated yet. + with: + ref: ${{ github.sha }} + path: ./tooling + fetch-depth: 1 + filter: blob:none + sparse-checkout: | + scripts + persist-credentials: false + + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: "Resolve the latest versions outside the cooldown window" + id: update + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + uv run --no-project ./tooling/scripts/update_pinned_deps.py \ + --repo-root . \ + --cooldown-days "${COOLDOWN_DAYS}" \ + --summary-file "${RUNNER_TEMP}/summary.md" + + - name: "Lint the updated configuration" + if: steps.update.outputs.updated == 'true' + run: | + uv tool install pre-commit --with pre-commit-uv + # Pull requests opened with GITHUB_TOKEN do not trigger other + # workflows, so the checks that would run on the pull request are run + # here instead - on the changed files only, because `--all-files` + # would walk all 800k+ files in the index. + git diff --name-only -z | xargs -0 pre-commit run --color always --files + + - name: "Open the update pull request" + if: steps.update.outputs.updated == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "${UPDATE_BRANCH}" + git commit -a -m "Update pinned GitHub Actions and pre-commit hooks" + git push --force "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"; \ + "HEAD:refs/heads/${UPDATE_BRANCH}" + if [[ -n "$(gh pr list --head "${UPDATE_BRANCH}" --base "${TARGET_BRANCH}" \ + --state open --json number --jq '.[].number')" ]]; then + echo "Pull request already open - the force push updated it" + exit 0 + fi + { + echo "Automated update of the SHA-pinned GitHub Actions and pre-commit hooks," + echo "honouring a ${COOLDOWN_DAYS}-day cooldown on new releases." + echo + cat "${RUNNER_TEMP}/summary.md" + echo + echo "Opened by [\`update-pinned-deps.yml\`](.github/workflows/update-pinned-deps.yml)." + echo "Pre-commit ran on the changed files in that workflow run, because pull" + echo "requests opened with \`GITHUB_TOKEN\` do not trigger \`Basic checks\`." + } > "${RUNNER_TEMP}/pr-body.md" + gh pr create --base "${TARGET_BRANCH}" --head "${UPDATE_BRANCH}" \ + --title "Update pinned GitHub Actions and pre-commit hooks (${TARGET_BRANCH})" \ + --body-file "${RUNNER_TEMP}/pr-body.md" diff --git a/scripts/update_pinned_deps.py b/scripts/update_pinned_deps.py new file mode 100755 index 00000000000..7fd47a0819b --- /dev/null +++ b/scripts/update_pinned_deps.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# dependencies = [] +# /// +"""Update SHA-pinned GitHub Actions and pre-commit hooks to their latest versions. + +This replaces Dependabot for the `github-actions` and `pre-commit` ecosystems. +Dependabot clones the whole repository before its file fetcher runs, and this +repository is far too big for that (800k+ files, 47+ GB), so every Dependabot +job times out and updates never land. This script only ever reads +`.github/workflows`, `.github/actions` and `.pre-commit-config.yaml`, so it runs +from a sparse checkout in seconds. + +Only pinned references are touched, and the pinning style is preserved: + + uses: actions/checkout@<sha> # v7.0.1 -> newest release tag + uses: apache/some-action@<sha> # main -> newest commit on that branch + rev: <sha> # frozen: v1.29.0 -> newest release tag + +A cooldown window (7 days by default, matching the Dependabot config this +replaces) keeps us off releases that are still hot. + +Set GITHUB_TOKEN (or GH_TOKEN) to avoid the unauthenticated API rate limit. +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import re +import sys +import urllib.error +import urllib.request +from datetime import UTC, datetime, timedelta +from pathlib import Path +from time import sleep + +API_ROOT = "https://api.github.com"; +API_ATTEMPTS = 3 +# Enough history to find the newest release outside the cooldown window without +# pulling megabytes of release notes for repositories that publish a lot. +API_PAGE_SIZE = 30 + +# `uses: owner/repo[/sub/path]@<40-hex sha> # <tag-or-branch>` +USES_RE = re.compile( + r"(?P<prefix>uses:\s+)" + r"(?P<repo>[A-Za-z0-9._-]+/[A-Za-z0-9._-]+)" + r"(?P<subpath>(?:/[^@\s]+)?)" + r"@(?P<sha>[0-9a-f]{40})" + r"(?P<gap>\s+#\s+)" + r"(?P<comment>\S+)" +) +# `- repo: https://github.com/owner/repo` +PRE_COMMIT_REPO_RE = re.compile(r"^\s*-\s+repo:\s+https://github\.com/(?P<repo>[^\s/]+/[^\s/]+?)(?:\.git)?\s*$") +# `rev: <sha> # frozen: v1.29.0` +PRE_COMMIT_REV_RE = re.compile(r"^(?P<prefix>\s*rev:\s+)(?P<rev>\S+)(?P<gap>\s+#\s+frozen:\s+)(?P<tag>\S+)\s*$") +# Strict release versions only - no prereleases, no moving major/minor tags. +VERSION_RE = re.compile(r"^v?(?P<version>\d+(?:\.\d+)*)$") + + +class GitHubError(RuntimeError): + """Raised when the GitHub API cannot answer a question we need answered.""" + + +def api(path: str, **params: str | int) -> list | dict | None: + """Call the GitHub REST API, returning None for a 404. + + Transient failures (dropped connections, truncated responses, 5xx) are + retried - a single flaky read should not fail the whole update run. + """ + url = f"{API_ROOT}/{path}" + if params: + url += "?" + "&".join(f"{key}={value}" for key, value in params.items()) + request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + request.add_header("Authorization", f"Bearer {token}") + for attempt in range(1, API_ATTEMPTS + 1): + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read()) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + if error.code < 500 or attempt == API_ATTEMPTS: + raise GitHubError(f"GET {url} failed: {error.code} {error.reason}") from error + except (urllib.error.URLError, http.client.HTTPException, TimeoutError) as error: + if attempt == API_ATTEMPTS: + raise GitHubError(f"GET {url} failed: {error}") from error + sleep(attempt) + return None + + +def parse_version(tag: str) -> tuple[int, ...] | None: + """Turn `v1.29.0` into `(1, 29, 0)`, or return None if it is not a release version.""" + match = VERSION_RE.match(tag) + if not match: + return None + return tuple(int(part) for part in match.group("version").split(".")) + + +def released_at(entry: dict) -> datetime: + return datetime.fromisoformat(entry["published_at"]) + + +def latest_release_tag(repo: str, cutoff: datetime) -> str | None: + """Newest release tag of `repo` that was published before `cutoff`. + + Releases are preferred over tags because they carry a publication date and + exclude drafts, prereleases and moving major tags such as `v7`. Repositories + that do not publish releases fall back to tags, dated by their commit. + """ + releases = api(f"repos/{repo}/releases", per_page=API_PAGE_SIZE) or [] + candidates = [ + (version, release["tag_name"]) + for release in releases + if not release["draft"] and not release["prerelease"] and released_at(release) <= cutoff + for version in [parse_version(release["tag_name"])] + if version + ] + if candidates: + return max(candidates)[1] + + tags = api(f"repos/{repo}/tags", per_page=API_PAGE_SIZE) or [] + dated = sorted( + ((version, tag["name"], tag["commit"]["sha"]) for tag in tags for version in [parse_version(tag["name"])] if version), + reverse=True, + ) + for _, name, sha in dated: + commit = api(f"repos/{repo}/commits/{sha}") + if commit and datetime.fromisoformat(commit["commit"]["committer"]["date"]) <= cutoff: + return name + return None + + +def tag_sha(repo: str, tag: str) -> str: + """Commit SHA a tag points at, dereferencing annotated tags.""" + reference = api(f"repos/{repo}/git/ref/tags/{tag}") + if not reference: + raise GitHubError(f"tag {tag} not found in {repo}") + target = reference["object"] + if target["type"] == "tag": + target = api(f"repos/{repo}/git/tags/{target['sha']}")["object"] + return target["sha"] + + +def latest_branch_sha(repo: str, branch: str, cutoff: datetime) -> str | None: + """Newest commit on `branch` that is older than `cutoff`.""" + commits = api(f"repos/{repo}/commits", sha=branch, until=cutoff.strftime("%Y-%m-%dT%H:%M:%SZ"), per_page=1) + return commits[0]["sha"] if commits else None + + +class Updater: + """Resolves and applies pin updates, collecting a report as it goes.""" + + def __init__(self, cutoff: datetime, dry_run: bool = False) -> None: + self.cutoff = cutoff + self.dry_run = dry_run + self.updates: list[tuple[str, str, str]] = [] + self.resolved: dict[tuple[str, str], tuple[str, str] | None] = {} + + def write(self, path: Path, content: str) -> None: + if not self.dry_run: + path.write_text(content) + + def resolve(self, repo: str, current_tag: str, current_sha: str) -> tuple[str, str] | None: + """Return the (tag, sha) this pin should move to, or None to leave it alone.""" + key = (repo, current_tag) + if key not in self.resolved: + self.resolved[key] = self._resolve(repo, current_tag, current_sha) + target = self.resolved[key] + return target if target and target[1] != current_sha else None + + def _resolve(self, repo: str, current_tag: str, current_sha: str) -> tuple[str, str] | None: + current_version = parse_version(current_tag) + if current_version is None: + # A branch pin such as `# main` - move it along the branch instead. + sha = latest_branch_sha(repo, current_tag, self.cutoff) + return (current_tag, sha) if sha else None + tag = latest_release_tag(repo, self.cutoff) + if tag is None: + print(f" ! {repo}: no release older than the cooldown window, left at {current_tag}") + return None + if parse_version(tag) < current_version: + print(f" ! {repo}: newest eligible release {tag} is older than the pinned {current_tag}, left alone") + return None + return tag, tag_sha(repo, tag) + + def record(self, name: str, before_tag: str, before_sha: str, after_tag: str, after_sha: str) -> None: + if before_tag == after_tag: + # A branch pin, or a tag that was moved to a different commit - the + # tag alone would read as a no-op, so show what actually changed. + before, after = f"{before_tag} ({before_sha[:7]})", f"{after_tag} ({after_sha[:7]})" + else: + before, after = before_tag, after_tag + # The same action is typically pinned in several places - report it once. + if (name, before, after) in self.updates: + return + self.updates.append((name, before, after)) + print(f" * {name}: {before} -> {after}") + + def update_workflows(self, paths: list[Path]) -> None: + for path in sorted(paths): + content = original = path.read_text() + for match in list(USES_RE.finditer(original)): + repo, current_tag, current_sha = match["repo"], match["comment"], match["sha"] + target = self.resolve(repo, current_tag, current_sha) + if not target: + continue + tag, sha = target + replacement = ( + f"{match['prefix']}{repo}{match['subpath']}@{sha}{match['gap']}{tag}" + ) + content = content.replace(match.group(0), replacement) + self.record(f"{repo}{match['subpath']}", current_tag, current_sha, tag, sha) + if content != original: + self.write(path, content) + + def update_pre_commit(self, path: Path) -> None: + if not path.exists(): + return + lines = path.read_text().splitlines(keepends=True) + repo: str | None = None + for index, line in enumerate(lines): + repo_match = PRE_COMMIT_REPO_RE.match(line) + if repo_match: + repo = repo_match["repo"] + continue + rev_match = PRE_COMMIT_REV_RE.match(line) + if not (repo and rev_match): + continue + target = self.resolve(repo, rev_match["tag"], rev_match["rev"]) + if not target: + continue + tag, sha = target + lines[index] = f"{rev_match['prefix']}{sha}{rev_match['gap']}{tag}\n" + self.record(repo, rev_match["tag"], rev_match["rev"], tag, sha) + self.write(path, "".join(lines)) + + def report(self) -> str: + if not self.updates: + return "No updates available outside the cooldown window.\n" + widths = [max(len(row[column]) for row in self.updates) for column in range(3)] + header = f"| {'Dependency':<{widths[0]}} | {'From':<{widths[1]}} | {'To':<{widths[2]}} |\n" + divider = f"|{'-' * (widths[0] + 2)}|{'-' * (widths[1] + 2)}|{'-' * (widths[2] + 2)}|\n" + rows = "".join( + f"| {name:<{widths[0]}} | {before:<{widths[1]}} | {after:<{widths[2]}} |\n" + for name, before, after in self.updates + ) + return header + divider + rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cooldown-days", type=int, default=7, help="ignore releases newer than this many days") + parser.add_argument("--repo-root", type=Path, default=Path(__file__).parent.parent, help="repository root") + parser.add_argument("--summary-file", type=Path, help="write the markdown summary table here") + parser.add_argument("--dry-run", action="store_true", help="report what would change without writing files") + args = parser.parse_args() + + now = datetime.now(UTC) + cutoff = now - timedelta(days=args.cooldown_days) + print(f"Updating pins to the newest versions released before {cutoff:%Y-%m-%d %H:%M} UTC " + f"({args.cooldown_days}-day cooldown)") + + workflows = sorted( + { + *(args.repo_root / ".github" / "workflows").glob("*.y*ml"), + *(args.repo_root / ".github" / "actions").rglob("*.y*ml"), + } + ) + pre_commit_config = args.repo_root / ".pre-commit-config.yaml" + + updater = Updater(cutoff, dry_run=args.dry_run) + try: + print("GitHub Actions:") + updater.update_workflows(workflows) + print("Pre-commit hooks:") + updater.update_pre_commit(pre_commit_config) + except GitHubError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + summary = updater.report() + print(summary, end="") + if args.summary_file: + args.summary_file.write_text(summary) + if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"): + Path(step_summary).write_text(f"### Pinned dependency updates\n\n{summary}") + if github_output := os.environ.get("GITHUB_OUTPUT"): + with Path(github_output).open("a") as output: + output.write(f"updated={'true' if updater.updates else 'false'}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main())
