commit: aad52cc7e99799777cb90ae371a4414022d39e20 Author: Thomas Bracht Laumann Jespersen <t <AT> laumann <DOT> xyz> AuthorDate: Sun Feb 8 06:43:16 2026 +0000 Commit: Michał Górny <mgorny <AT> gentoo <DOT> org> CommitDate: Tue Feb 10 08:17:12 2026 +0000 URL: https://gitweb.gentoo.org/proj/repo-mirror-ci.git/commit/?id=aad52cc7
pull-request: add scanning of codeberg PRs Extend scan-pull-requests.py to scan the configured Codeberg repo for PRs and push them into the same queue. The PRs from Codeberg take priority over the GitHub ones. Based on the forge prefix we then select which remote to fetch PR changes from for testing. Signed-off-by: Thomas Bracht Laumann Jespersen <t <AT> laumann.xyz> Part-of: https://github.com/gentoo/repo-mirror-ci/pull/13 Signed-off-by: Michał Górny <mgorny <AT> gentoo.org> pull-request/codebergapi.py | 122 +++++++++++++++------------ pull-request/pull-requests.bash | 58 ++++++++++--- pull-request/report-codeberg-pull-request.py | 111 ++++++++++++------------ pull-request/scan-pull-requests.py | 102 ++++++++++++++++++++-- pull-request/set-codeberg-pr-status.py | 4 +- 5 files changed, 269 insertions(+), 128 deletions(-) diff --git a/pull-request/codebergapi.py b/pull-request/codebergapi.py index d7932fb..85d8112 100644 --- a/pull-request/codebergapi.py +++ b/pull-request/codebergapi.py @@ -1,63 +1,74 @@ import requests -import json -from dataclasses import dataclass +from typing import Generator + -@dataclass class CodebergAPI: - owner: str - repo: str - token: str + def __init__(self, owner: str, repo: str, token: str): + self.owner = owner + self.repo = repo + self.token = token - @property - def headers(self): - return { - "Authorization": f"token {self.token}", - "Accept": "application/json" + def __enter__(self): + self.session = requests.Session() + self.session.headers.update( + { + "Authorization": f"token {self.token}", + "Content-Type": "application/json", + } + ) + self.session.hooks = { + "response": lambda r, *args, **kwargs: r.raise_for_status() } + return self + + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: + self.session.close() @property - def repos_baseurl(self): + def repos_baseurl(self) -> str: return f"https://codeberg.org/api/v1/repos/{self.owner}/{self.repo}"; - def _get(self, url, params={}): - resp = requests.get(url, params=params, headers=self.headers) - resp.raise_for_status() - return json.loads(resp.content) - - def _post(self, url, body): - resp = requests.post(url, headers=self.headers, json=body) - resp.raise_for_status() + def _get_paginated(self, url) -> Generator[None, dict, None]: + r = self.session.get(url, params={"limit": 100}) + yield from r.json() + if "next" not in r.links: + return + next_url = r.links["next"]["url"] + while True: + r = self.session.get(next_url) + yield from r.json() + if "next" not in r.links: + break + next_url = r.links["next"]["url"] - def _delete(self, url): - resp = requests.delete(url, headers=self.headers) - resp.raise_for_status() + def pulls(self, state="open") -> Generator[None, dict, None]: + """ + state must be one of: open, closed, all + """ + return self._get_paginated(f"{self.repos_baseurl}/pulls?state={state}") - def _patch(self, url, body): - resp = requests.patch(url, headers=self.headers, json=body) - resp.raise_for_status() + def set_pr_title(self, pr_id: int, title: str) -> None: + self.session.patch(f"{self.repos_baseurl}/pulls/{pr_id}", json={"title": title}) - def pulls_page(self, page): - return self._get( - f"{self.repos_baseurl}/pulls", - params={ - "page": page, - } + def add_pr_labels(self, pr_id: int, labels: list[int]) -> None: + self.session.patch( + f"{self.repos_baseurl}/pulls/{pr_id}", json=({"labels": labels}) ) - def pulls(self): - page = 1 - while True: - pulls = self.pulls_page(page) - if not pulls: - break - yield from pulls - page += 1 + def labels(self) -> list[dict]: + return self.session.get(f"{self.repos_baseurl}/labels").json() + + def commits(self, pr_id: int) -> list[dict]: + # https://codeberg.org/api/swagger#/repository/repoGetPullRequestCommits + return self.session.get(f"{self.repos_baseurl}/pulls/{pr_id}/commits").json() def commit_statuses(self, sha): # /repos/{owner}/{repo}/statuses/{sha} - return self._get(f"{self.repos_baseurl}/statuses/{sha}") + return self.session.get(f"{self.repos_baseurl}/statuses/{sha}").json() - def commit_set_status(self, sha, state, description=None, target_url=None, context=None): + def commit_set_status( + self, sha, state, description=None, target_url=None, context=None + ): # /repos/{owner}/{repo}/statuses/{sha} body = { "context": context, @@ -65,18 +76,23 @@ class CodebergAPI: "description": description, "target_url": target_url, } - return self._post(f"{self.repos_baseurl}/statuses/{sha}", body) + self.session.post(f"{self.repos_baseurl}/statuses/{sha}", json=body) - def get_reviews(self, pr_id): - return self._get(f"{self.repos_baseurl}/pulls/{pr_id}/reviews") + def files(self, pr_id: int) -> list[dict]: + return self.session.get(f"{self.repos_baseurl}/pulls/{pr_id}/files").json() - def create_review(self, pr_id, comment): + def get_reviews(self, pr_id: int) -> list[dict]: + return self.session.get(f"{self.repos_baseurl}/pulls/{pr_id}/reviews").json() + + def create_review(self, pr_id: int, comment: str) -> None: + # Does not appear to be possible to simply post comments # https://codeberg.org/api/swagger#/repository/repoCreatePullReview - url = f"{self.repos_baseurl}/pulls/{pr_id}/reviews" - body = { - "body": comment, - } - self._post(url, body) + self.session.post( + f"{self.repos_baseurl}/pulls/{pr_id}/reviews", + json={ + "body": comment, + }, + ) - def delete_review(self, pr_id, review_id): - self._delete(f"{self.repos_baseurl}/pulls/{pr_id}/reviews/{review_id}") + def delete_review(self, pr_id: int, review_id: int) -> None: + self.session.delete(f"{self.repos_baseurl}/pulls/{pr_id}/reviews/{review_id}") diff --git a/pull-request/pull-requests.bash b/pull-request/pull-requests.bash index f7c3138..0d40d95 100755 --- a/pull-request/pull-requests.bash +++ b/pull-request/pull-requests.bash @@ -11,19 +11,35 @@ gentooci=${GENTOO_CI_GIT} pull=${PULL_REQUEST_DIR} if [[ -s ${pull}/current-pr ]]; then - iid=$(<"${pull}"/current-pr) + pr=$(<"${pull}"/current-pr) + forge="${pr%/*}" + prid="${pr#*/}" cd -- "${sync}" - hash=$(git rev-parse "refs/pull/${iid}") - "${SCRIPT_DIR}"/pull-request/set-pull-request-status.py "${hash}" error \ + hash=$(git rev-parse "refs/pull/${pr}") + case ${forge} in + github) + pr_repo="${PULL_REQUEST_REPO}" + status_script="set-pull-request-status.py" + ;; + codeberg) + pr_repo="https://codeberg.org/${CODEBERG_REPO}"; + status_script="set-codeberg-pull-request-status.py" + ;; + *) + echo "unknown forge ${forge}" + exit 1 + ;; + esac + "${SCRIPT_DIR}"/pull-request/"${status_script}" "${hash}" error \ "QA checks crashed. Please rebase and check profile changes for syntax errors." sendmail "${CRONJOB_ADMIN_MAIL}" <<-EOF - Subject: Pull request crash: ${iid} + Subject: Pull request crash: ${pr} To: <${CRONJOB_ADMIN_MAIL}> Content-Type: text/plain; charset=utf8 - It seems that pull request check for ${iid} crashed [1]. + It seems that pull request check for ${pr} crashed [1]. - [1]:${PULL_REQUEST_REPO}/pull/${iid} + [1]:${pr_repo}/pull/${prid} EOF rm -f -- "${pull}"/current-pr fi @@ -61,11 +77,24 @@ if [[ -n ${pr} ]]; then echo "${pr}" > "${pull}"/current-pr cd -- "${sync}" + if ! git remote | grep -q codeberg; then + git remote add codeberg ssh://[email protected]/gentoo/gentoo + fi ref=refs/pull/${pr} case ${forge} in - github) remote="origin" ;; - *) echo "unknown forge ${forge}"; exit 1 ;; + github) + remote="origin" + report_script="report-pull-request.py" + ;; + codeberg) + remote="codeberg" + report_script="report-codeberg-pull-request.py" + ;; + *) + echo "unknown forge ${forge}" + exit 1 + ;; esac git fetch -f "${remote}" "refs/pull/${prid}/head:${ref}" @@ -154,9 +183,16 @@ if [[ -n ${pr} ]]; then fi fi - "${SCRIPT_DIR}"/pull-request/report-pull-request.py "${prid}" "${pr_hash}" \ - "${pull}"/gentoo-ci/borked.list .pre-merge.borked "${hash}" - + case ${forge} in + github) + "${SCRIPT_DIR}"/pull-request/report-pull-request.py "${prid}" "${pr_hash}" \ + "${pull}"/gentoo-ci/borked.list .pre-merge.borked "${hash}" + ;; + codeberg) + "${SCRIPT_DIR}"/pull-request/report-codeberg-pull-request.py "${prid}" "${pr_hash}" \ + "${pull}"/gentoo-ci/borked.list .pre-merge.borked "${hash}" + ;; + esac rm -f -- "${pull}"/current-pr rm -rf -- "${pull}"/tmp "${pull}"/gentoo-ci diff --git a/pull-request/report-codeberg-pull-request.py b/pull-request/report-codeberg-pull-request.py index 02f116c..848b89f 100755 --- a/pull-request/report-codeberg-pull-request.py +++ b/pull-request/report-codeberg-pull-request.py @@ -38,69 +38,68 @@ def main(prid, prhash, borked_path, pre_borked_path, commit_hash): with open(CODEBERG_TOKEN_FILE) as f: token = f.read().strip() - cb = CodebergAPI(owner, repo, token) - - # delete old results - had_broken = False - old_comments = [] - # note: technically we could have multiple leftover comments - for co in cb.get_reviews(prid): - if co['user']['login'] == CODEBERG_USERNAME: - body = co['body'] - if 'All QA issues have been fixed' in body: - had_broken = False - elif 'has found no issues' in body: - had_broken = False - elif 'No issues found' in body: - had_broken = False - elif 'New issues' in body: - had_broken = True - elif 'Issues already there' in body: - had_broken = True - elif 'Issues inherited from Gentoo' in body: - had_broken = True - else: - # skip comments that don't look like CI results - continue - old_comments.append(co) - - for co in old_comments: - cb.delete_review(prid, co['id']) - - report_url = REPORT_URI_PREFIX + '/' + prhash + '/output.html' - body = f'''## Pull request CI report + with CodebergAPI(owner, repo, token) as cb: + # delete old results + had_broken = False + old_comments = [] + # note: technically we could have multiple leftover comments + for co in cb.get_reviews(prid): + if co['user']['login'] == CODEBERG_USERNAME: + body = co['body'] + if 'All QA issues have been fixed' in body: + had_broken = False + elif 'has found no issues' in body: + had_broken = False + elif 'No issues found' in body: + had_broken = False + elif 'New issues' in body: + had_broken = True + elif 'Issues already there' in body: + had_broken = True + elif 'Issues inherited from Gentoo' in body: + had_broken = True + else: + # skip comments that don't look like CI results + continue + old_comments.append(co) + + for co in old_comments: + cb.delete_review(prid, co['id']) + + report_url = REPORT_URI_PREFIX + '/' + prhash + '/output.html' + body = f'''## Pull request CI report *Report generated at*: {datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%d %H:%M UTC')} *Newest commit scanned*: {commit_hash} *Status*: {':x: **broken**' if borked else ':white_check_mark: good'} ''' - if borked or pre_borked: - if borked: - if too_many_borked: - body += '\nThere are too many broken packages to determine whether the breakages were added by the pull request. If in doubt, please rebase.\n\nIssues:' - else: - body += '\nNew issues caused by PR:\n' - for url in borked: - body += url - if pre_borked: - body += f'\nThere are existing issues already. Please look into the report to make sure none of them affect the packages in question:\n{report_url}s\n' - elif had_broken: - body += '\nAll QA issues have been fixed!\n' - else: - body += '\nNo issues found\n' - - cb.create_review(prid, body) + if borked or pre_borked: + if borked: + if too_many_borked: + body += '\nThere are too many broken packages to determine whether the breakages were added by the pull request. If in doubt, please rebase.\n\nIssues:' + else: + body += '\nNew issues caused by PR:\n' + for url in borked: + body += url + if pre_borked: + body += f'\nThere are existing issues already. Please look into the report to make sure none of them affect the packages in question:\n{report_url}s\n' + elif had_broken: + body += '\nAll QA issues have been fixed!\n' + else: + body += '\nNo issues found\n' + + cb.create_review(prid, body) - if borked: - cb.commit_set_status(commit_hash, 'failure', description='PR introduced new issues', - target_url=report_url, context='gentoo-ci') - elif pre_borked: - cb.commit_set_status(commit_hash, 'success', description='No new issues found', - target_url=report_url, context='gentoo-ci') - else: - cb.commit_set_status(commit_hash, 'success', description='All pkgcheck QA checks passed', - target_url=report_url, context='gentoo-ci') + if borked: + cb.commit_set_status(commit_hash, 'failure', description='PR introduced new issues', + target_url=report_url, context='gentoo-ci') + elif pre_borked: + cb.commit_set_status(commit_hash, 'success', description='No new issues found', + target_url=report_url, context='gentoo-ci') + else: + cb.commit_set_status(commit_hash, 'success', description='All pkgcheck QA checks passed', + target_url=report_url, context='gentoo-ci') if __name__ == '__main__': diff --git a/pull-request/scan-pull-requests.py b/pull-request/scan-pull-requests.py index 21c378e..6a85105 100755 --- a/pull-request/scan-pull-requests.py +++ b/pull-request/scan-pull-requests.py @@ -8,11 +8,98 @@ import errno import os import pickle import sys +from datetime import datetime import github +from codebergapi import CodebergAPI -def scan_github(db: dict): +def scan_codeberg(db: dict): + CODEBERG_USERNAME = os.environ["CODEBERG_USERNAME"] + CODEBERG_TOKEN_FILE = os.environ["CODEBERG_TOKEN_FILE"] + (owner, repo) = os.environ["CODEBERG_REPO"].split("/") + + with open(CODEBERG_TOKEN_FILE) as f: + token = f.read().strip() + + with CodebergAPI(owner, repo, token) as cb: + to_process = [] + for pr in cb.pulls(): + pr_key = f"codeberg/{pr['number']}" + sha = pr["head"]["sha"] + + # skip PRs marked noci + if any(x["name"] == "noci" for x in pr["labels"]): + print(f"{pr_key}: noci", file=sys.stderr) + + # if it made it to the cache, we probably need to wipe + # pending status + if pr_key in db: + statuses = cb.commit_statuses(sha) + for status in statuses: + # skip foreign statuses + if status["creator"]["login"] != CODEBERG_USERNAME: + continue + # if it's pending, mark it done + if status["status"] == "pending": + cb.commit_set_status( + sha, + "success", + description="Checks skipped due to [noci] label", + context="gentoo-ci", + ) + break + del db[pr_key] + + continue + + # if it's not cached, get its status + if pr_key not in db: + print(f"{pr_key}: updating status ...", file=sys.stderr) + statuses = cb.commit_statuses(sha) + for status in statuses: + # skip foreign statuses + if status["creator"]["login"] != CODEBERG_USERNAME: + continue + # if it's not pending, mark it done + if status["status"] == "pending": + db[pr_key] = "" + print(f"{pr_key}: found pending", file=sys.stderr) + else: + db[pr_key] = sha + print(f"{pr_key}: at {sha}", file=sys.stderr) + break + else: + db[pr_key] = "" + print(f"{pr_key}: unprocessed", file=sys.stderr) + + if db.get(pr_key, "") != sha: + to_process.append(pr) + + to_process = sorted( + to_process, + key=lambda x: ( + not any(x["name"] == "priority-ci" for x in pr["labels"]), + datetime.fromisoformat(x["updated_at"]), + ), + ) + queue = [] + for i, pr in enumerate(to_process): + pr_key = f"codeberg/{pr['number']}" + sha = pr["head"]["sha"] + if i == 0: + desc = "QA checks in progress..." + db[pr_key] = sha + else: + desc = "QA checks pending. Currently {}. in queue.".format(i) + cb.commit_set_status(sha, "pending", description=desc, context="gentoo-ci") + + print(f"{pr_key}: {db.get(pr_key, '') or '(none)'} -> {sha}", file=sys.stderr) + queue.append(pr_key) + return queue + + +def scan_github(db: dict, queue_len: int): """ Given a db of knowns PRs, inspect open PRs, update commit statuses, and update the db accordingly. Return a list of @@ -89,6 +176,7 @@ def scan_github(db: dict): x.updated_at, ), ) + queue = [] for i, pr in enumerate(to_process): pr_key = f"github/{pr.number}" db_key = pr.number if pr.number in db else pr_key @@ -97,14 +185,15 @@ def scan_github(db: dict): desc = "QA checks in progress..." db[db_key] = commit.sha else: - desc = f"QA checks pending. Currently {i}. in queue." + desc = f"QA checks pending. Currently {i+queue_len}. in queue." commit.create_status(context="gentoo-ci", state="pending", description=desc) print( f"{pr_key}: {db.get(db_key, '') or '(none)'} -> {pr.head.sha}", file=sys.stderr ) + queue.append(pr_key) - return to_process + return queue def main(): @@ -118,14 +207,15 @@ def main(): if e.errno != errno.ENOENT: raise - to_process = scan_github(db) + queue = scan_codeberg(db) + queue.extend(scan_github(db, len(queue))) with open(PULL_REQUEST_DB + ".tmp", "wb") as f: pickle.dump(db, f) os.rename(PULL_REQUEST_DB + ".tmp", PULL_REQUEST_DB) - if to_process: - print(f"github/{to_process[0].number}") + if queue: + print(queue[0]) return 0 diff --git a/pull-request/set-codeberg-pr-status.py b/pull-request/set-codeberg-pr-status.py index 233346d..0743b07 100755 --- a/pull-request/set-codeberg-pr-status.py +++ b/pull-request/set-codeberg-pr-status.py @@ -14,8 +14,8 @@ def main(commit_hash, stat, desc): with open(CODEBERG_TOKEN_FILE) as f: token = f.read().strip() - c = CodebergAPI(owner, repo, token) - c.commit_set_status(commit_hash, stat, description=desc, context='gentoo-ci') + with CodebergAPI(owner, repo, token) as cb: + cb.commit_set_status(commit_hash, stat, description=desc, context='gentoo-ci') if __name__ == '__main__':
