potiuk commented on code in PR #74823: URL: https://github.com/apache/airflow/pull/74823#discussion_r4304447183
########## dev/breeze/src/airflow_breeze/utils/ci_audit.py: ########## @@ -0,0 +1,291 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Checks behind `breeze ci audit`: is this checkout and fork set up for fork CI (AIP-120)? + +Every check is read-only. A failing check offers the commands that fix it; `--fix` runs them after +confirmation. See "breeze ci audit" in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +from packaging.version import Version + +from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, MIN_GH_VERSION +from airflow_breeze.utils.git_remotes import find_remote, find_remote_for_repo, parse_remotes + +if TYPE_CHECKING: + from airflow_breeze.utils.fork_ci_gate import GateConfig + +UPSTREAM_URL = f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git"; +CI_WORKFLOW_FILE = "ci-amd.yml" +# Commits a fork's main may lag behind upstream main before a sync is suggested. +MAX_FORK_MAIN_BEHIND = 200 + +Runner = Callable[[Sequence[str]], subprocess.CompletedProcess] + + +class CheckStatus(str, Enum): + OK = "ok" + FAIL = "fail" + INFO = "info" + + +class Role(str, Enum): + COMMITTER = "committer" + COLLABORATOR = "collaborator" + EXEMPTED = "exempted organisation member" + PILOT = "fork CI pilot participant" + EXTERNAL = "external contributor" + + +@dataclass(frozen=True) +class CheckResult: + number: int + name: str + status: CheckStatus + finding: str + fix: tuple[str, ...] = () + + +def _ok(result: subprocess.CompletedProcess) -> bool: + return result.returncode == 0 + + +def _out(result: subprocess.CompletedProcess) -> str: + return (result.stdout or "").strip() + + +def get_role(*, login: str, run: Runner, config: GateConfig, collaborators: frozenset[str]) -> Role: + membership = run(["gh", "api", "user/memberships/orgs/apache", "--jq", ".state"]) + if _ok(membership) and _out(membership) == "active": + return Role.COMMITTER + if login.casefold() in {c.casefold() for c in collaborators}: + return Role.COLLABORATOR + if config.is_allowlisted(login): + return Role.EXEMPTED + if config.pilot_joined_at(login) is not None: Review Comment: Right, the audit now follows the gate's order: a `[pilot]` entry makes you a pilot participant whatever your role, so a committer in the pilot is told to enable the workflow. The doc's check table says so. Fixed in 2296029eeb. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/src/airflow_breeze/commands/ci_commands.py: ########## @@ -1683,3 +1656,56 @@ def fork_ci_reconcile(config_path: Path | None, output_file: Path, run_url: str, if github_output: with open(github_output, "a") as f: f.write(get_ga_output("has-alerts", has_alerts) + "\n") + + +@ci_group.command( + name="audit", + help="Check that your checkout and fork are set up for fork CI (AIP-120), and offer the fixes.", +) [email protected]("--fix", is_flag=True, help="Run the offered fix commands, each after confirmation.") +@option_verbose +@option_dry_run +@option_answer +def audit(fix: bool): + import shlex + + import yaml + from rich.table import Table + + from airflow_breeze.utils.ci_audit import CheckStatus, run_audit + from airflow_breeze.utils.fork_ci_gate import load_config + + def run(command): + return subprocess.run(list(command), capture_output=True, text=True, check=False) Review Comment: Fixed in 2296029eeb. The audit runs commands through `run_tool`, which reports a missing executable as exit code 127 instead of raising, so the "`gh` is not installed" row is reached. There's a test with a non-existent command. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/src/airflow_breeze/utils/ci_audit.py: ########## @@ -0,0 +1,291 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Checks behind `breeze ci audit`: is this checkout and fork set up for fork CI (AIP-120)? + +Every check is read-only. A failing check offers the commands that fix it; `--fix` runs them after +confirmation. See "breeze ci audit" in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +from packaging.version import Version + +from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, MIN_GH_VERSION +from airflow_breeze.utils.git_remotes import find_remote, find_remote_for_repo, parse_remotes + +if TYPE_CHECKING: + from airflow_breeze.utils.fork_ci_gate import GateConfig + +UPSTREAM_URL = f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git"; +CI_WORKFLOW_FILE = "ci-amd.yml" +# Commits a fork's main may lag behind upstream main before a sync is suggested. +MAX_FORK_MAIN_BEHIND = 200 + +Runner = Callable[[Sequence[str]], subprocess.CompletedProcess] + + +class CheckStatus(str, Enum): + OK = "ok" + FAIL = "fail" + INFO = "info" + + +class Role(str, Enum): + COMMITTER = "committer" + COLLABORATOR = "collaborator" + EXEMPTED = "exempted organisation member" + PILOT = "fork CI pilot participant" + EXTERNAL = "external contributor" + + +@dataclass(frozen=True) +class CheckResult: + number: int + name: str + status: CheckStatus + finding: str + fix: tuple[str, ...] = () + + +def _ok(result: subprocess.CompletedProcess) -> bool: + return result.returncode == 0 + + +def _out(result: subprocess.CompletedProcess) -> str: + return (result.stdout or "").strip() + + +def get_role(*, login: str, run: Runner, config: GateConfig, collaborators: frozenset[str]) -> Role: + membership = run(["gh", "api", "user/memberships/orgs/apache", "--jq", ".state"]) + if _ok(membership) and _out(membership) == "active": + return Role.COMMITTER + if login.casefold() in {c.casefold() for c in collaborators}: + return Role.COLLABORATOR + if config.is_allowlisted(login): + return Role.EXEMPTED + if config.pilot_joined_at(login) is not None: + return Role.PILOT + return Role.EXTERNAL + + +def fork_ci_required(role: Role, config: GateConfig) -> bool | None: + """True when fork CI must be on, False when it must be off, None when either is fine.""" + if role == Role.PILOT: + return True + if role == Role.EXTERNAL: + return True if config.enabled else None + return False + + +def run_audit(*, run: Runner, config: GateConfig, collaborators: frozenset[str]) -> list[CheckResult]: + results: list[CheckResult] = [] + + gh_version = run(["gh", "--version"]) + if not _ok(gh_version): + results.append( + CheckResult( + 1, "gh", CheckStatus.FAIL, "`gh` is not installed", ("echo See https://cli.github.com";,) + ) + ) + return results + version_match = re.search(r"gh version (\d+\.\d+\.\d+)", _out(gh_version)) + if version_match is None or Version(version_match.group(1)) < Version(MIN_GH_VERSION): + results.append( + CheckResult(1, "gh", CheckStatus.FAIL, f"`gh` is older than {MIN_GH_VERSION}; upgrade it") + ) + return results + user = run(["gh", "api", "user", "--jq", ".login"]) + if not _ok(user): + results.append( + CheckResult(1, "gh", CheckStatus.FAIL, "`gh` is not authenticated", ("gh auth login",)) + ) + return results + login = _out(user) + results.append(CheckResult(1, "gh", CheckStatus.OK, f"authenticated as {login}")) + + remotes = parse_remotes(_out(run(["git", "remote", "-v"]))) + upstream = find_remote(remotes, "upstream") + apache = find_remote_for_repo(remotes, APACHE_AIRFLOW_GITHUB_REPOSITORY) + origin = find_remote(remotes, "origin") + if ( + upstream is not None + and upstream.repo + and upstream.repo.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY + ): + if origin is not None and origin.repo and origin.repo.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY: + results.append( + CheckResult( + 2, + "remotes", + CheckStatus.FAIL, + "`origin` points at apache/airflow; it should be your fork", + (f"git remote set-url origin https://github.com/{login}/airflow.git";,), + ) + ) + else: + results.append(CheckResult(2, "remotes", CheckStatus.OK, "`upstream` is apache/airflow")) + elif apache is not None and upstream is None: + fix = [f"git remote rename {apache.name} upstream"] + if apache.name == "origin": + fix.append(f"git remote add origin https://github.com/{login}/airflow.git";) + results.append( + CheckResult( + 2, + "remotes", + CheckStatus.FAIL, + f"apache/airflow is the remote `{apache.name}`; Airflow's convention is `upstream`", + tuple(fix), + ) + ) + else: + results.append( + CheckResult( + 2, + "remotes", + CheckStatus.FAIL, + "no `upstream` remote pointing at apache/airflow", + (f"git remote add upstream {UPSTREAM_URL}",), + ) + ) + + fork = origin.repo if origin is not None and origin.repo else None + fork_ok = False + if fork is None or fork.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY: + results.append( + CheckResult( + 3, + "fork", + CheckStatus.FAIL, + "no `origin` remote pointing at your fork of apache/airflow", + ( + f"gh repo fork {APACHE_AIRFLOW_GITHUB_REPOSITORY} --remote=false", + f"git remote add origin https://github.com/{login}/airflow.git";, Review Comment: Fixed in 2296029eeb. When check 2's fix already adds `origin`, check 3 only offers `gh repo fork ... --remote=false`. Test added. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/src/airflow_breeze/utils/git_remotes.py: ########## @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Git remotes of the local checkout, as `breeze ci upgrade` and `breeze ci audit` understand them.""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass + +from airflow_breeze.utils.run_utils import run_command + +GITHUB_REMOTE_URL = re.compile( + r"^(?:https://(?:[^@/]+@)?github\.com/|git@github\.com:|ssh://git@github\.com/)" Review Comment: Not intended, thanks. `GITHUB_REMOTE_URL` now accepts `http` as well as `https`, `www.`, a port, `ssh://` with a port and any case of the host, while still requiring the repository to be the whole path, so `apache/airflow-site` still doesn't match. Tests for each form. 2296029eeb. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
