This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch backport-89ed9cf-v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9e23ac220d818afe48dce58fde9ecf888980f716 Author: Ayush Meel <[email protected]> AuthorDate: Sun Nov 15 20:52:15 2026 +0530 Optimize fork CI change detection and conflict handling (#76955) When running CI on personal forks, GitHub's compare API caps file lists at 300, marking large documentation or typing changes as truncated and needlessly triggering full-matrix runs. Additionally, branches with merge conflicts against upstream run through the full test suite before anyone notices the branch cannot be merged. This adds a local git diff fallback using the merge base SHA to resolve full change sets when compare is truncated, hardens base selection to strictly fetch upstream refs with retries, switches to null-delimited diff parsing to handle spaces and renames safely, and adds early conflict detection with shallow clone deepening to fail fast on merge conflicts. (cherry picked from commit 89ed9cfc4ebbd071d7c28916d733cbe7e561a300) --- .github/workflows/ci-amd.yml | 40 ++++++ .github/workflows/ci-arm.yml | 40 ++++++ dev/breeze/doc/ci/04_selective_checks.md | 8 +- .../doc/ci/08_fork_ci_for_external_contributors.md | 12 +- .../src/airflow_breeze/commands/ci_commands.py | 3 +- .../src/airflow_breeze/utils/fork_compare.py | 152 +++++++++++++++++++-- .../src/airflow_breeze/utils/selective_checks.py | 15 ++ dev/breeze/tests/test_ci_commands.py | 26 ++++ dev/breeze/tests/test_fork_compare.py | 152 ++++++++++++++++++++- dev/breeze/tests/test_selective_checks.py | 78 +++++++++-- 10 files changed, 499 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 10cd2a85960..42f1a581049 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -397,6 +397,46 @@ jobs: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false + - name: "Check clean integration with upstream target branch" + if: github.event_name == 'push' && github.repository != 'apache/airflow' + shell: bash + run: | + target_branch=$(grep -E '^AIRFLOW_BRANCH\s*=\s*' dev/breeze/src/airflow_breeze/branch_defaults.py | sed -E 's/.*"([^"]+)".*/\1/' || echo "main") + target_branch="${target_branch:-main}" + fetched=0 + for i in 1 2 3; do + if git fetch --no-tags --depth=100 https://github.com/apache/airflow.git "${target_branch}:refs/remotes/upstream/${target_branch}"; then + fetched=1 + break + fi + sleep "$i" + done + if [ "${fetched}" -eq 1 ] && git rev-parse --verify "refs/remotes/upstream/${target_branch}" >/dev/null 2>&1; then + merge_base=$(git merge-base "refs/remotes/upstream/${target_branch}" HEAD 2>/dev/null || true) + step=200 + max_deepen=5 + iter=0 + while [ -z "${merge_base}" ] && [ "${iter}" -lt "${max_deepen}" ]; do + iter=$((iter + 1)) + git fetch --deepen="${step}" origin "${GITHUB_REF_NAME:-HEAD}" >/dev/null 2>&1 || true + git fetch --deepen="${step}" https://github.com/apache/airflow.git "${target_branch}:refs/remotes/upstream/${target_branch}" >/dev/null 2>&1 || true + merge_base=$(git merge-base "refs/remotes/upstream/${target_branch}" HEAD 2>/dev/null || true) + done + if [ -n "${merge_base}" ]; then + conflict=0 + git merge-tree --write-tree --merge-base "${merge_base}" HEAD "refs/remotes/upstream/${target_branch}" >/dev/null 2>&1 || conflict=$? + if [ "${conflict}" -eq 1 ]; then + echo "::error::Your branch has merge conflicts with apache/airflow:${target_branch}. Please rebase on upstream/${target_branch} before running fork CI." + exit 1 + elif [ "${conflict}" -gt 1 ]; then + echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + fi + else + echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." + fi + else + echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." + fi - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 0c877c17767..8d02e795524 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -273,6 +273,46 @@ jobs: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false + - name: "Check clean integration with upstream target branch" + if: github.event_name == 'push' && github.repository != 'apache/airflow' + shell: bash + run: | + target_branch=$(grep -E '^AIRFLOW_BRANCH\s*=\s*' dev/breeze/src/airflow_breeze/branch_defaults.py | sed -E 's/.*"([^"]+)".*/\1/' || echo "main") + target_branch="${target_branch:-main}" + fetched=0 + for i in 1 2 3; do + if git fetch --no-tags --depth=100 https://github.com/apache/airflow.git "${target_branch}:refs/remotes/upstream/${target_branch}"; then + fetched=1 + break + fi + sleep "$i" + done + if [ "${fetched}" -eq 1 ] && git rev-parse --verify "refs/remotes/upstream/${target_branch}" >/dev/null 2>&1; then + merge_base=$(git merge-base "refs/remotes/upstream/${target_branch}" HEAD 2>/dev/null || true) + step=200 + max_deepen=5 + iter=0 + while [ -z "${merge_base}" ] && [ "${iter}" -lt "${max_deepen}" ]; do + iter=$((iter + 1)) + git fetch --deepen="${step}" origin "${GITHUB_REF_NAME:-HEAD}" >/dev/null 2>&1 || true + git fetch --deepen="${step}" https://github.com/apache/airflow.git "${target_branch}:refs/remotes/upstream/${target_branch}" >/dev/null 2>&1 || true + merge_base=$(git merge-base "refs/remotes/upstream/${target_branch}" HEAD 2>/dev/null || true) + done + if [ -n "${merge_base}" ]; then + conflict=0 + git merge-tree --write-tree --merge-base "${merge_base}" HEAD "refs/remotes/upstream/${target_branch}" >/dev/null 2>&1 || conflict=$? + if [ "${conflict}" -eq 1 ]; then + echo "::error::Your branch has merge conflicts with apache/airflow:${target_branch}. Please rebase on upstream/${target_branch} before running fork CI." + exit 1 + elif [ "${conflict}" -gt 1 ]; then + echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + fi + else + echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." + fi + else + echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." + fi - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 20401351701..cad95fe3bb8 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -362,10 +362,12 @@ flowchart TD not from `git diff-tree`, with a renamed file counted under both paths, and the run is selected exactly like a pull request with those files. `is_committer_build` is always `False` because forks have no self-hosted runners. When the file list reaches the compare API's 300-file cap, - or the compare fails, everything runs, including the UI E2E tests, which otherwise only follow - matching files or the `full tests needed` label. Checks that compare a file with its previous version + or the compare fails, a local git diff fallback resolves the full change set against upstream to avoid + truncation; if local git diff is unavailable, the change set is treated as truncated and everything runs, + including the UI E2E tests, which otherwise only follow matching files or the `full tests needed` label. Checks that compare a file with its previous version (`pyproject.toml` and the provider dependency bump check) read the previous version at the - compare's merge base in `apache/airflow`, not at the pushed commit's parent, which after a + compare's merge base in `apache/airflow` (or directly from local git with `git show` when + the merge-base commit is present locally), not at the pushed commit's parent, which after a merge of upstream `main` is the contributor's own previous tip. A file missing at the merge base counts as new; any other failure to read it fails the selective checks instead of skipping the check. The labels come from the open diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index cfee5eae88a..a1d932971a5 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -455,14 +455,20 @@ upstream yields no changed files, so such a run performs basic checks only. of the pushed code, breeze's default branch at that commit (`main`, or the release branch it was cut from), because a branch name such as `backport-<sha>-v3-1-test` cannot be trusted to tell. A renamed file counts under both paths. The API caps the file list at 300; at the cap, - and when the compare fails, the run falls back to "run everything", UI E2E tests included, - rather than failing. + and when the compare fails, the run falls back to a local git diff against upstream to avoid + truncation; if local git diff is unavailable, it falls back to running everything rather than failing. - **Previous versions** of changed files, for the `pyproject.toml` and provider dependency bump checks, are read at the compare's merge base in `apache/airflow` - (`GET /repos/apache/airflow/contents/<path>?ref=<merge-base>`). The pushed commit's parent + (`GET /repos/apache/airflow/contents/<path>?ref=<merge-base>`), or directly from local git via + `git show` when the merge-base commit is present locally. The pushed commit's parent would be the contributor's own previous tip after they merge upstream `main`, so a release manager's bump from `main` would count as theirs. A file missing at the merge base is new; any other read failure fails the run rather than letting the check pass unseen. +- **Early conflict check**: before the test matrix runs, `ci-amd.yml` verifies that the pushed + commit merges cleanly with upstream's target branch using `git merge-tree`. It fetches the + upstream target branch and deepens history in increments of 200 commits up to 1,000 commits if + needed to resolve the merge base; if a merge conflict is detected, the workflow fails early with + clear rebase instructions rather than spending runner hours on conflicting code. - **Labels** come from the open `apache/airflow` pull request whose head is `<fork-owner>:<branch>`, since a push event carries none. `full tests needed`, `allow provider dependency bump`, `skip common compat check` and the other labels work as on diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index ed17d4d16d8..81ea81a5228 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -334,9 +334,10 @@ def selective_check( labels = tuple(ast.literal_eval(pr_labels)) if pr_labels else () fork_owner = github_repository.split("/")[0] if fork_push and github_context_dict.get("ref_name"): - labels = get_fork_pull_request_labels( + fetched_labels = get_fork_pull_request_labels( fork_owner=fork_owner, branch=github_context_dict["ref_name"], base=default_branch ) + labels = tuple(sorted(set(labels) | set(fetched_labels))) if fork_push and commit_ref is not None: fork_changes = get_fork_changes(fork_owner=fork_owner, sha=commit_ref, base=default_branch) changed_files = fork_changes.files diff --git a/dev/breeze/src/airflow_breeze/utils/fork_compare.py b/dev/breeze/src/airflow_breeze/utils/fork_compare.py index 5ced24ec5e1..6ebf3896eea 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_compare.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_compare.py @@ -19,11 +19,14 @@ from __future__ import annotations import json +import time from dataclasses import dataclass from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, GithubEvents from airflow_breeze.utils.console import console_print from airflow_breeze.utils.github import run_gh_command +from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH +from airflow_breeze.utils.run_utils import run_command # The compare API lists at most this many files: a list that reaches it may be missing some, so # the change set is treated as truncated. @@ -41,6 +44,114 @@ def is_fork_push(github_event: GithubEvents, github_repository: str) -> bool: return github_event == GithubEvents.PUSH and github_repository != APACHE_AIRFLOW_GITHUB_REPOSITORY +def _get_local_git_changes(*, base: str, sha: str, known_merge_base: str | None = None) -> ForkChanges | None: + merge_base = None + if known_merge_base: + has_commit = run_command( + ["git", "cat-file", "-e", f"{known_merge_base}^{{commit}}"], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if has_commit.returncode == 0: + merge_base = known_merge_base + else: + for attempt in range(3): + fetch_mb = run_command( + [ + "git", + "fetch", + "--no-tags", + "--depth=1", + f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git", + known_merge_base, + ], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if fetch_mb.returncode == 0: + merge_base = known_merge_base + break + if attempt < 2: + time.sleep(attempt + 1) + + if merge_base is None: + base_ref = f"refs/remotes/upstream/{base}" + fetched = False + for attempt in range(3): + fetch_res = run_command( + [ + "git", + "fetch", + "--no-tags", + "--depth=100", + f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git", + f"{base}:{base_ref}", + ], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if fetch_res.returncode == 0: + fetched = True + break + if attempt < 2: + time.sleep(attempt + 1) + + if not fetched: + verify_ref = run_command( + ["git", "rev-parse", "--verify", base_ref], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if verify_ref.returncode != 0: + return None + + mb_res = run_command( + ["git", "merge-base", base_ref, sha], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if mb_res.returncode != 0: + return None + merge_base = mb_res.stdout.strip() + + diff_res = run_command( + ["git", "diff", "-z", "-M", "--name-status", merge_base, sha], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if diff_res.returncode != 0: + return None + files: list[str] = [] + tokens = diff_res.stdout.split("\0") + if tokens and not tokens[-1]: + tokens.pop() + i = 0 + while i < len(tokens): + status = tokens[i] + if status.startswith(("R", "C")): + if i + 2 < len(tokens): + files.append(tokens[i + 1]) + files.append(tokens[i + 2]) + i += 3 + else: + if i + 1 < len(tokens): + files.append(tokens[i + 1]) + i += 2 + return ForkChanges(files=tuple(dict.fromkeys(files)), truncated=False, merge_base=merge_base) + + def get_fork_changes(*, fork_owner: str, sha: str, base: str) -> ForkChanges: """ Three-dot compare of the pushed commit against the upstream base branch. @@ -49,8 +160,8 @@ def get_fork_changes(*, fork_owner: str, sha: str, base: str) -> ForkChanges: so a fork whose ``main`` is behind upstream does not inflate the change set. ``base`` is the branch line the pushed code belongs to (breeze's default branch at that commit), which a branch name cannot be trusted to tell. A renamed file counts under both paths, as in - ``git diff``. When the compare fails the change set is reported as truncated, so the run - covers everything instead of failing. + ``git diff``. When the compare fails or reaches the API file cap, local git diff is used + as a fallback before falling back to truncated change set. ``merge_base`` is the upstream commit the changes are measured from. Checks that compare a file with its previous version use it instead of the pushed commit's parent, which after a @@ -68,18 +179,39 @@ def get_fork_changes(*, fork_owner: str, sha: str, base: str) -> ForkChanges: capture_output=True, text=True, ) - if result.returncode != 0: + compare_merge_base = None + if result.returncode == 0: + compare = json.loads(result.stdout or "{}") + entries = compare.get("files", []) + files = tuple(path for entry in entries for path in entry) + compare_merge_base = compare.get("merge_base") + if len(entries) < COMPARE_FILE_CAP: + return ForkChanges(files=files, truncated=False, merge_base=compare_merge_base) + console_print( + f"[warning]Compare API returned {len(entries)} files (capped at {COMPARE_FILE_CAP}), " + "falling back to local git diff to avoid truncation[/]" + ) + else: + files = () console_print( f"[warning]Could not compare {fork_owner}:{sha} with {APACHE_AIRFLOW_GITHUB_REPOSITORY}:{base}, " - f"so everything runs: {result.stderr.strip()}[/]" + f"trying local git diff: {result.stderr.strip()}[/]" ) - return ForkChanges(files=(), truncated=True) - compare = json.loads(result.stdout or "{}") - entries = compare.get("files", []) - files = tuple(path for entry in entries for path in entry) - return ForkChanges( - files=files, truncated=len(entries) >= COMPARE_FILE_CAP, merge_base=compare.get("merge_base") + + local_changes = _get_local_git_changes( + base=base, + sha=sha, + known_merge_base=compare_merge_base if result.returncode == 0 else None, ) + if local_changes is not None: + console_print( + f"[info]Local git diff resolved {len(local_changes.files)} changed files " + f"(merge base: {local_changes.merge_base})[/]" + ) + return local_changes + + console_print("[warning]Local git diff fallback unavailable, treating change set as truncated[/]") + return ForkChanges(files=files, truncated=True, merge_base=compare_merge_base) def get_fork_pull_request_labels(*, fork_owner: str, branch: str, base: str) -> tuple[str, ...]: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 2e6da054806..47e19b7e219 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1148,6 +1148,21 @@ class SelectiveChecks: def _get_previous_file_content(self, path: str) -> str | None: if self._fork_merge_base: + commit_check = run_command( + ["git", "cat-file", "-e", f"{self._fork_merge_base}^{{commit}}"], + capture_output=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + if commit_check.returncode == 0: + result = run_command( + ["git", "show", f"{self._fork_merge_base}:{path}"], + capture_output=True, + text=True, + cwd=AIRFLOW_ROOT_PATH, + check=False, + ) + return result.stdout if result.returncode == 0 else None return get_upstream_file(path=path, ref=self._fork_merge_base) result = run_command( ["git", "show", f"{self._commit_ref}^:{path}"], diff --git a/dev/breeze/tests/test_ci_commands.py b/dev/breeze/tests/test_ci_commands.py index ca64a30b0b1..56eb7e77443 100644 --- a/dev/breeze/tests/test_ci_commands.py +++ b/dev/breeze/tests/test_ci_commands.py @@ -23,6 +23,7 @@ from unittest.mock import patch import pytest from click.testing import CliRunner +import airflow_breeze.utils.selective_checks # noqa: F401 from airflow_breeze.commands.ci_commands import get_changed_files_since, selective_check from airflow_breeze.utils.fork_compare import ForkChanges @@ -152,6 +153,31 @@ def test_fork_push_takes_changed_files_from_compare( assert kwargs["pr_labels"] == ("full tests needed",) +@patch("airflow_breeze.commands.ci_commands.get_fork_pull_request_labels", autospec=True) +@patch("airflow_breeze.commands.ci_commands.get_changed_files", autospec=True) +@patch("airflow_breeze.commands.ci_commands.get_fork_changes", autospec=True) +@patch("airflow_breeze.utils.selective_checks.SelectiveChecks", autospec=True) +def test_fork_push_merges_passed_labels_with_fetched_labels( + mock_selective_checks, mock_fork_changes, mock_changed_files, mock_labels +): + mock_fork_changes.return_value = ForkChanges(files=("README.md",), truncated=False, merge_base="base-sha") + mock_labels.return_value = ("full tests needed",) + + _invoke_selective_check( + "--github-repository", + "someone/airflow", + "--github-context", + json.dumps({"ref_name": "backport-abc-v3-1-test"}), + "--default-branch", + "v3-1-test", + "--pr-labels", + "['allow provider dependency bump']", + ) + + kwargs = mock_selective_checks.call_args.kwargs + assert kwargs["pr_labels"] == ("allow provider dependency bump", "full tests needed") + + @patch("airflow_breeze.commands.ci_commands.get_fork_pull_request_labels", autospec=True) @patch("airflow_breeze.commands.ci_commands.get_changed_files", autospec=True, return_value=("README.md",)) @patch("airflow_breeze.commands.ci_commands.get_fork_changes", autospec=True) diff --git a/dev/breeze/tests/test_fork_compare.py b/dev/breeze/tests/test_fork_compare.py index 4d745b84a95..ebee101c8b0 100644 --- a/dev/breeze/tests/test_fork_compare.py +++ b/dev/breeze/tests/test_fork_compare.py @@ -27,6 +27,7 @@ from airflow_breeze.utils.fork_compare import ( COMPARE_FILE_CAP, ForkChanges, UpstreamFileError, + _get_local_git_changes, get_fork_changes, get_fork_pull_request_labels, get_upstream_file, @@ -46,6 +47,7 @@ def test_is_fork_push(event, repository, expected): assert is_fork_push(event, repository) is expected [email protected]("airflow_breeze.utils.fork_compare._get_local_git_changes", autospec=True) @mock.patch("airflow_breeze.utils.fork_compare.run_gh_command", autospec=True) @pytest.mark.parametrize( ("files", "expected"), @@ -64,7 +66,8 @@ def test_is_fork_push(event, repository, expected): ), ], ) -def test_get_fork_changes(mock_gh, files, expected): +def test_get_fork_changes(mock_gh, mock_local_git, files, expected): + mock_local_git.return_value = None mock_gh.return_value = subprocess.CompletedProcess( [], 0, stdout=json.dumps({"files": files, "merge_base": None}), stderr="" ) @@ -74,8 +77,10 @@ def test_get_fork_changes(mock_gh, files, expected): assert mock_gh.call_args.args[0][:3] == ["gh", "api", "repos/apache/airflow/compare/main...someone:abc"] [email protected]("airflow_breeze.utils.fork_compare._get_local_git_changes", autospec=True) @mock.patch("airflow_breeze.utils.fork_compare.run_gh_command", autospec=True) -def test_get_fork_changes_runs_everything_when_the_compare_fails(mock_gh): +def test_get_fork_changes_runs_everything_when_the_compare_fails(mock_gh, mock_local_git): + mock_local_git.return_value = None mock_gh.return_value = subprocess.CompletedProcess([], 1, stdout="", stderr="Not Found") assert get_fork_changes(fork_owner="someone", sha="abc", base="main") == ForkChanges( @@ -92,6 +97,149 @@ def test_get_fork_changes_returns_the_merge_base(mock_gh): assert get_fork_changes(fork_owner="someone", sha="abc", base="main").merge_base == "base-sha" [email protected]("airflow_breeze.utils.fork_compare._get_local_git_changes", autospec=True) [email protected]("airflow_breeze.utils.fork_compare.run_gh_command", autospec=True) +def test_get_fork_changes_falls_back_to_local_git_at_cap(mock_gh, mock_local_git): + mock_gh.return_value = subprocess.CompletedProcess( + [], + 0, + stdout=json.dumps( + {"files": [[f"f{i}.py"] for i in range(COMPARE_FILE_CAP)], "merge_base": "base-sha"} + ), + stderr="", + ) + mock_local_git.return_value = ForkChanges( + files=tuple(f"f{i}.py" for i in range(350)), truncated=False, merge_base="local-mb-sha" + ) + + result = get_fork_changes(fork_owner="someone", sha="abc", base="main") + + assert result == ForkChanges( + files=tuple(f"f{i}.py" for i in range(350)), truncated=False, merge_base="local-mb-sha" + ) + mock_local_git.assert_called_once_with(base="main", sha="abc", known_merge_base="base-sha") + + [email protected]("airflow_breeze.utils.fork_compare._get_local_git_changes", autospec=True) [email protected]("airflow_breeze.utils.fork_compare.run_gh_command", autospec=True) +def test_get_fork_changes_falls_back_to_local_git_when_gh_fails(mock_gh, mock_local_git): + mock_gh.return_value = subprocess.CompletedProcess([], 1, stdout="", stderr="API rate limit exceeded") + mock_local_git.return_value = ForkChanges( + files=("file1.py", "file2.py"), truncated=False, merge_base="local-mb-sha" + ) + + result = get_fork_changes(fork_owner="someone", sha="abc", base="main") + + assert result == ForkChanges(files=("file1.py", "file2.py"), truncated=False, merge_base="local-mb-sha") + mock_local_git.assert_called_once_with(base="main", sha="abc", known_merge_base=None) + + [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_uses_known_merge_base(mock_run_cmd): + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 0, stdout="M\0foo.py\0", stderr=""), + ] + + result = _get_local_git_changes(base="main", sha="head-sha", known_merge_base="known-mb") + + assert result == ForkChanges(files=("foo.py",), truncated=False, merge_base="known-mb") + + [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_success(mock_run_cmd): + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 0, stdout="mb-sha-123\n", stderr=""), + subprocess.CompletedProcess([], 0, stdout="M\0foo.py\0R100\0old.py\0new.py\0", stderr=""), + ] + + result = _get_local_git_changes(base="main", sha="head-sha") + + assert result == ForkChanges( + files=("foo.py", "old.py", "new.py"), truncated=False, merge_base="mb-sha-123" + ) + + [email protected]("time.sleep", autospec=True) [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_retries_fetch(mock_run_cmd, mock_sleep): + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 1, stdout="", stderr="transient network error"), + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 0, stdout="mb-sha\n", stderr=""), + subprocess.CompletedProcess([], 0, stdout="M\0bar.py\0", stderr=""), + ] + + result = _get_local_git_changes(base="main", sha="head-sha") + + assert result == ForkChanges(files=("bar.py",), truncated=False, merge_base="mb-sha") + mock_sleep.assert_called_once_with(1) + + [email protected]("time.sleep", autospec=True) [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_falls_back_to_existing_upstream_ref(mock_run_cmd, mock_sleep): + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 1, stdout="", stderr="network error 1"), + subprocess.CompletedProcess([], 1, stdout="", stderr="network error 2"), + subprocess.CompletedProcess([], 1, stdout="", stderr="network error 3"), + subprocess.CompletedProcess([], 0, stdout="refs/remotes/upstream/main\n", stderr=""), + subprocess.CompletedProcess([], 0, stdout="mb-sha\n", stderr=""), + subprocess.CompletedProcess([], 0, stdout="M\0cached.py\0", stderr=""), + ] + + result = _get_local_git_changes(base="main", sha="head-sha") + + assert result == ForkChanges(files=("cached.py",), truncated=False, merge_base="mb-sha") + assert mock_sleep.call_count == 2 + + [email protected]("time.sleep", autospec=True) [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_returns_none_when_fetch_and_ref_fail(mock_run_cmd, mock_sleep): + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 1, stdout="", stderr="error 1"), + subprocess.CompletedProcess([], 1, stdout="", stderr="error 2"), + subprocess.CompletedProcess([], 1, stdout="", stderr="error 3"), + subprocess.CompletedProcess([], 1, stdout="", stderr="ref missing"), + ] + + result = _get_local_git_changes(base="main", sha="head-sha") + + assert result is None + + [email protected]("airflow_breeze.utils.fork_compare.run_command", autospec=True) +def test_get_local_git_changes_parses_null_delimited_with_spaces_and_copies(mock_run_cmd): + diff_output = ( + "M\0path with spaces/file 1.py\0" + "C100\0source path/original.py\0dest path/copy.py\0" + "R090\0old dir/rename.py\0new dir/renamed.py\0" + "D\0deleted.py\0" + ) + mock_run_cmd.side_effect = [ + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 0, stdout="mb-sha\n", stderr=""), + subprocess.CompletedProcess([], 0, stdout=diff_output, stderr=""), + ] + + result = _get_local_git_changes(base="main", sha="head-sha") + + assert result == ForkChanges( + files=( + "path with spaces/file 1.py", + "source path/original.py", + "dest path/copy.py", + "old dir/rename.py", + "new dir/renamed.py", + "deleted.py", + ), + truncated=False, + merge_base="mb-sha", + ) + + @mock.patch("airflow_breeze.utils.fork_compare.run_gh_command", autospec=True) @pytest.mark.parametrize( ("returncode", "stdout", "expected"), diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index a7069398b78..5dc9dfedb5a 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -45,6 +45,7 @@ from airflow_breeze.global_constants import ( get_airflow_version, get_provider_split_test_types, ) +from airflow_breeze.utils.fork_compare import UpstreamFileError from airflow_breeze.utils.functools_cache import clearable_cache from airflow_breeze.utils.packages import get_available_distributions from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH @@ -4657,14 +4658,20 @@ def test_provider_dependency_bump_on_a_fork_push_compares_with_the_merge_base( so a release manager's bump from main would look like theirs. """ bumped = '[project]\ndependencies = ["apache-airflow-providers-common-sql>=1.1.0"]\n' - mock_run_command.side_effect = lambda *args, **kwargs: Mock( - returncode=0, - stdout=( - '[project]\ndependencies = ["apache-airflow-providers-common-sql>=1.0.0"]\n' - if "^:" in args[0][2] - else bumped - ), - ) + + def mock_cmd(*args, **kwargs): + if len(args[0]) > 1 and args[0][1] == "cat-file": + return Mock(returncode=1) + return Mock( + returncode=0, + stdout=( + '[project]\ndependencies = ["apache-airflow-providers-common-sql>=1.0.0"]\n' + if len(args[0]) > 2 and "^:" in args[0][2] + else bumped + ), + ) + + mock_run_command.side_effect = mock_cmd mock_upstream_file.return_value = bumped assert not SelectiveChecks( @@ -4680,6 +4687,61 @@ def test_provider_dependency_bump_on_a_fork_push_compares_with_the_merge_base( mock_upstream_file.assert_called_once_with(path="providers/amazon/pyproject.toml", ref="base-sha") +@patch("airflow_breeze.utils.selective_checks.get_upstream_file", autospec=True) +@patch("airflow_breeze.utils.selective_checks.run_command") +def test_provider_dependency_bump_on_a_fork_push_reads_from_local_git_when_merge_base_present( + mock_run_command, mock_upstream_file +): + bumped = '[project]\ndependencies = ["apache-airflow-providers-common-sql>=1.1.0"]\n' + + def mock_cmd(*args, **kwargs): + if len(args[0]) > 1 and args[0][1] == "cat-file": + return Mock(returncode=0) + return Mock(returncode=0, stdout=bumped) + + mock_run_command.side_effect = mock_cmd + + assert not SelectiveChecks( + files=("providers/amazon/pyproject.toml",), + commit_ref=NEUTRAL_COMMIT, + pr_labels=(), + github_event=GithubEvents.PUSH, + default_branch="main", + fork_push=True, + fork_merge_base="base-sha", + ).provider_dependency_bump + + mock_upstream_file.assert_not_called() + + +@patch("airflow_breeze.utils.selective_checks.get_upstream_file", autospec=True) +@patch("airflow_breeze.utils.selective_checks.run_command") +def test_pyproject_toml_changed_on_fork_push_with_local_git_fallback_does_not_call_api( + mock_run_command, mock_upstream_file +): + mock_upstream_file.side_effect = UpstreamFileError("API rate limit exceeded") + old_content = '[project]\nname = "apache-airflow"\n' + + def mock_cmd(*args, **kwargs): + if len(args[0]) > 1 and args[0][1] == "cat-file": + return Mock(returncode=0) + return Mock(returncode=0, stdout=old_content) + + mock_run_command.side_effect = mock_cmd + + checks = SelectiveChecks( + files=("pyproject.toml",), + commit_ref=NEUTRAL_COMMIT, + pr_labels=(), + github_event=GithubEvents.PUSH, + default_branch="main", + fork_push=True, + fork_merge_base="local-mb-sha", + ) + assert checks.pyproject_toml_changed + mock_upstream_file.assert_not_called() + + @patch("airflow_breeze.utils.selective_checks.run_command") def test_provider_dependency_bump_check_skipped_on_release_branch(mock_run_command): """Test that provider dependency bump check is a no-op on release branches (v3-X-test)."""
