This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fork-ci-status-latency-and-detail in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6eb5a48923bfac71e74784c14fcdfaa80ce7e39e Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Nov 10 05:46:00 2026 -0700 Run the fork CI monitor every 5 minutes Fork CI reaches a pull request only when the monitor runs: the fork run is in another repository, so no event in apache/airflow reports its start or end. At a 15-minute cadence plus GitHub's delay of scheduled workflows, the start, failure or success of a fork run showed up on the pull request 10 to 20 minutes late, and so did `CI run started`, which every pull request waits for before it can merge. The monitor now runs every 5 minutes. The per-run write budgets and the number of runs before a Slack note are derived from MONITOR_INTERVAL, so their hourly totals stay as they were; a test checks that the interval matches the workflow's cron. Unchanged pull requests cost no writes; the cost is about three times the monitor's runner time and reads. Generated-by: Claude Opus 5 --- .github/workflows/fork-ci-monitor.yml | 5 +- contributing-docs/05_pull_requests.rst | 2 +- dev/AGENTS.md | 4 +- dev/breeze/doc/08_ci_tasks.rst | 2 +- .../doc/ci/08_fork_ci_for_external_contributors.md | 56 +++++++++++++--------- .../src/airflow_breeze/utils/fork_ci_reconcile.py | 27 +++++++---- dev/breeze/tests/test_fork_ci_reconcile.py | 44 ++++++++++++----- 7 files changed, 91 insertions(+), 49 deletions(-) diff --git a/.github/workflows/fork-ci-monitor.yml b/.github/workflows/fork-ci-monitor.yml index e95becc9e27..a2d09f59fb0 100644 --- a/.github/workflows/fork-ci-monitor.yml +++ b/.github/workflows/fork-ci-monitor.yml @@ -27,7 +27,8 @@ name: "Fork CI monitor" on: # yamllint disable-line rule:truthy schedule: - - cron: '*/15 * * * *' + # MONITOR_INTERVAL in fork_ci_reconcile.py, whose per-run write budgets assume this cadence. + - cron: '*/5 * * * *' workflow_dispatch: inputs: dry-run: @@ -59,7 +60,7 @@ jobs: timeout-minutes: 20 steps: # What earlier runs reported, so that a problem that persists is posted to Slack once - # rather than every 15 minutes. Losing it, for example to cache eviction, re-posts once. + # rather than on every run. Losing it, for example to cache eviction, re-posts once. # Restored before anything that can fail, so a failing setup is reported once too, and # outside the checkout, where nothing but the steps below reads it. - name: "Restore what earlier runs reported" diff --git a/contributing-docs/05_pull_requests.rst b/contributing-docs/05_pull_requests.rst index 7320241b18c..d927b218f7c 100644 --- a/contributing-docs/05_pull_requests.rst +++ b/contributing-docs/05_pull_requests.rst @@ -272,7 +272,7 @@ and approves them automatically when all of these hold: If your PR's CI runs in your fork (it carries the ``fork ci required`` label), the CI run in ``apache/airflow`` only checks that, so none of the rules above apply. The ``CI run started`` check -waits until the fork CI monitor sees your fork's run, usually within 15 minutes, and then links it. Its runs are approved +waits until the fork CI monitor sees your fork's run, usually within 5 to 10 minutes, and then links it. Its runs are approved automatically unless the PR changes files that decide which CI jobs run and on which runners: anything under ``.github/``, ``dev/breeze/src/`` or ``scripts/ci/prek/``, or ``dev/breeze/pyproject.toml``, ``dev/breeze/uv.lock`` or ``scripts/ci/install_breeze.sh``. Then diff --git a/dev/AGENTS.md b/dev/AGENTS.md index f9e999acfc4..27e8736208b 100644 --- a/dev/AGENTS.md +++ b/dev/AGENTS.md @@ -54,9 +54,9 @@ workflow of `apache/airflow` shares (see "Write budget" in If the budget is exceeded, turn the job statuses off: set `job_statuses = false` in `.github/fork-ci-gate.toml`. Signs that it is exceeded are the monitor's Slack note that job -statuses "have not fit in the budget" (posted only after three runs in a row deferred them) +statuses "have not fit in the budget" (posted only after about 45 minutes of runs in a row deferred them) staying for hours, pull requests listed because their fork run finished more than an hour ago with -job statuses still not posted, its Slack note that GitHub rate-limited it in two runs in a row, the +job statuses still not posted, its Slack note that GitHub rate-limited it in runs in a row for about 30 minutes, the "Rate limits left" line of its run log approaching zero, or other workflows of the repository failing on secondary rate limits. The aggregate `Fork CI / Tests (AMD)` status and `CI passed` are posted either way, so turning job statuses off loses detail, not gating. diff --git a/dev/breeze/doc/08_ci_tasks.rst b/dev/breeze/doc/08_ci_tasks.rst index 03d61fed826..c4001b658c0 100644 --- a/dev/breeze/doc/08_ci_tasks.rst +++ b/dev/breeze/doc/08_ci_tasks.rst @@ -185,7 +185,7 @@ Reconciling pull requests gated to fork CI ------------------------------------------ Pull requests from external contributors can run their CI in the contributor's fork instead of in -``apache/airflow`` (AIP-120). The ``fork-ci-reconcile`` command, run every 15 minutes by the +``apache/airflow`` (AIP-120). The ``fork-ci-reconcile`` command, run every 5 minutes by the ``fork-ci-monitor.yml`` workflow, is the only place that changes those pull requests: it labels newly gated pull requests ``fork ci required``, keeps them in draft until the fork run is green, maintains one state comment and a ``Fork CI / Tests (AMD)`` commit status linking to the run in the fork, closes diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index 3014ab1c345..1e1a304e6d6 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -87,7 +87,7 @@ Non-goals: │ ├─ build-info: fork mode │ │ (build-info and everything after │ │ │ changed files from │ │ are skipped for gated PRs) │ │ │ compare API vs upstream │ │ │ - │ └─ public runners only │ │ fork-ci-monitor.yml (every 15 minutes) │ + │ └─ public runners only │ │ fork-ci-monitor.yml (every 5 minutes) │ └──────────────────────────────────┘ │ └─ breeze ci fork-ci-reconcile │ ▲ │ ├─ discover new gated PRs -> label │ │ GraphQL: checkSuites │ ├─ per labelled PR: fork state -> │ @@ -488,7 +488,7 @@ reads too: the default branch's comes from the reconciler's checkout, any other from the contents API once per run. Those with a gated reason get the label. A base branch with neither gate file is never gated, as its gate job runs project CI (`base_without_gate`), and one with only one of the two files is reported as unknown. The -24-hour window overlaps many 15-minute cadences, so a lost scheduled run changes nothing. A +24-hour window overlaps many monitor cadences, so a lost scheduled run changes nothing. A pull request whose author later becomes a committer keeps the label until a maintainer removes it or adds `use project ci`; the reconciler does not un-gate on its own. @@ -722,7 +722,7 @@ every event that runs the workflow files of a pull request's merge commit: `pull requests by head repository and branch. Looking at every recent run of the branch, rather than only at the head commit, is what stops a contributor from pushing a tampered commit and then a clean one to hide the tampered run. The 6-hour window spans many -15-minute cadences. Runs created before the most recent removal of `use project ci` are +monitor cadences. Runs created before the most recent removal of `use project ci` are ignored, because they ran legitimately while the label was on. So are runs created before the most recent reopen of the pull request by someone other than its author, which is how a maintainer brings back a pull request closed for tampering; reopening one whose gate is still @@ -920,7 +920,7 @@ suites); its jobs come from `GET /repos/<fork>/actions/runs/<id>/jobs?filter=lat - Only finished jobs are reported: a successful one as `success`, and a failed, timed-out, cancelled or unstartable one as `failure`. Queued and running jobs are not, so each job costs one write instead of two; `Fork CI / Tests (AMD)` shows the run as running, linking it, and with a - 15-minute cadence a per-job "running" status was behind the fork's own run page anyway. A job of + 5-minute cadence a per-job "running" status was behind the fork's own run page anyway. A job of a re-run in the fork keeps the result of the earlier attempt until it finishes. Skipped and neutral jobs, and the jobs every run has (`Fork CI gate`, `Check if the run is needed`, `Platform: AMD` and `CI passed`), are not reported. @@ -928,12 +928,12 @@ suites); its jobs come from `GET /repos/<fork>/actions/runs/<id>/jobs?filter=lat (`GET /repos/apache/airflow/commits/<sha>/status`), and only differences are posted, failures first. GraphQL has no mutation for commit statuses (`createCheckRun` exists, but see above for why check runs do not fit), so each one is a REST call, and they count towards GitHub's limits on - content-creating requests: a reconciler run posts at most `MAX_JOB_STATUSES_PER_RUN` (40) of + content-creating requests: a reconciler run posts at most `MAX_JOB_STATUSES_PER_RUN` (13) of them, each write charged as it is made. What does not fit is posted by the next run, and once the budget is spent no further pull request's jobs or statuses are read in that run. One fork run of the full matrix (about 130 jobs) needs more than a run's budget on its own, which the next run catches up on, so deferring alone is not a problem: the run log lists the deferred pull - requests, and only when three runs in a row deferred, about 45 minutes of the budget not + requests, and only when nine runs in a row deferred, about 45 minutes of the budget not keeping up, is a note added to the Slack message. The count is kept in the record the monitor carries between runs (`--reported-file`); without a readable record the note is added at once. Its wording is fixed, so it is posted once while deferrals persist. A pull request whose fork run @@ -953,7 +953,7 @@ suites); its jobs come from `GET /repos/<fork>/actions/runs/<id>/jobs?filter=lat classified, so for example a `CI passed` that no longer holds is still revoked. Reading a fork run's jobs fails now and then on GitHub server errors (HTTP 502, a reset HTTP/2 stream) that the next run gets past, so each failure is only logged, and the pull request is listed with the - unknown items for Slack once its job statuses failed in three runs in a row. The per pull request + unknown items for Slack once its job statuses failed in nine runs in a row, about 45 minutes. The per pull request count is kept in the same record as that of deferred job statuses; without a readable record it starts again. The item's wording is fixed whatever the error, so it is posted once while the failure persists, and the run log has each run's error. @@ -968,7 +968,7 @@ records this for agents. Each run logs what is left of the token's REST and GraphQL quotas (`GET /rate_limit`, which does not count against them) and how many of its calls GitHub rate-limited; the client retries those with -backoff. Rate limiting in two runs in a row adds a note to the Slack message, with fixed wording so +backoff. Rate limiting in six runs in a row, about 30 minutes, adds a note to the Slack message, with fixed wording so it is posted once while it persists; the count is kept in the same record as that of deferred job statuses. @@ -997,14 +997,23 @@ requests a minute and 500 an hour. Assuming about 35 reported jobs for an averag monitor runs. - **Gate enabled**, assuming about half of the pull request runs come from gated pull requests (about 5-6 pushes an hour): about 200 writes an hour without a cap, 40% of the content-creating - limit and more at peaks. The cap of 40 writes a run, four runs an hour, holds job statuses to at - most 160 an hour: 32% of the content-creating limit, leaving the rest to the reconciler's other + limit and more at peaks. The cap of 13 writes a run, twelve runs an hour, holds job statuses to at + most 156 an hour: 31% of the content-creating limit, leaving the rest to the reconciler's other writes (labels, comments, the aggregate statuses) and to the other workflows that share it. Each - run posts at most 40 statuses at about one a second, well under the 80-a-minute limit. Job + run posts at most 13 statuses at about one a second, well under the 80-a-minute limit. Job statuses then lag behind at busy times, the aggregate status does not. The `CI run started` - statuses of pull requests whose CI runs in `apache/airflow` come out of the same 40 writes a run, - first and at most 20 of them (see "The `CI run started` check"), so they do not raise the total; - at about 11 pull request runs an hour they take about 3 writes a run. + statuses of pull requests whose CI runs in `apache/airflow` come out of the same 13 writes a run, + first and at most 6 of them (see "The `CI run started` check"), so they do not raise the total; + at about 11 pull request runs an hour they take about 1 write a run. + +The per-run caps and the run counts before a Slack note are derived from `MONITOR_INTERVAL` in +`fork_ci_reconcile.py`, which must match the monitor's cron (a test checks it), so changing the +cadence keeps the hourly totals above. The cadence was lowered from 15 to 5 minutes because fork CI +reaches a pull request only when the monitor runs: a fork run's start, failure or success showed up +to 15 minutes late, plus GitHub's delay of scheduled workflows. Unchanged pull requests cost no +writes, so the faster cadence adds reads, about 45 a run, well within the token's quotas, and +runner time: each run takes about two minutes on a GitHub-hosted runner, mostly installing +Breeze, so the monitor uses about 24 runner minutes an hour instead of 8. These are estimates: the share of gated runs, the jobs per push and the pushes per pull request are assumptions, not measurements. Watch the monitor's call counts in its run log after enabling @@ -1043,7 +1052,7 @@ is the source the ruleset accepts. ### The `CI run started` check -Fork CI reports to the pull request only when the scheduled monitor runs, about every 15 minutes +Fork CI reports to the pull request only when the scheduled monitor runs, about every 5 minutes and later when GitHub delays scheduled workflows, and until then a gated pull request shows only skipped checks. The gate cannot say more itself: it runs on the contributor's `pull_request` event with a read-only token. Instead both CI rulesets in `.asf.yaml` also require a @@ -1069,8 +1078,8 @@ with state `success` linking the run: Folding the check into the scheduled monitor costs no runner per push, at the price of a pull request waiting for the next monitor run before it can merge, which a CI run takes longer than anyway. Each new head commit needs one write, taken first out of the per-run budget of job -statuses (see "Job statuses"), since merging waits for it; at most 20 such writes per run, so job -statuses keep at least half of the budget, and after the check became required the open pull +statuses (see "Job statuses"), since merging waits for it; at most `MAX_CI_RUN_STARTED_PER_RUN` (6) such +writes per run, so job statuses keep at least half of the budget, and after the check became required the open pull requests are caught up a batch at a time. A release branch has no monitor of its own, but `main`'s monitor covers pull requests to every base branch. @@ -1154,14 +1163,15 @@ change. Job statuses add a jobs call and a status call per pull request with a f progress or a completed run not yet reported, and up to two status mutations per job, at most `MAX_JOB_STATUSES_PER_RUN` per run; see "Write budget". Pull requests that pass 1 labelled and the search does not list yet add one GraphQL call per 25. Well inside -the `GITHUB_TOKEN` limits at a 15-minute cadence. The reconciler logs its call counts so the +the `GITHUB_TOKEN` limits at a 5-minute cadence. The reconciler logs its call counts so the budget stays observable. ## Monitor workflow `.github/workflows/fork-ci-monitor.yml`: -- `on: schedule: cron '*/15 * * * *'` and `workflow_dispatch` with a `dry-run` input. +- `on: schedule: cron '*/5 * * * *'` (`MONITOR_INTERVAL`) and `workflow_dispatch` with a `dry-run` + input. - Two jobs, each with its own `permissions` block and a justification comment; the workflow grants nothing by default. - The `reconcile` job: `actions: write, contents: read, pull-requests: write, statuses: write`. @@ -1333,9 +1343,11 @@ as a failed check rather than a crash. ## Known limitations -- Drafting, the setup comment and the commit status arrive within one cadence of the event - they reflect, not instantly. The cadence can be lowered to 5 minutes without changing the - budget analysis, since unchanged pull requests cost no mutations. +- Drafting, the setup comment and the commit status arrive within one cadence (5 minutes) of the + event they reflect, plus GitHub's delay of scheduled workflows, not instantly. The fork runs in + another repository, so no event in `apache/airflow` reports its start or end; a `workflow_run` + trigger on project CI's gate run would only cover a push, at the cost of a monitor run per + pull request push. - The workflows API lists workflows from the fork's default branch. A fork whose `main` predates `ci-amd.yml` reports it missing; the setup comment covers this with `gh repo sync`. diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index 1e6c9e0e211..b308ad08b87 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -127,8 +127,19 @@ BOT_LOGINS = frozenset({"github-actions", "github-actions[bot]"}) DISCOVERY_WINDOW = timedelta(hours=24) PASS_TWO_PAGE_SIZE = 25 FORK_BATCH_SIZE = 50 +# The cron interval of fork-ci-monitor.yml. Fork CI reaches a pull request only when the monitor +# runs, so this is how late it can be; the write budgets and alert thresholds below are per run and +# derived from it, to keep their hourly totals when it changes. +MONITOR_INTERVAL = timedelta(minutes=5) +RUNS_PER_HOUR = timedelta(hours=1) // MONITOR_INTERVAL + + +def count_runs_in(duration: timedelta) -> int: + return max(duration // MONITOR_INTERVAL, 1) + + # Project CI runs created this long ago or later are checked for gate tampering. Several hours -# overlap the 15-minute cadence, so a tampered run is seen even after the pull request was +# overlap the monitor's cadence, so a tampered run is seen even after the pull request was # pushed again with the gate restored. TAMPER_WINDOW = timedelta(hours=6) # A ci-amd.yml run in which only the gate executed finishes in about a minute; only longer runs @@ -169,27 +180,27 @@ UNREPORTED_JOB_NAMES = GATED_RUN_JOB_NAMES | {GATE_JOB_NAME, "Platform: AMD"} RUNNING_JOB_STATUSES = frozenset({"queued", "waiting", "pending", "requested", "in_progress"}) JOB_STATE_RANKS = {"failure": 0, "pending": 1, "success": 2} # Commit statuses count towards GitHub's limit of 500 content-creating requests per hour, shared by -# every workflow of the repository. At four runs an hour this keeps job statuses to at most about -# half of it; the rest follow on the next run, failures first. See "Job statuses" in the design doc. -MAX_JOB_STATUSES_PER_RUN = 40 +# every workflow of the repository. This keeps job statuses to about a third of it; the rest follow +# on the next run, failures first. See "Job statuses" in the design doc. +MAX_JOB_STATUSES_PER_RUN = 160 // RUNS_PER_HOUR # `CI run started` statuses of pull requests whose CI runs in apache/airflow come out of that budget # first: one write per new head commit, which merging waits for, while job statuses are informational. # At most this many, so job statuses keep the rest; the first runs after the check became required # catch up on the open pull requests a batch at a time, most recently updated first. -MAX_CI_RUN_STARTED_PER_RUN = 20 +MAX_CI_RUN_STARTED_PER_RUN = 80 // RUNS_PER_HOUR # The GitHub Actions app, whose check suites are the workflow runs of a commit. GITHUB_ACTIONS_APP_ID = 15368 # A single large fork run can need more writes than one run's budget, which the next run catches up # on, so deferring is only reported when it lasts this many consecutive runs: then the budget does # not keep up with the pull requests. -DEFERRED_RUNS_BEFORE_ALERT = 3 +DEFERRED_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=45)) # GitHub answering with rate limits in this many consecutive runs is reported: a single burst is # retried, while a persisting one means the repository's shared limits are close to exhausted. -RATE_LIMITED_RUNS_BEFORE_ALERT = 2 +RATE_LIMITED_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=30)) # Reading a fork run's jobs fails now and then on GitHub server errors (HTTP 502, a reset HTTP/2 # stream) that the next run gets past, so a pull request whose job statuses failed is only reported # when it failed in this many consecutive runs. -JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT = 3 +JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT = count_runs_in(timedelta(minutes=45)) # Fixed text, so the item is posted to Slack once while the failure persists, whatever the error; # the run log has each run's error. JOB_STATUSES_FAILED_REASON = ( diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index 140078f5dda..ee1751aedb0 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -23,6 +23,7 @@ from datetime import UTC, datetime, timedelta, timezone from unittest import mock import pytest +import yaml from click.testing import CliRunner from airflow_breeze.commands.ci_commands import fork_ci_reconcile @@ -42,9 +43,15 @@ from airflow_breeze.utils.fork_ci_messages import ( render_marker, ) from airflow_breeze.utils.fork_ci_reconcile import ( + DEFERRED_RUNS_BEFORE_ALERT, + JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT, JOB_STATUSES_DEFERRED_NOTE, JOB_STATUSES_FAILED_REASON, + MAX_CI_RUN_STARTED_PER_RUN, + MAX_JOB_STATUSES_PER_RUN, + MONITOR_INTERVAL, RATE_LIMITED_NOTE, + RATE_LIMITED_RUNS_BEFORE_ALERT, BaseGateConfigs, BypassFinder, DraftChange, @@ -2821,23 +2828,23 @@ def test_run_reconcile_reports_failing_job_statuses_once_when_failing_persists( reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] + failures = [GitHubError("HTTP 502"), GitHubError("stream ID 1; CANCEL")] errors = [ GitHubError("HTTP 502"), GitHubError("HTTP 502"), None, - GitHubError("HTTP 502"), - GitHubError("stream ID 1; CANCEL"), - GitHubError("HTTP 502"), - GitHubError("stream ID 1; CANCEL"), + *(failures[run % 2] for run in range(JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 1)), ] for error in errors: mock_reporter.return_value.report.side_effect = error mock_reporter.return_value.report.return_value = None alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, False, False, True, False] + assert alerts == [*[False] * (JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 2), True, False] assert JOB_STATUSES_FAILED_REASON in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["job_status_failure_runs"] == {"1": 4} + assert json.loads(reported_file.read_text())["job_status_failure_runs"] == { + "1": JOB_STATUS_FAILURE_RUNS_BEFORE_ALERT + 1 + } @pytest.mark.parametrize( @@ -2912,14 +2919,14 @@ def test_run_reconcile_reports_deferred_job_statuses_only_when_deferring_persist reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] - for deferred in ([1], [1], [], [1], [1], [1]): + for deferred in ([1], [1], [], *[[1]] * DEFERRED_RUNS_BEFORE_ALERT): mock_reporter.return_value.deferred = deferred mock_reporter.return_value.delayed = [] alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, False, False, True] + assert alerts == [*[False] * (DEFERRED_RUNS_BEFORE_ALERT + 2), True] assert JOB_STATUSES_DEFERRED_NOTE in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["deferred_runs"] == 3 + assert json.loads(reported_file.read_text())["deferred_runs"] == DEFERRED_RUNS_BEFORE_ALERT def test_build_job_statuses_treats_contexts_case_insensitively(): @@ -3531,7 +3538,10 @@ def test_run_reconcile_retries_a_pull_request_whose_head_moved(client, tmp_path) @pytest.mark.parametrize( ("fails", "job_status_budget"), - [pytest.param(False, 38, id="reported"), pytest.param(True, 20, id="fails")], + [ + pytest.param(False, MAX_JOB_STATUSES_PER_RUN - 2, id="reported"), + pytest.param(True, MAX_JOB_STATUSES_PER_RUN - MAX_CI_RUN_STARTED_PER_RUN, id="fails"), + ], ) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.report_project_ci_runs_started", autospec=True) @@ -3649,13 +3659,13 @@ def test_run_reconcile_reports_rate_limiting_only_when_it_persists( reported_file = tmp_path / "reported.json" reported_file.write_text("{}") alerts = [] - for rate_limited in (1, 0, 3, 2): + for rate_limited in (1, 0, *[3] * RATE_LIMITED_RUNS_BEFORE_ALERT): client.rate_limited = rate_limited alerts.append(_reconcile_with_reporter(client, tmp_path, reported_file=reported_file)) - assert alerts == [False, False, False, True] + assert alerts == [*[False] * (RATE_LIMITED_RUNS_BEFORE_ALERT + 1), True] assert RATE_LIMITED_NOTE in (tmp_path / "s.json").read_text() - assert json.loads(reported_file.read_text())["rate_limited_runs"] == 2 + assert json.loads(reported_file.read_text())["rate_limited_runs"] == RATE_LIMITED_RUNS_BEFORE_ALERT @pytest.mark.parametrize( @@ -3816,3 +3826,11 @@ def test_state_comment_is_left_for_the_next_run_when_failed_jobs_cannot_be_read( assert plan.status == ("failure", RUN_URL, "Fork CI failed - see Details") assert plan.remove_labels == ["ready for maintainer review"] assert plan.marker_comment is None + + +def test_monitor_interval_is_the_monitor_workflow_cadence(): + workflow = yaml.safe_load((AIRFLOW_ROOT_PATH / ".github/workflows/fork-ci-monitor.yml").read_text()) + + crons = [entry["cron"] for entry in workflow[True]["schedule"]] + + assert crons == [f"*/{MONITOR_INTERVAL // timedelta(minutes=1)} * * * *"]
