This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch fork-ci-time-based-status-budget
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 551a4c0cf4ceac8bdeb66fcd4353aeb76624ace9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 15 23:36:39 2026 +0900

    Measure the job status budget between monitor run starts
    
    Counting from when reconciling starts credited each run's setup time
    again, and a re-run credited the time its first attempt already had,
    so the hourly budget could be exceeded.
    
    Generated-by: Claude Opus 5
    Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq
---
 .../doc/ci/08_fork_ci_for_external_contributors.md | 10 ++--
 .../src/airflow_breeze/utils/fork_ci_reconcile.py  | 36 +++++++++----
 dev/breeze/tests/test_fork_ci_reconcile.py         | 60 +++++++++++++++++-----
 3 files changed, 80 insertions(+), 26 deletions(-)

diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md 
b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
index f9ca257532c..71c4a648a45 100644
--- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
+++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
@@ -1001,11 +1001,13 @@ requests a minute and 500 an hour. Assuming about 35 
reported jobs for an averag
   run (see "The `CI run started` check"), so they do not raise the total; at 
about 11 pull
   request runs an hour they take about 1 write a run.
 
-A run gets the hourly budget for the time since the previous monitor run 
started, not one twelfth
-of it. That time is read from the monitor's runs (`GET
+A run gets the hourly budget for the time between the previous monitor run's 
start and its own,
+not one twelfth of it. Both are read from the monitor's runs (`GET
 /repos/apache/airflow/actions/workflows/fork-ci-monitor.yml/runs`, runs still 
waiting in its
-concurrency group left out), not from the record the monitor carries between 
runs, which a failed
-run or a failed Slack post leaves unchanged although that run may have posted 
statuses. GitHub
+concurrency group left out), so they are the same boundary of each run, setup 
included, and not
+from the record the monitor carries between runs, which a failed run or a 
failed Slack post leaves
+unchanged although that run may have posted statuses. A re-run counts from 
when its run was
+created, so the time its first attempt had is not credited again. GitHub
 starts scheduled workflows later than their cron: on 2026-11-15 the 5-minute 
monitor ran every 9
 to 10 minutes, so a fixed 13 writes a run (160 an hour at the cron's cadence) 
posted only about 82
 an hour, and four full-matrix fork runs at once took about six hours to catch 
up. Runs queued
diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py 
b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
index d1501093389..010edf8e05d 100644
--- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
+++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
@@ -2831,16 +2831,21 @@ def read_run_count(reported_file: Path | None, key: 
str) -> int | None:
     return count if isinstance(count, int) and not isinstance(count, bool) 
else 0
 
 
-def find_previous_monitor_run_start(client: GitHubClient, repo: str, *, 
run_url: str) -> datetime | None:
+def find_time_since_previous_monitor_run(
+    client: GitHubClient, repo: str, *, run_url: str
+) -> timedelta | None:
     """
-    When the monitor's latest run other than this one started; None when it 
cannot be told.
+    The time between the previous monitor run's start and this run's; None 
when it cannot be told.
 
-    Read from GitHub rather than from the record the monitor carries between 
runs, which a failed run
-    or a failed Slack post leaves unchanged although that run may have posted 
statuses.
+    Both starts are read from GitHub, so they are the same boundary of each 
run, setup included, and a
+    failed run or a failed Slack post, which leave the record the monitor 
carries between runs
+    unchanged, still counts. A re-run of this run counts from when the run was 
created, the earliest
+    its first attempt can have posted.
     """
     match = re.search(r"/actions/runs/(\d+)", run_url)
     if match is None:
         return None
+    run_id = int(match[1])
     try:
         runs = client.rest_get(
             
f"repos/{repo}/actions/workflows/{MONITOR_WORKFLOW_FILE}/runs?per_page={MONITOR_RUNS_READ}"
@@ -2848,13 +2853,25 @@ def find_previous_monitor_run_start(client: 
GitHubClient, repo: str, *, run_url:
     except Exception as e:
         console_print(f"[warning]Could not read the monitor's earlier runs: 
{e}[/]")
         return None
+    current = next((run for run in runs if run["id"] == run_id), None)
+    started = parse_github_datetime(current.get("run_started_at")) if current 
else None
+    if current is None or started is None:
+        return None
     # Runs waiting in the concurrency group have not started, so they posted 
nothing.
-    starts = [
-        parse_github_datetime(run.get("run_started_at"))
+    boundaries = [
+        start
         for run in runs
-        if run["id"] != int(match[1]) and run.get("status") not in 
NOT_STARTED_RUN_STATUSES
+        if run["id"] != run_id and run.get("status") not in 
NOT_STARTED_RUN_STATUSES
+        for start in [parse_github_datetime(run.get("run_started_at"))]
+        if start is not None and start <= started
     ]
-    return max((start for start in starts if start is not None), default=None)
+    if (current.get("run_attempt") or 1) > 1:
+        created = parse_github_datetime(current.get("created_at"))
+        if created is not None:
+            boundaries.append(created)
+    if not boundaries:
+        return None
+    return started - max(boundaries)
 
 
 def read_job_status_failure_runs(reported_file: Path | None) -> dict[int, int]:
@@ -2985,9 +3002,8 @@ def run_reconcile(
             f"listing the pull requests with `{LABEL_USE_PROJECT_CI}` failed: 
{type(e).__name__}"
         )
     read_jobs: ReadJobs = {}
-    previous_run_at = find_previous_monitor_run_start(client, repo, 
run_url=run_url)
     job_status_budget = compute_job_status_budget(
-        now - previous_run_at if previous_run_at is not None else None
+        find_time_since_previous_monitor_run(client, repo, run_url=run_url)
     )
     job_reporter = JobStatusReporter(
         client,
diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py 
b/dev/breeze/tests/test_fork_ci_reconcile.py
index 51221295180..7c850503798 100644
--- a/dev/breeze/tests/test_fork_ci_reconcile.py
+++ b/dev/breeze/tests/test_fork_ci_reconcile.py
@@ -87,11 +87,11 @@ from airflow_breeze.utils.fork_ci_reconcile import (
     find_failed_jobs,
     find_heads_shared_across_bases,
     find_passed_jobs,
-    find_previous_monitor_run_start,
     find_project_ci_label_heads,
     find_project_ci_passed_url,
     find_project_ci_run_url,
     find_state_comment,
+    find_time_since_previous_monitor_run,
     is_cancelled_gate_run,
     is_gated_skip_run,
     is_own_pilot_change,
@@ -3462,8 +3462,14 @@ def test_job_status_budget_after_a_gap_is_capped():
 MONITOR_RUN_URL = "https://github.com/apache/airflow/actions/runs/100";
 
 
-def _monitor_run(run_id, started, status="completed"):
-    return {"id": run_id, "status": status, "run_started_at": started}
+def _monitor_run(run_id, started, status="completed", attempt=1, created=None):
+    return {
+        "id": run_id,
+        "status": status,
+        "run_started_at": started,
+        "run_attempt": attempt,
+        "created_at": created or started,
+    }
 
 
 @pytest.mark.parametrize(
@@ -3476,33 +3482,63 @@ def _monitor_run(run_id, started, status="completed"):
                 _monitor_run(99, "2026-11-10T11:51:00Z"),
                 _monitor_run(98, "2026-11-10T11:42:00Z"),
             ],
-            datetime(2026, 11, 10, 11, 51, tzinfo=UTC),
+            timedelta(minutes=7),
             id="previous-started-run",
         ),
+        pytest.param(
+            [
+                _monitor_run(101, "2026-11-10T11:59:00Z"),
+                _monitor_run(
+                    100,
+                    "2026-11-10T12:05:00Z",
+                    status="in_progress",
+                    attempt=2,
+                    created="2026-11-10T11:58:00Z",
+                ),
+                _monitor_run(99, "2026-11-10T11:51:00Z"),
+            ],
+            timedelta(minutes=6),
+            id="re-run-after-a-later-run",
+        ),
+        pytest.param(
+            [
+                _monitor_run(
+                    100,
+                    "2026-11-10T12:05:00Z",
+                    status="in_progress",
+                    attempt=2,
+                    created="2026-11-10T11:58:00Z",
+                ),
+                _monitor_run(99, "2026-11-10T11:51:00Z"),
+            ],
+            timedelta(minutes=7),
+            id="re-run-counts-from-its-first-attempt",
+        ),
         pytest.param([_monitor_run(100, "2026-11-10T11:58:00Z")], None, 
id="first-run"),
+        pytest.param([_monitor_run(99, "2026-11-10T11:51:00Z")], None, 
id="this-run-not-listed"),
     ],
 )
-def test_find_previous_monitor_run_start(client, runs, expected):
+def test_find_time_since_previous_monitor_run(client, runs, expected):
     client.rest_get.return_value = {"workflow_runs": runs}
 
-    assert find_previous_monitor_run_start(client, REPO, 
run_url=MONITOR_RUN_URL) == expected
+    assert find_time_since_previous_monitor_run(client, REPO, 
run_url=MONITOR_RUN_URL) == expected
     client.rest_get.assert_called_once_with(
         f"repos/{REPO}/actions/workflows/fork-ci-monitor.yml/runs?per_page=10"
     )
 
 
-def test_find_previous_monitor_run_start_without_a_run_id(client):
-    assert find_previous_monitor_run_start(client, REPO, 
run_url="https://run";) is None
+def test_find_time_since_previous_monitor_run_without_a_run_id(client):
+    assert find_time_since_previous_monitor_run(client, REPO, 
run_url="https://run";) is None
     client.rest_get.assert_not_called()
 
 
-def test_find_previous_monitor_run_start_when_the_runs_cannot_be_read(client):
+def 
test_find_time_since_previous_monitor_run_when_the_runs_cannot_be_read(client):
     client.rest_get.side_effect = GitHubError("gh api failed: HTTP 502")
 
-    assert find_previous_monitor_run_start(client, REPO, 
run_url=MONITOR_RUN_URL) is None
+    assert find_time_since_previous_monitor_run(client, REPO, 
run_url=MONITOR_RUN_URL) is None
 
 
[email protected]("airflow_breeze.utils.fork_ci_reconcile.find_previous_monitor_run_start",
 autospec=True)
[email protected]("airflow_breeze.utils.fork_ci_reconcile.find_time_since_previous_monitor_run",
 autospec=True)
 @mock.patch("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", 
autospec=True)
 
@mock.patch("airflow_breeze.utils.fork_ci_reconcile.report_project_ci_runs_started",
 autospec=True)
 @mock.patch("airflow_breeze.utils.fork_ci_reconcile.classify", autospec=True)
@@ -3514,7 +3550,7 @@ def 
test_run_reconcile_budgets_job_statuses_for_the_time_since_the_previous_run(
     mock_reporter.return_value.report.return_value = None
     mock_reporter.return_value.deferred = []
     mock_reporter.return_value.delayed = []
-    mock_previous.return_value = NOW - timedelta(minutes=9)
+    mock_previous.return_value = timedelta(minutes=9)
 
     _reconcile_with_reporter(client, tmp_path)
 

Reply via email to