This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f242c8e0513 Skip CI duration alerts when the newest run is stale
(#73831)
f242c8e0513 is described below
commit f242c8e05132165daa2df55f09282644b2907e2e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Sep 28 12:19:06 2026 +0200
Skip CI duration alerts when the newest run is stale (#73831)
On 2026-09-28 the duration monitor posted a Static checks regression whose
"latest" run was a five-week-old scheduled run, although the same runs query
made afterwards returns runs from two days earlier. The Actions API was
degraded at the time, so it most likely returned an old page of runs, and
nothing checked that the runs treated as "latest" are actually recent.
Generated-by: Claude Opus 5
---
scripts/ci/analyze_ci_job_durations.py | 24 +++++++++++-
scripts/tests/ci/test_analyze_ci_job_durations.py | 46 +++++++++++++++++++++++
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/scripts/ci/analyze_ci_job_durations.py
b/scripts/ci/analyze_ci_job_durations.py
index 202dfb449e1..9ac2ca59003 100644
--- a/scripts/ci/analyze_ci_job_durations.py
+++ b/scripts/ci/analyze_ci_job_durations.py
@@ -59,6 +59,8 @@ Environment variables (optional):
JOB_MIN_ABS_INCREASE_MINUTES - Absolute floor for per-job alerts (default: 3)
IMAGE_BUILD_PERSISTENCE_DAYS - Only report a slow image build once it has
stayed
elevated for at least this many days (default: 2)
+ MAX_LATEST_RUN_AGE_DAYS - Skip the analysis when the newest usable run is
older than
+ this many days (default: 3)
ANALYZE_JOBS - Whether to fetch per-job durations
("true"/"false", default: true)
ONLY_SUCCESSFUL - Only consider runs that concluded "success"
(default: true)
SLACK_CHANNEL - Slack channel for the message payload (default:
internal-airflow-ci-cd)
@@ -73,7 +75,7 @@ import json
import os
import subprocess
import sys
-from datetime import datetime
+from datetime import datetime, timezone
from pathlib import Path
from typing import TypedDict
@@ -167,6 +169,14 @@ def parse_iso(timestamp: str | None) -> datetime | None:
return None
+def compute_run_age_days(run: dict, now: datetime) -> float | None:
+ """Return how many days before *now* the run was created, or None if
unparsable."""
+ created = parse_iso(run.get("created_at"))
+ if created is None:
+ return None
+ return (now - created).total_seconds() / 86400
+
+
def duration_seconds(start: str | None, end: str | None) -> float | None:
"""Return the number of seconds between two ISO timestamps, or None if
unparsable."""
start_dt = parse_iso(start)
@@ -702,6 +712,7 @@ def main() -> None:
min_abs_increase_seconds = env_float("MIN_ABS_INCREASE_MINUTES", 5.0) * 60
job_min_abs_increase_seconds = env_float("JOB_MIN_ABS_INCREASE_MINUTES",
3.0) * 60
image_build_persistence_days = env_float("IMAGE_BUILD_PERSISTENCE_DAYS",
2.0)
+ max_latest_run_age_days = env_float("MAX_LATEST_RUN_AGE_DAYS", 3.0)
do_analyze_jobs = env_bool("ANALYZE_JOBS", True)
only_successful = env_bool("ONLY_SUCCESSFUL", True)
channel = os.environ.get("SLACK_CHANNEL", "internal-airflow-ci-cd")
@@ -712,6 +723,17 @@ def main() -> None:
print(f"Window: up to {max_runs} completed runs; latest
{latest_runs_count} vs baseline.")
runs = get_recent_runs(repo, workflow, branch, max_runs, only_successful,
event)
+ # A degraded Actions API can return an old page of runs, and without this
check its
+ # weeks-old durations were reported as "latest" and raised a false alert.
+ newest_age_days = compute_run_age_days(runs[0],
datetime.now(timezone.utc)) if runs else None
+ if newest_age_days is not None and newest_age_days >
max_latest_run_age_days:
+ print(
+ f"::warning::The newest usable run (#{runs[0]['run_number']}) is "
+ f"{newest_age_days:.1f} days old, over
MAX_LATEST_RUN_AGE_DAYS={max_latest_run_age_days:g}: "
+ f"the run listing is stale or {branch} has had no usable run
since. Skipping."
+ )
+ _write_outputs(False, False, 0, False)
+ sys.exit(0)
if len(runs) < latest_runs_count + min_baseline_runs:
print(
f"Not enough runs to establish a trend "
diff --git a/scripts/tests/ci/test_analyze_ci_job_durations.py
b/scripts/tests/ci/test_analyze_ci_job_durations.py
index be5b3cbc2e1..c6bfdd23ecd 100644
--- a/scripts/tests/ci/test_analyze_ci_job_durations.py
+++ b/scripts/tests/ci/test_analyze_ci_job_durations.py
@@ -659,3 +659,49 @@ class TestFormatSlackMessage:
channel="internal-airflow-ci-cd",
)
assert "image build slow" not in json.dumps(msg).lower()
+
+
+class TestComputeRunAgeDays:
+ @pytest.mark.parametrize(
+ ("created_at", "expected"),
+ [
+ pytest.param("2026-09-28T09:00:00Z", 0.0, id="same-moment"),
+ pytest.param("2026-09-26T21:00:00Z", 1.5, id="fractional-days"),
+ pytest.param("2026-08-21T09:00:00Z", 38.0, id="weeks-old"),
+ pytest.param("", None, id="missing"),
+ pytest.param("not-a-date", None, id="unparsable"),
+ ],
+ )
+ def test_age(self, durations_module, created_at, expected):
+ now = durations_module.parse_iso("2026-09-28T09:00:00Z")
+ assert durations_module.compute_run_age_days({"created_at":
created_at}, now) == expected
+
+
+class TestMainSkipsStaleRuns:
+ def test_skips_analysis_when_newest_run_is_stale(self, durations_module,
tmp_path, monkeypatch):
+ stale_runs = [
+ {
+ "id": run_id,
+ "run_number": run_id,
+ "created_at": "2020-01-01T02:00:00Z",
+ "conclusion": "success",
+ "event": "schedule",
+ "html_url": f"https://example/{run_id}";,
+ "duration": 6000,
+ }
+ for run_id in range(10, 4, -1)
+ ]
+ github_output = tmp_path / "github_output"
+ output_file = tmp_path / "slack-message.json"
+ monkeypatch.setenv("GITHUB_OUTPUT", str(github_output))
+ monkeypatch.setenv("OUTPUT_FILE", str(output_file))
+ with (
+ patch.object(durations_module, "get_recent_runs", autospec=True,
return_value=stale_runs),
+ patch.object(durations_module, "fetch_run_jobs_map",
autospec=True) as mock_fetch_jobs,
+ pytest.raises(SystemExit) as exc_info,
+ ):
+ durations_module.main()
+ assert exc_info.value.code == 0
+ mock_fetch_jobs.assert_not_called()
+ assert not output_file.exists()
+ assert "has-regression=false" in github_output.read_text()
