This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e109bfdb45e Skip CI runs deleted while the Spot interruption notices
handle them (#76249)
e109bfdb45e is described below
commit e109bfdb45ec5a23f1d2afbbf4b0f23df8164f5a
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 1 21:19:18 2026 +0100
Skip CI runs deleted while the Spot interruption notices handle them
(#76249)
A run can be deleted between listing the runs and reading its jobs. The 404
that follows failed the whole scheduled job, although nothing is left to report
on a deleted run. A 404 still fails the job when the run exists, as that points
at a real problem.
Generated-by: Claude Opus 5
---
scripts/ci/spot_interruptions.py | 16 +++++++++++
scripts/tests/ci/test_spot_interruptions.py | 44 +++++++++++++++++++++++++++--
2 files changed, 57 insertions(+), 3 deletions(-)
diff --git a/scripts/ci/spot_interruptions.py b/scripts/ci/spot_interruptions.py
index e78344fe074..d344e423c47 100755
--- a/scripts/ci/spot_interruptions.py
+++ b/scripts/ci/spot_interruptions.py
@@ -84,6 +84,7 @@ FAILED_CONCLUSIONS = ("failure", "cancelled", "timed_out")
TRANSIENT_ERROR = re.compile(
r"\(HTTP 5\d\d\)|stream error|connection reset by peer|unexpected EOF|TLS
handshake timeout|i/o timeout"
)
+NOT_FOUND_ERROR = re.compile(r"\(HTTP 404\)")
GH_API_ATTEMPTS = 3
GH_API_RETRY_DELAY_SECONDS = 5
@@ -101,6 +102,10 @@ def is_transient(error: subprocess.CalledProcessError) ->
bool:
return bool(TRANSIENT_ERROR.search(error.stderr or ""))
+def is_not_found(error: subprocess.CalledProcessError) -> bool:
+ return bool(NOT_FOUND_ERROR.search(error.stderr or ""))
+
+
def gh_api(*args: str) -> str:
command = ["gh", "api", *args]
# A POST that failed with a server error may still have created the
comment.
@@ -384,6 +389,14 @@ def update_notice_on(repo: str, pr_number: int, run: dict,
jobs: list[FailedJob]
write_comment(repo, pr_number, existing, body)
+def is_run_deleted(repo: str, run_id: int) -> bool:
+ try:
+ gh_api(f"repos/{repo}/actions/runs/{run_id}", "--jq", ".id")
+ except subprocess.CalledProcessError as e:
+ return is_not_found(e)
+ return False
+
+
def describe_error(error: subprocess.CalledProcessError) -> str:
return " ".join((error.stderr or str(error)).split())
@@ -409,6 +422,9 @@ def run_pr_comments(repo: str, now: datetime) -> bool:
except subprocess.CalledProcessError as e:
if is_transient(e):
print(f"::warning::Skipped run {run['html_url']}, GitHub keeps
failing: {describe_error(e)}")
+ elif is_not_found(e) and is_run_deleted(repo, run["id"]):
+ # Runs are deleted at any time, also between listing them and
reading their jobs.
+ print(f"Skipped run {run['html_url']}: it was deleted")
else:
print(f"Could not handle run {run['html_url']}:
{describe_error(e)}", file=sys.stderr)
ok = False
diff --git a/scripts/tests/ci/test_spot_interruptions.py
b/scripts/tests/ci/test_spot_interruptions.py
index d516890e658..72cfee8b3ad 100644
--- a/scripts/tests/ci/test_spot_interruptions.py
+++ b/scripts/tests/ci/test_spot_interruptions.py
@@ -303,20 +303,58 @@ STREAM_ERROR = subprocess.CalledProcessError(
@pytest.mark.parametrize(
- ("error", "expected_ok"), [(SERVER_ERROR, True), (STREAM_ERROR, True),
(NOT_FOUND_ERROR, False)]
+ ("error", "run_deleted", "expected_ok", "expected_out", "expected_err"),
+ [
+ pytest.param(SERVER_ERROR, None, True, "::warning::Skipped run", "",
id="server-error"),
+ pytest.param(STREAM_ERROR, None, True, "::warning::Skipped run", "",
id="dropped-connection"),
+ pytest.param(
+ NOT_FOUND_ERROR, True, True, f"Skipped run {RUN_URL}: it was
deleted", "", id="deleted-run"
+ ),
+ pytest.param(
+ NOT_FOUND_ERROR,
+ False,
+ False,
+ "",
+ f"Could not handle run {RUN_URL}",
+ id="not-found-in-existing-run",
+ ),
+ ],
)
-def test_run_pr_comments_carries_on_past_a_failing_run(spot, capsys, error,
expected_ok):
+def test_run_pr_comments_carries_on_past_a_failing_run(
+ spot, capsys, error, run_deleted, expected_ok, expected_out, expected_err
+):
runs = [_run(branch="a"), _run(branch="b")]
with (
mock.patch.object(spot, "list_pull_request_runs", autospec=True,
return_value=runs),
mock.patch.object(spot, "update_notice", autospec=True,
side_effect=[error, None]) as mock_update,
+ mock.patch.object(spot, "is_run_deleted", autospec=True,
return_value=run_deleted) as mock_deleted,
):
ok = spot.run_pr_comments("apache/airflow", datetime(2026, 10, 25,
tzinfo=UTC))
assert ok is expected_ok
assert mock_update.call_count == 2
- assert capsys.readouterr().out.startswith("::warning::") is expected_ok
+ captured = capsys.readouterr()
+ assert captured.out.startswith(expected_out) if expected_out else
captured.out == ""
+ assert captured.err.startswith(expected_err) if expected_err else
captured.err == ""
+ if run_deleted is None:
+ mock_deleted.assert_not_called()
+ else:
+ mock_deleted.assert_called_once_with("apache/airflow", 7)
+
+
[email protected](
+ ("error", "expected"),
+ [
+ pytest.param(None, False, id="run-exists"),
+ pytest.param(NOT_FOUND_ERROR, True, id="run-deleted"),
+ pytest.param(SERVER_ERROR, False, id="unknown"),
+ ],
+)
+def test_is_run_deleted(spot, error, expected):
+ with mock.patch.object(spot, "gh_api", autospec=True, side_effect=error,
return_value="7") as mock_gh:
+ assert spot.is_run_deleted("apache/airflow", 7) is expected
+ mock_gh.assert_called_once_with("repos/apache/airflow/actions/runs/7",
"--jq", ".id")
@pytest.mark.parametrize(("error", "expected_ok"), [(SERVER_ERROR, True),
(NOT_FOUND_ERROR, None)])
