This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2ae00f1606a Retry GitHub calls that fail to connect in the fork CI
monitor (#75701)
2ae00f1606a is described below
commit 2ae00f1606ab4fefb12952270826efc2e07b8697
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Oct 27 05:00:11 2026 +0100
Retry GitHub calls that fail to connect in the fork CI monitor (#75701)
A monitor run failed to report job statuses for a PR because a POST to the
statuses API ended in a TLS handshake timeout. Network failures are as
transient as the server errors already retried, and one that happens before the
request reaches GitHub cannot have taken effect, so it is safe to repeat even
for calls such as creating a comment.
Generated-by: Claude Opus 5
---
.../src/airflow_breeze/utils/fork_ci_github.py | 25 +++++++++++++++----
dev/breeze/tests/test_fork_ci_github.py | 29 +++++++++++++++++++++-
2 files changed, 48 insertions(+), 6 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py
b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py
index df00485a2b1..721ecc7873d 100644
--- a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py
+++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py
@@ -33,8 +33,11 @@ from airflow_breeze.utils.fork_ci_messages import
STATUS_CONTEXT
RATE_LIMIT_MARKERS = ("rate limit", "secondary rate", "abuse detection")
MAX_ATTEMPTS = 4
# GitHub answers with these while overloaded, and GraphQL searches that take
too long end in HTTP 504;
-# the same call usually succeeds shortly after.
-SERVER_ERROR = re.compile(r"\bHTTP 5\d\d\b")
+# the same call usually succeeds shortly after. A connection dropped
mid-request is as transient, and as
+# unclear about whether the request took effect.
+SERVER_ERROR = re.compile(r"\bHTTP 5\d\d\b|connection reset by peer|unexpected
EOF")
+# Network failures before the request reached GitHub, so repeating it is safe
for any call.
+CONNECTION_ERROR = re.compile(r"TLS handshake timeout|dial tcp")
SERVER_ERROR_WAIT_SECONDS = 10
# A hung `gh` would otherwise hold the job until its timeout-minutes.
GH_TIMEOUT_SECONDS = 120
@@ -64,7 +67,8 @@ class GitHubClient:
graphql`` exits non-zero when some fields of the response errored,
with the data GitHub
could resolve still on stdout.
:param retry_server_errors: repeat a call that failed with a server
error. Off for a call that
- may have taken effect anyway and must not be repeated, such as
creating a comment.
+ may have taken effect anyway and must not be repeated, such as
creating a comment. A call
+ that failed to connect is repeated either way.
"""
payload = json.dumps(body) if body is not None else None
rate_limited = False
@@ -84,10 +88,21 @@ class GitHubClient:
if result.returncode == 0:
return json.loads(result.stdout) if result.stdout.strip() else
None
rate_limited = any(marker in result.stderr.lower() for marker in
RATE_LIMIT_MARKERS)
- if rate_limited or (retry_server_errors and
SERVER_ERROR.search(result.stderr)):
+ connection_error = bool(CONNECTION_ERROR.search(result.stderr))
+ if (
+ rate_limited
+ or connection_error
+ or (retry_server_errors and SERVER_ERROR.search(result.stderr))
+ ):
if attempt == MAX_ATTEMPTS - 1:
break
- reason = "rate limit" if rate_limited else "server error"
+ reason = (
+ "rate limit"
+ if rate_limited
+ else "connection error"
+ if connection_error
+ else "server error"
+ )
wait = (30 if rate_limited else SERVER_ERROR_WAIT_SECONDS) *
2**attempt
console_print(f"[warning]GitHub {reason}, retrying in
{wait}s[/]")
time.sleep(wait)
diff --git a/dev/breeze/tests/test_fork_ci_github.py
b/dev/breeze/tests/test_fork_ci_github.py
index 1bd85bcb749..9f6505e2954 100644
--- a/dev/breeze/tests/test_fork_ci_github.py
+++ b/dev/breeze/tests/test_fork_ci_github.py
@@ -53,7 +53,17 @@ def
test_gives_up_after_the_last_attempt_without_waiting(mock_run, mock_sleep):
assert [c.args[0] for c in mock_sleep.call_args_list] == [30, 60, 120]
[email protected]("stderr", ["gh: HTTP 504", "gh: Server Error (HTTP
502)"])
[email protected](
+ "stderr",
+ [
+ "gh: HTTP 504",
+ "gh: Server Error (HTTP 502)",
+ 'Post "https://api.github.com/graphql": read tcp
10.1.0.4:51234->140.82.112.6:443: read: connection reset by peer',
+ 'Post "https://api.github.com/graphql": unexpected EOF',
+ 'Post "https://api.github.com/graphql": net/http: TLS handshake
timeout',
+ 'Post "https://api.github.com/graphql": dial tcp 140.82.112.6:443: i/o
timeout',
+ ],
+)
@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run",
autospec=True)
def test_retries_on_server_errors(mock_run, mock_sleep, stderr):
@@ -90,6 +100,23 @@ def
test_does_not_repeat_creating_a_comment_after_a_server_error(mock_run, mock_
mock_sleep.assert_not_called()
[email protected](
+ "stderr",
+ [
+ 'Post "https://api.github.com/repos/x/y/issues/10/comments": net/http:
TLS handshake timeout',
+ 'Post "https://api.github.com/repos/x/y/issues/10/comments": dial tcp:
lookup api.github.com: no such host',
+ ],
+)
[email protected]("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
[email protected]("airflow_breeze.utils.fork_ci_github.subprocess.run",
autospec=True)
+def test_repeats_creating_a_comment_that_did_not_connect(mock_run, mock_sleep,
stderr):
+ mock_run.side_effect = [_completed(1, stderr=stderr), _completed(0,
stdout="{}")]
+
+ GitHubClient().create_comment(10, "body")
+ assert mock_run.call_count == 2
+ mock_sleep.assert_called_once_with(10)
+
+
@mock.patch(
"airflow_breeze.utils.fork_ci_github.subprocess.run",
autospec=True,
