This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new c9c8f8e0f fix(ci): add retry logic to Bazel downloads to prevent flaky
CI failures (#2733)
c9c8f8e0f is described below
commit c9c8f8e0fd40ac9ac428602ca739b06c379fcb7b
Author: Sanyam Suyal <[email protected]>
AuthorDate: Fri Oct 10 09:12:41 2025 +0530
fix(ci): add retry logic to Bazel downloads to prevent flaky CI failures
(#2733)
### Problem
Fixes #2731
Python CI is unstable because of downloading Bazel error. The CI
randomly fails with:
could not download Bazel: failed to download bazel: stream error: stream
ID 1; INTERNAL_ERROR; received from peer
### Solution
Added retry logic with exponential backoff (5 attempts with 2s, 4s, 8s,
16s, 32s delays) to handle transient network errors from GitHub CDN.
### Changes
- ci/run_ci.sh: Added retry loop in install_bazel() function
- ci/tasks/common.py: Added urlretrieve_with_retries() helper function
### Impact
- No breaking changes
- Fully backward compatible
- Reduces CI flakiness significantly
Fixes #2731
---
ci/run_ci.sh | 18 +++++++++++++++++-
ci/tasks/common.py | 33 ++++++++++++++++++++++++++++++---
2 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index bbc09feeb..f27835341 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -96,7 +96,23 @@ install_bazel() {
BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}";
echo "Downloading bazel from: $BINARY_URL"
- curl -L -sSf -o "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to
download bazel"; exit 1; }
+ # Retry download with exponential backoff to avoid transient network errors
in CI
+ MAX_ATTEMPTS=5
+ SLEEP_SECONDS=2
+ attempt=1
+ while [ $attempt -le $MAX_ATTEMPTS ]; do
+ if curl -L -sSf -o "$BAZEL_DIR/bazel" "$BINARY_URL"; then
+ break
+ fi
+ echo "Attempt $attempt to download bazel failed."
+ if [ $attempt -eq $MAX_ATTEMPTS ]; then
+ echo "Failed to download bazel after $MAX_ATTEMPTS attempts"
+ exit 1
+ fi
+ sleep $SLEEP_SECONDS
+ SLEEP_SECONDS=$((SLEEP_SECONDS * 2))
+ attempt=$((attempt + 1))
+ done
chmod +x "$BAZEL_DIR/bazel"
# Add to current shell's PATH
diff --git a/ci/tasks/common.py b/ci/tasks/common.py
index 76f321735..c1393ce86 100644
--- a/ci/tasks/common.py
+++ b/ci/tasks/common.py
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-import shutil
import subprocess
import platform
import urllib.request as ulib
@@ -88,6 +87,34 @@ def get_bazel_download_url():
return
f"{download_url_base}/bazel-{bazel_version}-{get_os_name_lower()}-{get_os_machine()}"
+def urlretrieve_with_retries(url, filename, max_attempts=5, initial_delay=2):
+ """Download a URL to filename with retries and exponential backoff.
+
+ Raises the last exception if all attempts fail.
+ """
+ attempt = 1
+ delay = initial_delay
+ last_exc = None
+ while attempt <= max_attempts:
+ try:
+ logging.info(f"Downloading (attempt {attempt}) {url} ->
{filename}")
+ ulib.urlretrieve(url, filename)
+ return
+ except Exception as e:
+ logging.error(f"Download attempt {attempt} failed: {e}")
+ last_exc = e
+ if attempt == max_attempts:
+ break
+ logging.info(f"Retrying in {delay} seconds...")
+ import time
+
+ time.sleep(delay)
+ delay *= 2
+ attempt += 1
+ logging.error(f"All {max_attempts} download attempts failed for URL:
{url}")
+ raise last_exc
+
+
def cd_project_subdir(subdir):
"""Change to a subdirectory of the project."""
os.chdir(os.path.join(PROJECT_ROOT_DIR, subdir))
@@ -124,7 +151,7 @@ def install_bazel():
# For Windows, download the installer and add it to PATH
local_name = "bazel.exe"
try:
- ulib.urlretrieve(bazel_download_url, local_name)
+ urlretrieve_with_retries(bazel_download_url, local_name)
except Exception as e:
logging.error(f"Failed to download bazel: {e}")
logging.error(f"URL: {bazel_download_url}")
@@ -142,7 +169,7 @@ def install_bazel():
bazel_path = os.path.join(home_bin, "bazel")
try:
- ulib.urlretrieve(bazel_download_url, bazel_path)
+ urlretrieve_with_retries(bazel_download_url, bazel_path)
except Exception as e:
logging.error(f"Failed to download bazel: {e}")
logging.error(f"URL: {bazel_download_url}")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
