This is an automated email from the ASF dual-hosted git repository.
netudima pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-ccm.git
The following commit(s) were added to refs/heads/trunk by this push:
new e8508d1 Retry git clone and fetch from remote repositories, and abort
stalled git transfers
e8508d1 is described below
commit e8508d167e8538fc7213b83cbd1fc8cbba5dc175
Author: Dmitry Konstantinov <[email protected]>
AuthorDate: Sun Nov 1 13:00:12 2026 +0000
Retry git clone and fetch from remote repositories, and abort stalled git
transfers
Make up to 5 attempts for git clone --bare and git fetch when the
repository is remote, waiting 15s after
the first failure and doubling the wait after each further failure. Local
paths and file:// urls get one attempt.
Abort a transfer that stays below 1000 bytes/s for 60s instead of waiting
for the server to close the connection
(unless GIT_HTTP_LOW_SPEED_LIMIT or GIT_HTTP_LOW_SPEED_TIME is already set).
After a remote command fails every attempt, later remote commands get one
attempt until one succeeds.
Include git's error lines in the "Could not do a git clone" and "Could not
update git" messages.
patch by Dmitry Konstantinov; reviewed by Brandon Williams for
CASSANDRA-21834
---
ccmlib/repository.py | 69 ++++++++++++++++++++++-----
tests/test_repository.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 177 insertions(+), 13 deletions(-)
diff --git a/ccmlib/repository.py b/ccmlib/repository.py
index 55af470..78ad8cd 100644
--- a/ccmlib/repository.py
+++ b/ccmlib/repository.py
@@ -52,12 +52,20 @@ MIRRORS = ["https://dlcdn.apache.org/cassandra";,
"https://downloads.apache.org/c
DOWNLOAD_TIMEOUT = 45 # seconds a download may wait for data before the
attempt is abandoned
DOWNLOAD_ATTEMPTS = 3 # attempts per url
DOWNLOAD_RETRY_DELAY = 5 # seconds, multiplied by the attempt number
+GIT_ATTEMPTS = 5 # attempts for the git commands that reach a remote
repository
+GIT_RETRY_DELAY_SEC = 15 # doubled after each attempt, so the attempts span
about 4 minutes
+GIT_STALL_BYTES_PER_SEC = 1000 # a git transfer slower than this counts as
stalled
+GIT_STALL_TIMEOUT_SEC = 60 # how long a git transfer may stay stalled before
the attempt is abandoned
GIT_REPO = "https://github.com/apache/cassandra.git";
GITHUB_REPO = "https://github.com/apache/cassandra";
GITHUB_TAGS = "https://api.github.com/repos/apache/cassandra/git/refs/tags";
CCM_CONFIG = ConfigParser.RawConfigParser()
CCM_CONFIG.read(os.path.join(os.path.expanduser("~"), ".ccm", "config"))
+# set when a remote git command failed every attempt, cleared when one
succeeds; meanwhile each command
+# is tried once, so a remote that stays unreachable does not add minutes to
every clone_development
+_git_remote_down = False
+
def setup(version, verbose=False):
(cdir, version, fallback) = __setup(version, verbose)
@@ -173,20 +181,15 @@ def clone_development(git_repo, version, verbose=False,
alias=False):
# remote fetches we need to perform:
if not os.path.exists(local_git_cache):
common.info("Cloning Cassandra...")
- process = subprocess.Popen(
- ['git', 'clone', '--bare',
- git_repo, local_git_cache],
- cwd=__get_dir(), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
- out, _, _ = log_info(process, logger)
- assert out == 0, "Could not do a git clone"
+ out, error = __git_remote(['clone', '--bare', git_repo,
local_git_cache],
+ git_repo, __get_dir(), logger)
+ assert out == 0, "Could not do a git clone: {}".format(error)
else:
common.info("Fetching Cassandra updates...")
- process = subprocess.Popen(
- ['git', 'fetch', '-fup', 'origin',
- '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*'],
- cwd=local_git_cache, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
- out, _, _ = log_info(process, logger)
- assert out == 0, "Could not update git"
+ out, error = __git_remote(['fetch', '-fup', 'origin',
+ '+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*'],
+ git_repo, local_git_cache, logger)
+ assert out == 0, "Could not update git: {}".format(error)
# Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
@@ -282,6 +285,48 @@ def clone_development(git_repo, version, verbose=False,
alias=False):
raise e
+def __git_remote(args, git_repo, cwd, logger):
+ """
+ Run a git command that reaches git_repo, retrying it when git_repo is
remote.
+ Returns git's exit code and the errors it printed on the last attempt.
+ """
+ global _git_remote_down
+ remote = __is_remote(git_repo)
+ attempts = GIT_ATTEMPTS if remote and not _git_remote_down else 1
+ # without a low speed limit git waits on a stalled http transfer until the
server drops the connection
+ env = {'GIT_HTTP_LOW_SPEED_LIMIT': str(GIT_STALL_BYTES_PER_SEC),
+ 'GIT_HTTP_LOW_SPEED_TIME': str(GIT_STALL_TIMEOUT_SEC)}
+ env.update(os.environ)
+ for attempt in range(1, attempts + 1):
+ process = subprocess.Popen(['git'] + args, cwd=cwd, env=env,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, _, stderr = log_info(process, logger)
+ if out == 0:
+ if remote:
+ _git_remote_down = False
+ return out, None
+ error = __git_errors(stderr)
+ if attempt < attempts:
+ common.warning("git {} {} failed (attempt {} of {}): {}".format(
+ args[0], git_repo, attempt, attempts, error))
+ time.sleep(GIT_RETRY_DELAY_SEC * 2 ** (attempt - 1))
+ if attempts > 1:
+ _git_remote_down = True
+ return out, error
+
+
+def __is_remote(git_repo):
+ # local paths and file:// urls are not retried; scp-like addresses are,
e.g. [email protected]:apache/cassandra.git
+ if '://' in git_repo:
+ return not git_repo.startswith('file://')
+ return re.match(r'[\w.-]+@[\w.-]+:', git_repo) is not None
+
+
+def __git_errors(stderr):
+ # git also prints ref updates to stderr, so keep its error lines
+ lines = [line.strip() for line in stderr.decode().splitlines() if
line.strip()]
+ return ' | '.join([line for line in lines if line.startswith(('error:',
'fatal:'))] or lines[-3:])
+
+
def download_version(version, url=None, verbose=False, binary=False):
"""Download, extract, and build Cassandra tarball.
diff --git a/tests/test_repository.py b/tests/test_repository.py
index 4b1031c..701708e 100644
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -16,13 +16,15 @@
import configparser
+import functools
import io
import os
import ssl
+import subprocess
import tarfile
import threading
import time
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from http.server import BaseHTTPRequestHandler, SimpleHTTPRequestHandler,
ThreadingHTTPServer
import pytest
from six.moves import urllib
@@ -106,10 +108,67 @@ def untrusted_mirror():
yield from _serve(_MirrorServer(context))
+class _GitRemoteHandler(SimpleHTTPRequestHandler):
+ """
+ Serves git repositories over git's dumb http protocol. Every git command
starts by requesting info/refs,
+ and that request misbehaves as the next entry of server.misbehave says.
+ """
+
+ def do_GET(self):
+ if '/info/refs' in self.path:
+ self.server.refs_requests += 1
+ behaviour = self.server.misbehave.pop(0) if self.server.misbehave
else None
+ if behaviour == 'error':
+ self.send_error(500)
+ return
+ if behaviour == 'stall':
+ self.send_response(200)
+ self.send_header('Content-Length', '1000000')
+ self.end_headers()
+ time.sleep(STALL_SECONDS)
+ return
+ SimpleHTTPRequestHandler.do_GET(self)
+
+ def log_message(self, *args):
+ pass
+
+
+class _GitRemoteServer(ThreadingHTTPServer):
+ block_on_close = False # do not wait for stalled handlers
+
+ def __init__(self, directory):
+ ThreadingHTTPServer.__init__(self, ('127.0.0.1', 0),
functools.partial(_GitRemoteHandler, directory=directory))
+ self.refs_requests = 0
+ self.misbehave = []
+ self.url =
'http://127.0.0.1:{}/cassandra.git'.format(self.server_address[1])
+
+
+def _git(*args):
+ subprocess.check_call(('git',) + args, stdout=subprocess.DEVNULL)
+
+
[email protected]
+def git_remote(tmp_path, monkeypatch):
+ """A repository with a cassandra-4.0 branch served over http, with ccm's
repository directory in tmp_path"""
+ source, served = str(tmp_path / 'source'), str(tmp_path / 'served' /
'cassandra.git')
+ _git('init', '-q', source)
+ _git('-C', source, 'symbolic-ref', 'HEAD', 'refs/heads/cassandra-4.0')
+ _git('-C', source, '-c', 'user.name=ccm', '-c',
'[email protected]', '-c', 'commit.gpgsign=false',
+ 'commit', '-q', '--allow-empty', '-m', 'initial')
+ _git('clone', '-q', '--bare', source, served)
+ _git('-C', served, 'update-server-info')
+ monkeypatch.setattr(repository, 'get_default_path', lambda: str(tmp_path))
+ monkeypatch.setattr(repository, 'compile_version', lambda *args: None)
+ yield from _serve(_GitRemoteServer(str(tmp_path / 'served')))
+
+
@pytest.fixture(autouse=True)
def fast_retries(monkeypatch):
monkeypatch.setattr(repository, 'DOWNLOAD_TIMEOUT', 0.5)
monkeypatch.setattr(repository, 'DOWNLOAD_RETRY_DELAY', 0)
+ monkeypatch.setattr(repository, 'GIT_RETRY_DELAY_SEC', 0)
+ monkeypatch.setattr(repository, 'GIT_STALL_TIMEOUT_SEC', 2)
+ monkeypatch.setattr(repository, '_git_remote_down', False)
def _read(path):
@@ -195,3 +254,63 @@ def test_download_version_tries_mirrors_in_order(mirror,
tmp_path, monkeypatch):
installed = tmp_path / 'repository' / '9.9.9'
assert _read(str(installed / 'bin' / 'cassandra')) == PAYLOAD
assert _read(str(installed / '0.version.txt')) == b'9.9.9'
+
+
+GITHUB_VERSION = 'github:apache/cassandra-4.0'
+
+
+def test_git_clone_from_remote_is_retried(git_remote):
+ git_remote.misbehave = ['error']
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+
+ assert git_remote.refs_requests == 2
+ assert
os.path.isdir(os.path.join(repository.directory_name(GITHUB_VERSION), '.git'))
+
+
+def test_stalled_git_fetch_is_abandoned_and_retried(git_remote):
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+ git_remote.misbehave = ['stall']
+ start = time.time()
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+
+ assert git_remote.refs_requests == 3
+ assert time.time() - start < STALL_SECONDS
+
+
+def test_git_error_is_reported_once_attempts_run_out(git_remote):
+ git_remote.misbehave = ['error'] * repository.GIT_ATTEMPTS
+ with pytest.raises(AssertionError) as e:
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+
+ assert str(e.value).startswith("Could not do a git clone: fatal: unable to
access '{}/'".format(git_remote.url))
+ assert str(e.value).endswith('500')
+ assert git_remote.refs_requests == repository.GIT_ATTEMPTS
+
+
+def test_unreachable_remote_is_tried_once_until_it_works_again(git_remote):
+ git_remote.misbehave = ['error'] * (repository.GIT_ATTEMPTS + 1)
+ with pytest.raises(AssertionError):
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+ with pytest.raises(AssertionError):
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+ assert git_remote.refs_requests == repository.GIT_ATTEMPTS + 1
+
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+ assert git_remote.refs_requests == repository.GIT_ATTEMPTS + 2
+
+ # the remote works again, so its next failure is retried, this time by the
fetch into the cache
+ git_remote.misbehave = ['error']
+ repository.clone_development(git_remote.url, GITHUB_VERSION)
+ assert git_remote.refs_requests == repository.GIT_ATTEMPTS + 4
+
+
+def test_local_repository_is_not_retried(tmp_path, monkeypatch):
+ monkeypatch.setattr(repository, 'get_default_path', lambda: str(tmp_path))
+ monkeypatch.setattr(repository, 'GIT_RETRY_DELAY_SEC', 60)
+ missing = str(tmp_path / 'missing')
+ start = time.time()
+ with pytest.raises(AssertionError) as e:
+ repository.clone_development(missing, 'local:{}:trunk'.format(missing))
+
+ assert 'does not exist' in str(e.value)
+ assert time.time() - start < STALL_SECONDS
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
