This is an automated email from the ASF dual-hosted git repository. tomaz pushed a commit to branch understand-ai-intelligent-retry in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 997b177490cc08da566c518d84a28c3063296cdf Author: Veith Röthlingshöfer <[email protected]> AuthorDate: Wed Nov 3 12:09:43 2021 +0100 Add parameter to turn off infinite retry on rate limiting --- libcloud/utils/retry.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libcloud/utils/retry.py b/libcloud/utils/retry.py index a8ccf5f..dd62dbd 100644 --- a/libcloud/utils/retry.py +++ b/libcloud/utils/retry.py @@ -18,6 +18,7 @@ import ssl import time from datetime import datetime, timedelta from functools import wraps +import logging from libcloud.utils.py3 import httplib from libcloud.common.exceptions import RateLimitReachedError @@ -26,7 +27,7 @@ __all__ = [ 'Retry' ] - +_logger = logging.getLogger(__name__) # Error message which indicates a transient SSL error upon which request # can be retried TRANSIENT_SSL_ERROR = 'The read operation timed out' @@ -42,6 +43,7 @@ class TransientSSLError(ssl.SSLError): DEFAULT_TIMEOUT = 30 # default retry timeout DEFAULT_DELAY = 1 # default sleep delay used in each iterator DEFAULT_BACKOFF = 1 # retry backup multiplier +DEFAULT_MAX_RATE_LIMIT_RETRIES = float("inf") # default max number of times to retry on rate limit RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror, httplib.NotConnected, httplib.ImproperConnectionState, TransientSSLError) @@ -50,15 +52,18 @@ RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror, class MinimalRetry: def __init__(self, retry_delay=DEFAULT_DELAY, - timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF): + timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF, + max_rate_limit_retries=DEFAULT_MAX_RATE_LIMIT_RETRIES): """ Wrapper around retrying that helps to handle common transient exceptions. This minimalistic version only retries SSL errors and rate limiting. - :param retry_exceptions: types of exceptions to retry on. :param retry_delay: retry delay between the attempts. :param timeout: maximum time to wait. :param backoff: multiplier added to delay between attempts. + :param max_rate_limit_retries: The maximum number of retries to do when being rate limited by the server. + Set to `float("inf")` if retrying forever is desired. + Being rate limited does not count towards the timeout. :Example: @@ -72,12 +77,15 @@ class MinimalRetry: timeout = DEFAULT_TIMEOUT if backoff is None: backoff = DEFAULT_BACKOFF + if max_rate_limit_retries is None: + max_rate_limit_retries = DEFAULT_MAX_RATE_LIMIT_RETRIES timeout = max(timeout, 0) self.retry_delay = retry_delay self.timeout = timeout self.backoff = backoff + self.max_rate_limit_retries = max_rate_limit_retries def __call__(self, func): def transform_ssl_error(function, *args, **kwargs): @@ -93,19 +101,21 @@ class MinimalRetry: def retry_loop(*args, **kwargs): current_delay = self.retry_delay end = datetime.now() + timedelta(seconds=self.timeout) + number_rate_limited_retries = 0 while True: try: return transform_ssl_error(func, *args, **kwargs) except Exception as exc: - if isinstance(exc, RateLimitReachedError): + if isinstance(exc, RateLimitReachedError) and number_rate_limited_retries <= self.max_rate_limit_retries: + _logger.debug("You are being rate limited, backing off...") time.sleep(exc.retry_after) - # Reset retries if we're told to wait due to rate # limiting current_delay = self.retry_delay end = datetime.now() + timedelta( seconds=exc.retry_after + self.timeout) + number_rate_limited_retries += 1 elif datetime.now() >= end: raise elif self.should_retry(exc):
