Repository: libcloud Updated Branches: refs/heads/trunk e3bb83801 -> c626d9820
Add SSLError to retry decorator exceptions Closes #556 Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/5733de36 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/5733de36 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/5733de36 Branch: refs/heads/trunk Commit: 5733de362d198e70ae5c81082f96de83f307d4b2 Parents: e3bb838 Author: Scott Kruger <[email protected]> Authored: Thu Jul 30 10:02:31 2015 -0500 Committer: Tomaz Muraus <[email protected]> Committed: Sun Jan 10 16:31:52 2016 +0100 ---------------------------------------------------------------------- libcloud/test/common/test_retry_limit.py | 15 +++++ libcloud/utils/misc.py | 83 +++++++++++++++++++-------- 2 files changed, 74 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/5733de36/libcloud/test/common/test_retry_limit.py ---------------------------------------------------------------------- diff --git a/libcloud/test/common/test_retry_limit.py b/libcloud/test/common/test_retry_limit.py index 72e3710..06eac2a 100644 --- a/libcloud/test/common/test_retry_limit.py +++ b/libcloud/test/common/test_retry_limit.py @@ -15,10 +15,12 @@ import socket import tempfile +import ssl from mock import Mock, patch, MagicMock from libcloud.utils.py3 import httplib +from libcloud.utils.misc import TRANSIENT_SSL_ERROR from libcloud.common.base import Connection from libcloud.common.base import Response from libcloud.common.exceptions import RateLimitReachedError @@ -31,6 +33,7 @@ CONFLICT_RESPONSE_STATUS = [ SIMPLE_RESPONSE_STATUS = ('HTTP/1.1', 429, 'CONFLICT') +@patch('os.environ', {'LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS': True}) class FailedRequestRetryTestCase(unittest.TestCase): def _raise_socket_error(self): @@ -50,6 +53,18 @@ class FailedRequestRetryTestCase(unittest.TestCase): except Exception: self.fail('Failed to raise socket exception') + def test_retry_connection_ssl_error(self): + conn = Connection(timeout=1, retry_delay=0.1) + + with patch.object(conn, 'connect', Mock()): + with patch.object(conn, 'connection') as connection: + connection.request = MagicMock( + __name__='request', + side_effect=ssl.SSLError(TRANSIENT_SSL_ERROR)) + + self.assertRaises(ssl.SSLError, conn.request, '/') + self.assertGreater(connection.request.call_count, 1) + def test_rate_limit_error(self): sock = Mock() con = Connection() http://git-wip-us.apache.org/repos/asf/libcloud/blob/5733de36/libcloud/utils/misc.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/misc.py b/libcloud/utils/misc.py index 970feaa..1902d23 100644 --- a/libcloud/utils/misc.py +++ b/libcloud/utils/misc.py @@ -22,14 +22,25 @@ from libcloud.utils.py3 import httplib import socket from datetime import datetime, timedelta import time +import ssl from libcloud.common.exceptions import RateLimitReachedError + +TRANSIENT_SSL_ERROR = 'The read operation timed out' + + +class TransientSSLError(ssl.SSLError): + """Represent transient SSL errors, e.g. timeouts""" + pass + + DEFAULT_TIMEOUT = 30 -DEFAULT_SLEEP = 1 -DEFAULT_BACKCOFF = 1 -EXCEPTION_TYPES = (RateLimitReachedError, socket.error, socket.gaierror, - httplib.NotConnected, httplib.ImproperConnectionState) +DEFAULT_DELAY = 1 +DEFAULT_BACKOFF = 1 +RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror, + httplib.NotConnected, httplib.ImproperConnectionState, + TransientSSLError) __all__ = [ 'find', @@ -298,10 +309,10 @@ class ReprMixin(object): return str(self.__repr__()) -def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None, - timeout=None, backoff=None): +def retry(retry_exceptions=None, retry_delay=None, timeout=None, + backoff=None): """ - Retry method that helps to handle common exception. + Retry decorator that helps to handle common transient exceptions. :param retry_exceptions: types of exceptions to retry on. :param retry_delay: retry delay between the attempts. @@ -313,29 +324,53 @@ def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None, retry_request = retry(timeout=1, retry_delay=1, backoff=1) retry_request(self.connection.request)() """ - def deco_retry(func): + if retry_exceptions is None: + retry_exceptions = RETRY_EXCEPTIONS + if retry_delay is None: + retry_delay = DEFAULT_DELAY + if timeout is None: + timeout = DEFAULT_TIMEOUT + if backoff is None: + backoff = DEFAULT_BACKOFF + + timeout = max(timeout, 0) + + def transform_ssl_error(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except ssl.SSLError: + exc = sys.exc_info()[1] + + if TRANSIENT_SSL_ERROR in str(exc): + raise TransientSSLError(*exc.args) + + raise exc + + def decorator(func): @wraps(func) def retry_loop(*args, **kwargs): - delay = retry_delay + current_delay = retry_delay end = datetime.now() + timedelta(seconds=timeout) - exc_info = None - while datetime.now() < end: + + while True: try: - result = func(*args, **kwargs) - return result + return transform_ssl_error(func, *args, **kwargs) except retry_exceptions: - e = sys.exc_info()[1] + exc = sys.exc_info()[1] - if isinstance(e, RateLimitReachedError): - time.sleep(e.retry_after) + if isinstance(exc, RateLimitReachedError): + time.sleep(exc.retry_after) + + # Reset retries if we're told to wait due to rate + # limiting + current_delay = retry_delay end = datetime.now() + timedelta( - seconds=e.retry_after + timeout) + seconds=exc.retry_after + timeout) + elif datetime.now() >= end: + raise else: - exc_info = e - time.sleep(delay) - delay *= backoff - if exc_info: - raise exc_info - return func(*args, **kwargs) + time.sleep(current_delay) + current_delay *= backoff + return retry_loop - return deco_retry + return decorator
