Repository: libcloud
Updated Branches:
  refs/heads/trunk 5c8bab406 -> c61db674e


Retry on rate limit and connection timeout.

Closes #515

Signed-off-by: Tomaz Muraus <[email protected]>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/fe81fef8
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/fe81fef8
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/fe81fef8

Branch: refs/heads/trunk
Commit: fe81fef807d0f26b0fe94d55aa59b6979d133ae8
Parents: 5c8bab4
Author: Arpith Kuppur Prakash <[email protected]>
Authored: Wed May 6 14:02:17 2015 -0500
Committer: Tomaz Muraus <[email protected]>
Committed: Sun Jul 19 17:16:26 2015 +0800

----------------------------------------------------------------------
 CHANGES.rst                               |  6 +++
 libcloud/common/abiquo.py                 |  7 ++-
 libcloud/common/aws.py                    | 15 ++++--
 libcloud/common/base.py                   | 57 +++++++++++++++-----
 libcloud/common/exceptions.py             | 72 +++++++++++++++++++++++++
 libcloud/common/gandi.py                  |  8 ++-
 libcloud/common/openstack.py              |  6 ++-
 libcloud/compute/drivers/cloudframes.py   |  7 ++-
 libcloud/compute/drivers/rimuhosting.py   |  6 ++-
 libcloud/test/compute/test_retry_limit.py | 75 ++++++++++++++++++++++++++
 libcloud/test/test_connection.py          | 58 ++++++++++++++++++--
 libcloud/utils/misc.py                    | 54 +++++++++++++++++++
 12 files changed, 339 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index 1a7e459..a42a067 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -31,6 +31,12 @@ General
   (GITHUB-343, GITHUB-498, LIBCLOUD-601, LIBCLOUD-686)
   [Nicolas Fraison, Samuel Marks]
 
+- Add support for retrying failed HTTP requests.
+
+  Retrying is off by default and can be enabled by setting
+  ``LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS`` environment variable.
+  (GITHUB-515, LIBCLOUD-360, LIBCLOUD-709)
+
 Compute
 ~~~~~~~
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/abiquo.py
----------------------------------------------------------------------
diff --git a/libcloud/common/abiquo.py b/libcloud/common/abiquo.py
index 0d52fd7..c0a52fd 100644
--- a/libcloud/common/abiquo.py
+++ b/libcloud/common/abiquo.py
@@ -180,11 +180,14 @@ class AbiquoConnection(ConnectionUserAndKey, 
PollingConnection):
     responseCls = AbiquoResponse
 
     def __init__(self, user_id, key, secure=True, host=None, port=None,
-                 url=None, timeout=None):
+                 url=None, timeout=None,
+                 retry_delay=None, backoff=None):
         super(AbiquoConnection, self).__init__(user_id=user_id, key=key,
                                                secure=secure,
                                                host=host, port=port,
-                                               url=url, timeout=timeout)
+                                               url=url, timeout=timeout,
+                                               retry_delay=retry_delay,
+                                               backoff=backoff)
 
         # This attribute stores data cached across multiple request
         self.cache = {}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/aws.py
----------------------------------------------------------------------
diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py
index 45239e6..7a48e24 100644
--- a/libcloud/common/aws.py
+++ b/libcloud/common/aws.py
@@ -128,11 +128,14 @@ class AWSGenericResponse(AWSBaseResponse):
 
 class AWSTokenConnection(ConnectionUserAndKey):
     def __init__(self, user_id, key, secure=True,
-                 host=None, port=None, url=None, timeout=None, token=None):
+                 host=None, port=None, url=None, timeout=None, token=None,
+                 retry_delay=None, backoff=None):
         self.token = token
         super(AWSTokenConnection, self).__init__(user_id, key, secure=secure,
                                                  host=host, port=port, url=url,
-                                                 timeout=timeout)
+                                                 timeout=timeout,
+                                                 retry_delay=retry_delay,
+                                                 backoff=backoff)
 
     def add_default_params(self, params):
         # Even though we are adding it to the headers, we need it here too
@@ -325,12 +328,14 @@ class AWSRequestSignerAlgorithmV4(AWSRequestSigner):
 
 class SignedAWSConnection(AWSTokenConnection):
     def __init__(self, user_id, key, secure=True, host=None, port=None,
-                 url=None, timeout=None, token=None,
-                 signature_version=DEFAULT_SIGNATURE_VERSION):
+                 url=None, timeout=None, token=None, retry_delay=None,
+                 backoff=None, signature_version=DEFAULT_SIGNATURE_VERSION):
         super(SignedAWSConnection, self).__init__(user_id=user_id, key=key,
                                                   secure=secure, host=host,
                                                   port=port, url=url,
-                                                  timeout=timeout, token=token)
+                                                  timeout=timeout, token=token,
+                                                  retry_delay=retry_delay,
+                                                  backoff=backoff)
         self.signature_version = str(signature_version)
 
         if self.signature_version == '2':

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index d147ee3..4663e69 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -44,10 +44,11 @@ from libcloud.utils.py3 import StringIO
 from libcloud.utils.py3 import u
 from libcloud.utils.py3 import b
 
-from libcloud.utils.misc import lowercase_keys
+from libcloud.utils.misc import lowercase_keys, retry
 from libcloud.utils.compression import decompress_data
-from libcloud.common.types import LibcloudError, MalformedResponseError
 
+from libcloud.common.exceptions import exception_from_message
+from libcloud.common.types import LibcloudError, MalformedResponseError
 from libcloud.httplib_ssl import LibcloudHTTPConnection
 from libcloud.httplib_ssl import LibcloudHTTPSConnection
 
@@ -114,7 +115,9 @@ class Response(object):
             self.body = b(self.body).decode('utf-8')
 
         if not self.success():
-            raise Exception(self.parse_error())
+            raise exception_from_message(code=self.status,
+                                         message=self.parse_error(),
+                                         headers=self.headers)
 
         self.object = self.parse_body()
 
@@ -460,11 +463,13 @@ class Connection(object):
     driver = None
     action = None
     cache_busting = False
+    backoff = None
+    retry_delay = None
 
     allow_insecure = True
 
     def __init__(self, secure=True, host=None, port=None, url=None,
-                 timeout=None, proxy_url=None):
+                 timeout=None, proxy_url=None, retry_delay=None, backoff=None):
         self.secure = secure and 1 or 0
         self.ua = []
         self.context = {}
@@ -492,9 +497,11 @@ class Connection(object):
             (self.host, self.port, self.secure,
              self.request_path) = self._tuple_from_url(url)
 
-        if timeout:
-            self.timeout = timeout
-
+        if timeout is None:
+            timeout = self.__class__.timeout
+        self.timeout = timeout
+        self.retry_delay = retry_delay
+        self.backoff = backoff
         self.proxy_url = proxy_url
 
     def set_http_proxy(self, proxy_url):
@@ -675,6 +682,9 @@ class Connection(object):
         else:
             headers = copy.copy(headers)
 
+        retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS',
+                                       False)
+
         action = self.morph_action_hook(action)
         self.action = action
         self.method = method
@@ -738,8 +748,17 @@ class Connection(object):
 
                 self.connection.endheaders()
             else:
-                self.connection.request(method=method, url=url, body=data,
-                                        headers=headers)
+                if retry_enabled:
+                    retry_request = retry(timeout=self.timeout,
+                                          retry_delay=self.retry_delay,
+                                          backoff=self.backoff)
+                    retry_request(self.connection.request)(method=method,
+                                                           url=url,
+                                                           body=data,
+                                                           headers=headers)
+                else:
+                    self.connection.request(method=method, url=url, body=data,
+                                            headers=headers)
         except ssl.SSLError:
             e = sys.exc_info()[1]
             self.reset_context()
@@ -946,7 +965,7 @@ class ConnectionKey(Connection):
     Base connection class which accepts a single ``key`` argument.
     """
     def __init__(self, key, secure=True, host=None, port=None, url=None,
-                 timeout=None, proxy_url=None):
+                 timeout=None, proxy_url=None, backoff=None, retry_delay=None):
         """
         Initialize `user_id` and `key`; set `secure` to an ``int`` based on
         passed value.
@@ -954,7 +973,9 @@ class ConnectionKey(Connection):
         super(ConnectionKey, self).__init__(secure=secure, host=host,
                                             port=port, url=url,
                                             timeout=timeout,
-                                            proxy_url=proxy_url)
+                                            proxy_url=proxy_url,
+                                            backoff=backoff,
+                                            retry_delay=retry_delay)
         self.key = key
 
 
@@ -963,14 +984,16 @@ class CertificateConnection(Connection):
     Base connection class which accepts a single ``cert_file`` argument.
     """
     def __init__(self, cert_file, secure=True, host=None, port=None, url=None,
-                 timeout=None):
+                 timeout=None, backoff=None, retry_delay=None):
         """
         Initialize `cert_file`; set `secure` to an ``int`` based on
         passed value.
         """
         super(CertificateConnection, self).__init__(secure=secure, host=host,
                                                     port=port, url=url,
-                                                    timeout=timeout)
+                                                    timeout=timeout,
+                                                    backoff=backoff,
+                                                    retry_delay=retry_delay)
 
         self.cert_file = cert_file
 
@@ -983,10 +1006,13 @@ class ConnectionUserAndKey(ConnectionKey):
     user_id = None
 
     def __init__(self, user_id, key, secure=True, host=None, port=None,
-                 url=None, timeout=None, proxy_url=None):
+                 url=None, timeout=None, proxy_url=None,
+                 backoff=None, retry_delay=None):
         super(ConnectionUserAndKey, self).__init__(key, secure=secure,
                                                    host=host, port=port,
                                                    url=url, timeout=timeout,
+                                                   backoff=backoff,
+                                                   retry_delay=retry_delay,
                                                    proxy_url=proxy_url)
         self.user_id = user_id
 
@@ -1048,6 +1074,9 @@ class BaseDriver(object):
         self.region = region
 
         conn_kwargs = self._ex_connection_class_kwargs()
+        conn_kwargs.update({'timeout': kwargs.pop('timeout', None),
+                            'retry_delay': kwargs.pop('retry_delay', None),
+                            'backoff': kwargs.pop('backoff', None)})
         self.connection = self.connectionCls(*args, **conn_kwargs)
 
         self.connection.driver = self

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/exceptions.py
----------------------------------------------------------------------
diff --git a/libcloud/common/exceptions.py b/libcloud/common/exceptions.py
new file mode 100644
index 0000000..6768945
--- /dev/null
+++ b/libcloud/common/exceptions.py
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class BaseHTTPException(Exception):
+
+    """
+    The base exception class for all exception raises.
+    """
+
+    def __init__(self, code, message, headers=None):
+
+        self.message = message
+        self.code = code
+        self.headers = headers
+        # preserve old exception behavior for tests that
+        # look for e.args[0]
+        super(BaseHTTPException, self).__init__(message)
+
+    def __str__(self):
+        return self.message
+
+
+class RateLimit(BaseHTTPException):
+    """
+    HTTP 429 - Rate limit: you've sent too many requests for this time period.
+    """
+    code = 429
+    message = "{code} Rate limit exceeded".format(code=code)
+
+    def __init__(self, *args, **kwargs):
+        try:
+            self.retry_after = int(kwargs.pop('retry_after'))
+        except (KeyError, ValueError):
+            self.retry_after = 0
+
+
+_error_classes = [RateLimit]
+_code_map = dict((c.code, c) for c in _error_classes)
+
+
+def exception_from_message(code, message, headers=None):
+    """
+    Return an instance of BaseHTTPException or subclass based on response code.
+
+    Usage::
+        raise exception_from_message(code=self.status,
+                                     message=self.parse_error())
+    """
+    kwargs = {
+        'code': code,
+        'message': message,
+        'headers': headers
+    }
+
+    if headers and 'retry_after' in headers:
+        kwargs['retry_after'] = headers['retry_after']
+
+    cls = _code_map.get(code, BaseHTTPException)
+    return cls(**kwargs)

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/gandi.py
----------------------------------------------------------------------
diff --git a/libcloud/common/gandi.py b/libcloud/common/gandi.py
index 11e9622..7155f4a 100644
--- a/libcloud/common/gandi.py
+++ b/libcloud/common/gandi.py
@@ -57,12 +57,16 @@ class GandiConnection(XMLRPCConnection, ConnectionKey):
     host = 'rpc.gandi.net'
     endpoint = '/xmlrpc/'
 
-    def __init__(self, key, secure=True):
+    def __init__(self, key, secure=True, timeout=None,
+                 retry_delay=None, backoff=None):
         # Note: Method resolution order in this case is
         # XMLRPCConnection -> Connection and Connection doesn't take key as the
         # first argument so we specify a keyword argument instead.
         # Previously it was GandiConnection -> ConnectionKey so it worked fine.
-        super(GandiConnection, self).__init__(key=key, secure=secure)
+        super(GandiConnection, self).__init__(key=key, secure=secure,
+                                              timeout=timeout,
+                                              retry_delay=retry_delay,
+                                              backoff=backoff)
         self.driver = BaseGandiDriver
 
     def request(self, method, *args):

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/openstack.py
----------------------------------------------------------------------
diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py
index e44d144..4e8d6fc 100644
--- a/libcloud/common/openstack.py
+++ b/libcloud/common/openstack.py
@@ -130,9 +130,11 @@ class OpenStackBaseConnection(ConnectionUserAndKey):
                  ex_tenant_name=None,
                  ex_force_service_type=None,
                  ex_force_service_name=None,
-                 ex_force_service_region=None):
+                 ex_force_service_region=None,
+                 retry_delay=None, backoff=None):
         super(OpenStackBaseConnection, self).__init__(
-            user_id, key, secure=secure, timeout=timeout)
+            user_id, key, secure=secure, timeout=timeout,
+            retry_delay=retry_delay, backoff=backoff)
 
         if ex_force_auth_version:
             self._auth_version = ex_force_auth_version

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/compute/drivers/cloudframes.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/cloudframes.py 
b/libcloud/compute/drivers/cloudframes.py
index 9902f5d..6aca784 100644
--- a/libcloud/compute/drivers/cloudframes.py
+++ b/libcloud/compute/drivers/cloudframes.py
@@ -116,7 +116,8 @@ class CloudFramesConnection(XMLRPCConnection, 
ConnectionKey):
     base_url = None
 
     def __init__(self, key=None, secret=None, secure=True,
-                 host=None, port=None, url=None, timeout=None):
+                 host=None, port=None, url=None, timeout=None,
+                 retry_delay=None, backoff=None):
         """
         :param    key:    The username to connect with to the cloudapi
         :type     key:    ``str``
@@ -139,7 +140,9 @@ class CloudFramesConnection(XMLRPCConnection, 
ConnectionKey):
 
         super(CloudFramesConnection, self).__init__(key=key, secure=secure,
                                                     host=host, port=port,
-                                                    url=url, timeout=timeout)
+                                                    url=url, timeout=timeout,
+                                                    retry_delay=retry_delay,
+                                                    backoff=backoff)
         self._auth = base64.b64encode(
             b('%s:%s' % (key, secret))).decode('utf-8')
         self.endpoint = url

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/compute/drivers/rimuhosting.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/rimuhosting.py 
b/libcloud/compute/drivers/rimuhosting.py
index f7cfdd7..4cf8457 100644
--- a/libcloud/compute/drivers/rimuhosting.py
+++ b/libcloud/compute/drivers/rimuhosting.py
@@ -75,9 +75,11 @@ class RimuHostingConnection(ConnectionKey):
     port = 443
     responseCls = RimuHostingResponse
 
-    def __init__(self, key, secure=True):
+    def __init__(self, key, secure=True, retry_delay=None,
+                 backoff=None, timeout=None):
         # override __init__ so that we can set secure of False for testing
-        ConnectionKey.__init__(self, key, secure)
+        ConnectionKey.__init__(self, key, secure, timeout=timeout,
+                               retry_delay=retry_delay, backoff=backoff)
 
     def add_default_headers(self, headers):
         # We want JSON back from the server. Could be application/xml

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/test/compute/test_retry_limit.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_retry_limit.py 
b/libcloud/test/compute/test_retry_limit.py
new file mode 100644
index 0000000..d285c80
--- /dev/null
+++ b/libcloud/test/compute/test_retry_limit.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import socket
+import tempfile
+from libcloud.utils.py3 import httplib
+
+from mock import Mock, patch, MagicMock
+
+from libcloud.common.base import Connection
+from libcloud.common.base import Response
+from libcloud.common.exceptions import RateLimit
+from libcloud.test import unittest
+
+CONFLICT_RESPONSE_STATUS = [
+    ('status', '429'), ('reason', 'CONFLICT'),
+    ('retry_after', '3'),
+    ('content-type', 'application/json')]
+SIMPLE_RESPONSE_STATUS = ('HTTP/1.1', 429, 'CONFLICT')
+
+
+class RateLimitTestCase(unittest.TestCase):
+
+    def _raise_socket_error(self):
+        raise socket.gaierror('')
+
+    def test_retry_connection(self):
+        con = Connection(timeout=1, retry_delay=0.1)
+        con.connection = Mock()
+        connect_method = 'libcloud.common.base.Connection.request'
+
+        with patch(connect_method) as mock_connect:
+            try:
+                mock_connect.side_effect = socket.gaierror('')
+                con.request('/')
+            except socket.gaierror:
+                pass
+            except Exception:
+                self.fail('Failed to raise socket exception')
+
+    def test_rate_limit_error(self):
+        sock = Mock()
+        con = Connection()
+
+        try:
+            with patch('libcloud.utils.py3.httplib.HTTPResponse.getheaders',
+                       MagicMock(return_value=CONFLICT_RESPONSE_STATUS)):
+                with patch(
+                        'libcloud.utils.py3.httplib.HTTPResponse._read_status',
+                        MagicMock(return_value=SIMPLE_RESPONSE_STATUS)):
+                    with tempfile.TemporaryFile(mode='w+b') as f:
+                        f.write('HTTP/1.1 429 CONFLICT\n'.encode())
+                        f.flush()
+                        sock.makefile = Mock(return_value=f)
+                        mock_obj = httplib.HTTPResponse(sock)
+                        mock_obj.begin()
+                        Response(mock_obj, con)
+        except RateLimit:
+            pass
+        except Exception:
+            self.fail('Failed to raise Rate Limit exception')
+
+if __name__ == '__main__':
+    unittest.main()

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/test/test_connection.py
----------------------------------------------------------------------
diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py
index 821425d..96e86ce 100644
--- a/libcloud/test/test_connection.py
+++ b/libcloud/test/test_connection.py
@@ -15,16 +15,18 @@
 # limitations under the License.
 
 import os
+import socket
 import sys
 import ssl
 
-from mock import Mock, call
+from mock import Mock, call, patch
 
 from libcloud.test import unittest
 from libcloud.common.base import Connection
 from libcloud.common.base import LoggingConnection
 from libcloud.httplib_ssl import LibcloudBaseConnection
 from libcloud.httplib_ssl import LibcloudHTTPConnection
+from libcloud.utils.misc import retry
 
 
 class BaseConnectionClassTestCase(unittest.TestCase):
@@ -230,12 +232,11 @@ class ConnectionClassTestCase(unittest.TestCase):
         self.assertEqual(con.context, {})
 
         # Context should also be reset if a method inside request throws
-        con = Connection()
+        con = Connection(timeout=1, retry_delay=0.1)
         con.connection = Mock()
 
         con.set_context(context)
         self.assertEqual(con.context, context)
-
         con.connection.request = Mock(side_effect=ssl.SSLError())
 
         try:
@@ -278,5 +279,56 @@ class ConnectionClassTestCase(unittest.TestCase):
         cmd = con._log_curl(method='HEAD', url=url, body=body, headers=headers)
         self.assertEqual(cmd, 'curl -i --head --compress 
http://example.com:80/test/path')
 
+    def _raise_socket_error(self):
+        raise socket.gaierror('')
+
+    def test_retry_with_sleep(self):
+        con = Connection()
+        con.connection = Mock()
+        connect_method = 'libcloud.common.base.Connection.request'
+
+        with patch(connect_method) as mock_connect:
+            mock_connect.__name__ = 'mock_connect'
+            with self.assertRaises(socket.gaierror):
+                mock_connect.side_effect = socket.gaierror('')
+                retry_request = retry(timeout=1, retry_delay=.1,
+                                      backoff=1)
+                retry_request(con.request)(action='/')
+
+            self.assertGreater(mock_connect.call_count, 1,
+                               'Retry logic failed')
+
+    def test_retry_with_timeout(self):
+        con = Connection()
+        con.connection = Mock()
+        connect_method = 'libcloud.common.base.Connection.request'
+
+        with patch(connect_method) as mock_connect:
+            mock_connect.__name__ = 'mock_connect'
+            with self.assertRaises(socket.gaierror):
+                mock_connect.side_effect = socket.gaierror('')
+                retry_request = retry(timeout=2, retry_delay=.1,
+                                      backoff=1)
+                retry_request(con.request)(action='/')
+
+            self.assertGreater(mock_connect.call_count, 1,
+                               'Retry logic failed')
+
+    def test_retry_with_backoff(self):
+        con = Connection()
+        con.connection = Mock()
+        connect_method = 'libcloud.common.base.Connection.request'
+
+        with patch(connect_method) as mock_connect:
+            mock_connect.__name__ = 'mock_connect'
+            with self.assertRaises(socket.gaierror):
+                mock_connect.side_effect = socket.gaierror('')
+                retry_request = retry(timeout=2, retry_delay=.1,
+                                      backoff=1)
+                retry_request(con.request)(action='/')
+
+            self.assertGreater(mock_connect.call_count, 1,
+                               'Retry logic failed')
+
 if __name__ == '__main__':
     sys.exit(unittest.main())

http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/utils/misc.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/misc.py b/libcloud/utils/misc.py
index 49c1a0c..6389827 100644
--- a/libcloud/utils/misc.py
+++ b/libcloud/utils/misc.py
@@ -12,11 +12,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from functools import wraps
 
 import os
 import sys
 import binascii
+from libcloud.utils.py3 import httplib
 
+import socket
+from datetime import datetime, timedelta
+import time
+
+from libcloud.common.exceptions import RateLimit
+
+DEFAULT_TIMEOUT = 30
+DEFAULT_SLEEP = 1
+DEFAULT_BACKCOFF = 1
+EXCEPTION_TYPES = (RateLimit, socket.error, socket.gaierror,
+                   httplib.NotConnected, httplib.ImproperConnectionState)
 
 __all__ = [
     'find',
@@ -282,3 +295,44 @@ class ReprMixin(object):
 
     def __str__(self):
         return str(self.__repr__())
+
+
+def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None,
+          timeout=None, backoff=None):
+    """
+    Retry method that helps to handle common exception.
+    :param func: the function to execute.
+    :param timeout: maximum time to wait.
+    :param retry_delay: retry delay between the attempts.
+    :param backoff: multiplier added to delay between attempts.
+    :param retry_exceptions: types of exceptions to retry on.
+
+    :Example:
+
+    retry_request = retry(timeout=1, retry_delay=1, backoff=1)
+    retry_request(self.connection.request)()
+    """
+    def deco_retry(func):
+        @wraps(func)
+        def retry_loop(*args, **kwargs):
+            delay = retry_delay
+            end = datetime.now() + timedelta(seconds=timeout)
+            exc_info = None
+            while datetime.now() < end:
+                try:
+                    result = func(*args, **kwargs)
+                    return result
+                except retry_exceptions as exc:
+                    if isinstance(exc, RateLimit):
+                        time.sleep(exc.retry_after)
+                        end = datetime.now() + timedelta(
+                            seconds=exc.retry_after + timeout)
+                    else:
+                        exc_info = sys.exc_info()
+                        time.sleep(delay)
+                        delay *= backoff
+            if exc_info:
+                raise exc_info[1]
+            return func(*args, **kwargs)
+        return retry_loop
+    return deco_retry

Reply via email to