Repository: libcloud Updated Branches: refs/heads/trunk 5c8bab406 -> c61db674e
Retry on rate limit and connection timeout. Closes #515 Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/fe81fef8 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/fe81fef8 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/fe81fef8 Branch: refs/heads/trunk Commit: fe81fef807d0f26b0fe94d55aa59b6979d133ae8 Parents: 5c8bab4 Author: Arpith Kuppur Prakash <[email protected]> Authored: Wed May 6 14:02:17 2015 -0500 Committer: Tomaz Muraus <[email protected]> Committed: Sun Jul 19 17:16:26 2015 +0800 ---------------------------------------------------------------------- CHANGES.rst | 6 +++ libcloud/common/abiquo.py | 7 ++- libcloud/common/aws.py | 15 ++++-- libcloud/common/base.py | 57 +++++++++++++++----- libcloud/common/exceptions.py | 72 +++++++++++++++++++++++++ libcloud/common/gandi.py | 8 ++- libcloud/common/openstack.py | 6 ++- libcloud/compute/drivers/cloudframes.py | 7 ++- libcloud/compute/drivers/rimuhosting.py | 6 ++- libcloud/test/compute/test_retry_limit.py | 75 ++++++++++++++++++++++++++ libcloud/test/test_connection.py | 58 ++++++++++++++++++-- libcloud/utils/misc.py | 54 +++++++++++++++++++ 12 files changed, 339 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index 1a7e459..a42a067 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -31,6 +31,12 @@ General (GITHUB-343, GITHUB-498, LIBCLOUD-601, LIBCLOUD-686) [Nicolas Fraison, Samuel Marks] +- Add support for retrying failed HTTP requests. + + Retrying is off by default and can be enabled by setting + ``LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS`` environment variable. + (GITHUB-515, LIBCLOUD-360, LIBCLOUD-709) + Compute ~~~~~~~ http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/abiquo.py ---------------------------------------------------------------------- diff --git a/libcloud/common/abiquo.py b/libcloud/common/abiquo.py index 0d52fd7..c0a52fd 100644 --- a/libcloud/common/abiquo.py +++ b/libcloud/common/abiquo.py @@ -180,11 +180,14 @@ class AbiquoConnection(ConnectionUserAndKey, PollingConnection): responseCls = AbiquoResponse def __init__(self, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None): + url=None, timeout=None, + retry_delay=None, backoff=None): super(AbiquoConnection, self).__init__(user_id=user_id, key=key, secure=secure, host=host, port=port, - url=url, timeout=timeout) + url=url, timeout=timeout, + retry_delay=retry_delay, + backoff=backoff) # This attribute stores data cached across multiple request self.cache = {} http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/aws.py ---------------------------------------------------------------------- diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py index 45239e6..7a48e24 100644 --- a/libcloud/common/aws.py +++ b/libcloud/common/aws.py @@ -128,11 +128,14 @@ class AWSGenericResponse(AWSBaseResponse): class AWSTokenConnection(ConnectionUserAndKey): def __init__(self, user_id, key, secure=True, - host=None, port=None, url=None, timeout=None, token=None): + host=None, port=None, url=None, timeout=None, token=None, + retry_delay=None, backoff=None): self.token = token super(AWSTokenConnection, self).__init__(user_id, key, secure=secure, host=host, port=port, url=url, - timeout=timeout) + timeout=timeout, + retry_delay=retry_delay, + backoff=backoff) def add_default_params(self, params): # Even though we are adding it to the headers, we need it here too @@ -325,12 +328,14 @@ class AWSRequestSignerAlgorithmV4(AWSRequestSigner): class SignedAWSConnection(AWSTokenConnection): def __init__(self, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None, token=None, - signature_version=DEFAULT_SIGNATURE_VERSION): + url=None, timeout=None, token=None, retry_delay=None, + backoff=None, signature_version=DEFAULT_SIGNATURE_VERSION): super(SignedAWSConnection, self).__init__(user_id=user_id, key=key, secure=secure, host=host, port=port, url=url, - timeout=timeout, token=token) + timeout=timeout, token=token, + retry_delay=retry_delay, + backoff=backoff) self.signature_version = str(signature_version) if self.signature_version == '2': http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index d147ee3..4663e69 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -44,10 +44,11 @@ from libcloud.utils.py3 import StringIO from libcloud.utils.py3 import u from libcloud.utils.py3 import b -from libcloud.utils.misc import lowercase_keys +from libcloud.utils.misc import lowercase_keys, retry from libcloud.utils.compression import decompress_data -from libcloud.common.types import LibcloudError, MalformedResponseError +from libcloud.common.exceptions import exception_from_message +from libcloud.common.types import LibcloudError, MalformedResponseError from libcloud.httplib_ssl import LibcloudHTTPConnection from libcloud.httplib_ssl import LibcloudHTTPSConnection @@ -114,7 +115,9 @@ class Response(object): self.body = b(self.body).decode('utf-8') if not self.success(): - raise Exception(self.parse_error()) + raise exception_from_message(code=self.status, + message=self.parse_error(), + headers=self.headers) self.object = self.parse_body() @@ -460,11 +463,13 @@ class Connection(object): driver = None action = None cache_busting = False + backoff = None + retry_delay = None allow_insecure = True def __init__(self, secure=True, host=None, port=None, url=None, - timeout=None, proxy_url=None): + timeout=None, proxy_url=None, retry_delay=None, backoff=None): self.secure = secure and 1 or 0 self.ua = [] self.context = {} @@ -492,9 +497,11 @@ class Connection(object): (self.host, self.port, self.secure, self.request_path) = self._tuple_from_url(url) - if timeout: - self.timeout = timeout - + if timeout is None: + timeout = self.__class__.timeout + self.timeout = timeout + self.retry_delay = retry_delay + self.backoff = backoff self.proxy_url = proxy_url def set_http_proxy(self, proxy_url): @@ -675,6 +682,9 @@ class Connection(object): else: headers = copy.copy(headers) + retry_enabled = os.environ.get('LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS', + False) + action = self.morph_action_hook(action) self.action = action self.method = method @@ -738,8 +748,17 @@ class Connection(object): self.connection.endheaders() else: - self.connection.request(method=method, url=url, body=data, - headers=headers) + if retry_enabled: + retry_request = retry(timeout=self.timeout, + retry_delay=self.retry_delay, + backoff=self.backoff) + retry_request(self.connection.request)(method=method, + url=url, + body=data, + headers=headers) + else: + self.connection.request(method=method, url=url, body=data, + headers=headers) except ssl.SSLError: e = sys.exc_info()[1] self.reset_context() @@ -946,7 +965,7 @@ class ConnectionKey(Connection): Base connection class which accepts a single ``key`` argument. """ def __init__(self, key, secure=True, host=None, port=None, url=None, - timeout=None, proxy_url=None): + timeout=None, proxy_url=None, backoff=None, retry_delay=None): """ Initialize `user_id` and `key`; set `secure` to an ``int`` based on passed value. @@ -954,7 +973,9 @@ class ConnectionKey(Connection): super(ConnectionKey, self).__init__(secure=secure, host=host, port=port, url=url, timeout=timeout, - proxy_url=proxy_url) + proxy_url=proxy_url, + backoff=backoff, + retry_delay=retry_delay) self.key = key @@ -963,14 +984,16 @@ class CertificateConnection(Connection): Base connection class which accepts a single ``cert_file`` argument. """ def __init__(self, cert_file, secure=True, host=None, port=None, url=None, - timeout=None): + timeout=None, backoff=None, retry_delay=None): """ Initialize `cert_file`; set `secure` to an ``int`` based on passed value. """ super(CertificateConnection, self).__init__(secure=secure, host=host, port=port, url=url, - timeout=timeout) + timeout=timeout, + backoff=backoff, + retry_delay=retry_delay) self.cert_file = cert_file @@ -983,10 +1006,13 @@ class ConnectionUserAndKey(ConnectionKey): user_id = None def __init__(self, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None, proxy_url=None): + url=None, timeout=None, proxy_url=None, + backoff=None, retry_delay=None): super(ConnectionUserAndKey, self).__init__(key, secure=secure, host=host, port=port, url=url, timeout=timeout, + backoff=backoff, + retry_delay=retry_delay, proxy_url=proxy_url) self.user_id = user_id @@ -1048,6 +1074,9 @@ class BaseDriver(object): self.region = region conn_kwargs = self._ex_connection_class_kwargs() + conn_kwargs.update({'timeout': kwargs.pop('timeout', None), + 'retry_delay': kwargs.pop('retry_delay', None), + 'backoff': kwargs.pop('backoff', None)}) self.connection = self.connectionCls(*args, **conn_kwargs) self.connection.driver = self http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/exceptions.py ---------------------------------------------------------------------- diff --git a/libcloud/common/exceptions.py b/libcloud/common/exceptions.py new file mode 100644 index 0000000..6768945 --- /dev/null +++ b/libcloud/common/exceptions.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseHTTPException(Exception): + + """ + The base exception class for all exception raises. + """ + + def __init__(self, code, message, headers=None): + + self.message = message + self.code = code + self.headers = headers + # preserve old exception behavior for tests that + # look for e.args[0] + super(BaseHTTPException, self).__init__(message) + + def __str__(self): + return self.message + + +class RateLimit(BaseHTTPException): + """ + HTTP 429 - Rate limit: you've sent too many requests for this time period. + """ + code = 429 + message = "{code} Rate limit exceeded".format(code=code) + + def __init__(self, *args, **kwargs): + try: + self.retry_after = int(kwargs.pop('retry_after')) + except (KeyError, ValueError): + self.retry_after = 0 + + +_error_classes = [RateLimit] +_code_map = dict((c.code, c) for c in _error_classes) + + +def exception_from_message(code, message, headers=None): + """ + Return an instance of BaseHTTPException or subclass based on response code. + + Usage:: + raise exception_from_message(code=self.status, + message=self.parse_error()) + """ + kwargs = { + 'code': code, + 'message': message, + 'headers': headers + } + + if headers and 'retry_after' in headers: + kwargs['retry_after'] = headers['retry_after'] + + cls = _code_map.get(code, BaseHTTPException) + return cls(**kwargs) http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/gandi.py ---------------------------------------------------------------------- diff --git a/libcloud/common/gandi.py b/libcloud/common/gandi.py index 11e9622..7155f4a 100644 --- a/libcloud/common/gandi.py +++ b/libcloud/common/gandi.py @@ -57,12 +57,16 @@ class GandiConnection(XMLRPCConnection, ConnectionKey): host = 'rpc.gandi.net' endpoint = '/xmlrpc/' - def __init__(self, key, secure=True): + def __init__(self, key, secure=True, timeout=None, + retry_delay=None, backoff=None): # Note: Method resolution order in this case is # XMLRPCConnection -> Connection and Connection doesn't take key as the # first argument so we specify a keyword argument instead. # Previously it was GandiConnection -> ConnectionKey so it worked fine. - super(GandiConnection, self).__init__(key=key, secure=secure) + super(GandiConnection, self).__init__(key=key, secure=secure, + timeout=timeout, + retry_delay=retry_delay, + backoff=backoff) self.driver = BaseGandiDriver def request(self, method, *args): http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/common/openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py index e44d144..4e8d6fc 100644 --- a/libcloud/common/openstack.py +++ b/libcloud/common/openstack.py @@ -130,9 +130,11 @@ class OpenStackBaseConnection(ConnectionUserAndKey): ex_tenant_name=None, ex_force_service_type=None, ex_force_service_name=None, - ex_force_service_region=None): + ex_force_service_region=None, + retry_delay=None, backoff=None): super(OpenStackBaseConnection, self).__init__( - user_id, key, secure=secure, timeout=timeout) + user_id, key, secure=secure, timeout=timeout, + retry_delay=retry_delay, backoff=backoff) if ex_force_auth_version: self._auth_version = ex_force_auth_version http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/compute/drivers/cloudframes.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/cloudframes.py b/libcloud/compute/drivers/cloudframes.py index 9902f5d..6aca784 100644 --- a/libcloud/compute/drivers/cloudframes.py +++ b/libcloud/compute/drivers/cloudframes.py @@ -116,7 +116,8 @@ class CloudFramesConnection(XMLRPCConnection, ConnectionKey): base_url = None def __init__(self, key=None, secret=None, secure=True, - host=None, port=None, url=None, timeout=None): + host=None, port=None, url=None, timeout=None, + retry_delay=None, backoff=None): """ :param key: The username to connect with to the cloudapi :type key: ``str`` @@ -139,7 +140,9 @@ class CloudFramesConnection(XMLRPCConnection, ConnectionKey): super(CloudFramesConnection, self).__init__(key=key, secure=secure, host=host, port=port, - url=url, timeout=timeout) + url=url, timeout=timeout, + retry_delay=retry_delay, + backoff=backoff) self._auth = base64.b64encode( b('%s:%s' % (key, secret))).decode('utf-8') self.endpoint = url http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/compute/drivers/rimuhosting.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/rimuhosting.py b/libcloud/compute/drivers/rimuhosting.py index f7cfdd7..4cf8457 100644 --- a/libcloud/compute/drivers/rimuhosting.py +++ b/libcloud/compute/drivers/rimuhosting.py @@ -75,9 +75,11 @@ class RimuHostingConnection(ConnectionKey): port = 443 responseCls = RimuHostingResponse - def __init__(self, key, secure=True): + def __init__(self, key, secure=True, retry_delay=None, + backoff=None, timeout=None): # override __init__ so that we can set secure of False for testing - ConnectionKey.__init__(self, key, secure) + ConnectionKey.__init__(self, key, secure, timeout=timeout, + retry_delay=retry_delay, backoff=backoff) def add_default_headers(self, headers): # We want JSON back from the server. Could be application/xml http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/test/compute/test_retry_limit.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_retry_limit.py b/libcloud/test/compute/test_retry_limit.py new file mode 100644 index 0000000..d285c80 --- /dev/null +++ b/libcloud/test/compute/test_retry_limit.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import socket +import tempfile +from libcloud.utils.py3 import httplib + +from mock import Mock, patch, MagicMock + +from libcloud.common.base import Connection +from libcloud.common.base import Response +from libcloud.common.exceptions import RateLimit +from libcloud.test import unittest + +CONFLICT_RESPONSE_STATUS = [ + ('status', '429'), ('reason', 'CONFLICT'), + ('retry_after', '3'), + ('content-type', 'application/json')] +SIMPLE_RESPONSE_STATUS = ('HTTP/1.1', 429, 'CONFLICT') + + +class RateLimitTestCase(unittest.TestCase): + + def _raise_socket_error(self): + raise socket.gaierror('') + + def test_retry_connection(self): + con = Connection(timeout=1, retry_delay=0.1) + con.connection = Mock() + connect_method = 'libcloud.common.base.Connection.request' + + with patch(connect_method) as mock_connect: + try: + mock_connect.side_effect = socket.gaierror('') + con.request('/') + except socket.gaierror: + pass + except Exception: + self.fail('Failed to raise socket exception') + + def test_rate_limit_error(self): + sock = Mock() + con = Connection() + + try: + with patch('libcloud.utils.py3.httplib.HTTPResponse.getheaders', + MagicMock(return_value=CONFLICT_RESPONSE_STATUS)): + with patch( + 'libcloud.utils.py3.httplib.HTTPResponse._read_status', + MagicMock(return_value=SIMPLE_RESPONSE_STATUS)): + with tempfile.TemporaryFile(mode='w+b') as f: + f.write('HTTP/1.1 429 CONFLICT\n'.encode()) + f.flush() + sock.makefile = Mock(return_value=f) + mock_obj = httplib.HTTPResponse(sock) + mock_obj.begin() + Response(mock_obj, con) + except RateLimit: + pass + except Exception: + self.fail('Failed to raise Rate Limit exception') + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/test/test_connection.py ---------------------------------------------------------------------- diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py index 821425d..96e86ce 100644 --- a/libcloud/test/test_connection.py +++ b/libcloud/test/test_connection.py @@ -15,16 +15,18 @@ # limitations under the License. import os +import socket import sys import ssl -from mock import Mock, call +from mock import Mock, call, patch from libcloud.test import unittest from libcloud.common.base import Connection from libcloud.common.base import LoggingConnection from libcloud.httplib_ssl import LibcloudBaseConnection from libcloud.httplib_ssl import LibcloudHTTPConnection +from libcloud.utils.misc import retry class BaseConnectionClassTestCase(unittest.TestCase): @@ -230,12 +232,11 @@ class ConnectionClassTestCase(unittest.TestCase): self.assertEqual(con.context, {}) # Context should also be reset if a method inside request throws - con = Connection() + con = Connection(timeout=1, retry_delay=0.1) con.connection = Mock() con.set_context(context) self.assertEqual(con.context, context) - con.connection.request = Mock(side_effect=ssl.SSLError()) try: @@ -278,5 +279,56 @@ class ConnectionClassTestCase(unittest.TestCase): cmd = con._log_curl(method='HEAD', url=url, body=body, headers=headers) self.assertEqual(cmd, 'curl -i --head --compress http://example.com:80/test/path') + def _raise_socket_error(self): + raise socket.gaierror('') + + def test_retry_with_sleep(self): + con = Connection() + con.connection = Mock() + connect_method = 'libcloud.common.base.Connection.request' + + with patch(connect_method) as mock_connect: + mock_connect.__name__ = 'mock_connect' + with self.assertRaises(socket.gaierror): + mock_connect.side_effect = socket.gaierror('') + retry_request = retry(timeout=1, retry_delay=.1, + backoff=1) + retry_request(con.request)(action='/') + + self.assertGreater(mock_connect.call_count, 1, + 'Retry logic failed') + + def test_retry_with_timeout(self): + con = Connection() + con.connection = Mock() + connect_method = 'libcloud.common.base.Connection.request' + + with patch(connect_method) as mock_connect: + mock_connect.__name__ = 'mock_connect' + with self.assertRaises(socket.gaierror): + mock_connect.side_effect = socket.gaierror('') + retry_request = retry(timeout=2, retry_delay=.1, + backoff=1) + retry_request(con.request)(action='/') + + self.assertGreater(mock_connect.call_count, 1, + 'Retry logic failed') + + def test_retry_with_backoff(self): + con = Connection() + con.connection = Mock() + connect_method = 'libcloud.common.base.Connection.request' + + with patch(connect_method) as mock_connect: + mock_connect.__name__ = 'mock_connect' + with self.assertRaises(socket.gaierror): + mock_connect.side_effect = socket.gaierror('') + retry_request = retry(timeout=2, retry_delay=.1, + backoff=1) + retry_request(con.request)(action='/') + + self.assertGreater(mock_connect.call_count, 1, + 'Retry logic failed') + if __name__ == '__main__': sys.exit(unittest.main()) http://git-wip-us.apache.org/repos/asf/libcloud/blob/fe81fef8/libcloud/utils/misc.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/misc.py b/libcloud/utils/misc.py index 49c1a0c..6389827 100644 --- a/libcloud/utils/misc.py +++ b/libcloud/utils/misc.py @@ -12,11 +12,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from functools import wraps import os import sys import binascii +from libcloud.utils.py3 import httplib +import socket +from datetime import datetime, timedelta +import time + +from libcloud.common.exceptions import RateLimit + +DEFAULT_TIMEOUT = 30 +DEFAULT_SLEEP = 1 +DEFAULT_BACKCOFF = 1 +EXCEPTION_TYPES = (RateLimit, socket.error, socket.gaierror, + httplib.NotConnected, httplib.ImproperConnectionState) __all__ = [ 'find', @@ -282,3 +295,44 @@ class ReprMixin(object): def __str__(self): return str(self.__repr__()) + + +def retry(retry_exceptions=EXCEPTION_TYPES, retry_delay=None, + timeout=None, backoff=None): + """ + Retry method that helps to handle common exception. + :param func: the function to execute. + :param timeout: maximum time to wait. + :param retry_delay: retry delay between the attempts. + :param backoff: multiplier added to delay between attempts. + :param retry_exceptions: types of exceptions to retry on. + + :Example: + + retry_request = retry(timeout=1, retry_delay=1, backoff=1) + retry_request(self.connection.request)() + """ + def deco_retry(func): + @wraps(func) + def retry_loop(*args, **kwargs): + delay = retry_delay + end = datetime.now() + timedelta(seconds=timeout) + exc_info = None + while datetime.now() < end: + try: + result = func(*args, **kwargs) + return result + except retry_exceptions as exc: + if isinstance(exc, RateLimit): + time.sleep(exc.retry_after) + end = datetime.now() + timedelta( + seconds=exc.retry_after + timeout) + else: + exc_info = sys.exc_info() + time.sleep(delay) + delay *= backoff + if exc_info: + raise exc_info[1] + return func(*args, **kwargs) + return retry_loop + return deco_retry
