Retry requests that have been rate-limited in Vultr compute driver Closes #1058
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/d19574e3 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/d19574e3 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/d19574e3 Branch: refs/heads/trunk Commit: d19574e3464d86c1770ec0b08b4ca520d80a9e66 Parents: e597775 Author: Francisco Ros <[email protected]> Authored: Wed May 24 09:03:10 2017 +0200 Committer: Anthony Shaw <[email protected]> Committed: Sun Jun 18 12:24:13 2017 +1000 ---------------------------------------------------------------------- libcloud/compute/drivers/vultr.py | 51 ++++++++++++++++++-- .../compute/fixtures/vultr/error_rate_limit.txt | 1 + libcloud/test/compute/test_vultr.py | 11 +++++ 3 files changed, 60 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/compute/drivers/vultr.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/vultr.py b/libcloud/compute/drivers/vultr.py index 7c31dfb..1cabd46 100644 --- a/libcloud/compute/drivers/vultr.py +++ b/libcloud/compute/drivers/vultr.py @@ -17,17 +17,58 @@ Vultr Driver """ import time +from functools import update_wrapper from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlencode from libcloud.common.base import ConnectionKey, JsonResponse from libcloud.compute.types import Provider, NodeState -from libcloud.common.types import LibcloudError, InvalidCredsError +from libcloud.common.types import InvalidCredsError +from libcloud.common.types import LibcloudError +from libcloud.common.types import ServiceUnavailableError from libcloud.compute.base import NodeDriver from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation +class rate_limited: + """ + Decorator for retrying Vultr calls that are rate-limited. + + :param int sleep: Seconds to sleep after being rate-limited. + :param int retries: Number of retries. + """ + + def __init__(self, sleep=1, retries=1): + self.sleep = sleep + self.retries = retries + + def __call__(self, call): + """ + Run ``call`` method until it's not rate-limited. + + The method is invoked while it returns 503 Service Unavailable or the + allowed number of retries is reached. + + :param callable call: Method to be decorated. + """ + + def wrapper(*args, **kwargs): + last_exception = None + + for i in range(self.retries + 1): + try: + return call(*args, **kwargs) + except ServiceUnavailableError as e: + last_exception = e + time.sleep(self.sleep) # hit by rate limit, let's sleep + + raise last_exception + + update_wrapper(wrapper, call) + return wrapper + + class VultrResponse(JsonResponse): def parse_error(self): if self.status == httplib.OK: @@ -35,6 +76,8 @@ class VultrResponse(JsonResponse): return body elif self.status == httplib.FORBIDDEN: raise InvalidCredsError(self.body) + elif self.status == httplib.SERVICE_UNAVAILABLE: + raise ServiceUnavailableError(self.body) else: raise LibcloudError(self.body) @@ -57,7 +100,7 @@ class VultrConnection(ConnectionKey): host = 'api.vultr.com' responseCls = VultrResponse - unauthenticated_endpoints = { # {path: actions} + unauthenticated_endpoints = { # {action: methods} '/v1/app/list': ['GET'], '/v1/os/list': ['GET'], '/v1/plans/list': ['GET'], @@ -82,9 +125,11 @@ class VultrConnection(ConnectionKey): def encode_data(self, data): return urlencode(data) + @rate_limited() def get(self, url): return self.request(url) + @rate_limited() def post(self, url, data): headers = {'Content-Type': 'application/x-www-form-urlencoded'} return self.request(url, data=data, headers=headers, method='POST') @@ -99,7 +144,7 @@ class VultrConnection(ConnectionKey): try: return self.method \ - not in self.unauthenticated_endpoints[self.action] + not in self.unauthenticated_endpoints[self.action] except KeyError: return True http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt b/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt new file mode 100644 index 0000000..27def76 --- /dev/null +++ b/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt @@ -0,0 +1 @@ +Rate limit reached - please try your request again later. Current rate limit: 2 requests/sec \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/test/compute/test_vultr.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_vultr.py b/libcloud/test/compute/test_vultr.py index 303f206..9d94fb4 100644 --- a/libcloud/test/compute/test_vultr.py +++ b/libcloud/test/compute/test_vultr.py @@ -22,6 +22,8 @@ except ImportError: from libcloud.utils.py3 import httplib +from libcloud.common.types import ServiceUnavailableError + from libcloud.compute.drivers.vultr import VultrNodeDriver from libcloud.test import LibcloudTestCase, MockHttp @@ -116,6 +118,10 @@ class VultrTests(LibcloudTestCase): res = self.driver.delete_key_pair(key_pair) self.assertTrue(res) + def test_rate_limit(self): + VultrMockHttp.type = 'SERVICE_UNAVAILABLE' + self.assertRaises(ServiceUnavailableError, self.driver.list_nodes) + class VultrMockHttp(MockHttp): fixtures = ComputeFileFixtures('vultr') @@ -136,6 +142,11 @@ class VultrMockHttp(MockHttp): body = self.fixtures.load('list_nodes.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _v1_server_list_SERVICE_UNAVAILABLE(self, method, url, body, headers): + body = self.fixtures.load('error_rate_limit.txt') + return (httplib.SERVICE_UNAVAILABLE, body, {}, + httplib.responses[httplib.SERVICE_UNAVAILABLE]) + def _v1_server_create(self, method, url, body, headers): body = self.fixtures.load('create_node.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK])
