Repository: libcloud Updated Branches: refs/heads/trunk a4d0081b2 -> 1a3ebe5d6
list calls paginate in OpenStackNodeDriver the default max_limits for OpenStack is 1000. If you have more than 1000 resources (i.e. snapshots) then everything but the newest 1000 will not be listed. If you set the max_limits lower even less will not be returned, etc. This change implements pagination for ex_list_snapshots, ex_list_ports, list_volumes and list_nodes in the OpenStack_2_NodeDriver. Signed-off-by: Rick van de Loo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/1aaeff45 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/1aaeff45 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/1aaeff45 Branch: refs/heads/trunk Commit: 1aaeff4537dbe23886cff505ef3f5c708d823995 Parents: a4d0081 Author: Rick van de Loo <[email protected]> Authored: Wed Nov 28 17:29:45 2018 +0100 Committer: Rick van de Loo <[email protected]> Committed: Tue Dec 4 10:49:30 2018 +0100 ---------------------------------------------------------------------- libcloud/compute/drivers/openstack.py | 84 ++++++++++++++++++-- .../openstack_v1.1/_v2_0__snapshots.json | 2 +- .../_v2_0__snapshots_paginate_start.json | 52 ++++++++++++ libcloud/test/compute/test_openstack.py | 36 ++++++++- 4 files changed, 163 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/compute/drivers/openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 5affde6..1d2ba4b 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -31,6 +31,7 @@ from libcloud.utils.py3 import httplib from libcloud.utils.py3 import b from libcloud.utils.py3 import next from libcloud.utils.py3 import urlparse +from libcloud.utils.py3 import parse_qs from libcloud.common.openstack import OpenStackBaseConnection @@ -70,6 +71,8 @@ ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"; DEFAULT_API_VERSION = '1.1' +PAGINATION_LIMIT = 1000 + class OpenStackComputeConnection(OpenStackBaseConnection): # default config for http://devstack.org/ @@ -169,6 +172,57 @@ class OpenStackNodeDriver(NodeDriver, OpenStackDriverMixin): OpenStackDriverMixin.__init__(self, **kwargs) super(OpenStackNodeDriver, self).__init__(*args, **kwargs) + @staticmethod + def _paginated_request(url, obj, connection, params=None): + """ + Perform multiple calls in order to have a full list of elements when + the API responses are paginated. + + :param url: API endpoint + :type url: ``str`` + + :param obj: Result object key + :type obj: ``str`` + + :param connection: The API connection to use to perform the request + :type connection: ``obj`` + + :param params: Any request parameters + :type params: ``dict`` + + :return: ``list`` of API response objects + :rtype: ``list`` + """ + params = params or {} + objects = list() + loop_count = 0 + while True: + data = connection.request(url, params=params) + values = data.object.get(obj, list()) + objects.extend(values) + links = data.object.get('%s_links' % obj, list()) + next_links = [n for n in links if n['rel'] == 'next'] + if next_links: + next_link = next_links[0] + query = urlparse.urlparse(next_link['href']) + # The query[4] references the query parameters from the url + params.update(parse_qs(query[4])) + else: + break + + # Prevent the pagination from looping indefinitely in case + # the API returns a loop for some reason. + loop_count += 1 + if loop_count > PAGINATION_LIMIT: + raise OpenStackException( + 'Pagination limit reached for %s, the limit is %d. ' + 'This might indicate that your API is returning a ' + 'looping next target for pagination!' % ( + url, PAGINATION_LIMIT + ), None + ) + return {obj: objects} + def destroy_node(self, node): uri = '/servers/%s' % (node.id) resp = self.connection.request(uri, method='DELETE') @@ -2699,6 +2753,21 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): ) ) + def list_nodes(self, ex_all_tenants=False): + """ + List the nodes in a tenant + + :param ex_all_tenants: List nodes for all the tenants. Note: Your user + must have admin privileges for this + functionality to work. + :type ex_all_tenants: ``bool`` + """ + params = {} + if ex_all_tenants: + params = {'all_tenants': 1} + return self._to_nodes(self._paginated_request( + '/servers/detail', 'servers', self.connection, params=params)) + def get_image(self, image_id): """ Get a NodeImage using the V2 Glance API @@ -2955,10 +3024,9 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): :rtype: ``list`` of :class:`OpenStack_2_PortInterface` """ - response = self.network_connection.request( - '/v2.0/ports' - ) - return [self._to_port(port) for port in response.object['ports']] + response = self._paginated_request( + '/v2.0/ports', 'ports', self.network_connection) + return [self._to_port(port) for port in response['ports']] def ex_delete_port(self, port): """ @@ -3069,8 +3137,8 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): :rtype: ``list`` of :class:`StorageVolume` """ - return self._to_volumes( - self.volumev2_connection.request('/volumes/detail').object) + return self._to_volumes(self._paginated_request( + '/volumes/detail', 'volumes', self.volumev2_connection)) def ex_get_volume(self, volumeId): """ @@ -3152,8 +3220,8 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): :rtype: ``list`` of :class:`VolumeSnapshot` """ - return self._to_snapshots( - self.volumev2_connection.request('/snapshots/detail').object) + return self._to_snapshots(self._paginated_request( + '/snapshots/detail', 'snapshots', self.volumev2_connection)) def create_volume_snapshot(self, volume, name=None, ex_description=None, ex_force=True): http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json index 763831c..1fad9da 100644 --- a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json @@ -43,4 +43,4 @@ "description": "volume snapshot" } ] -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json new file mode 100644 index 0000000..8603f67 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json @@ -0,0 +1,52 @@ +{ + "snapshots_links": [ + { + "href": "https://api.example.com:8776/v2/abcdec85bee34bb0a44ab8255eb36fbd/snapshots?&marker=abcde279-4cc0-4b47-a4ee-ec9c9e474b1b";, + "rel": "next" + } + ], + "snapshots": [ + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "snap-101", + "volume_id": "473f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2012-02-29T03:50:07Z", + "size": 1, + "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test2" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot2", + "volume_id": "7edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2015-11-29T02:25:51.000000", + "size": 1, + "id": "5fbbdccf-e058-6502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test2" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot", + "volume_id": "473f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2013-02-29T03:50:07Z", + "size": 1, + "id": "2fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + } + ] +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/test_openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index e567f17..d626474 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -20,6 +20,7 @@ import sys import unittest import datetime import pytest + from libcloud.utils.iso8601 import UTC try: @@ -47,7 +48,8 @@ from libcloud.compute.drivers.openstack import ( OpenStack_1_1_FloatingIpAddress, OpenStackKeyPair, OpenStack_1_0_Connection, OpenStack_2_FloatingIpPool, OpenStackNodeDriver, - OpenStack_2_NodeDriver, OpenStack_2_PortInterfaceState, OpenStackNetwork) + OpenStack_2_NodeDriver, OpenStack_2_PortInterfaceState, OpenStackNetwork, + OpenStackException) from libcloud.compute.base import Node, NodeImage, NodeSize from libcloud.pricing import set_pricing, clear_pricing_data @@ -1628,6 +1630,32 @@ class OpenStack_2_Tests(OpenStack_1_1_Tests): # normally authentication happens lazily, but we force it here self.driver.volumev2_connection._populate_hosts_and_request_paths() + def test__paginated_request_single_page(self): + snapshots = self.driver._paginated_request( + '/snapshots/detail', 'snapshots', + self.driver.volumev2_connection + )['snapshots'] + + self.assertEqual(len(snapshots), 3) + self.assertEqual(snapshots[0]['name'], 'snap-001') + + def test__paginated_request_two_pages(self): + snapshots = self.driver._paginated_request( + '/snapshots/detail?unit_test=paginate', 'snapshots', + self.driver.volumev2_connection + )['snapshots'] + + self.assertEqual(len(snapshots), 6) + self.assertEqual(snapshots[0]['name'], 'snap-101') + self.assertEqual(snapshots[3]['name'], 'snap-001') + + def test__paginated_request_raises_if_stuck_in_a_loop(self): + with pytest.raises(OpenStackException): + self.driver._paginated_request( + '/snapshots/detail?unit_test=pagination_loop', 'snapshots', + self.driver.volumev2_connection + ) + def test_ex_force_auth_token_passed_to_connection(self): base_url = 'https://servers.api.rackspacecloud.com/v1.1/slug' kwargs = { @@ -2485,7 +2513,11 @@ class OpenStack_1_1_MockHttp(MockHttp, unittest.TestCase): return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_snapshots_detail(self, method, url, body, headers): - body = self.fixtures.load('_v2_0__snapshots.json') + if ('unit_test=paginate' in url and 'marker' not in url) or \ + 'unit_test=pagination_loop' in url: + body = self.fixtures.load('_v2_0__snapshots_paginate_start.json') + else: + body = self.fixtures.load('_v2_0__snapshots.json') return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) def _v2_1337_snapshots(self, method, url, body, headers):
