Repository: libcloud Updated Branches: refs/heads/trunk 582633929 -> 1ed502045
[google|compute] Add support for GCE list paging and filtering GCE will return a maximum of 500 resources in a single list. This change adds an iterator that allows filtering and/or paging of list results. Closes #491 Signed-off-by: Eric Johnson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/1ed50204 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/1ed50204 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/1ed50204 Branch: refs/heads/trunk Commit: 1ed5020450e6e8deb0361239bfb91ba795447e87 Parents: 5826339 Author: Lee Verberne <[email protected]> Authored: Tue Jan 6 09:54:05 2015 -0800 Committer: Eric Johnson <[email protected]> Committed: Wed May 13 13:52:46 2015 +0000 ---------------------------------------------------------------------- CHANGES.rst | 4 + libcloud/compute/drivers/gce.py | 174 ++++++++++++++++++- .../compute/fixtures/gce/regions-paged-1.json | 97 +++++++++++ .../compute/fixtures/gce/regions-paged-2.json | 52 ++++++ libcloud/test/compute/fixtures/gce/zones.json | 12 +- libcloud/test/compute/test_gce.py | 63 ++++++- 6 files changed, 393 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index 98ba7a4..5b7ef6f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,6 +25,10 @@ General Compute ~~~~~~~ +- Google Compute now supports paginated lists including filtering. + (GITHUB-491) + [Lee Verberne] + - OpenStackNodeSize objects now support optional, additional fields that are supported in OpenStack 2.1: `ephemeral_disk`, `swap`, `extra`. (GITHUB-488, LIBCLOUD-682) http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/compute/drivers/gce.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py index 666f1c1..e1e7549 100644 --- a/libcloud/compute/drivers/gce.py +++ b/libcloud/compute/drivers/gce.py @@ -64,7 +64,28 @@ class GCEResponse(GoogleResponse): class GCEConnection(GoogleBaseConnection): - """Connection class for the GCE driver.""" + """ + Connection class for the GCE driver. + + GCEConnection extends :class:`google.GoogleBaseConnection` for 2 reasons: + 1. modify request_path for GCE URI. + 2. Implement gce_params functionality described below. + + If the parameter gce_params is set to a dict prior to calling request(), + the URL parameters will be updated to include those key/values FOR A + SINGLE REQUEST. If the response contains a nextPageToken, + gce_params['pageToken'] will be set to its value. This can be used to + implement paging in list: + + >>> params, more_results = {'maxResults': 2}, True + >>> while more_results: + ... driver.connection.gce_params=params + ... driver.ex_list_urlmaps() + ... more_results = 'pageToken' in params + ... + [<GCEUrlMap id="..." name="cli-map">, <GCEUrlMap id="..." name="lc-map">] + [<GCEUrlMap id="..." name="web-map">] + """ host = 'www.googleapis.com' responseCls = GCEResponse @@ -76,6 +97,138 @@ class GCEConnection(GoogleBaseConnection): **kwargs) self.request_path = '/compute/%s/projects/%s' % (API_VERSION, project) + self.gce_params = None + + def pre_connect_hook(self, params, headers): + """ + Update URL parameters with values from self.gce_params. + + @inherits: :class:`GoogleBaseConnection.pre_connect_hook` + """ + params, headers = super(GCEConnection, self).pre_connect_hook(params, + headers) + if self.gce_params: + params.update(self.gce_params) + return params, headers + + def request(self, *args, **kwargs): + """ + Perform request then do GCE-specific processing of URL params. + + @inherits: :class:`GoogleBaseConnection.request` + """ + response = super(GCEConnection, self).request(*args, **kwargs) + + # If gce_params has been set, then update the pageToken with the + # nextPageToken so it can be used in the next request. + if self.gce_params: + if 'nextPageToken' in response.object: + self.gce_params['pageToken'] = response.object['nextPageToken'] + elif 'pageToken' in self.gce_params: + del self.gce_params['pageToken'] + self.gce_params = None + + return response + + +class GCEList(object): + """ + An Iterator that wraps list functions to provide additional features. + + GCE enforces a limit on the number of objects returned by a list operation, + so users with more than 500 objects of a particular type will need to use + filter(), page() or both. + + >>> l=GCEList(driver, driver.ex_list_urlmaps) + >>> for sublist in l.filter('name eq ...-map').page(1): + ... sublist + ... + [<GCEUrlMap id="..." name="cli-map">] + [<GCEUrlMap id="..." name="web-map">] + + One can create a GCEList manually, but it's slightly easier to use the + ex_list() method of :class:`GCENodeDriver`. + """ + + def __init__(self, driver, list_fn, **kwargs): + """ + :param driver: An initialized :class:``GCENodeDriver`` + :type driver: :class:``GCENodeDriver`` + + :param list_fn: A bound list method from :class:`GCENodeDriver`. + :type list_fn: ``instancemethod`` + """ + self.driver = driver + self.list_fn = list_fn + self.kwargs = kwargs + self.params = {} + + def __iter__(self): + list_fn = self.list_fn + more_results = True + while more_results: + self.driver.connection.gce_params = self.params + yield list_fn(**self.kwargs) + more_results = 'pageToken' in self.params + + def __repr__(self): + return '<GCEList list="%s" params="%s">' % ( + self.list_fn.__name__, repr(self.params)) + + def filter(self, expression): + """ + Filter results of a list operation. + + GCE supports server-side filtering of resources returned by a list + operation. Syntax of the filter expression is fully descripted in the + GCE API reference doc, but in brief it is:: + + FIELD_NAME COMPARISON_STRING LITERAL_STRING + + where FIELD_NAME is the resource's property name, COMPARISON_STRING is + 'eq' or 'ne', and LITERAL_STRING is a regular expression in RE2 syntax. + + >>> for sublist in l.filter('name eq ...-map'): + ... sublist + ... + [<GCEUrlMap id="..." name="cli-map">, \ + <GCEUrlMap id="..." name="web-map">] + + API reference: https://cloud.google.com/compute/docs/reference/latest/ + RE2 syntax: https://github.com/google/re2/blob/master/doc/syntax.txt + + :param expression: Filter expression described above. + :type expression: ``str`` + + :return: This :class:`GCEList` instance + :rtype: :class:`GCEList` + """ + self.params['filter'] = expression + return self + + def page(self, max_results=500): + """ + Limit the number of results by each iteration. + + This implements the paging functionality of the GCE list methods and + returns this GCEList instance so that results can be chained: + + >>> for sublist in GCEList(driver, driver.ex_list_urlmaps).page(2): + ... sublist + ... + [<GCEUrlMap id="..." name="cli-map">, \ + <GCEUrlMap id="..." name="lc-map">] + [<GCEUrlMap id="..." name="web-map">] + + :keyword max_results: Maximum number of results to return per + iteration. Defaults to the GCE default of 500. + :type max_results: ``int`` + + :return: This :class:`GCEList` instance + :rtype: :class:`GCEList` + """ + self.params['maxResults'] = max_results + return self class GCELicense(UuidMixin): @@ -1032,6 +1185,25 @@ class GCENodeDriver(NodeDriver): response = self.connection.request(request, method='GET').object return response['contents'] + def ex_list(self, list_fn, **kwargs): + """ + Wrap a list method in a :class:`GCEList` iterator. + + >>> for sublist in driver.ex_list(driver.ex_list_urlmaps).page(1): + ... sublist + ... + [<GCEUrlMap id="..." name="cli-map">] + [<GCEUrlMap id="..." name="lc-map">] + [<GCEUrlMap id="..." name="web-map">] + + :param list_fn: A bound list method from :class:`GCENodeDriver`. + :type list_fn: ``instancemethod`` + + :return: An iterator that returns sublists from list_fn. + :rtype: :class:`GCEList` + """ + return GCEList(driver=self, list_fn=list_fn, **kwargs) + def ex_list_disktypes(self, zone=None): """ Return a list of DiskTypes for a zone or all. http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/regions-paged-1.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/gce/regions-paged-1.json b/libcloud/test/compute/fixtures/gce/regions-paged-1.json new file mode 100644 index 0000000..790f31b --- /dev/null +++ b/libcloud/test/compute/fixtures/gce/regions-paged-1.json @@ -0,0 +1,97 @@ +{ + "kind": "compute#regionList", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions";, + "id": "projects/project_name/regions", + "items": [ + { + "kind": "compute#region", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/asia-east1";, + "id": "1220", + "creationTimestamp": "2014-04-11T13:47:12.495-07:00", + "name": "asia-east1", + "description": "asia-east1", + "status": "UP", + "zones": [ + "https://www.googleapis.com/compute/v1/projects/project_name/zones/asia-east1-a"; + ], + "quotas": [ + { + "metric": "CPUS", + "limit": 24.0, + "usage": 0.0 + }, + { + "metric": "DISKS_TOTAL_GB", + "limit": 5120.0, + "usage": 0.0 + }, + { + "metric": "STATIC_ADDRESSES", + "limit": 7.0, + "usage": 0.0 + }, + { + "metric": "IN_USE_ADDRESSES", + "limit": 23.0, + "usage": 0.0 + }, + { + "metric": "SSD_TOTAL_GB", + "limit": 1024.0, + "usage": 0.0 + }, + { + "metric": "LOCAL_SSD_TOTAL_GB", + "limit": 1500.0, + "usage": 0.0 + } + ] + }, + { + "kind": "compute#region", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/europe-west1";, + "id": "1100", + "creationTimestamp": "2014-04-11T13:47:12.495-07:00", + "name": "europe-west1", + "description": "europe-west1", + "status": "UP", + "zones": [ + "https://www.googleapis.com/compute/v1/projects/project_name/zones/europe-west1-a";, + "https://www.googleapis.com/compute/v1/projects/project_name/zones/europe-west1-b"; + ], + "quotas": [ + { + "metric": "CPUS", + "limit": 24.0, + "usage": 0.0 + }, + { + "metric": "DISKS_TOTAL_GB", + "limit": 5120.0, + "usage": 0.0 + }, + { + "metric": "STATIC_ADDRESSES", + "limit": 7.0, + "usage": 0.0 + }, + { + "metric": "IN_USE_ADDRESSES", + "limit": 23.0, + "usage": 0.0 + }, + { + "metric": "SSD_TOTAL_GB", + "limit": 1024.0, + "usage": 0.0 + }, + { + "metric": "LOCAL_SSD_TOTAL_GB", + "limit": 1500.0, + "usage": 0.0 + } + ] + } + ], + "nextPageToken": "CjQIz5W-w6HRxAI6KQoCGAEKAiAACgIYAQoCIAAKAhgTCg4qDGV1cm9wZS13ZXN0MQoDIMwI" +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/regions-paged-2.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/gce/regions-paged-2.json b/libcloud/test/compute/fixtures/gce/regions-paged-2.json new file mode 100644 index 0000000..d99ea6d --- /dev/null +++ b/libcloud/test/compute/fixtures/gce/regions-paged-2.json @@ -0,0 +1,52 @@ +{ + "kind": "compute#regionList", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions";, + "id": "projects/project_name/regions", + "items": [ + { + "kind": "compute#region", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/us-central1";, + "id": "1000", + "creationTimestamp": "2014-04-11T13:47:12.495-07:00", + "name": "us-central1", + "description": "us-central1", + "status": "UP", + "zones": [ + "https://www.googleapis.com/compute/v1/projects/project_name/zones/us-central1-a";, + "https://www.googleapis.com/compute/v1/projects/project_name/zones/us-central1-b"; + ], + "quotas": [ + { + "metric": "CPUS", + "limit": 24.0, + "usage": 1.0 + }, + { + "metric": "DISKS_TOTAL_GB", + "limit": 5120.0, + "usage": 60.0 + }, + { + "metric": "STATIC_ADDRESSES", + "limit": 7.0, + "usage": 1.0 + }, + { + "metric": "IN_USE_ADDRESSES", + "limit": 23.0, + "usage": 1.0 + }, + { + "metric": "SSD_TOTAL_GB", + "limit": 1024.0, + "usage": 0.0 + }, + { + "metric": "LOCAL_SSD_TOTAL_GB", + "limit": 1500.0, + "usage": 0.0 + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/zones.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/gce/zones.json b/libcloud/test/compute/fixtures/gce/zones.json index 0d564e7..1ecf5ac 100644 --- a/libcloud/test/compute/fixtures/gce/zones.json +++ b/libcloud/test/compute/fixtures/gce/zones.json @@ -2,6 +2,16 @@ "id": "projects/project_name/zones", "items": [ { + "kind": "compute#zone", + "selfLink": "https://www.googleapis.com/compute/v1/projects/verb-test/zones/asia-east1-a";, + "id": "2220", + "creationTimestamp": "2014-05-30T18:35:16.575-07:00", + "name": "asia-east1-a", + "description": "asia-east1-a", + "status": "UP", + "region": "https://www.googleapis.com/compute/v1/projects/verb-test/regions/asia-east1"; + }, + { "creationTimestamp": "2013-02-05T16:19:23.254-08:00", "description": "europe-west1-a", "id": "13416642339679437530", @@ -82,4 +92,4 @@ ], "kind": "compute#zoneList", "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/zones"; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/test_gce.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py index 6759656..09466c9 100644 --- a/libcloud/test/compute/test_gce.py +++ b/libcloud/test/compute/test_gce.py @@ -121,6 +121,45 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): self.assertTrue(self.driver.ex_get_serial_output(node), 'This is some serial\r\noutput for you.') + def test_ex_list(self): + d = self.driver + # Test the default case for all list methods + # (except list_volume_snapshots, which requires an arg) + for list_fn in (d.ex_list_addresses, + d.ex_list_backendservices, + d.ex_list_disktypes, + d.ex_list_firewalls, + d.ex_list_forwarding_rules, + d.ex_list_healthchecks, + d.ex_list_networks, + d.ex_list_project_images, + d.ex_list_regions, + d.ex_list_routes, + d.ex_list_snapshots, + d.ex_list_targethttpproxies, + d.ex_list_targetinstances, + d.ex_list_targetpools, + d.ex_list_urlmaps, + d.ex_list_zones, + d.list_images, + d.list_locations, + d.list_nodes, + d.list_sizes, + d.list_volumes): + full_list = [item.name for item in list_fn()] + li = d.ex_list(list_fn) + iter_list = [item.name for sublist in li for item in sublist] + self.assertEqual(full_list, iter_list) + + # Test paging & filtering with a single list function as they require + # additional test fixtures + list_fn = d.ex_list_regions + for count, sublist in zip((2, 1), d.ex_list(list_fn).page(2)): + self.assertTrue(len(sublist) == count) + for sublist in d.ex_list(list_fn).filter('name eq us-central1'): + self.assertTrue(len(sublist) == 1) + self.assertEqual(sublist[0].name, 'us-central1') + def test_ex_list_addresses(self): address_list = self.driver.ex_list_addresses() address_list_all = self.driver.ex_list_addresses('all') @@ -190,8 +229,8 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): def test_list_locations(self): locations = self.driver.list_locations() - self.assertEqual(len(locations), 5) - self.assertEqual(locations[0].name, 'europe-west1-a') + self.assertEqual(len(locations), 6) + self.assertEqual(locations[0].name, 'asia-east1-a') def test_ex_list_routes(self): routes = self.driver.ex_list_routes() @@ -306,8 +345,8 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): def test_ex_list_zones(self): zones = self.driver.ex_list_zones() - self.assertEqual(len(zones), 5) - self.assertEqual(zones[0].name, 'europe-west1-a') + self.assertEqual(len(zones), 6) + self.assertEqual(zones[0].name, 'asia-east1-a') def test_ex_create_address_global(self): address_name = 'lcaddressglobal' @@ -1398,8 +1437,10 @@ class GCEMockHttp(MockHttpTestCase): if method == 'POST': body = self.fixtures.load('global_backendServices_post.json') else: + backend_name = getattr(self.test, 'backendservices_mock', + 'web-service') body = self.fixtures.load('global_backendServices-%s.json' % - self.test.backendservices_mock) + backend_name) return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK]) def _global_backendServices_no_backends(self, method, url, body, headers): @@ -1987,8 +2028,12 @@ class GCEMockHttp(MockHttpTestCase): return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK]) def _regions(self, method, url, body, headers): - body = self.fixtures.load( - 'regions.json') + if 'pageToken' in url or 'filter' in url: + body = self.fixtures.load('regions-paged-2.json') + elif 'maxResults' in url: + body = self.fixtures.load('regions-paged-1.json') + else: + body = self.fixtures.load('regions.json') return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK]) def _global_addresses(self, method, url, body, headers): @@ -2147,6 +2192,10 @@ class GCEMockHttp(MockHttpTestCase): body = self.fixtures.load('zones.json') return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK]) + def _zones_asia_east_1a(self, method, url, body, headers): + body = self.fixtures.load('zones_asia-east1-a.json') + return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK]) + def _zones_us_central1_a_diskTypes(self, method, url, body, headers): body = self.fixtures.load('zones_us-central1-a_diskTypes.json') return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
