Repository: libcloud Updated Branches: refs/heads/trunk 69d27cd5b -> f30b3e37f
[google storage] Google Storage permissions support Closes #860 Signed-off-by: Eric Johnson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f30b3e37 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f30b3e37 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f30b3e37 Branch: refs/heads/trunk Commit: f30b3e37f61b5c31c1e5db6dbe47067dc3b40aad Parents: 69d27cd Author: Scott Crunkleton <[email protected]> Authored: Thu Sep 29 11:30:08 2016 -0700 Committer: Eric Johnson <[email protected]> Committed: Fri Sep 30 06:27:08 2016 +0000 ---------------------------------------------------------------------- CHANGES.rst | 7 + libcloud/storage/drivers/google_storage.py | 255 +++++++++++++- .../fixtures/google_storage/get_container.json | 13 + .../fixtures/google_storage/get_object.json | 18 + .../google_storage/list_container_acl.json | 74 ++++ .../google_storage/list_object_acl.json | 86 +++++ libcloud/test/storage/test_google_storage.py | 353 +++++++++++++++++-- 7 files changed, 765 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index 781d22d..9fb2618 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -51,6 +51,13 @@ Load Balancer (GITHUB-848) [Anton Kozyrev] +Storage +~~~~~~~ + +- Added storage permissions for Google Cloud Storage + (GITHUB-860) + [Scott Crunkleton] + Changes in Apache Libcloud 1.2.1 -------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/storage/drivers/google_storage.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/google_storage.py b/libcloud/storage/drivers/google_storage.py index 580a29c..91e117e 100644 --- a/libcloud/storage/drivers/google_storage.py +++ b/libcloud/storage/drivers/google_storage.py @@ -14,16 +14,20 @@ # limitations under the License. import copy +import json import email.utils from libcloud.common.base import ConnectionUserAndKey from libcloud.common.google import GoogleAuthType from libcloud.common.google import GoogleOAuth2Credential +from libcloud.common.google import GoogleResponse +from libcloud.common.types import ProviderError from libcloud.storage.drivers.s3 import BaseS3Connection from libcloud.storage.drivers.s3 import BaseS3StorageDriver from libcloud.storage.drivers.s3 import S3RawResponse from libcloud.storage.drivers.s3 import S3Response +from libcloud.utils.py3 import httplib # Docs are a lie. Actual namespace returned is different that the one listed # in the docs. @@ -32,6 +36,21 @@ API_VERSION = '2006-03-01' NAMESPACE = 'http://doc.s3.amazonaws.com/%s' % (API_VERSION) +class ContainerPermissions(object): + values = ['NONE', 'READER', 'WRITER', 'OWNER'] + NONE = 0 + READER = 1 + WRITER = 2 + OWNER = 3 + + +class ObjectPermissions(object): + values = ['NONE', 'READER', 'OWNER'] + NONE = 0 + READER = 1 + OWNER = 2 + + class GoogleStorageConnection(ConnectionUserAndKey): """ Represents a single connection to the Google storage API endpoint. @@ -45,7 +64,7 @@ class GoogleStorageConnection(ConnectionUserAndKey): rawResponseCls = S3RawResponse PROJECT_ID_HEADER = 'x-goog-project-id' - def __init__(self, user_id, key, secure, auth_type=None, + def __init__(self, user_id, key, secure=True, auth_type=None, credential_file=None, **kwargs): self.auth_type = auth_type or GoogleAuthType.guess_type(user_id) if GoogleAuthType.is_oauth2(self.auth_type): @@ -65,7 +84,7 @@ class GoogleStorageConnection(ConnectionUserAndKey): return headers def get_project(self): - return getattr(self.driver, 'project') + return getattr(self.driver, 'project', None) def pre_connect_hook(self, params, headers): if self.auth_type == GoogleAuthType.GCS_S3: @@ -102,6 +121,28 @@ class GoogleStorageConnection(ConnectionUserAndKey): vendor_prefix=GoogleStorageDriver.http_vendor_prefix) +class GCSResponse(GoogleResponse): + pass + + +class GoogleStorageJSONConnection(GoogleStorageConnection): + """ + Represents a single connection to the Google storage JSON API endpoint. + + This can either authenticate via the Google OAuth2 methods or via + the S3 HMAC interoperability method. + """ + host = 'www.googleapis.com' + responseCls = GCSResponse + rawResponseCls = None + + def add_default_headers(self, headers): + headers = super(GoogleStorageJSONConnection, self).add_default_headers( + headers) + headers['Content-Type'] = 'application/json' + return headers + + class GoogleStorageDriver(BaseS3StorageDriver): """ Driver for Google Cloud Storage. @@ -121,7 +162,7 @@ class GoogleStorageDriver(BaseS3StorageDriver): From GCE instance:: - driver = GoogleStorageDriver(key=foo , secret=bar, ...) + driver = GoogleStorageDriver(key=foo, secret=bar, ...) Can also authenticate via Google Cloud Storage's S3 HMAC interoperability API. S3 user keys are 20 alphanumeric characters, starting with GOOG. @@ -131,9 +172,10 @@ class GoogleStorageDriver(BaseS3StorageDriver): driver = GoogleStorageDriver(key='GOOG0123456789ABCXYZ', secret=key_secret) """ - name = 'Google Storage' - website = 'http://cloud.google.com/' + name = 'Google Cloud Storage' + website = 'http://cloud.google.com/storage' connectionCls = GoogleStorageConnection + jsonConnectionCls = GoogleStorageJSONConnection hash_type = 'md5' namespace = NAMESPACE supports_chunked_encoding = False @@ -141,5 +183,206 @@ class GoogleStorageDriver(BaseS3StorageDriver): http_vendor_prefix = 'x-goog' def __init__(self, key, secret=None, project=None, **kwargs): - self.project = project super(GoogleStorageDriver, self).__init__(key, secret, **kwargs) + self.project = project + + self.json_connection = GoogleStorageJSONConnection( + key, secret, **kwargs) + + def _get_container_permissions(self, container_name): + """ + Return the container permissions for the current authenticated user. + + :param container_name: The container name. + :param container_name: ``str`` + + :return: The permissions on the container. + :rtype: ``int`` from ContainerPermissions + """ + # Try OWNER permissions first: try listing the bucket ACL. + # FORBIDDEN -> exists, but not an OWNER. + # NOT_FOUND -> bucket DNE, return NONE. + try: + self.json_connection.request( + '/storage/v1/b/%s/acl' % container_name) + return ContainerPermissions.OWNER + except ProviderError as e: + if e.http_code == httplib.FORBIDDEN: + pass + elif e.http_code == httplib.NOT_FOUND: + return ContainerPermissions.NONE + else: + raise + + # Try WRITER permissions with a noop request: try delete with an + # impossible precondition. Authorization is checked before file + # existence or preconditions. So, if we get a NOT_FOUND or a + # PRECONDITION_FAILED, then we must be authorized. + try: + self.json_connection.request( + '/storage/v1/b/%s/o/writecheck' % container_name, + headers={'x-goog-if-generation-match': '0'}, method='DELETE') + except ProviderError as e: + if e.http_code in [httplib.NOT_FOUND, httplib.PRECONDITION_FAILED]: + return ContainerPermissions.WRITER + elif e.http_code != httplib.FORBIDDEN: + raise + + # Last, try READER permissions: try getting container metadata. + try: + self.json_connection.request('/storage/v1/b/%s' % container_name) + return ContainerPermissions.READER + except ProviderError as e: + if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + raise + + return ContainerPermissions.NONE + + def _get_user(self): + """Gets this drivers' authenticated user, if any.""" + oauth2_creds = getattr(self.connection, 'oauth2_credential') + if oauth2_creds: + return oauth2_creds.user_id + else: + return None + + def _get_object_permissions(self, container_name, object_name): + """ + Return the object permissions for the current authenticated user. + If the object does not exist, or no object_name is given, return the + default object permissions. + + :param container_name: The container name. + :type container_name: ``str`` + + :param object_name: The object name. + :type object_name: ``str`` + + :return: The permissions on the object or default object permissions. + :rtype: ``int`` from ObjectPermissions + """ + # Try OWNER permissions first: try listing the object ACL. + try: + self.json_connection.request( + '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name)) + return ObjectPermissions.OWNER + except ProviderError as e: + if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + raise + + # Try READER permissions: try getting the object. + try: + self.json_connection.request( + '/storage/v1/b/%s/o/%s' % (container_name, object_name)) + return ObjectPermissions.READER + except ProviderError as e: + if e.http_code not in [httplib.FORBIDDEN, httplib.NOT_FOUND]: + raise + + return ObjectPermissions.NONE + + def ex_delete_permissions(self, container_name, object_name=None, + entity=None): + """ + Delete permissions for an ACL entity on a container or object. + + :param container_name: The container name. + :type container_name: ``str`` + + :param object_name: The object name. Optional. Not providing an object + will delete a container permission. + :type object_name: ``str`` + + :param entity: The entity to whose permission will be deleted. + Optional. If not provided, the role will be applied to the + authenticated user, if using an OAuth2 authentication scheme. + :type entity: ``str`` or ``None`` + """ + if not entity: + user_id = self._get_user() + if not user_id: + raise ValueError( + 'Must provide an entity. Driver is not using an ' + 'authenticated user.') + else: + entity = 'user-%s' % user_id + + if object_name: + url = ('/storage/v1/b/%s/o/%s/acl/%s' % + (container_name, object_name, entity)) + else: + url = '/storage/v1/b/%s/acl/%s' % (container_name, entity) + + self.json_connection.request(url, method='DELETE') + + def ex_get_permissions(self, container_name, object_name=None): + """ + Return the permissions for the currently authenticated user. + + :param container_name: The container name. + :type container_name: ``str`` + + :param object_name: The object name. Optional. Not providing an object + will return only container permissions. + :type object_name: ``str`` or ``None`` + + :return: A tuple of container and object permissions. + :rtype: ``tuple`` of (``int``, ``int`` or ``None``) from + ContainerPermissions and ObjectPermissions, respectively. + """ + obj_perms = self._get_object_permissions( + container_name, object_name) if object_name else None + return self._get_container_permissions(container_name), obj_perms + + def ex_set_permissions(self, container_name, object_name=None, + entity=None, role=None): + """ + Set the permissions for an ACL entity on a container or an object. + + :param container_name: The container name. + :type container_name: ``str`` + + :param object_name: The object name. Optional. Not providing an object + will apply the acl to the container. + :type object_name: ``str`` + + :param entity: The entity to which apply the role. Optional. If not + provided, the role will be applied to the authenticated user, if + using an OAuth2 authentication scheme. + :type entity: ``str`` + + :param role: The permission/role to set on the entity. + :type role: ``int`` from ContainerPermissions or ObjectPermissions + or ``str``. + + :raises ValueError: If no entity was given, but was required. Or if + the role isn't valid for the bucket or object. + """ + if isinstance(role, int): + perms = ObjectPermissions if object_name else ContainerPermissions + try: + role = perms.values[role] + except IndexError: + raise ValueError( + '%s is not a valid role level for container=%s object=%s' % + (role, container_name, object_name)) + elif not isinstance(role, str): + raise ValueError('%s is not a valid permission.' % role) + + if not entity: + user_id = self._get_user() + if not user_id: + raise ValueError( + 'Must provide an entity. Driver is not using an ' + 'authenticated user.') + else: + entity = 'user-%s' % user_id + + if object_name: + url = '/storage/v1/b/%s/o/%s/acl' % (container_name, object_name) + else: + url = '/storage/v1/b/%s/acl' % container_name + + self.json_connection.request( + url, method='POST', + data=json.dumps({'role': role, 'entity': entity})) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/test/storage/fixtures/google_storage/get_container.json ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/google_storage/get_container.json b/libcloud/test/storage/fixtures/google_storage/get_container.json new file mode 100644 index 0000000..9eb85b1 --- /dev/null +++ b/libcloud/test/storage/fixtures/google_storage/get_container.json @@ -0,0 +1,13 @@ +{ + "kind": "storage#bucket", + "id": "test-bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket";, + "projectNumber": "123", + "name": "test-bucket", + "timeCreated": "2015-12-17T21:38:31.413Z", + "updated": "2015-12-17T21:38:31.413Z", + "metageneration": "123", + "location": "US", + "storageClass": "STANDARD", + "etag": "etag" +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/test/storage/fixtures/google_storage/get_object.json ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/google_storage/get_object.json b/libcloud/test/storage/fixtures/google_storage/get_object.json new file mode 100644 index 0000000..324228d --- /dev/null +++ b/libcloud/test/storage/fixtures/google_storage/get_object.json @@ -0,0 +1,18 @@ +{ + "kind": "storage#object", + "id": "test-bucket/test-object/12345", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object ", + "name": "test-object", + "bucket": "test-bucket", + "generation": "12345", + "metageneration": "67890", + "contentType": "text/plain", + "timeCreated": "2016-09-14T01:19:33.850Z", + "updated": "2016-09-14T01:19:33.850Z", + "storageClass": "STANDARD", + "size": "123", + "md5Hash": "wVenkDHhxA+FkxgpvF/FUg==", + "mediaLink": "https://www.googleapis.com/download/storage/v1/b/test-bucket/o/test-object?generation=12345&alt=media";, + "crc32": "+x0GyA==", + "etag": "etag" +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/test/storage/fixtures/google_storage/list_container_acl.json ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/google_storage/list_container_acl.json b/libcloud/test/storage/fixtures/google_storage/list_container_acl.json new file mode 100644 index 0000000..97f57be --- /dev/null +++ b/libcloud/test/storage/fixtures/google_storage/list_container_acl.json @@ -0,0 +1,74 @@ +{ + "kind": "storage#bucketAccessControls", + "items": [ + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/project-owners-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/project-owners-123 ", + "bucket": "test-bucket", + "entity": "project-owners-123", + "role": "OWNER", + "projectTeam": { + "projectNumber": "123", + "team": "owners" + }, + "etag": "etag" + }, + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/project-editors-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/project-editors-123 ", + "bucket": "test-bucket", + "entity": "project-editors-123", + "role": "WRITER", + "projectTeam": { + "projectNumber": "123", + "team": "editors" + }, + "etag": "etag" + }, + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/project-viewers-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/project-viewers-123 ", + "bucket": "test-bucket", + "entity": "project-viewers-123", + "role": "READER", + "projectTeam": { + "projectNumber": "123", + "team": "viewers" + }, + "etag": "etag" + }, + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/[email protected]", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/[email protected] ", + "bucket": "test-bucket", + "entity": "[email protected]", + "role": "OWNER", + "email": "[email protected]", + "etag": "etag" + }, + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/domain-test.com", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/domain-test.com ", + "bucket": "test-bucket", + "entity": "domain-test.com", + "role": "OWNER", + "domain": "test.com", + "etag": "etag" + }, + { + "kind": "storage#bucketAccessControl", + "id": "test-bucket/[email protected]", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/acl/[email protected] ", + "bucket": "test-bucket", + "entity": "[email protected]", + "role": "OWNER", + "email": "[email protected]", + "etag": "etag" + } + ] +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/test/storage/fixtures/google_storage/list_object_acl.json ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/google_storage/list_object_acl.json b/libcloud/test/storage/fixtures/google_storage/list_object_acl.json new file mode 100644 index 0000000..d620139 --- /dev/null +++ b/libcloud/test/storage/fixtures/google_storage/list_object_acl.json @@ -0,0 +1,86 @@ +{ + "kind": "storage#objectAccessControls", + "items": [ + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/project-owners-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/project-owners-123 ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "project-owners-123", + "role": "OWNER", + "projectTeam": { + "projectNumber": "123", + "team": "owners" + }, + "etag": "etag" + }, + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/project-editors-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/project-editors-123 ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "project-editors-123", + "role": "OWNER", + "projectTeam": { + "projectNumber": "123", + "team": "editors" + }, + "etag": "etag" + }, + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/project-viewers-123", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/project-viewers-123 ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "project-viewers-123", + "role": "READER", + "projectTeam": { + "projectNumber": "123", + "team": "viewers" + }, + "etag": "etag" + }, + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/[email protected]", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/[email protected] ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "[email protected]", + "role": "OWNER", + "email": "[email protected]", + "etag": "etag" + }, + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/domain-test.com", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/domain-test.com ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "domain-test.com", + "role": "OWNER", + "domain": "test.com", + "etag": "etag" + }, + { + "kind": "storage#objectAccessControl", + "id": "test-bucket/test-object/12345/[email protected]", + "selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object/acl/[email protected] ", + "bucket": "test-bucket", + "object": "test-object", + "generation": "12345", + "entity": "[email protected]", + "role": "OWNER", + "email": "[email protected]", + "etag": "etag" + } + ] +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/f30b3e37/libcloud/test/storage/test_google_storage.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_google_storage.py b/libcloud/test/storage/test_google_storage.py index 2668c50..a2ba5fe 100644 --- a/libcloud/test/storage/test_google_storage.py +++ b/libcloud/test/storage/test_google_storage.py @@ -14,7 +14,9 @@ # limitations under the License. import copy +import json import mock +import re import sys import unittest @@ -22,6 +24,7 @@ import email.utils from libcloud.common.google import GoogleAuthType from libcloud.storage.drivers import google_storage +from libcloud.test import StorageMockHttp from libcloud.test.common.test_google import GoogleTestCase from libcloud.test.file_fixtures import StorageFileFixtures from libcloud.test.secrets import STORAGE_GOOGLE_STORAGE_PARAMS @@ -29,11 +32,27 @@ from libcloud.test.storage.test_s3 import S3Tests, S3MockHttp from libcloud.utils.py3 import httplib CONN_CLS = google_storage.GoogleStorageConnection -STORAGE_CLS = google_storage.GoogleStorageDriver +JSON_CONN_CLS = google_storage.GoogleStorageJSONConnection TODAY = email.utils.formatdate(usegmt=True) +def _error_helper(code, headers): + message = httplib.responses[code] + body = { + 'error': { + 'errors': [ + { + 'code': code, + 'message': message, + 'reason': message, + }, + ], + }, + } + return code, json.dumps(body), headers, httplib.responses[code] + + class GoogleStorageMockHttp(S3MockHttp): fixtures = StorageFileFixtures('google_storage') @@ -49,12 +68,115 @@ class GoogleStorageMockHttp(S3MockHttp): 'last-modified': 'Thu, 13 Sep 2012 07:13:22 GMT' } - return ( - httplib.OK, - body, - headers, - httplib.responses[httplib.OK] - ) + return httplib.OK, body, headers, httplib.responses[httplib.OK] + + +class GoogleStorageJSONMockHttp(StorageMockHttp): + """ + Extracts bucket and object out of requests and routes to methods of the + forms (bucket, object, entity, and type are sanitized values + {'-', '.', '/' are replaced with '_'}): + + _<bucket>[_<type>] + _<bucket>_acl[_entity][_<type>] + _<bucket>_defaultObjectAcl[_<entity>][_<type>] + _<bucket>_<object>[_<type>] + _<bucket>_<object>_acl[_<entity>][_<type>] + + Ugly example: + /storage/v1/b/test-bucket/o/test-object/acl/test-entity + with type='FOO' yields + _test_bucket_test_object_acl_test_entity_FOO + """ + fixtures = StorageFileFixtures('google_storage') + base_headers = {} + + # Path regex captures bucket, object, defaultObjectAcl, and acl values. + path_rgx = re.compile( + r'/storage/[^/]+/b/([^/]+)' + r'(?:/(defaultObjectAcl(?:/[^/]+)?$)|' + r'(?:/o/(.+?))?(?:/(acl(?:/[^/]+)?))?$)') + + # Permissions to use when handling requests. + bucket_perms = google_storage.ContainerPermissions.NONE + object_perms = google_storage.ObjectPermissions.NONE + + _FORBIDDEN = _error_helper(httplib.FORBIDDEN, base_headers) + _NOT_FOUND = _error_helper(httplib.NOT_FOUND, base_headers) + _PRECONDITION_FAILED = _error_helper( + httplib.PRECONDITION_FAILED, base_headers) + + def _get_method_name(self, type, use_param, qs, path): + match = self.path_rgx.match(path) + if not match: + raise ValueError('%s is not a valid path.' % path) + + joined_groups = '_'.join([g for g in match.groups() if g]) + if type: + meth_name = '_%s_%s' % (joined_groups, type) + else: + meth_name = '_%s' % joined_groups + # Return sanitized method name. + return meth_name.replace('/', '_').replace('.', '_').replace('-', '_') + + def _response_helper(self, fixture): + body = self.fixtures.load(fixture) + return httplib.OK, body, {}, httplib.responses[httplib.OK] + + #################### + # Request handlers # + #################### + def _test_bucket(self, method, url, body, headers): + """Bucket request.""" + if method != 'GET': + raise NotImplementedError('%s is not implemented.' % method) + + if self.bucket_perms < google_storage.ContainerPermissions.READER: + return self._FORBIDDEN + else: + return self._response_helper('get_container.json') + + def _test_bucket_acl(self, method, url, body, headers): + """Bucket list ACL request.""" + if method != 'GET': + raise NotImplementedError('%s is not implemented.' % method) + + if self.bucket_perms < google_storage.ContainerPermissions.OWNER: + return self._FORBIDDEN + else: + return self._response_helper('list_container_acl.json') + + def _test_bucket_test_object(self, method, url, body, headers): + """Object request.""" + if method != 'GET': + raise NotImplementedError('%s is not implemented.' % method) + + if self.object_perms < google_storage.ObjectPermissions.READER: + return self._FORBIDDEN + else: + return self._response_helper('get_object.json') + + def _test_bucket_test_object_acl(self, method, url, body, headers): + """Object list ACL request.""" + if method != 'GET': + raise NotImplementedError('%s is not implemented.' % method) + + if self.object_perms < google_storage.ObjectPermissions.OWNER: + return self._FORBIDDEN + else: + return self._response_helper('list_object_acl.json') + + def _test_bucket_writecheck(self, method, url, body, headers): + gen_match = headers.get('x-goog-if-generation-match') + if method != 'DELETE' or gen_match != '0': + msg = ('Improper write check delete strategy. method: %s, ' + 'headers: %s' % (method, headers)) + raise ValueError(msg) + + if self.bucket_perms < google_storage.ContainerPermissions.WRITER: + return self._FORBIDDEN + else: + return self._PRECONDITION_FAILED class GoogleStorageConnectionTest(GoogleTestCase): @@ -66,23 +188,25 @@ class GoogleStorageConnectionTest(GoogleTestCase): project = 'foo-project' # Modify headers when there is no project. - conn = CONN_CLS('foo_user', 'bar_key', secure=True, - auth_type=GoogleAuthType.GCS_S3) - conn.get_project = lambda: None + conn = CONN_CLS( + 'foo_user', 'bar_key', secure=True, + auth_type=GoogleAuthType.GCS_S3) + conn.get_project = mock.Mock(return_value=None) headers = dict(starting_headers) headers['Date'] = TODAY - self.assertEqual(conn.add_default_headers(dict(starting_headers)), - headers) + self.assertEqual( + conn.add_default_headers(dict(starting_headers)), headers) # Modify headers when there is a project. - conn = CONN_CLS('foo_user', 'bar_key', secure=True, - auth_type=GoogleAuthType.GCS_S3) - conn.get_project = lambda: project + conn = CONN_CLS( + 'foo_user', 'bar_key', secure=True, + auth_type=GoogleAuthType.GCS_S3) + conn.get_project = mock.Mock(return_value=project) headers = dict(starting_headers) headers['Date'] = TODAY headers[CONN_CLS.PROJECT_ID_HEADER] = project - self.assertEqual(conn.add_default_headers(dict(starting_headers)), - headers) + self.assertEqual( + conn.add_default_headers(dict(starting_headers)), headers) @mock.patch('libcloud.storage.drivers.s3.' 'BaseS3Connection.get_auth_signature') @@ -105,20 +229,17 @@ class GoogleStorageConnectionTest(GoogleTestCase): 'other': 'lower this!' } - conn = CONN_CLS('foo_user', 'bar_key', secure=True, - auth_type=GoogleAuthType.GCS_S3) + conn = CONN_CLS( + 'foo_user', 'bar_key', secure=True, + auth_type=GoogleAuthType.GCS_S3) conn.method = 'GET' conn.action = '/path' result = conn._get_s3_auth_signature(starting_params, starting_headers) self.assertNotEqual(starting_headers, modified_headers) self.assertEqual(result, 'mock signature!') mock_s3_auth_sig_method.assert_called_once_with( - method='GET', - headers=modified_headers, - params=starting_params, - expires=None, - secret_key='bar_key', - path='/path', + method='GET', headers=modified_headers, params=starting_params, + expires=None, secret_key='bar_key', path='/path', vendor_prefix='x-goog' ) @@ -131,15 +252,15 @@ class GoogleStorageConnectionTest(GoogleTestCase): starting_params = {'starting': 'params'} starting_headers = {'starting': 'headers'} - conn = CONN_CLS('foo_user', 'bar_key', secure=True, - auth_type=GoogleAuthType.GCE) + conn = CONN_CLS( + 'foo_user', 'bar_key', secure=True, auth_type=GoogleAuthType.GCE) conn._get_s3_auth_signature = mock.Mock() conn.oauth2_credential = mock.Mock() conn.oauth2_credential.access_token = 'Access_Token!' expected_headers = dict(starting_headers) expected_headers['Authorization'] = 'Bearer Access_Token!' - result = conn.pre_connect_hook(dict(starting_params), - dict(starting_headers)) + result = conn.pre_connect_hook( + dict(starting_params), dict(starting_headers)) self.assertEqual(result, (starting_params, expected_headers)) def test_pre_connect_hook_hmac(self): @@ -155,8 +276,9 @@ class GoogleStorageConnectionTest(GoogleTestCase): fake_hmac_method.headers_passed = copy.deepcopy(headers) return 'fake signature!' - conn = CONN_CLS('foo_user', 'bar_key', secure=True, - auth_type=GoogleAuthType.GCS_S3) + conn = CONN_CLS( + 'foo_user', 'bar_key', secure=True, + auth_type=GoogleAuthType.GCS_S3) conn._get_s3_auth_signature = fake_hmac_method conn.action = 'GET' conn.method = '/foo' @@ -165,8 +287,8 @@ class GoogleStorageConnectionTest(GoogleTestCase): '%s %s:%s' % (google_storage.SIGNATURE_IDENTIFIER, 'foo_user', 'fake signature!') ) - result = conn.pre_connect_hook(dict(starting_params), - dict(starting_headers)) + result = conn.pre_connect_hook( + dict(starting_params), dict(starting_headers)) self.assertEqual(result, (dict(starting_params), expected_headers)) self.assertEqual(fake_hmac_method.params_passed, starting_params) self.assertEqual(fake_hmac_method.headers_passed, starting_headers) @@ -174,10 +296,14 @@ class GoogleStorageConnectionTest(GoogleTestCase): class GoogleStorageTests(S3Tests, GoogleTestCase): - driver_type = STORAGE_CLS + driver_type = google_storage.GoogleStorageDriver driver_args = STORAGE_GOOGLE_STORAGE_PARAMS mock_response_klass = GoogleStorageMockHttp - driver = google_storage.GoogleStorageDriver + + def setUp(self): + super(GoogleStorageTests, self).setUp() + self.driver_type.jsonConnectionCls.conn_classes = ( + None, GoogleStorageJSONMockHttp) def test_billing_not_enabled(self): # TODO @@ -187,6 +313,163 @@ class GoogleStorageTests(S3Tests, GoogleTestCase): # Not supported on Google Storage pass + def test_delete_permissions(self): + mock_request = mock.Mock() + self.driver.json_connection.request = mock_request + + # Test deleting object permissions. + self.driver.ex_delete_permissions( + 'bucket', 'object', entity='user-foo') + url = '/storage/v1/b/bucket/o/object/acl/user-foo' + mock_request.assert_called_once_with(url, method='DELETE') + + # Test deleting bucket permissions. + mock_request.reset_mock() + self.driver.ex_delete_permissions('bucket', entity='user-foo') + url = '/storage/v1/b/bucket/acl/user-foo' + mock_request.assert_called_once_with(url, method='DELETE') + + def test_delete_permissions_no_entity(self): + mock_request = mock.Mock() + mock_get_user = mock.Mock(return_value=None) + self.driver._get_user = mock_get_user + self.driver.json_connection.request = mock_request + + # Test deleting permissions on an object with no entity. + self.assertRaises( + ValueError, self.driver.ex_delete_permissions, 'bucket', 'object') + + # Test deleting permissions on an bucket with no entity. + self.assertRaises( + ValueError, self.driver.ex_delete_permissions, 'bucket') + + mock_request.assert_not_called() + + # Test deleting permissions on an object with a default entity. + mock_get_user.return_value = '[email protected]' + self.driver.ex_delete_permissions('bucket', 'object') + url = '/storage/v1/b/bucket/o/object/acl/[email protected]' + mock_request.assert_called_once_with(url, method='DELETE') + + # Test deleting permissions on an bucket with a default entity. + mock_request.reset_mock() + mock_get_user.return_value = '[email protected]' + self.driver.ex_delete_permissions('bucket') + url = '/storage/v1/b/bucket/acl/[email protected]' + mock_request.assert_called_once_with(url, method='DELETE') + + def test_get_permissions(self): + def test_permission_config(bucket_perms, object_perms): + GoogleStorageJSONMockHttp.bucket_perms = bucket_perms + GoogleStorageJSONMockHttp.object_perms = object_perms + + perms = self.driver.ex_get_permissions( + 'test-bucket', 'test-object') + self.assertEqual(perms, (bucket_perms, object_perms)) + + bucket_levels = range(len(google_storage.ContainerPermissions.values)) + object_levels = range(len(google_storage.ObjectPermissions.values)) + for bucket_perms in bucket_levels: + for object_perms in object_levels: + test_permission_config(bucket_perms, object_perms) + + def test_set_permissions(self): + mock_request = mock.Mock() + self.driver.json_connection.request = mock_request + + # Test setting object permissions. + self.driver.ex_set_permissions( + 'bucket', 'object', entity='user-foo', role='OWNER') + url = '/storage/v1/b/bucket/o/object/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': 'user-foo'})) + + # Test setting object permissions with an ObjectPermissions value. + mock_request.reset_mock() + self.driver.ex_set_permissions( + 'bucket', 'object', entity='user-foo', + role=google_storage.ObjectPermissions.OWNER) + url = '/storage/v1/b/bucket/o/object/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': 'user-foo'})) + + # Test setting bucket permissions. + mock_request.reset_mock() + self.driver.ex_set_permissions( + 'bucket', entity='user-foo', role='OWNER') + url = '/storage/v1/b/bucket/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': 'user-foo'})) + + # Test setting bucket permissions with a ContainerPermissions value. + mock_request.reset_mock() + self.driver.ex_set_permissions( + 'bucket', entity='user-foo', + role=google_storage.ContainerPermissions.OWNER) + url = '/storage/v1/b/bucket/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': 'user-foo'})) + + def test_set_permissions_bad_roles(self): + mock_request = mock.Mock() + self.driver.json_connection.request = mock_request + + # Test forgetting a role. + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket', 'object') + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket') + mock_request.assert_not_called() + + # Test container permissions on an object. + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket', 'object', + role=google_storage.ContainerPermissions.OWNER) + mock_request.assert_not_called() + + # Test object permissions on a container. + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket', + role=google_storage.ObjectPermissions.OWNER) + mock_request.assert_not_called() + + def test_set_permissions_no_entity(self): + mock_request = mock.Mock() + mock_get_user = mock.Mock(return_value=None) + self.driver._get_user = mock_get_user + self.driver.json_connection.request = mock_request + + # Test for setting object permissions with no entity. + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket', 'object', + role='OWNER') + + # Test for setting bucket permissions with no entity. + self.assertRaises( + ValueError, self.driver.ex_set_permissions, 'bucket', role='OWNER') + + mock_request.assert_not_called() + + # Test for setting object permissions with a default entity. + mock_get_user.return_value = '[email protected]' + self.driver.ex_set_permissions('bucket', 'object', role='OWNER') + url = '/storage/v1/b/bucket/o/object/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': '[email protected]'})) + + # Test for setting bucket permissions with a default entity. + mock_request.reset_mock() + mock_get_user.return_value = '[email protected]' + self.driver.ex_set_permissions('bucket', role='OWNER') + url = '/storage/v1/b/bucket/acl' + mock_request.assert_called_once_with( + url, method='POST', + data=json.dumps({'role': 'OWNER', 'entity': '[email protected]'})) if __name__ == '__main__': sys.exit(unittest.main())
