This is an automated email from the ASF dual-hosted git repository. tomaz pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 74b0f671da92ccbfc48aa7787d2829415a4854dc Author: Aaron Virshup <[email protected]> AuthorDate: Fri May 1 10:49:53 2020 -0700 Implement pre-signed URL for AWS SignatureV4 connections only --- libcloud/storage/drivers/s3.py | 120 ++++++++++++++++++----------------- libcloud/test/storage/test_aurora.py | 10 +++ libcloud/test/storage/test_s3.py | 18 ++++-- 3 files changed, 83 insertions(+), 65 deletions(-) diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 2784f52..69e07f4 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -372,65 +372,6 @@ class BaseS3StorageDriver(StorageDriver): raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) - def get_object_cdn_url(self, obj, - ex_expiry=S3_CDN_URL_EXPIRY_HOURS): - """ - Return a "presigned URL" for read-only access to object - - :param obj: Object instance. - :type obj: :class:`Object` - - :param ex_expiry: The number of hours after which the URL expires. - Defaults to 24 hours or the value of the environment - variable "LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS", - if set. - :type ex_expiry: ``float`` - - :return: Presigned URL for the object. - :rtype: ``str`` - """ - - # assemble data for the request we want to pre-sign - # see: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html # noqa - object_path = self._get_object_path(obj.container, obj.name) - now = datetime.utcnow() - duration_seconds = int(ex_expiry * 3600) - credparts = ( - self.key, - now.strftime(S3_CDN_URL_DATE_FORMAT), - self.region, - 's3', - 'aws4_request') - params_to_sign = { - 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', - 'X-Amz-Credential': '/'.join(credparts), - 'X-Amz-Date': now.strftime(S3_CDN_URL_DATETIME_FORMAT), - 'X-Amz-Expires': duration_seconds, - 'X-Amz-SignedHeaders': 'host'} - headers_to_sign = {'host': self.connection.host} - - # generate signature for the pre-signed request - signature = self.connection.signer._get_signature( - params=params_to_sign, - headers=headers_to_sign, - dt=now, - method='GET', - path=object_path, - data=UnsignedPayloadSentinel - ) - - # Create final params for pre-signed URL - params = params_to_sign.copy() - params['X-Amz-Signature'] = signature - - return '{scheme}://{host}:{port}{path}?{params}'.format( - scheme='https' if self.secure else 'http', - host=self.connection.host, - port=self.connection.port, - path=object_path, - params=urlencode(params), - ) - def _get_container_path(self, container): """ Return a container path @@ -1180,6 +1121,67 @@ class S3StorageDriver(AWSDriver, BaseS3StorageDriver): def list_regions(self): return REGION_TO_HOST_MAP.keys() + def get_object_cdn_url(self, obj, + ex_expiry=S3_CDN_URL_EXPIRY_HOURS): + """ + Return a "presigned URL" for read-only access to object + + AWS only - requires AWS signature V4 authenticaiton. + + :param obj: Object instance. + :type obj: :class:`Object` + + :param ex_expiry: The number of hours after which the URL expires. + Defaults to 24 hours or the value of the environment + variable "LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS", + if set. + :type ex_expiry: ``float`` + + :return: Presigned URL for the object. + :rtype: ``str`` + """ + + # assemble data for the request we want to pre-sign + # see: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html # noqa + object_path = self._get_object_path(obj.container, obj.name) + now = datetime.utcnow() + duration_seconds = int(ex_expiry * 3600) + credparts = ( + self.key, + now.strftime(S3_CDN_URL_DATE_FORMAT), + self.region, + 's3', + 'aws4_request') + params_to_sign = { + 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', + 'X-Amz-Credential': '/'.join(credparts), + 'X-Amz-Date': now.strftime(S3_CDN_URL_DATETIME_FORMAT), + 'X-Amz-Expires': duration_seconds, + 'X-Amz-SignedHeaders': 'host'} + headers_to_sign = {'host': self.connection.host} + + # generate signature for the pre-signed request + signature = self.connection.signer._get_signature( + params=params_to_sign, + headers=headers_to_sign, + dt=now, + method='GET', + path=object_path, + data=UnsignedPayloadSentinel + ) + + # Create final params for pre-signed URL + params = params_to_sign.copy() + params['X-Amz-Signature'] = signature + + return '{scheme}://{host}:{port}{path}?{params}'.format( + scheme='https' if self.secure else 'http', + host=self.connection.host, + port=self.connection.port, + path=object_path, + params=urlencode(params), + ) + class S3USEast2Connection(S3SignatureV4Connection): host = S3_US_EAST2_HOST diff --git a/libcloud/test/storage/test_aurora.py b/libcloud/test/storage/test_aurora.py index 87728b7..7ef0d17 100644 --- a/libcloud/test/storage/test_aurora.py +++ b/libcloud/test/storage/test_aurora.py @@ -16,6 +16,7 @@ import sys import unittest +from libcloud.common.types import LibcloudError from libcloud.storage.drivers.auroraobjects import AuroraObjectsStorageDriver from libcloud.test.storage.test_s3 import S3MockHttp, S3Tests @@ -30,6 +31,15 @@ class AuroraObjectsTests(S3Tests, unittest.TestCase): S3MockHttp.type = None self.driver = self.create_driver() + def test_get_object_cdn_url(self): + self.mock_response_klass.type = 'get_object' + obj = self.driver.get_object(container_name='test2', + object_name='test') + + with self.assertRaises(LibcloudError): + self.driver.get_object_cdn_url(obj) + + if __name__ == '__main__': sys.exit(unittest.main()) diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index 5adfcf6..6d1af26 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -43,7 +43,7 @@ from libcloud.storage.types import ContainerIsNotEmptyError from libcloud.storage.types import InvalidContainerNameError from libcloud.storage.types import ObjectDoesNotExistError from libcloud.storage.types import ObjectHashMismatchError -from libcloud.storage.drivers.s3 import BaseS3Connection +from libcloud.storage.drivers.s3 import BaseS3Connection, S3SignatureV4Connection from libcloud.storage.drivers.s3 import S3StorageDriver, S3USWestStorageDriver from libcloud.storage.drivers.s3 import CHUNK_SIZE from libcloud.utils.py3 import b @@ -533,12 +533,18 @@ class S3Tests(unittest.TestCase): obj = self.driver.get_object(container_name='test2', object_name='test') - url = urlparse.urlparse(self.driver.get_object_cdn_url(obj, ex_expiry=12)) - query = urlparse.parse_qs(url.query) + # cdn urls can only be generated using a V4 connection + if issubclass(self.driver.connectionCls, S3SignatureV4Connection): + cdn_url = self.driver.get_object_cdn_url(obj, ex_expiry=12) + url = urlparse.urlparse(cdn_url) + query = urlparse.parse_qs(url.query) - self.assertEqual(len(query['X-Amz-Signature']), 1) - self.assertGreater(len(query['X-Amz-Signature'][0]), 0) - self.assertEqual(query['X-Amz-Expires'], ['43200']) + self.assertEqual(len(query['X-Amz-Signature']), 1) + self.assertGreater(len(query['X-Amz-Signature'][0]), 0) + self.assertEqual(query['X-Amz-Expires'], ['43200']) + else: + with self.assertRaises(NotImplementedError): + self.driver.get_object_cdn_url(obj) def test_get_object_container_doesnt_exist(self): # This method makes two requests which makes mocking the response a bit
