This is an automated email from the ASF dual-hosted git repository. tomaz pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 59ce3b6b0c358012f1000691f57bbd1e76668327 Author: Aaron Virshup <[email protected]> AuthorDate: Thu Apr 30 23:17:11 2020 -0700 Implement s3 get_object_cdn_url using pre-signed urls --- CHANGES.rst | 12 +++++++ libcloud/common/aws.py | 6 ++++ libcloud/storage/drivers/s3.py | 70 +++++++++++++++++++++++++++++++++++++++- libcloud/test/storage/test_s3.py | 12 +++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9648526..05b088f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -422,6 +422,18 @@ Storage (GITHUB-1410) [Clemens Wolff - @c-w] +- [AWS S3] Implement ``get_object_cdn_url`` for the AWS storage driver. + + The AWS storage driver can now be used to generate temporary URLs that + grant clients read access to objects. The URLs expire after a certain + period of time, either configured via the ``ex_expiry`` argument or the + ``LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS`` environment variable + (default: 24 hours). + + Reported by @rvolykh. + (GITHUB-1403) + [Aaron Virshup - @avirshup] + DNS ~~~ diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py index ac9ba9b..0dba374 100644 --- a/libcloud/common/aws.py +++ b/libcloud/common/aws.py @@ -340,6 +340,8 @@ class AWSRequestSignerAlgorithmV4(AWSRequestSigner): for k, v in sorted(headers.items())]) + '\n' def _get_payload_hash(self, method, data=None): + if data is UnsignedPayloadSentinel: + return UNSIGNED_PAYLOAD if method in ('POST', 'PUT'): if data: if hasattr(data, 'next') or hasattr(data, '__next__'): @@ -368,6 +370,10 @@ class AWSRequestSignerAlgorithmV4(AWSRequestSigner): ]) +class UnsignedPayloadSentinel: + pass + + class SignedAWSConnection(AWSTokenConnection): version = None # type: Optional[str] diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 17a7d08..2784f52 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -17,6 +17,8 @@ import base64 import hmac import time from hashlib import sha1 +import os +from datetime import datetime import libcloud.utils.py3 @@ -32,13 +34,14 @@ from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote from libcloud.utils.py3 import b from libcloud.utils.py3 import tostring +from libcloud.utils.py3 import urlencode from libcloud.utils.xml import fixxpath, findtext from libcloud.utils.files import read_in_chunks from libcloud.common.types import InvalidCredsError, LibcloudError from libcloud.common.base import ConnectionUserAndKey, RawResponse from libcloud.common.aws import AWSBaseResponse, AWSDriver, \ - AWSTokenConnection, SignedAWSConnection + AWSTokenConnection, SignedAWSConnection, UnsignedPayloadSentinel from libcloud.storage.base import Object, Container, StorageDriver from libcloud.storage.types import ContainerError @@ -108,6 +111,12 @@ CHUNK_SIZE = 5 * 1024 * 1024 # ex_iterate_multipart_uploads. RESPONSES_PER_REQUEST = 100 +S3_CDN_URL_DATETIME_FORMAT = '%Y%m%dT%H%M%SZ' +S3_CDN_URL_DATE_FORMAT = '%Y%m%d' +S3_CDN_URL_EXPIRY_HOURS = float( + os.getenv('LIBCLOUD_S3_CDN_URL_EXPIRY_HOURS', '24') +) + class S3Response(AWSBaseResponse): namespace = None @@ -363,6 +372,65 @@ class BaseS3StorageDriver(StorageDriver): raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) + def get_object_cdn_url(self, obj, + ex_expiry=S3_CDN_URL_EXPIRY_HOURS): + """ + Return a "presigned URL" for read-only access to object + + :param obj: Object instance. + :type obj: :class:`Object` + + :param ex_expiry: The number of hours after which the URL expires. + Defaults to 24 hours or the value of the environment + variable "LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS", + if set. + :type ex_expiry: ``float`` + + :return: Presigned URL for the object. + :rtype: ``str`` + """ + + # assemble data for the request we want to pre-sign + # see: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html # noqa + object_path = self._get_object_path(obj.container, obj.name) + now = datetime.utcnow() + duration_seconds = int(ex_expiry * 3600) + credparts = ( + self.key, + now.strftime(S3_CDN_URL_DATE_FORMAT), + self.region, + 's3', + 'aws4_request') + params_to_sign = { + 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', + 'X-Amz-Credential': '/'.join(credparts), + 'X-Amz-Date': now.strftime(S3_CDN_URL_DATETIME_FORMAT), + 'X-Amz-Expires': duration_seconds, + 'X-Amz-SignedHeaders': 'host'} + headers_to_sign = {'host': self.connection.host} + + # generate signature for the pre-signed request + signature = self.connection.signer._get_signature( + params=params_to_sign, + headers=headers_to_sign, + dt=now, + method='GET', + path=object_path, + data=UnsignedPayloadSentinel + ) + + # Create final params for pre-signed URL + params = params_to_sign.copy() + params['X-Amz-Signature'] = signature + + return '{scheme}://{host}:{port}{path}?{params}'.format( + scheme='https' if self.secure else 'http', + host=self.connection.host, + port=self.connection.port, + path=object_path, + params=urlencode(params), + ) + def _get_container_path(self, container): """ Return a container path diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index d605bd5..5adfcf6 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -528,6 +528,18 @@ class S3Tests(unittest.TestCase): container = self.driver.get_container(container_name='test1') self.assertTrue(container.name, 'test1') + def test_get_object_cdn_url(self): + self.mock_response_klass.type = 'get_object' + obj = self.driver.get_object(container_name='test2', + object_name='test') + + url = urlparse.urlparse(self.driver.get_object_cdn_url(obj, ex_expiry=12)) + query = urlparse.parse_qs(url.query) + + self.assertEqual(len(query['X-Amz-Signature']), 1) + self.assertGreater(len(query['X-Amz-Signature'][0]), 0) + self.assertEqual(query['X-Amz-Expires'], ['43200']) + def test_get_object_container_doesnt_exist(self): # This method makes two requests which makes mocking the response a bit # trickier
