Repository: libcloud Updated Branches: refs/heads/trunk a4323ca7d -> a19d1df1d
Allow user to pass "headers" argument to the upload_* methods in the storage API. This way user can specify CORS headers with the providers which supported that. Closes #404 Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a19d1df1 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a19d1df1 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a19d1df1 Branch: refs/heads/trunk Commit: a19d1df1d093975eb0ef2d3c829dd107bada872f Parents: a4323ca Author: Peter Schmidt <[email protected]> Authored: Thu Nov 27 11:20:38 2014 +1100 Committer: Tomaz Muraus <[email protected]> Committed: Mon Dec 1 06:28:49 2014 +0100 ---------------------------------------------------------------------- CHANGES.rst | 10 ++++ docs/development.rst | 4 +- libcloud/storage/base.py | 71 ++++++++++++++++----------- libcloud/storage/drivers/cloudfiles.py | 14 +++--- libcloud/test/storage/test_cloudfiles.py | 49 +++++++++++++++++- 5 files changed, 109 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index f55ebea..012f5e9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -61,6 +61,16 @@ Compute (GITHUB-401) [Eric Johnson] +Storage +~~~~~~~ + +- Allow user to pass ``headers`` argument to the ``upload_object`` and + ``upload_object_via_stream`` method. + + This way user can specify CORS headers with the drivers which support that. + (GITHUB-403, GITHUB-404) + [Peter Schmidt] + Changes with Apache Libcloud 0.16.0 ----------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/docs/development.rst ---------------------------------------------------------------------- diff --git a/docs/development.rst b/docs/development.rst index 8899df9..09cfe2b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -36,9 +36,9 @@ Code style guide * Use 79 characters in a line * Make sure edited file doesn't contain any trailing whitespace * You can verify that your modifications don't break any rules by running the - ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py.`` or + ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py`` or ``tox -e lint``. - Second command fill run flake8 on all the files in the repository. + Second command will run flake8 on all the files in the repository. And most importantly, follow the existing style in the file you are editing and **be consistent**. http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/base.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 5e4f09a..66e5d44 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -361,7 +361,7 @@ class StorageDriver(BaseDriver): 'download_object_as_stream not implemented for this driver') def upload_object(self, file_path, container, object_name, extra=None, - verify_hash=True): + verify_hash=True, headers=None): """ Upload an object currently located on a disk. @@ -380,6 +380,11 @@ class StorageDriver(BaseDriver): :param extra: Extra attributes (driver specific). (optional) :type extra: ``dict`` + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` + :rtype: :class:`Object` """ raise NotImplementedError( @@ -387,7 +392,8 @@ class StorageDriver(BaseDriver): def upload_object_via_stream(self, iterator, container, object_name, - extra=None): + extra=None, + headers=None): """ Upload an object using an iterator. @@ -405,19 +411,24 @@ class StorageDriver(BaseDriver): function which uses fs.stat function to determine the file size and it doesn't need to buffer whole object in the memory. - :type iterator: :class:`object` :param iterator: An object which implements the iterator interface. + :type iterator: :class:`object` - :type container: :class:`Container` :param container: Destination container. + :type container: :class:`Container` - :type object_name: ``str`` :param object_name: Object name. + :type object_name: ``str`` - :type extra: ``dict`` :param extra: (optional) Extra attributes (driver specific). Note: This dictionary must contain a 'content_type' key which represents a content type of the stored object. + :type extra: ``dict`` + + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` :rtype: ``object`` """ @@ -428,8 +439,8 @@ class StorageDriver(BaseDriver): """ Delete an object. - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` :return: ``bool`` True on success. :rtype: ``bool`` @@ -441,8 +452,8 @@ class StorageDriver(BaseDriver): """ Create a new container. - :type container_name: ``str`` :param container_name: Container name. + :type container_name: ``str`` :return: Container instance on success. :rtype: :class:`Container` @@ -454,8 +465,8 @@ class StorageDriver(BaseDriver): """ Delete a container. - :type container: :class:`Container` :param container: Container instance + :type container: :class:`Container` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -468,23 +479,23 @@ class StorageDriver(BaseDriver): """ Call passed callback and start transfer of the object' - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` - :type callback: :class:`function` :param callback: Function which is called with the passed callback_kwargs + :type callback: :class:`function` - :type callback_kwargs: ``dict`` :param callback_kwargs: Keyword arguments which are passed to the callback. + :type callback_kwargs: ``dict`` - :typed response: :class:`Response` :param response: Response instance. + :type response: :class:`Response` - :type success_status_code: ``int`` :param success_status_code: Status code which represents a successful transfer (defaults to httplib.OK) + :type success_status_code: ``int`` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -507,26 +518,26 @@ class StorageDriver(BaseDriver): """ Save object to the provided path. - :type response: :class:`RawResponse` :param response: RawResponse instance. + :type response: :class:`RawResponse` - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` - :type destination_path: ``str`` :param destination_path: Destination directory. + :type destination_path: ``str`` - :type delete_on_failure: ``bool`` :param delete_on_failure: True to delete partially downloaded object if the download fails. + :type delete_on_failure: ``bool`` - :type overwrite_existing: ``bool`` :param overwrite_existing: True to overwrite a local path if it already exists. + :type overwrite_existing: ``bool`` - :type chunk_size: ``int`` :param chunk_size: Optional chunk size (defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb) + :type chunk_size: ``int`` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -660,20 +671,20 @@ class StorageDriver(BaseDriver): """ Upload data stored in a string. - :type response: :class:`RawResponse` :param response: RawResponse object. + :type response: :class:`RawResponse` - :type data: ``str`` :param data: Data to upload. + :type data: ``str`` - :type calculate_hash: ``bool`` :param calculate_hash: True to calculate hash of the transferred data. - (defauls to True). + (defaults to True). + :type calculate_hash: ``bool`` - :rtype: ``tuple`` :return: First item is a boolean indicator of success, second one is the uploaded data MD5 hash and the third one is the number of transferred bytes. + :rtype: ``tuple`` """ bytes_transferred = 0 data_hash = None @@ -701,23 +712,23 @@ class StorageDriver(BaseDriver): """ Stream a data over an http connection. - :type response: :class:`RawResponse` :param response: RawResponse object. + :type response: :class:`RawResponse` - :type iterator: :class:`object` :param response: An object which implements an iterator interface or a File like object with read method. + :type iterator: :class:`object` - :type chunked: ``bool`` :param chunked: True if the chunked transfer encoding should be used (defauls to False). + :type chunked: ``bool`` - :type calculate_hash: ``bool`` :param calculate_hash: True to calculate hash of the transferred data. (defauls to True). + :type calculate_hash: ``bool`` - :type chunk_size: ``int`` :param chunk_size: Optional chunk size (defaults to ``CHUNK_SIZE``) + :type chunk_size: ``int`` :rtype: ``tuple`` :return: First item is a boolean indicator of success, second http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/drivers/cloudfiles.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index a2189c1..314586b 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -414,7 +414,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - verify_hash=True): + verify_hash=True, headers=None): """ Upload an object. @@ -427,10 +427,11 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin): upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - verify_hash=verify_hash) + verify_hash=verify_hash, headers=headers) def upload_object_via_stream(self, iterator, - container, object_name, extra=None): + container, object_name, extra=None, + headers=None): if isinstance(iterator, file): iterator = iter(iterator) @@ -440,7 +441,8 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin): return self._put_object(container=container, object_name=object_name, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, - extra=extra, iterator=iterator) + extra=extra, iterator=iterator, + headers=headers) def delete_object(self, obj): container_name = self._encode_container_name(obj.container.name) @@ -752,7 +754,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin): def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, verify_hash=True): + iterator=None, verify_hash=True, headers=None): extra = extra or {} container_name_encoded = self._encode_container_name(container.name) object_name_encoded = self._encode_object_name(object_name) @@ -760,7 +762,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin): meta_data = extra.get('meta_data', None) content_disposition = extra.get('content_disposition', None) - headers = {} + headers = headers or {} if meta_data: for key, value in list(meta_data.items()): key = 'X-Object-Meta-%s' % (key) http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/test/storage/test_cloudfiles.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 70e52c8..6590270 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -20,7 +20,6 @@ import os.path # pylint: disable-msg=W0404 import math import sys import copy -import unittest import mock @@ -45,6 +44,7 @@ from libcloud.storage.drivers.dummy import DummyIterator from libcloud.test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 from libcloud.test import MockHttpTestCase # pylint: disable-msg=E0611 +from libcloud.test import unittest from libcloud.test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 @@ -635,6 +635,53 @@ class CloudFilesTests(unittest.TestCase): self.assertEqual(func_kwargs['object_name'], expected_name) self.assertEqual(func_kwargs['container'], container) + def test_upload_object_via_stream_with_cors_headers(self): + """ + Test we can add some ``Cross-origin resource sharing`` headers + to the request about to be sent. + """ + cors_headers = { + 'Access-Control-Allow-Origin': 'http://mozilla.com', + 'Origin': 'http://storage.clouddrive.com', + } + expected_headers = { + # Automatically added headers + 'Content-Type': 'application/octet-stream', + 'Transfer-Encoding': 'chunked', + } + expected_headers.update(cors_headers) + + def intercept_request(request_path, + method=None, data=None, + headers=None, raw=True): + + # What we're actually testing + self.assertDictEqual(expected_headers, headers) + + raise NotImplementedError('oops') + self.driver.connection.request = intercept_request + + container = Container(name='CORS', extra={}, driver=self.driver) + + try: + self.driver.upload_object_via_stream( + # We never reach the Python 3 only bytes vs int error + # currently at libcloud/utils/py3.py:89 + # raise TypeError("Invalid argument %r for b()" % (s,)) + # because I raise a NotImplementedError. + iterator=iter(b'blob data like an image or video'), + container=container, + object_name="test_object", + headers=cors_headers, + ) + except NotImplementedError: + # Don't care about the response we'd have to mock anyway + # as long as we intercepted the request and checked its headers + pass + else: + self.fail('Expected NotImplementedError to be thrown to ' + 'verify we actually checked the expected headers') + def test__upload_object_manifest(self): hash_function = self.driver._get_hash_function() hash_function.update(b(''))
