Repository: libcloud Updated Branches: refs/heads/trunk f813ea17b -> 8fe3e4055
[LIBCLOUD-639] Fix upload_object_via_stream so it works correctly under Python 3.x if user manually passes an iterator to the method. Closes #408 Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/8fe3e405 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/8fe3e405 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/8fe3e405 Branch: refs/heads/trunk Commit: 8fe3e4055d8479c2ccaebff45bf0071890639f8c Parents: f813ea1 Author: Peter Schmidt <[email protected]> Authored: Mon Dec 1 17:28:20 2014 +1100 Committer: Tomaz Muraus <[email protected]> Committed: Tue Dec 9 13:34:13 2014 +0100 ---------------------------------------------------------------------- CHANGES.rst | 5 ++ libcloud/storage/base.py | 3 +- libcloud/test/storage/test_cloudfiles.py | 69 +++++++++++++++++++++++++-- libcloud/utils/files.py | 2 +- libcloud/utils/py3.py | 2 + 5 files changed, 74 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index e4858e7..3a2282d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -79,6 +79,11 @@ Storage (GITHUB-403, GITHUB-404) [Peter Schmidt] +- Fix upload_object_via_stream so it works correctly under Python 3.x if user + manually passes an iterator to the method. + (GITHUB-408, LIBCLOUD-639) + [Peter Schmidt] + Changes with Apache Libcloud 0.16.0 ----------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/storage/base.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 66e5d44..ead1545 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -742,7 +742,8 @@ class StorageDriver(BaseDriver): if calculate_hash: data_hash = self._get_hash_function() - generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size) + generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size, + fill_size=True) bytes_transferred = 0 try: http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/test/storage/test_cloudfiles.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 6590270..3e911da 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -30,7 +30,7 @@ from libcloud.utils.py3 import httplib from libcloud.utils.py3 import urlquote from libcloud.common.types import LibcloudError, MalformedResponseError -from libcloud.storage.base import Container, Object +from libcloud.storage.base import CHUNK_SIZE, Container, Object from libcloud.storage.types import ContainerAlreadyExistsError from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ContainerIsNotEmptyError @@ -665,10 +665,6 @@ class CloudFilesTests(unittest.TestCase): try: self.driver.upload_object_via_stream( - # We never reach the Python 3 only bytes vs int error - # currently at libcloud/utils/py3.py:89 - # raise TypeError("Invalid argument %r for b()" % (s,)) - # because I raise a NotImplementedError. iterator=iter(b'blob data like an image or video'), container=container, object_name="test_object", @@ -682,6 +678,64 @@ class CloudFilesTests(unittest.TestCase): self.fail('Expected NotImplementedError to be thrown to ' 'verify we actually checked the expected headers') + def test_upload_object_via_stream_python3_bytes_error(self): + container = Container(name='py3', extra={}, driver=self.driver) + bytes_blob = b'blob data like an image or video' + + # This is mostly to check we didn't discover other errors along the way + mocked_response = container.upload_object_via_stream( + iterator=iter(bytes_blob), + object_name="img_or_vid", + ) + self.assertEqual(len(bytes_blob), mocked_response.size) + + def test_upload_object_via_stream_chunked_encoding(self): + + # Create enough bytes it should get split into two chunks + bytes_blob = ''.join(['\0' for _ in range(CHUNK_SIZE + 1)]) + hex_chunk_size = ('%X' % CHUNK_SIZE).encode('utf8') + expected = [ + # Chunk 1 + hex_chunk_size + b'\r\n', + bytes(bytes_blob[:CHUNK_SIZE].encode('utf8')), + b'\r\n', + + # Chunk 2 + b'1\r\n', + bytes(bytes_blob[CHUNK_SIZE:].encode('utf8')), + b'\r\n', + + # If chunked, also send a final message + b'0\r\n\r\n', + ] + logged_data = [] + + class InterceptResponse(CloudFilesMockRawResponse): + def __init__(self, connection): + super(InterceptResponse, self).__init__(connection=connection) + old_send = self.connection.connection.send + + def intercept_send(data): + old_send(data) + logged_data.append(data) + self.connection.connection.send = intercept_send + + def _v1_MossoCloudFS_py3_img_or_vid2(self, + method, url, body, headers): + headers = {'etag': 'd79fb00c27b50494a463e680d459c90c'} + headers.update(self.base_headers) + _201 = httplib.CREATED + return _201, '', headers, httplib.responses[_201] + + self.driver_klass.connectionCls.rawResponseCls = InterceptResponse + + container = Container(name='py3', extra={}, driver=self.driver) + container.upload_object_via_stream( + iterator=iter(bytes_blob), + object_name="img_or_vid2", + ) + self.assertListEqual(expected, logged_data) + def test__upload_object_manifest(self): hash_function = self.driver._get_hash_function() hash_function.update(b('')) @@ -1087,6 +1141,11 @@ class CloudFilesMockRawResponse(MockRawResponse): fixtures = StorageFileFixtures('cloudfiles') base_headers = {'content-type': 'application/json; charset=UTF-8'} + def _v1_MossoCloudFS_py3_img_or_vid(self, method, url, body, headers): + headers = {'etag': 'e2378cace8712661ce7beec3d9362ef6'} + headers.update(self.base_headers) + return httplib.CREATED, '', headers, httplib.responses[httplib.CREATED] + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( self, method, url, body, headers): # test_object_upload_success http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/files.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/files.py b/libcloud/utils/files.py index a71e1c4..663677a 100644 --- a/libcloud/utils/files.py +++ b/libcloud/utils/files.py @@ -38,7 +38,7 @@ def read_in_chunks(iterator, chunk_size=None, fill_size=False, """ Return a generator which yields data in chunks. - :param terator: An object which implements an iterator interface + :param iterator: An object which implements an iterator interface or a File like object with read method. :type iterator: :class:`object` which implements iterator interface. http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/py3.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py index 4e05419..2b695a4 100644 --- a/libcloud/utils/py3.py +++ b/libcloud/utils/py3.py @@ -85,6 +85,8 @@ if PY3: return s.encode('utf-8') elif isinstance(s, bytes): return s + elif isinstance(s, int): + return bytes([s]) else: raise TypeError("Invalid argument %r for b()" % (s,))
