Update the code to use the same code path in case iterator object has next() method. next() and __next__() should be equivalent so we should follow the same code path in such scenario (no need to fall back to while loop at the end of method).
Also add a test case for it. Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/484e02a6 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/484e02a6 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/484e02a6 Branch: refs/heads/trunk Commit: 484e02a6395f3f05989a90b88bdb4c9b583ab4d7 Parents: 0710d23 Author: Tomaz Muraus <[email protected]> Authored: Mon Jan 8 12:04:09 2018 +0100 Committer: Tomaz Muraus <[email protected]> Committed: Mon Jan 8 12:04:09 2018 +0100 ---------------------------------------------------------------------- libcloud/storage/base.py | 7 +++++- libcloud/test/storage/test_base.py | 39 ++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/484e02a6/libcloud/storage/base.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 617cfed..052725b 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -642,22 +642,27 @@ class StorageDriver(BaseDriver): def _hash_buffered_stream(self, stream, hasher, blocksize=65536): total_len = 0 - if hasattr(stream, '__next__'): + + if hasattr(stream, '__next__') or hasattr(stream, 'next'): for chunk in libcloud.utils.files.read_in_chunks(iterator=stream): hasher.update(b(chunk)) total_len += len(chunk) + return (hasher.hexdigest(), total_len) + if not hasattr(stream, '__exit__'): for s in stream: hasher.update(s) total_len = total_len + len(s) return (hasher.hexdigest(), total_len) + with stream: buf = stream.read(blocksize) while len(buf) > 0: total_len = total_len + len(buf) hasher.update(buf) buf = stream.read(blocksize) + return (hasher.hexdigest(), total_len) def _get_hash_function(self): http://git-wip-us.apache.org/repos/asf/libcloud/blob/484e02a6/libcloud/test/storage/test_base.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_base.py b/libcloud/test/storage/test_base.py index dabc0f5..87a6954 100644 --- a/libcloud/test/storage/test_base.py +++ b/libcloud/test/storage/test_base.py @@ -142,16 +142,18 @@ class BaseStorageTests(unittest.TestCase): # object has when iterator has __next__ method, but instead read and calculate hash in chunks size = 100 + self.driver1.connection = Mock() + + # stream has __next__ method and next() method mock_read_in_chunks.return_value = 'a' * size iterator = BodyStream('a' * size) + self.assertTrue(hasattr(iterator, '__next__')) + self.assertTrue(hasattr(iterator, 'next')) upload_func = Mock() upload_func.return_value = True, '', size - # strict_mode is disabled, default content type should be used - self.driver1.connection = Mock() - self.assertEqual(mock_read_in_chunks.call_count, 0) self.assertEqual(mock_exhaust_iterator.call_count, 0) @@ -175,6 +177,37 @@ class BaseStorageTests(unittest.TestCase): self.assertEqual(mock_read_in_chunks.call_count, 1) self.assertEqual(mock_exhaust_iterator.call_count, 0) + # stream has only has next() method + mock_read_in_chunks.return_value = 'b' * size + + iterator = iter([str(v) for v in ['b' * size]]) + + self.assertFalse(hasattr(iterator, '__next__')) + self.assertTrue(hasattr(iterator, 'next')) + + self.assertEqual(mock_read_in_chunks.call_count, 1) + self.assertEqual(mock_exhaust_iterator.call_count, 0) + + result = self.driver1._upload_object(object_name='test', + content_type=None, + upload_func=upload_func, + upload_func_kwargs={}, + request_path='/', + stream=iterator) + + hasher = hashlib.md5() + hasher.update('b' * size) + expected_hash = hasher.hexdigest() + + self.assertEqual(result['data_hash'], expected_hash) + self.assertEqual(result['bytes_transferred'], size) + + headers = self.driver1.connection.request.call_args[-1]['headers'] + self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE) + + self.assertEqual(mock_read_in_chunks.call_count, 2) + self.assertEqual(mock_exhaust_iterator.call_count, 0) + if __name__ == '__main__': sys.exit(unittest.main())
