update test interfaces to work with new modules
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/7ec75c19 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/7ec75c19 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/7ec75c19 Branch: refs/heads/trunk Commit: 7ec75c19cd86f5fb46ee16ab8f5098778503c3c0 Parents: 610c624 Author: Anthony Shaw <anthonys...@apache.org> Authored: Fri Jan 6 16:26:54 2017 +1100 Committer: Anthony Shaw <anthonys...@apache.org> Committed: Fri Jan 6 16:26:54 2017 +1100 ---------------------------------------------------------------------- libcloud/common/base.py | 3 +- libcloud/storage/drivers/dummy.py | 5 + libcloud/storage/drivers/s3.py | 175 +-------------------------------- libcloud/test/__init__.py | 8 +- 4 files changed, 18 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index d893cf0..15233a9 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -154,7 +154,8 @@ class Response(object): self.request = response.request self.body = response.text.strip() \ - if response.text is not None else '' + if response.text is not None and hasattr(response.text, 'strip') \ + else '' if not self.success(): raise exception_from_message(code=self.status, http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/storage/drivers/dummy.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/dummy.py b/libcloud/storage/drivers/dummy.py index affd265..4065123 100644 --- a/libcloud/storage/drivers/dummy.py +++ b/libcloud/storage/drivers/dummy.py @@ -75,6 +75,11 @@ class DummyIterator(object): def __next__(self): return self.next() + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + pass class DummyStorageDriver(StorageDriver): """ http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/storage/drivers/s3.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 34fc455..28e205e 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -435,173 +435,6 @@ class BaseS3StorageDriver(StorageDriver): verify_hash=verify_hash, storage_class=ex_storage_class) - def _upload_multipart(self, response, data, iterator, container, - object_name, calculate_hash=True): - """ - Callback invoked for uploading data to S3 using Amazon's - multipart upload mechanism - - :param response: Response object from the initial POST request - :type response: :class:`S3RawResponse` - - :param data: Any data from the initial POST request - :type data: ``str`` - - :param iterator: The generator for fetching the upload data - :type iterator: ``generator`` - - :param container: The container owning the object to which data is - being uploaded - :type container: :class:`Container` - - :param object_name: The name of the object to which we are uploading - :type object_name: ``str`` - - :keyword calculate_hash: Indicates if we must calculate the data hash - :type calculate_hash: ``bool`` - - :return: A tuple of (status, checksum, bytes transferred) - :rtype: ``tuple`` - """ - - object_path = self._get_object_path(container, object_name) - - # Get the upload id from the response xml - response.body = response.response.read() - body = response.parse_body() - upload_id = body.find(fixxpath(xpath='UploadId', - namespace=self.namespace)).text - - try: - # Upload the data through the iterator - result = self._upload_from_iterator(iterator, object_path, - upload_id, calculate_hash) - (chunks, data_hash, bytes_transferred) = result - - # Commit the chunk info and complete the upload - etag = self._commit_multipart(object_path, upload_id, chunks) - except Exception: - exc = sys.exc_info()[1] - # Amazon provides a mechanism for aborting an upload. - self._abort_multipart(object_path, upload_id) - raise exc - - # Modify the response header of the first request. This is used - # by other functions once the callback is done - response.headers['etag'] = etag - - return (True, data_hash, bytes_transferred) - - def _upload_from_iterator(self, iterator, object_path, upload_id, - calculate_hash=True): - """ - Uploads data from an iterator in fixed sized chunks to S3 - - :param iterator: The generator for fetching the upload data - :type iterator: ``generator`` - - :param object_path: The path of the object to which we are uploading - :type object_name: ``str`` - - :param upload_id: The upload id allocated for this multipart upload - :type upload_id: ``str`` - - :keyword calculate_hash: Indicates if we must calculate the data hash - :type calculate_hash: ``bool`` - - :return: A tuple of (chunk info, checksum, bytes transferred) - :rtype: ``tuple`` - """ - - data_hash = None - if calculate_hash: - data_hash = self._get_hash_function() - - bytes_transferred = 0 - count = 1 - chunks = [] - params = {'uploadId': upload_id} - - # Read the input data in chunk sizes suitable for AWS - for data in read_in_chunks(iterator, chunk_size=CHUNK_SIZE, - fill_size=True, yield_empty=True): - bytes_transferred += len(data) - - if calculate_hash: - data_hash.update(data) - - chunk_hash = self._get_hash_function() - chunk_hash.update(data) - chunk_hash = base64.b64encode(chunk_hash.digest()).decode('utf-8') - - # This provides an extra level of data check and is recommended - # by amazon - headers = {'Content-MD5': chunk_hash} - params['partNumber'] = count - - request_path = '?'.join((object_path, urlencode(params))) - - resp = self.connection.request(request_path, method='PUT', - data=data, headers=headers) - - if resp.status != httplib.OK: - raise LibcloudError('Error uploading chunk', driver=self) - - server_hash = resp.headers['etag'] - - # Keep this data for a later commit - chunks.append((count, server_hash)) - count += 1 - - if calculate_hash: - data_hash = data_hash.hexdigest() - - return (chunks, data_hash, bytes_transferred) - - def _commit_multipart(self, object_path, upload_id, chunks): - """ - Makes a final commit of the data. - - :param object_path: Server side object path. - :type object_path: ``str`` - - :param upload_id: ID of the multipart upload. - :type upload_id: ``str`` - - :param upload_id: A list of (chunk_number, chunk_hash) tuples. - :type upload_id: ``list`` - """ - - root = Element('CompleteMultipartUpload') - - for (count, etag) in chunks: - part = SubElement(root, 'Part') - part_no = SubElement(part, 'PartNumber') - part_no.text = str(count) - - etag_id = SubElement(part, 'ETag') - etag_id.text = str(etag) - - data = tostring(root) - - params = {'uploadId': upload_id} - request_path = '?'.join((object_path, urlencode(params))) - response = self.connection.request(request_path, data=data, - method='POST') - - if response.status != httplib.OK: - element = response.object - # pylint: disable=maybe-no-member - code, message = response._parse_error_details(element=element) - msg = 'Error in multipart commit: %s (%s)' % (message, code) - raise LibcloudError(msg, driver=self) - - # Get the server's etag to be passed back to the caller - body = response.parse_body() - server_hash = body.find(fixxpath(xpath='ETag', - namespace=self.namespace)).text - return server_hash - def _abort_multipart(self, object_path, upload_id): """ Aborts an already initiated multipart upload @@ -637,9 +470,8 @@ class BaseS3StorageDriver(StorageDriver): # Amazon provides a different (complex?) mechanism to do multipart # uploads if self.supports_s3_multipart_upload: - method = 'POST' - params = 'uploads' - + # @TODO: This needs implementing again from scratch. + pass return self._put_object(container=container, object_name=object_name, extra=extra, method=method, query_args=params, stream=iterator, verify_hash=False, @@ -808,7 +640,8 @@ class BaseS3StorageDriver(StorageDriver): if (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( - value='MD5 hash checksum does not match', + value='MD5 hash {0} checksum does not match {1}'.format( + server_hash, result_dict['data_hash']), object_name=object_name, driver=self) elif response.status == httplib.OK: obj = Object( http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/test/__init__.py ---------------------------------------------------------------------- diff --git a/libcloud/test/__init__.py b/libcloud/test/__init__.py index 2024f30..c4db57c 100644 --- a/libcloud/test/__init__.py +++ b/libcloud/test/__init__.py @@ -88,7 +88,13 @@ class MockResponse(object): self.body = body self.headers = headers or self.headers self.reason = reason or self.reason - self.body_iter = iter(self.body) if self.body is not None else None + if self.body: + if not hasattr(self.body, '__next__'): + self.body_iter = iter(self.body) + else: + self.body_iter = self.body + else: + self.body_iter = iter('') def read(self, *args, **kwargs): return self.body
