download_object_as_stream now returns a proper iterator
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/109ecad9 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/109ecad9 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/109ecad9 Branch: refs/heads/trunk Commit: 109ecad94257615b57f196f3c36d13d7b3c8e8fd Parents: 5528f5b Author: Anthony Shaw <[email protected]> Authored: Fri Jan 6 19:52:51 2017 +1100 Committer: Anthony Shaw <[email protected]> Committed: Fri Jan 6 19:52:51 2017 +1100 ---------------------------------------------------------------------- libcloud/common/base.py | 15 +++++++++++---- libcloud/httplib_ssl.py | 7 ++++--- libcloud/storage/drivers/s3.py | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/common/base.py ---------------------------------------------------------------------- diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 15233a9..5266163 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -152,6 +152,7 @@ class Response(object): self.error = response.reason self.status = response.status_code self.request = response.request + self.iter_content = response.iter_content self.body = response.text.strip() \ if response.text is not None and hasattr(response.text, 'strip') \ @@ -501,7 +502,7 @@ class Connection(object): self.ua.append(token) def request(self, action, params=None, data=None, headers=None, - method='GET', raw=False): + method='GET', raw=False, stream=False): """ Request a given `action`. @@ -531,6 +532,10 @@ class Connection(object): and use the rawResponseCls class. This is used with storage API when uploading a file. + :type stream: ``bool`` + :param stream: True to return an iterator in Response.iter_content and allow + streaming of the response data (for downloading large files) + :return: An :class:`Response` instance. :rtype: :class:`Response` instance @@ -601,7 +606,8 @@ class Connection(object): method=method, url=url, body=data, - headers=headers) + headers=headers, + stream=stream) else: if retry_enabled: retry_request = retry(timeout=self.timeout, @@ -610,10 +616,11 @@ class Connection(object): retry_request(self.connection.request)(method=method, url=url, body=data, - headers=headers) + headers=headers, + stream=stream) else: self.connection.request(method=method, url=url, body=data, - headers=headers) + headers=headers, stream=stream) except socket.gaierror: e = sys.exc_info()[1] message = str(e) http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/httplib_ssl.py ---------------------------------------------------------------------- diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py index 640d31a..ac20ee7 100644 --- a/libcloud/httplib_ssl.py +++ b/libcloud/httplib_ssl.py @@ -180,19 +180,20 @@ class LibcloudConnection(LibcloudBaseConnection): self.set_http_proxy(proxy_url=proxy_url) self.session.timeout = kwargs.get('timeout', 60) - def request(self, method, url, body=None, headers=None, raw=False): + def request(self, method, url, body=None, headers=None, raw=False, + stream=False): self.response = self.session.request( method=method.lower(), url=''.join([self.host, url]), data=body, headers=headers, allow_redirects=1, - stream=raw, + stream=stream, verify=self.ca_cert if self.ca_cert is not None else self.verify ) def prepared_request(self, method, url, body=None, - headers=None, raw=False): + headers=None, raw=False, stream=False): req = requests.Request(method, ''.join([self.host, url]), data=body, headers=headers) http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/storage/drivers/s3.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 28e205e..b0cbd82 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -414,11 +414,11 @@ class BaseS3StorageDriver(StorageDriver): def download_object_as_stream(self, obj, chunk_size=None): obj_path = self._get_object_path(obj.container, obj.name) - response = self.connection.request(obj_path, method='GET', raw=True) + response = self.connection.request(obj_path, method='GET', stream=True) return self._get_object(obj=obj, callback=read_in_chunks, response=response, - callback_kwargs={'iterator': response.response, + callback_kwargs={'iterator': response.iter_content, 'chunk_size': chunk_size}, success_status_code=httplib.OK)
