Author: tomaz
Date: Thu Jul 21 01:07:26 2011
New Revision: 1148998
URL: http://svn.apache.org/viewvc?rev=1148998&view=rev
Log:
Add Ninefold storage driver. Patch has been contributed by Benno Rice and is
part of GITHUB-19.
Added:
libcloud/trunk/libcloud/storage/drivers/atmos.py
libcloud/trunk/libcloud/storage/drivers/ninefold.py
libcloud/trunk/test/storage/fixtures/atmos/
libcloud/trunk/test/storage/fixtures/atmos/already_exists.xml
libcloud/trunk/test/storage/fixtures/atmos/empty_directory_listing.xml
libcloud/trunk/test/storage/fixtures/atmos/list_containers.xml
libcloud/trunk/test/storage/fixtures/atmos/not_empty.xml
libcloud/trunk/test/storage/fixtures/atmos/not_found.xml
libcloud/trunk/test/storage/test_atmos.py
Modified:
libcloud/trunk/libcloud/common/base.py
libcloud/trunk/libcloud/storage/providers.py
libcloud/trunk/libcloud/storage/types.py
libcloud/trunk/libcloud/utils.py
libcloud/trunk/test/test_utils.py
Modified: libcloud/trunk/libcloud/common/base.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/common/base.py?rev=1148998&r1=1148997&r2=1148998&view=diff
==============================================================================
--- libcloud/trunk/libcloud/common/base.py (original)
+++ libcloud/trunk/libcloud/common/base.py Thu Jul 21 01:07:26 2011
@@ -382,7 +382,7 @@ class ConnectionKey(object):
self.connection.putrequest(method, url)
for key, value in headers.iteritems():
- self.connection.putheader(key, value)
+ self.connection.putheader(key, str(value))
self.connection.endheaders()
else:
Added: libcloud/trunk/libcloud/storage/drivers/atmos.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/atmos.py?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/atmos.py (added)
+++ libcloud/trunk/libcloud/storage/drivers/atmos.py Thu Jul 21 01:07:26 2011
@@ -0,0 +1,417 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import hmac
+import httplib
+import time
+import urllib
+import urlparse
+
+from xml.etree import ElementTree
+
+from libcloud import utils
+from libcloud.common.base import ConnectionUserAndKey, Response
+from libcloud.common.types import LazyList
+
+from libcloud.storage.base import Object, Container, StorageDriver, CHUNK_SIZE
+from libcloud.storage.types import ContainerAlreadyExistsError, \
+ ContainerDoesNotExistError, \
+ ContainerIsNotEmptyError, \
+ ObjectDoesNotExistError
+
+def collapse(s):
+ return ' '.join([x for x in s.split(' ') if x])
+
+class AtmosError(Exception):
+ def __init__(self, code, message):
+ self.code = code
+ self.message = message
+
+class AtmosResponse(Response):
+ def success(self):
+ return self.status in (httplib.OK, httplib.CREATED, httplib.NO_CONTENT,
+ httplib.PARTIAL_CONTENT)
+
+ def parse_body(self):
+ if not self.body:
+ return None
+ tree = ElementTree.fromstring(self.body)
+ return tree
+
+ def parse_error(self):
+ if not self.body:
+ return None
+ tree = ElementTree.fromstring(self.body)
+ code = int(tree.find('Code').text)
+ message = tree.find('Message').text
+ raise AtmosError(code, message)
+
+class AtmosConnection(ConnectionUserAndKey):
+ responseCls = AtmosResponse
+
+ def add_default_headers(self, headers):
+ headers['x-emc-uid'] = self.user_id
+ headers['Date'] = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
+ time.gmtime())
+ headers['x-emc-date'] = headers['Date']
+
+ if 'Content-Type' not in headers:
+ headers['Content-Type'] = 'application/octet-stream'
+ if 'Accept' not in headers:
+ headers['Accept'] = '*/*'
+
+ return headers
+
+ def pre_connect_hook(self, params, headers):
+ headers['x-emc-signature'] = self._calculate_signature(params, headers)
+
+ return params, headers
+
+ def _calculate_signature(self, params, headers):
+ pathstring = self.action
+ if pathstring.startswith(self.driver.path):
+ pathstring = pathstring[len(self.driver.path):]
+ if params:
+ if type(params) is dict:
+ params = params.items()
+ pathstring += '?' + urllib.urlencode(params)
+ pathstring = pathstring.lower()
+
+ xhdrs = [(k, v) for k, v in headers.items() if k.startswith('x-emc-')]
+ xhdrs.sort(key=lambda x: x[0])
+
+ signature = [
+ self.method,
+ headers.get('Content-Type', ''),
+ headers.get('Range', ''),
+ headers.get('Date', ''),
+ pathstring,
+ ]
+ signature.extend([k + ':' + collapse(v) for k, v in xhdrs])
+ signature = '\n'.join(signature)
+ key = base64.b64decode(self.key)
+ signature = hmac.new(key, signature, hashlib.sha1).digest()
+ return base64.b64encode(signature)
+
+class AtmosDriver(StorageDriver):
+ connectionCls = AtmosConnection
+
+ host = None
+ path = None
+ api_name = 'atmos'
+
+ DEFAULT_CDN_TTL = 60 * 60 * 24 * 7 # 1 week
+
+ def __init__(self, key, secret=None, secure=True, host=None, port=None):
+ host = host or self.host
+ super(AtmosDriver, self).__init__(key, secret, secure, host, port)
+
+ def list_containers(self):
+ result = self.connection.request(self._namespace_path(''))
+ entries = self._list_objects(result.object, object_type='directory')
+ containers = []
+ for entry in entries:
+ extra = {
+ 'object_id': entry['id']
+ }
+ containers.append(Container(entry['name'], extra, self))
+ return containers
+
+ def get_container(self, container_name):
+ path = self._namespace_path(container_name + '/?metadata/system')
+ try:
+ result = self.connection.request(path)
+ except AtmosError, e:
+ if e.code != 1003:
+ raise
+ raise ContainerDoesNotExistError(e, self, container_name)
+ meta = self._emc_meta(result)
+ extra = {
+ 'object_id': meta['objectid']
+ }
+ return Container(container_name, extra, self)
+
+ def create_container(self, container_name):
+ path = self._namespace_path(container_name + '/')
+ try:
+ result = self.connection.request(path, method='POST')
+ except AtmosError, e:
+ if e.code != 1016:
+ raise
+ raise ContainerAlreadyExistsError(e, self, container_name)
+ return self.get_container(container_name)
+
+ def delete_container(self, container):
+ try:
+ self.connection.request(self._namespace_path(container.name + '/'),
+ method='DELETE')
+ except AtmosError, e:
+ if e.code == 1003:
+ raise ContainerDoesNotExistError(e, self, container.name)
+ elif e.code == 1023:
+ raise ContainerIsNotEmptyError(e, self, container.name)
+ return True
+
+ def get_object(self, container_name, object_name):
+ container = self.get_container(container_name)
+ path = container_name + '/' + object_name
+ path = self._namespace_path(path)
+
+ try:
+ result = self.connection.request(path + '?metadata/system')
+ system_meta = self._emc_meta(result)
+
+ result = self.connection.request(path + '?metadata/user')
+ user_meta = self._emc_meta(result)
+ except AtmosError, e:
+ if e.code != 1003:
+ raise
+ raise ObjectDoesNotExistError(e, self, object_name)
+
+ last_modified = time.strptime(system_meta['mtime'],
+ '%Y-%m-%dT%H:%M:%SZ')
+ last_modified = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
+ last_modified)
+ extra = {
+ 'object_id': system_meta['objectid'],
+ 'last_modified': last_modified
+ }
+ data_hash = user_meta.pop('md5', '')
+ return Object(object_name, int(system_meta['size']), data_hash, extra,
+ user_meta, container, self)
+
+ def upload_object(self, file_path, container, object_name, extra=None,
+ verify_hash=True):
+ upload_func = self._upload_file
+ upload_func_kwargs = { 'file_path': file_path }
+ method = 'PUT'
+
+ extra = extra or {}
+ request_path = container.name + '/' + object_name
+ request_path = self._namespace_path(request_path)
+ content_type = extra.get('content_type', None)
+
+ try:
+ self.connection.request(request_path + '?metadata/system')
+ except AtmosError, e:
+ if e.code != 1003:
+ raise
+ method = 'POST'
+
+ result_dict = self._upload_object(object_name=object_name,
+ content_type=content_type,
+ upload_func=upload_func,
+
upload_func_kwargs=upload_func_kwargs,
+ request_path=request_path,
+ request_method=method,
+ headers={}, file_path=file_path)
+
+ response = result_dict['response'].response
+ bytes_transferred = result_dict['bytes_transferred']
+
+ if extra is None:
+ meta_data = {}
+ else:
+ meta_data = extra.get('meta_data', {})
+ meta_data['md5'] = result_dict['data_hash']
+ user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()])
+ self.connection.request(request_path + '?metadata/user', method='POST',
+ headers={'x-emc-meta': user_meta})
+ result = self.connection.request(request_path + '?metadata/system')
+ meta = self._emc_meta(result)
+ del meta_data['md5']
+ extra = {
+ 'object_id': meta['objectid'],
+ 'meta_data': meta_data,
+ }
+
+ return Object(object_name, bytes_transferred, result_dict['data_hash'],
+ extra, meta_data, container, self)
+
+ def upload_object_via_stream(self, iterator, container, object_name,
+ extra=None):
+ if isinstance(iterator, file):
+ iterator = iter(iterator)
+
+ data_hash = hashlib.md5()
+ generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True)
+ bytes_transferred = 0
+ try:
+ chunk = generator.next()
+ except StopIteration:
+ chunk = ''
+
+ path = self._namespace_path(container.name + '/' + object_name)
+
+ while True:
+ end = bytes_transferred + len(chunk) - 1
+ data_hash.update(chunk)
+ headers = {
+ 'x-emc-meta': 'md5=' + data_hash.hexdigest(),
+ }
+ if len(chunk) > 0:
+ headers['Range'] = 'Bytes=%d-%d' % (bytes_transferred, end)
+ result = self.connection.request(path, method='PUT', data=chunk,
+ headers=headers)
+ bytes_transferred += len(chunk)
+
+ try:
+ chunk = generator.next()
+ except StopIteration:
+ break
+ if len(chunk) == 0:
+ break
+
+ data_hash = data_hash.hexdigest()
+
+ if extra is None:
+ meta_data = {}
+ else:
+ meta_data = extra.get('meta_data', {})
+ meta_data['md5'] = data_hash
+ user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()])
+ self.connection.request(path + '?metadata/user', method='POST',
+ headers={'x-emc-meta': user_meta})
+
+ result = self.connection.request(path + '?metadata/system')
+
+ meta = self._emc_meta(result)
+ extra = {
+ 'object_id': meta['objectid'],
+ 'meta_data': meta_data,
+ }
+
+ return Object(object_name, bytes_transferred, data_hash, extra,
+ meta_data, container, self)
+
+ def download_object(self, obj, destination_path, overwrite_existing=False,
+ delete_on_failure=True):
+ path = self._namespace_path(obj.container.name + '/' + obj.name)
+ response = self.connection.request(path, method='GET', raw=True)
+
+ return self._get_object(obj=obj, callback=self._save_object,
+ response=response,
+ callback_kwargs={
+ 'obj': obj,
+ 'response': response.response,
+ 'destination_path': destination_path,
+ 'overwrite_existing': overwrite_existing,
+ 'delete_on_failure': delete_on_failure
+ },
+ success_status_code=httplib.OK)
+
+ def download_object_as_stream(self, obj, chunk_size=None):
+ path = self._namespace_path(obj.container.name + '/' + obj.name)
+ response = self.connection.request(path, method='GET', raw=True)
+
+ return self._get_object(obj=obj, callback=utils.read_in_chunks,
+ response=response,
+ callback_kwargs={
+ 'iterator': response.response,
+ 'chunk_size': chunk_size
+ },
+ success_status_code=httplib.OK)
+
+ def delete_object(self, obj):
+ path = self._namespace_path(obj.container.name + '/' + obj.name)
+ try:
+ self.connection.request(path, method='DELETE')
+ except AtmosError, e:
+ if e.code != 1003:
+ raise
+ raise ObjectDoesNotExistError(e, self, obj.name)
+ return True
+
+ def list_container_objects(self, container):
+ value_dict = {'container': container}
+ return LazyList(get_more=self._get_more, value_dict=value_dict)
+
+ def enable_object_cdn(self, obj):
+ return True
+
+ def get_object_cdn_url(self, obj, expiry=None, use_object=False):
+ if use_object:
+ path = '/rest/objects' + obj.meta_data['object_id']
+ else:
+ path = '/rest/namespace/' + obj.container.name + '/' + obj.name
+
+ if self.secure:
+ protocol = 'https'
+ else:
+ protocol = 'http'
+
+ expiry = str(expiry or int(time.time()) + self.DEFAULT_CDN_TTL)
+ params = [
+ ('uid', self.key),
+ ('expires', expiry),
+ ]
+ params.append(('signature', self._cdn_signature(path, params)))
+
+ params = urllib.urlencode(params)
+ path = self.path + path
+ return urlparse.urlunparse((protocol, self.host, path, '', params, ''))
+
+ def _cdn_signature(self, path, params):
+ key = base64.b64decode(self.secret)
+ signature = '\n'.join(['GET', path.lower(), self.key, expiry])
+ signature = hmac.new(key, signature, hashlib.sha1).digest()
+
+ return base64.b64encode(signature)
+
+ def _list_objects(self, tree, object_type=None):
+ listing = tree.find(self._emc_tag('DirectoryList'))
+ entries = []
+ for entry in listing.findall(self._emc_tag('DirectoryEntry')):
+ file_type = entry.find(self._emc_tag('FileType')).text
+ if object_type is not None and object_type != file_type:
+ continue
+ entries.append({
+ 'id': entry.find(self._emc_tag('ObjectID')).text,
+ 'type': file_type,
+ 'name': entry.find(self._emc_tag('Filename')).text
+ })
+ return entries
+
+ def _namespace_path(self, path):
+ return self.path + '/rest/namespace/' + path
+
+ def _object_path(self, object_id):
+ return self.path + '/rest/objects/' + object_id
+
+ @staticmethod
+ def _emc_tag(tag):
+ return '{http://www.emc.com/cos/}' + tag
+
+ def _emc_meta(self, response):
+ meta = response.headers.get('x-emc-meta', '')
+ if len(meta) == 0:
+ return {}
+ meta = meta.split(', ')
+ return dict([x.split('=', 1) for x in meta])
+
+ def _get_more(self, last_key, value_dict):
+ container = value_dict['container']
+ headers = {'x-emc-include-meta': '1'}
+ path = self._namespace_path(container.name + '/')
+ result = self.connection.request(path, headers=headers)
+ entries = self._list_objects(result.object, object_type='regular')
+ objects = []
+ for entry in entries:
+ metadata = {'object_id': entry['id']}
+ objects.append(Object(entry['name'], 0, '', {}, metadata,
container,
+ self))
+ return objects, None, True
Added: libcloud/trunk/libcloud/storage/drivers/ninefold.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/ninefold.py?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/ninefold.py (added)
+++ libcloud/trunk/libcloud/storage/drivers/ninefold.py Thu Jul 21 01:07:26 2011
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from libcloud.storage.providers import Provider
+from libcloud.storage.drivers.atmos import AtmosDriver
+
+class NinefoldStorageDriver(AtmosDriver):
+ host = 'api.ninefold.com'
+ path = '/storage/v1.0'
+
+ type = Provider.NINEFOLD
+ name = 'Ninefold'
Modified: libcloud/trunk/libcloud/storage/providers.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/providers.py?rev=1148998&r1=1148997&r2=1148998&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/providers.py (original)
+++ libcloud/trunk/libcloud/storage/providers.py Thu Jul 21 01:07:26 2011
@@ -33,6 +33,8 @@ DRIVERS = {
('libcloud.storage.drivers.s3', 'S3APSEStorageDriver'),
Provider.S3_AP_NORTHEAST:
('libcloud.storage.drivers.s3', 'S3APNEStorageDriver'),
+ Provider.NINEFOLD:
+ ('libcloud.storage.drivers.ninefold', 'NinefoldStorageDriver'),
}
def get_driver(provider):
Modified: libcloud/trunk/libcloud/storage/types.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/types.py?rev=1148998&r1=1148997&r2=1148998&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/types.py (original)
+++ libcloud/trunk/libcloud/storage/types.py Thu Jul 21 01:07:26 2011
@@ -37,6 +37,7 @@ class Provider(object):
@cvar S3_EU_WEST: Amazon S3 EU West (Ireland)
@cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore)
@cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo)
+ @cvar NINEFOLD: Ninefold
"""
DUMMY = 0
CLOUDFILES_US = 1
@@ -46,6 +47,7 @@ class Provider(object):
S3_EU_WEST = 5
S3_AP_SOUTHEAST = 6
S3_AP_NORTHEAST = 7
+ NINEFOLD = 8
class ContainerError(LibcloudError):
error_type = 'ContainerError'
Modified: libcloud/trunk/libcloud/utils.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/utils.py?rev=1148998&r1=1148997&r2=1148998&view=diff
==============================================================================
--- libcloud/trunk/libcloud/utils.py (original)
+++ libcloud/trunk/libcloud/utils.py Thu Jul 21 01:07:26 2011
@@ -22,7 +22,7 @@ SHOW_DEPRECATION_WARNING = True
SHOW_IN_DEVELOPMENT_WARNING = True
OLD_API_REMOVE_VERSION = '0.6.0'
-def read_in_chunks(iterator, chunk_size=None):
+def read_in_chunks(iterator, chunk_size=None, fill_size=False):
"""
Return a generator which yields data in chunks.
@@ -32,6 +32,10 @@ def read_in_chunks(iterator, chunk_size=
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
+
+ @type fill_size: C{bool}
+ @param fill_size: If True, make sure chunks are chunk_size in length
+ (except for last chunk).
"""
if isinstance(iterator, (file, HTTPResponse)):
@@ -41,13 +45,30 @@ def read_in_chunks(iterator, chunk_size=
get_data = iterator.next
args = ()
- while True:
- chunk = str(get_data(*args))
+ data = ''
+ empty = False
+
+ while not empty or len(data) > 0:
+ if not empty:
+ try:
+ chunk = str(get_data(*args))
+ if len(chunk) > 0:
+ data += chunk
+ else:
+ empty = True
+ except StopIteration:
+ empty = True
- if len(chunk) == 0:
+ if len(data) == 0:
raise StopIteration
- yield chunk
+ if fill_size:
+ if empty or len(data) >= chunk_size:
+ yield data[:chunk_size]
+ data = data[chunk_size:]
+ else:
+ yield data
+ data = ''
def guess_file_mime_type(file_path):
filename = os.path.basename(file_path)
Added: libcloud/trunk/test/storage/fixtures/atmos/already_exists.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/atmos/already_exists.xml?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/fixtures/atmos/already_exists.xml (added)
+++ libcloud/trunk/test/storage/fixtures/atmos/already_exists.xml Thu Jul 21
01:07:26 2011
@@ -0,0 +1,5 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Error>
+ <Code>1016</Code>
+ <Message>The resource you are trying to create already exists.</Message>
+</Error>
Added: libcloud/trunk/test/storage/fixtures/atmos/empty_directory_listing.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/atmos/empty_directory_listing.xml?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/fixtures/atmos/empty_directory_listing.xml
(added)
+++ libcloud/trunk/test/storage/fixtures/atmos/empty_directory_listing.xml Thu
Jul 21 01:07:26 2011
@@ -0,0 +1,4 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<ListDirectoryResponse xmlns='http://www.emc.com/cos/'>
+ <DirectoryList/>
+</ListDirectoryResponse>
Added: libcloud/trunk/test/storage/fixtures/atmos/list_containers.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/atmos/list_containers.xml?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/fixtures/atmos/list_containers.xml (added)
+++ libcloud/trunk/test/storage/fixtures/atmos/list_containers.xml Thu Jul 21
01:07:26 2011
@@ -0,0 +1,45 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<ListDirectoryResponse xmlns='http://www.emc.com/cos/'>
+ <DirectoryList>
+ <DirectoryEntry>
+
<ObjectID>b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container1</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>860855a4a445b5e45eeef4024371fd5c73ee3ada</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container2</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>651eae32634bf84529c74eabd555fda48c7cead6</ObjectID>
+ <FileType>regular</FileType>
+ <Filename>not-a-container1</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>089293be672782a255498b0b05c4877acf23ef9e</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container3</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>bd804e9f356b51844f93273ec8c94b2e274711d0</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container4</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>b40b0f3a17fad1d8c8b2085f668f8107bb400fa5</ObjectID>
+ <FileType>regular</FileType>
+ <Filename>not-a-container-2</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>10bd74388b55a3c8c329ff5dd6d21bd55bfb7370</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container5</Filename>
+ </DirectoryEntry>
+ <DirectoryEntry>
+
<ObjectID>c04fa4aa3d0adcdf104baa0cef7b6279680a23c3</ObjectID>
+ <FileType>directory</FileType>
+ <Filename>container6</Filename>
+ </DirectoryEntry>
+ </DirectoryList>
+</ListDirectoryResponse>
Added: libcloud/trunk/test/storage/fixtures/atmos/not_empty.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/atmos/not_empty.xml?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/fixtures/atmos/not_empty.xml (added)
+++ libcloud/trunk/test/storage/fixtures/atmos/not_empty.xml Thu Jul 21
01:07:26 2011
@@ -0,0 +1,5 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Error>
+ <Code>1023</Code>
+ <Message>The directory you are trying to delete is not empty.</Message>
+</Error>
Added: libcloud/trunk/test/storage/fixtures/atmos/not_found.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/atmos/not_found.xml?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/fixtures/atmos/not_found.xml (added)
+++ libcloud/trunk/test/storage/fixtures/atmos/not_found.xml Thu Jul 21
01:07:26 2011
@@ -0,0 +1,5 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Error>
+ <Code>1003</Code>
+ <Message>The requested object was not found.</Message>
+</Error>
Added: libcloud/trunk/test/storage/test_atmos.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/test_atmos.py?rev=1148998&view=auto
==============================================================================
--- libcloud/trunk/test/storage/test_atmos.py (added)
+++ libcloud/trunk/test/storage/test_atmos.py Thu Jul 21 01:07:26 2011
@@ -0,0 +1,570 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import httplib
+import os.path
+import sys
+import unittest
+import urlparse
+
+from xml.etree import ElementTree
+
+import libcloud.utils
+
+from libcloud.common.types import LibcloudError
+from libcloud.storage.base import Container, Object
+from libcloud.storage.types import ContainerAlreadyExistsError, \
+ ContainerDoesNotExistError, \
+ ContainerIsNotEmptyError, \
+ ObjectDoesNotExistError
+from libcloud.storage.drivers.atmos import AtmosConnection, AtmosDriver
+from libcloud.storage.drivers.dummy import DummyIterator
+
+from test import StorageMockHttp, MockRawResponse
+from test.file_fixtures import StorageFileFixtures
+
+class AtmosTests(unittest.TestCase):
+ def setUp(self):
+ AtmosDriver.connectionCls.conn_classes = (None, AtmosMockHttp)
+ AtmosDriver.connectionCls.rawResponseCls = AtmosMockRawResponse
+ AtmosDriver.path = ''
+ AtmosMockHttp.type = None
+ AtmosMockHttp.upload_created = False
+ AtmosMockRawResponse.type = None
+ self.driver = AtmosDriver('dummy', base64.b64encode('dummy'))
+ self._remove_test_file()
+
+ def tearDown(self):
+ self._remove_test_file()
+
+ def _remove_test_file(self):
+ file_path = os.path.abspath(__file__) + '.temp'
+
+ try:
+ os.unlink(file_path)
+ except OSError:
+ pass
+
+ def test_list_containers(self):
+ AtmosMockHttp.type = 'EMPTY'
+ containers = self.driver.list_containers()
+ self.assertEqual(len(containers), 0)
+
+ AtmosMockHttp.type = None
+ containers = self.driver.list_containers()
+ self.assertEqual(len(containers), 6)
+
+ def test_list_container_objects(self):
+ container = Container(name='test_container', extra={},
+ driver=self.driver)
+
+ AtmosMockHttp.type = 'EMPTY'
+ objects = self.driver.list_container_objects(container=container)
+ self.assertEqual(len(objects), 0)
+
+ AtmosMockHttp.type = None
+ objects = self.driver.list_container_objects(container=container)
+ self.assertEqual(len(objects), 2)
+
+ obj = [o for o in objects if o.name == 'not-a-container1'][0]
+ self.assertEqual(obj.meta_data['object_id'],
+ '651eae32634bf84529c74eabd555fda48c7cead6')
+ self.assertEqual(obj.container.name, 'test_container')
+
+ def test_get_container(self):
+ container = self.driver.get_container(container_name='test_container')
+ self.assertEqual(container.name, 'test_container')
+ self.assertEqual(container.extra['object_id'],
+ 'b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9')
+
+ def test_get_container_not_found(self):
+ try:
+ self.driver.get_container(container_name='not_found')
+ except ContainerDoesNotExistError:
+ pass
+ else:
+ self.fail('Exception was not thrown')
+
+ def test_create_container_success(self):
+ container = self.driver.create_container(
+ container_name='test_create_container')
+ self.assertTrue(isinstance(container, Container))
+ self.assertEqual(container.name, 'test_create_container')
+ self.assertEqual(container.extra['object_id'],
+ '31a27b593629a3fe59f887fd973fd953e80062ce')
+
+ def test_create_container_already_exists(self):
+ AtmosMockHttp.type = 'ALREADY_EXISTS'
+
+ try:
+ self.driver.create_container(
+ container_name='test_create_container')
+ except ContainerAlreadyExistsError:
+ pass
+ else:
+ self.fail(
+ 'Container already exists but an exception was not thrown')
+
+ def test_delete_container_success(self):
+ container = Container(name='foo_bar_container', extra={}, driver=self)
+ result = self.driver.delete_container(container=container)
+ self.assertTrue(result)
+
+ def test_delete_container_not_found(self):
+ AtmosMockHttp.type = 'NOT_FOUND'
+ container = Container(name='foo_bar_container', extra={}, driver=self)
+ try:
+ self.driver.delete_container(container=container)
+ except ContainerDoesNotExistError:
+ pass
+ else:
+ self.fail(
+ 'Container does not exist but an exception was not thrown')
+
+ def test_delete_container_not_empty(self):
+ AtmosMockHttp.type = 'NOT_EMPTY'
+ container = Container(name='foo_bar_container', extra={}, driver=self)
+ try:
+ self.driver.delete_container(container=container)
+ except ContainerIsNotEmptyError:
+ pass
+ else:
+ self.fail('Container is not empty but an exception was not thrown')
+
+ def test_get_object_success(self):
+ obj = self.driver.get_object(container_name='test_container',
+ object_name='test_object')
+ self.assertEqual(obj.container.name, 'test_container')
+ self.assertEqual(obj.size, 555)
+ self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17')
+ self.assertEqual(obj.extra['object_id'],
+ '322dce3763aadc41acc55ef47867b8d74e45c31d6643')
+ self.assertEqual(
+ obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT')
+ self.assertEqual(obj.meta_data['foo-bar'], 'test 1')
+ self.assertEqual(obj.meta_data['bar-foo'], 'test 2')
+
+
+ def test_get_object_not_found(self):
+ try:
+ self.driver.get_object(container_name='test_container',
+ object_name='not_found')
+ except ObjectDoesNotExistError:
+ pass
+ else:
+ self.fail('Exception was not thrown')
+
+ def test_delete_object_success(self):
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver)
+ status = self.driver.delete_object(obj=obj)
+ self.assertTrue(status)
+
+ def test_delete_object_not_found(self):
+ AtmosMockHttp.type = 'NOT_FOUND'
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver)
+ try:
+ self.driver.delete_object(obj=obj)
+ except ObjectDoesNotExistError:
+ pass
+ else:
+ self.fail('Object does not exist but an exception was not thrown')
+
+ def test_download_object_success(self):
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver)
+ destination_path = os.path.abspath(__file__) + '.temp'
+ result = self.driver.download_object(obj=obj,
+ destination_path=destination_path,
+ overwrite_existing=False,
+ delete_on_failure=True)
+ self.assertTrue(result)
+
+ def test_download_object_success_not_found(self):
+ AtmosMockRawResponse.type = 'NOT_FOUND'
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+
+ obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
+ container=container,
+ meta_data=None,
+ driver=self.driver)
+ destination_path = os.path.abspath(__file__) + '.temp'
+ try:
+ self.driver.download_object(
+ obj=obj,
+ destination_path=destination_path,
+ overwrite_existing=False,
+ delete_on_failure=True)
+ except ObjectDoesNotExistError:
+ pass
+ else:
+ self.fail('Object does not exist but an exception was not thrown')
+
+ def test_download_object_as_stream(self):
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver)
+
+ stream = self.driver.download_object_as_stream(obj=obj,
chunk_size=None)
+ self.assertTrue(hasattr(stream, '__iter__'))
+
+ def test_upload_object_success(self):
+ def upload_file(self, response, file_path, chunked=False,
+ calculate_hash=True):
+ return True, 'hash343hhash89h932439jsaa89', 1000
+
+ old_func = AtmosDriver._upload_file
+ AtmosDriver._upload_file = upload_file
+ path = os.path.abspath(__file__)
+ container = Container(name='fbc', extra={}, driver=self)
+ object_name = 'ftu'
+ extra = {'meta_data': { 'some-value': 'foobar'}}
+ obj = self.driver.upload_object(file_path=path, container=container,
+ extra=extra, object_name=object_name)
+ self.assertEqual(obj.name, 'ftu')
+ self.assertEqual(obj.size, 1000)
+ self.assertTrue('some-value' in obj.meta_data)
+ AtmosDriver._upload_file = old_func
+
+ def test_upload_object_no_content_type(self):
+ def no_content_type(name):
+ return None, None
+
+ old_func = libcloud.utils.guess_file_mime_type
+ libcloud.utils.guess_file_mime_type = no_content_type
+ file_path = os.path.abspath(__file__)
+ container = Container(name='fbc', extra={}, driver=self)
+ object_name = 'ftu'
+ try:
+ self.driver.upload_object(file_path=file_path, container=container,
+ object_name=object_name)
+ except AttributeError:
+ pass
+ else:
+ self.fail(
+ 'File content type not provided'
+ ' but an exception was not thrown')
+ finally:
+ libcloud.utils.guess_file_mime_type = old_func
+
+ def test_upload_object_error(self):
+ def dummy_content_type(name):
+ return 'application/zip', None
+
+ def send(instance):
+ raise Exception('')
+
+ old_func1 = libcloud.utils.guess_file_mime_type
+ libcloud.utils.guess_file_mime_type = dummy_content_type
+ old_func2 = AtmosMockHttp.send
+ AtmosMockHttp.send = send
+
+ file_path = os.path.abspath(__file__)
+ container = Container(name='fbc', extra={}, driver=self)
+ object_name = 'ftu'
+ try:
+ self.driver.upload_object(
+ file_path=file_path,
+ container=container,
+ object_name=object_name)
+ except LibcloudError:
+ pass
+ else:
+ self.fail('Timeout while uploading but an exception was not
thrown')
+ finally:
+ libcloud.utils.guess_file_mime_type = old_func1
+ AtmosMockHttp.send = old_func2
+
+ def test_upload_object_nonexistent_file(self):
+ def dummy_content_type(name):
+ return 'application/zip', None
+
+ old_func = libcloud.utils.guess_file_mime_type
+ libcloud.utils.guess_file_mime_type = dummy_content_type
+
+ file_path = os.path.abspath(__file__ + '.inexistent')
+ container = Container(name='fbc', extra={}, driver=self)
+ object_name = 'ftu'
+ try:
+ self.driver.upload_object(
+ file_path=file_path,
+ container=container,
+ object_name=object_name)
+ except OSError:
+ pass
+ else:
+ self.fail('Inesitent but an exception was not thrown')
+ finally:
+ libcloud.utils.guess_file_mime_type = old_func
+
+ def test_upload_object_via_stream(self):
+ def dummy_content_type(name):
+ return 'application/zip', None
+
+ old_func = libcloud.utils.guess_file_mime_type
+ libcloud.utils.guess_file_mime_type = dummy_content_type
+
+ container = Container(name='fbc', extra={}, driver=self)
+ object_name = 'ftsd'
+ iterator = DummyIterator(data=['2', '3', '5'])
+ try:
+ self.driver.upload_object_via_stream(container=container,
+ object_name=object_name,
+ iterator=iterator)
+ finally:
+ libcloud.utils.guess_file_mime_type = old_func
+
+ def test_signature_algorithm(self):
+ test_uid = 'fredsmagicuid'
+ test_key = base64.b64encode('ssssshhhhhmysecretkey')
+ test_date = 'Mon, 04 Jul 2011 07:39:19 GMT'
+ test_values = [
+ ('GET', '/rest/namespace/foo', '', {},
+ 'WfSASIA25TuqO2n0aO9k/dtg6S0='),
+ ('POST', '/rest/namespace/foo', '', {},
+ 'oYKdsF+1DOuUT7iX5CJCDym2EQk='),
+ ('PUT', '/rest/namespace/foo', '', {},
+ 'JleF9dpSWhaT3B2swZI3s41qqs4='),
+ ('DELETE', '/rest/namespace/foo', '', {},
+ '2IX+Bd5XZF5YY+g4P59qXV1uLpo='),
+ ('GET', '/rest/namespace/foo?metata/system', '', {},
+ 'zuHDEAgKM1winGnWn3WBsqnz4ks='),
+ ('POST', '/rest/namespace/foo?metadata/user', '', {
+ 'x-emc-meta': 'fakemeta=fake, othermeta=faketoo'
+ }, '7sLx1nxPIRAtocfv02jz9h1BjbU='),
+ ]
+
+ class FakeDriver(object):
+ path = ''
+
+ for method, action, api_path, headers, expected in test_values:
+ c = AtmosConnection(test_uid, test_key)
+ c.method = method
+ c.action = action
+ d = FakeDriver()
+ d.path = api_path
+ c.driver = d
+ headers = c.add_default_headers(headers)
+ headers['Date'] = headers['x-emc-date'] = test_date
+ self.assertEqual(c._calculate_signature({}, headers), expected)
+
+class AtmosMockHttp(StorageMockHttp, unittest.TestCase):
+ fixtures = StorageFileFixtures('atmos')
+ upload_created = False
+
+ def __init__(self, *args, **kwargs):
+ unittest.TestCase.__init__(self)
+
+ if kwargs.get('host', None) and kwargs.get('port', None):
+ StorageMockHttp.__init__(self, *args, **kwargs)
+
+ def runTest(self):
+ pass
+
+ def request(self, method, url, body=None, headers=None, raw=False):
+ headers = headers or {}
+ parsed = urlparse.urlparse(url)
+ if parsed.query.startswith('metadata/'):
+ parsed = list(parsed)
+ parsed[2] = parsed[2] + '/' + parsed[4]
+ parsed[4] = ''
+ url = urlparse.urlunparse(parsed)
+ return super(AtmosMockHttp, self).request(method, url, body, headers,
+ raw)
+
+ def _rest_namespace_EMPTY(self, method, url, body, headers):
+ body = self.fixtures.load('empty_directory_listing.xml')
+ return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace(self, method, url, body, headers):
+ body = self.fixtures.load('list_containers.xml')
+ return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_container_EMPTY(self, method, url, body, headers):
+ body = self.fixtures.load('empty_directory_listing.xml')
+ return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_container(self, method, url, body, headers):
+ body = self.fixtures.load('list_containers.xml')
+ return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_container__metadata_system(self, method, url,
body,
+ headers):
+ headers = {
+ 'x-emc-meta': 'objectid=b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9'
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+ def _rest_namespace_not_found__metadata_system(self, method, url, body,
+ headers):
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ def _rest_namespace_test_create_container(self, method, url, body,
headers):
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_create_container__metadata_system(self, method,
+ url, body,
+ headers):
+ headers = {
+ 'x-emc-meta': 'objectid=31a27b593629a3fe59f887fd973fd953e80062ce'
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_create_container_ALREADY_EXISTS(self, method, url,
+ body, headers):
+ body = self.fixtures.load('already_exists.xml')
+ return (httplib.BAD_REQUEST, body, {},
+ httplib.responses[httplib.BAD_REQUEST])
+
+ def _rest_namespace_foo_bar_container(self, method, url, body, headers):
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_foo_bar_container_NOT_FOUND(self, method, url, body,
+ headers):
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ def _rest_namespace_foo_bar_container_NOT_EMPTY(self, method, url, body,
+ headers):
+ body = self.fixtures.load('not_empty.xml')
+ return (httplib.BAD_REQUEST, body, {},
+ httplib.responses[httplib.BAD_REQUEST])
+
+ def _rest_namespace_test_container_test_object_metadata_system(self,
method,
+ url, body,
+ headers):
+ meta = {
+ 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643',
+ 'size': '555',
+ 'mtime': '2011-01-25T22:01:49Z'
+ }
+ headers = {
+ 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()])
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_container_test_object_metadata_user(self, method,
+ url, body,
+ headers):
+ meta = {
+ 'md5': '6b21c4a111ac178feacf9ec9d0c71f17',
+ 'foo-bar': 'test 1',
+ 'bar-foo': 'test 2',
+ }
+ headers = {
+ 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()])
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+ def _rest_namespace_test_container_not_found_metadata_system(self, method,
+ url, body,
+ headers):
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ def _rest_namespace_foo_bar_container_foo_bar_object(self, method, url,
+ body, headers):
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self,
method,
+ url, body,
+ headers):
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ def _rest_namespace_fbc_ftu_metadata_system(self, method, url, body,
+ headers):
+ if not self.upload_created:
+ self.__class__.upload_created = True
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ self.__class__.upload_created = False
+ meta = {
+ 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643',
+ 'size': '555',
+ 'mtime': '2011-01-25T22:01:49Z'
+ }
+ headers = {
+ 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()])
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+ def _rest_namespace_fbc_ftu_metadata_user(self, method, url, body,
headers):
+ self.assertTrue('x-emc-meta' in headers)
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_fbc_ftsd(self, method, url, body, headers):
+ self.assertTrue('Range' in headers)
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_fbc_ftsd_metadata_user(self, method, url, body,
+ headers):
+ self.assertTrue('x-emc-meta' in headers)
+ return (httplib.OK, '', {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_fbc_ftsd_metadata_system(self, method, url, body,
+ headers):
+ meta = {
+ 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643',
+ 'size': '555',
+ 'mtime': '2011-01-25T22:01:49Z'
+ }
+ headers = {
+ 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()])
+ }
+ return (httplib.OK, '', headers, httplib.responses[httplib.OK])
+
+class AtmosMockRawResponse(MockRawResponse):
+ fixtures = StorageFileFixtures('atmos')
+
+ def _rest_namespace_foo_bar_container_foo_bar_object(self, method, url,
+ body, headers):
+ body = 'test'
+ self._data = self._generate_random_data(1000)
+ return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+ def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self,
method,
+ url, body,
+ headers):
+ body = self.fixtures.load('not_found.xml')
+ return (httplib.NOT_FOUND, body, {},
+ httplib.responses[httplib.NOT_FOUND])
+
+ def _rest_namespace_fbc_ftu(self, method, url, body, headers):
+ return (httplib.CREATED, '', {}, httplib.responses[httplib.CREATED])
+
+if __name__ == '__main__':
+ sys.exit(unittest.main())
Modified: libcloud/trunk/test/test_utils.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/test_utils.py?rev=1148998&r1=1148997&r2=1148998&view=diff
==============================================================================
--- libcloud/trunk/test/test_utils.py (original)
+++ libcloud/trunk/test/test_utils.py Thu Jul 21 01:07:26 2011
@@ -88,5 +88,46 @@ class TestUtils(unittest.TestCase):
libcloud.utils.in_development_warning('test_module')
self.assertEqual(len(WARNINGS_BUFFER), 1)
+ def test_read_in_chunks_iterator(self):
+ def iterator():
+ for x in range(0, 1000):
+ yield 'aa'
+
+ for result in libcloud.utils.read_in_chunks(iterator(), chunk_size=10,
+ fill_size=False):
+ self.assertEqual(result, 'aa')
+
+ for result in libcloud.utils.read_in_chunks(iterator(), chunk_size=10,
+ fill_size=True):
+ self.assertEqual(result, 'aaaaaaaaaa')
+
+ def test_read_in_chunks_filelike(self):
+ class FakeFile(file):
+ def __init__(self):
+ self.remaining = 500
+
+ def read(self, size):
+ self.remaining -= 1
+ if self.remaining == 0:
+ return ''
+ return 'b' * (size + 1)
+
+ for index, result in enumerate(libcloud.utils.read_in_chunks(
+ FakeFile(), chunk_size=10,
+ fill_size=False)):
+ self.assertEqual(result, 'b' * 11)
+
+ self.assertEqual(index, 498)
+
+ for index, result in enumerate(libcloud.utils.read_in_chunks(
+ FakeFile(), chunk_size=10,
+ fill_size=True)):
+ if index != 548:
+ self.assertEqual(result, 'b' * 10)
+ else:
+ self.assertEqual(result, 'b' * 9)
+
+ self.assertEqual(index, 548)
+
if __name__ == '__main__':
sys.exit(unittest.main())
