Author: tomaz
Date: Sun Nov 18 19:25:54 2012
New Revision: 1410962
URL: http://svn.apache.org/viewvc?rev=1410962&view=rev
Log:
Add a new generator based method for listing / iterating over the
containers (iterate_containers).
Contributed by Mahendra M, part of LIBCLOUD-261.
Modified:
libcloud/trunk/CHANGES
libcloud/trunk/libcloud/storage/base.py
libcloud/trunk/libcloud/storage/drivers/atmos.py
libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
libcloud/trunk/libcloud/storage/drivers/dummy.py
libcloud/trunk/libcloud/storage/drivers/local.py
libcloud/trunk/libcloud/storage/drivers/nimbus.py
libcloud/trunk/libcloud/storage/drivers/s3.py
Modified: libcloud/trunk/CHANGES
URL:
http://svn.apache.org/viewvc/libcloud/trunk/CHANGES?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/CHANGES (original)
+++ libcloud/trunk/CHANGES Sun Nov 18 19:25:54 2012
@@ -75,6 +75,10 @@ Changes with Apache Libcloud in developm
driver. ; LIBCLOUD-257
[John Carr]
+ - Add a new generator based method for listing / iterating over the
+ containers (iterate_containers). ; LIBCLOUD-261
+ [Mahendra M]
+
*) DNS
- Update 'if type' checks in the update_record methods to behave correctly
Modified: libcloud/trunk/libcloud/storage/base.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/base.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/base.py (original)
+++ libcloud/trunk/libcloud/storage/base.py Sun Nov 18 19:25:54 2012
@@ -178,6 +178,16 @@ class StorageDriver(BaseDriver):
secure=secure, host=host,
port=port, **kwargs)
+ def iterate_containers(self):
+ """
+ Return a generator of containers for the given account
+
+ @return: A generator of Container instances.
+ @rtype: C{generator} of L{Container}
+ """
+ raise NotImplementedError(
+ 'iterate_containers not implemented for this driver')
+
def list_containers(self):
"""
Return a list of containers.
@@ -185,8 +195,7 @@ class StorageDriver(BaseDriver):
@return: A list of Container instances.
@rtype: C{list} of L{Container}
"""
- raise NotImplementedError(
- 'list_containers not implemented for this driver')
+ return list(self.iterate_containers())
def iterate_container_objects(self, container):
"""
Modified: libcloud/trunk/libcloud/storage/drivers/atmos.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/atmos.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/atmos.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/atmos.py Sun Nov 18 19:25:54 2012
@@ -133,16 +133,14 @@ class AtmosDriver(StorageDriver):
host = host or self.host
super(AtmosDriver, self).__init__(key, secret, secure, host, port)
- def list_containers(self):
+ def iterate_containers(self):
result = self.connection.request(self._namespace_path(''))
entries = self._list_objects(result.object, object_type='directory')
- containers = []
for entry in entries:
extra = {
'object_id': entry['id']
}
- containers.append(Container(entry['name'], extra, self))
- return containers
+ yield Container(entry['name'], extra, self)
def get_container(self, container_name):
path = self._namespace_path(container_name) + '/?metadata/system'
Modified: libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/cloudfiles.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/cloudfiles.py Sun Nov 18 19:25:54
2012
@@ -218,7 +218,7 @@ class CloudFilesStorageDriver(StorageDri
OpenStackDriverMixin.__init__(self, *args, **kwargs)
super(CloudFilesStorageDriver, self).__init__(*args, **kwargs)
- def list_containers(self):
+ def iterate_containers(self):
response = self.connection.request('')
if response.status == httplib.NO_CONTENT:
@@ -713,15 +713,10 @@ class CloudFilesStorageDriver(StorageDri
def _to_container_list(self, response):
# @TODO: Handle more then 10k containers - use "lazy list"?
- containers = []
-
for container in response:
extra = {'object_count': int(container['count']),
'size': int(container['bytes'])}
- containers.append(Container(name=container['name'], extra=extra,
- driver=self))
-
- return containers
+ yield Container(name=container['name'], extra=extra, driver=self)
def _to_object_list(self, response, container):
objects = []
Modified: libcloud/trunk/libcloud/storage/drivers/dummy.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/dummy.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/dummy.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/dummy.py Sun Nov 18 19:25:54 2012
@@ -141,10 +141,10 @@ class DummyStorageDriver(StorageDriver):
'object_count': int(object_count),
'bytes_used': int(bytes_used)}
- def list_containers(self):
+ def iterate_containers(self):
"""
>>> driver = DummyStorageDriver('key', 'secret')
- >>> driver.list_containers()
+ >>> list(driver.iterate_containers())
[]
>>> container = driver.create_container(container_name='test container
1')
>>> container
@@ -158,15 +158,15 @@ class DummyStorageDriver(StorageDriver):
... container_name='test container 2') #doctest:
+IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
ContainerAlreadyExistsError:
- >>> container_list=driver.list_containers()
+ >>> container_list=list(driver.iterate_containers())
>>> sorted([container.name for container in container_list])
['test container 1', 'test container 2']
- @inherits: L{StorageDriver.list_containers}
+ @inherits: L{StorageDriver.iterate_containers}
"""
- return [container['container'] for container in
- list(self._containers.values())]
+ for container in list(self._containers.values()):
+ yield container['container']
def list_container_objects(self, container):
container = self.get_container(container.name)
Modified: libcloud/trunk/libcloud/storage/drivers/local.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/local.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/local.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/local.py Sun Nov 18 19:25:54 2012
@@ -177,23 +177,19 @@ class LocalStorageDriver(StorageDriver):
driver=self, container=container, hash=None,
meta_data=None)
- def list_containers(self):
+ def iterate_containers(self):
"""
- Return a list of containers.
+ Return a generator of containers.
- @return: A list of Container instances.
- @rtype: C{list} of L{Container}
+ @return: A generator of Container instances.
+ @rtype: C{generator} of L{Container}
"""
- containers = []
-
for container_name in os.listdir(self.base_path):
full_path = os.path.join(self.base_path, container_name)
if not os.path.isdir(full_path):
continue
- containers.append(self._make_container(container_name))
-
- return containers
+ yield self._make_container(container_name)
def _get_objects(self, container):
"""
Modified: libcloud/trunk/libcloud/storage/drivers/nimbus.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/nimbus.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/nimbus.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/nimbus.py Sun Nov 18 19:25:54 2012
@@ -87,7 +87,7 @@ class NimbusStorageDriver(StorageDriver)
self.user_id = kwargs['user_id']
super(NimbusStorageDriver, self).__init__(*args, **kwargs)
- def list_containers(self):
+ def iterate_containers(self):
response = self.connection.request('/customers/%s/collections' %
(self.connection.user_id))
return self._to_containers(response.object)
@@ -101,7 +101,7 @@ class NimbusStorageDriver(StorageDriver)
return self._to_container(response.object)
def _to_containers(self, data):
- return [self._to_container(item) for item in data]
+ yield self._to_container(item) for item in data
def _to_container(self, data):
name = data[0]
Modified: libcloud/trunk/libcloud/storage/drivers/s3.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/s3.py?rev=1410962&r1=1410961&r2=1410962&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/s3.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/s3.py Sun Nov 18 19:25:54 2012
@@ -171,7 +171,7 @@ class S3StorageDriver(StorageDriver):
ex_location_name = ''
namespace = NAMESPACE
- def list_containers(self):
+ def iterate_containers(self):
response = self.connection.request('/')
if response.status == httplib.OK:
containers = self._to_containers(obj=response.object,
@@ -430,9 +430,9 @@ class S3StorageDriver(StorageDriver):
driver=self)
def _to_containers(self, obj, xpath):
- return [self._to_container(element) for element in
- obj.findall(fixxpath(
- xpath=xpath, namespace=self.namespace))]
+ for element in obj.findall(fixxpath(xpath=xpath,
+ namespace=self.namespace)):
+ yield self._to_container(element)
def _to_objs(self, obj, xpath, container):
return [self._to_obj(element, container) for element in
