Author: tomaz
Date: Sun Jun 26 10:24:39 2011
New Revision: 1139754
URL: http://svn.apache.org/viewvc?rev=1139754&view=rev
Log:
1. Add an implementation of LazyList with tests
2. Modify S3 and CloudFiles storage driver to return LazyList instead of a
normal list in the list_container_objects_method
Code has been contributed by Danny Clark and Wiktor Kolodziej.
This change is part of LIBCLOUD-78.
Added:
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml
libcloud/trunk/test/test_types.py
Modified:
libcloud/trunk/libcloud/common/types.py
libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
libcloud/trunk/libcloud/storage/drivers/s3.py
libcloud/trunk/test/storage/test_cloudfiles.py
libcloud/trunk/test/storage/test_s3.py
Modified: libcloud/trunk/libcloud/common/types.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/common/types.py?rev=1139754&r1=1139753&r2=1139754&view=diff
==============================================================================
--- libcloud/trunk/libcloud/common/types.py (original)
+++ libcloud/trunk/libcloud/common/types.py Sun Jun 26 10:24:39 2011
@@ -17,9 +17,11 @@ __all__ = [
"LibcloudError",
"MalformedResponseError",
"InvalidCredsError",
- "InvalidCredsException"
+ "InvalidCredsException",
+ "LazyList"
]
+
class LibcloudError(Exception):
"""The base class for other libcloud exceptions"""
@@ -30,9 +32,10 @@ class LibcloudError(Exception):
def __str__(self):
return ("<LibcloudError in "
+ repr(self.driver)
- +" "
+ + " "
+ repr(self.value) + ">")
+
class MalformedResponseError(LibcloudError):
"""Exception for the cases when a provider returns a malformed
response, e.g. you request JSON and provider returns
@@ -51,6 +54,7 @@ class MalformedResponseError(LibcloudErr
+ ">: "
+ repr(self.body))
+
class InvalidCredsError(LibcloudError):
"""Exception used when invalid credentials are used on a provider."""
@@ -58,8 +62,53 @@ class InvalidCredsError(LibcloudError):
driver=None):
self.value = value
self.driver = driver
+
def __str__(self):
return repr(self.value)
+
# Deprecated alias of L{InvalidCredsError}
InvalidCredsException = InvalidCredsError
+
+
+class LazyList(object):
+
+ def __init__(self, get_more, value_dict=None):
+ self._data = []
+ self._last_key = None
+ self._exhausted = False
+ self._all_loaded = False
+ self._get_more = get_more
+ self._value_dict = value_dict or {}
+
+ def __iter__(self):
+ if not self._all_loaded:
+ self._load_all()
+
+ data = self._data
+ for i in data:
+ yield i
+
+ def __getitem__(self, index):
+ if index >= len(self._data) and not self._all_loaded:
+ self._load_all()
+
+ return self._data[index]
+
+ def __len__(self):
+ self._load_all()
+ return len(self._data)
+
+ def __repr__(self):
+ self._load_all()
+ repr_string = ', ' .join([repr(item) for item in self._data])
+ repr_string = '[%s]' % (repr_string)
+ return repr_string
+
+ def _load_all(self):
+ while not self._exhausted:
+ newdata, self._last_key, self._exhausted = \
+ self._get_more(last_key=self._last_key,
+ value_dict=self._value_dict)
+ self._data.extend(newdata)
+ self._all_loaded = True
Modified: libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py?rev=1139754&r1=1139753&r2=1139754&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/cloudfiles.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/cloudfiles.py Sun Jun 26 10:24:39
2011
@@ -33,6 +33,7 @@ from libcloud.storage.types import Conta
from libcloud.storage.types import ObjectDoesNotExistError
from libcloud.storage.types import ObjectHashMismatchError
from libcloud.storage.types import InvalidContainerNameError
+from libcloud.common.types import LazyList
from libcloud.common.rackspace import (
AUTH_HOST_US, AUTH_HOST_UK, RackspaceBaseConnection)
@@ -161,15 +162,8 @@ class CloudFilesStorageDriver(StorageDri
raise LibcloudError('Unexpected status code: %s' % (response.status))
def list_container_objects(self, container):
- response = self.connection.request('/%s' % (container.name))
-
- if response.status == httplib.NO_CONTENT:
- # Empty or inexistent container
- return []
- elif response.status == httplib.OK:
- return self._to_object_list(json.loads(response.body), container)
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
+ value_dict = { 'container': container }
+ return LazyList(get_more=self._get_more, value_dict=value_dict)
def get_container(self, container_name):
response = self.connection.request('/%s' % (container_name),
@@ -354,6 +348,30 @@ class CloudFilesStorageDriver(StorageDri
raise LibcloudError('Unexpected status code: %s' % (response.status))
+ def _get_more(self, last_key, value_dict):
+ container = value_dict['container']
+ params = {}
+
+ if last_key:
+ params['marker'] = last_key
+
+ response = self.connection.request('/%s' % (container.name),
+ params=params)
+
+ if response.status == httplib.NO_CONTENT:
+ # Empty or inexistent container
+ return [], None, True
+ elif response.status == httplib.OK:
+ objects = self._to_object_list(json.loads(response.body),
container)
+
+ # TODO: Is this really needed?
+ if len(objects) == 0:
+ return [], None, True
+
+ return objects, objects[-1].name, False
+
+ raise LibcloudError('Unexpected status code: %s' % (response.status))
+
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
iterator=None, verify_hash=True):
Modified: libcloud/trunk/libcloud/storage/drivers/s3.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/libcloud/storage/drivers/s3.py?rev=1139754&r1=1139753&r2=1139754&view=diff
==============================================================================
--- libcloud/trunk/libcloud/storage/drivers/s3.py (original)
+++ libcloud/trunk/libcloud/storage/drivers/s3.py Sun Jun 26 10:24:39 2011
@@ -35,6 +35,7 @@ from libcloud.storage.types import Inval
from libcloud.storage.types import ContainerDoesNotExistError
from libcloud.storage.types import ObjectDoesNotExistError
from libcloud.storage.types import ObjectHashMismatchError
+from libcloud.common.types import LazyList
# How long before the token expires
EXPIRATION_SECONDS = 15 * 60
@@ -173,14 +174,8 @@ class S3StorageDriver(StorageDriver):
driver=self)
def list_container_objects(self, container):
- response = self.connection.request('/%s' % (container.name))
- if response.status == httplib.OK:
- objects = self._to_objs(obj=response.object,
- xpath='Contents', container=container)
- return objects
-
- raise LibcloudError('Unexpected status code: %s' % (response.status),
- driver=self)
+ value_dict = { 'container': container }
+ return LazyList(get_more=self._get_more, value_dict=value_dict)
def get_container(self, container_name):
# This is very inefficient, but afaik it's the only way to do it
@@ -328,6 +323,32 @@ class S3StorageDriver(StorageDriver):
name = urllib.quote(name)
return name
+ def _get_more(self, last_key, value_dict):
+ container = value_dict['container']
+ params = {}
+
+ if last_key:
+ params['marker'] = last_key
+
+ response = self.connection.request('/%s' % (container.name),
+ params=params)
+
+ if response.status == httplib.OK:
+ objects = self._to_objs(obj=response.object,
+ xpath='Contents', container=container)
+ is_truncated =
response.object.findtext(fixxpath(xpath='IsTruncated',
+
namespace=NAMESPACE)).lower()
+ exhausted = (is_truncated == 'false')
+
+ if (len(objects) > 0):
+ last_key = objects[-1].name
+ else:
+ last_key = None
+ return objects, last_key, exhausted
+
+ raise LibcloudError('Unexpected status code: %s' % (response.status),
+ driver=self)
+
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
iterator=None, verify_hash=True, storage_class=None):
Added:
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json?rev=1139754&view=auto
==============================================================================
---
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json
(added)
+++
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json
Sun Jun 26 10:24:39 2011
@@ -0,0 +1,11 @@
+[
+ {"name":"foo-test-1","hash":"16265549b5bda64ecdaa5156de4c97cc",
+ "bytes":1160520,"content_type":"application/zip",
+ "last_modified":"2011-01-25T22:01:50.351810"},
+ {"name":"foo-test-2","hash":"16265549b5bda64ecdaa5156de4c97bb",
+ "bytes":1160520,"content_type":"application/zip",
+ "last_modified":"2011-01-25T22:01:50.351810"},
+ {"name":"foo-test-3","hash":"16265549b5bda64ecdaa5156de4c97ee",
+ "bytes":1160520,"content_type":"application/zip",
+ "last_modified":"2011-01-25T22:01:46.549890"}
+]
Added:
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json?rev=1139754&view=auto
==============================================================================
---
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json
(added)
+++
libcloud/trunk/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json
Sun Jun 26 10:24:39 2011
@@ -0,0 +1,8 @@
+[
+ {"name":"foo-test-4","hash":"16265549b5bda64ecdaa5156de4c97cc",
+ "bytes":1160520,"content_type":"application/zip",
+ "last_modified":"2011-01-25T22:01:50.351810"},
+ {"name":"foo-test-5","hash":"16265549b5bda64ecdaa5156de4c97bb",
+ "bytes":1160520,"content_type":"application/zip",
+ "last_modified":"2011-01-25T22:01:50.351810"}
+]
Added:
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml?rev=1139754&view=auto
==============================================================================
---
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml
(added)
+++
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml
Sun Jun 26 10:24:39 2011
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>
+ <Name>test_container</Name>
+ <Prefix></Prefix>
+ <Marker></Marker>
+ <MaxKeys>1000</MaxKeys>
+ <IsTruncated>true</IsTruncated>
+ <Contents>
+ <Key>1.zip</Key>
+ <LastModified>2011-04-09T19:05:18.000Z</LastModified>
+ <ETag>"4397da7a7649e8085de9916c240e8166"</ETag>
+ <Size>1234567</Size>
+ <Owner>
+ <ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID>
+ </Owner>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+ <Contents>
+ <Key>2.zip</Key>
+ <LastModified>2011-04-09T19:05:18.000Z</LastModified>
+ <ETag>"4397da7a7649e8085de9916c240e8166"</ETag>
+ <Size>1234567</Size>
+ <Owner>
+ <ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID>
+ </Owner>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+ <Contents>
+ <Key>3.zip</Key>
+ <LastModified>2011-04-09T19:05:18.000Z</LastModified>
+ <ETag>"4397da7a7649e8085de9916c240e8166"</ETag>
+ <Size>1234567</Size>
+ <Owner>
+ <ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID>
+ </Owner>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+</ListBucketResult>
Added:
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml?rev=1139754&view=auto
==============================================================================
---
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml
(added)
+++
libcloud/trunk/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml
Sun Jun 26 10:24:39 2011
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>
+ <Name>test_container</Name>
+ <Prefix></Prefix>
+ <Marker></Marker>
+ <MaxKeys>3</MaxKeys>
+ <IsTruncated>false</IsTruncated>
+ <Contents>
+ <Key>4.zip</Key>
+ <LastModified>2011-04-09T19:05:18.000Z</LastModified>
+ <ETag>"4397da7a7649e8085de9916c240e8166"</ETag>
+ <Size>1234567</Size>
+ <Owner>
+ <ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID>
+ </Owner>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+ <Contents>
+ <Key>5.zip</Key>
+ <LastModified>2011-04-09T19:05:18.000Z</LastModified>
+ <ETag>"4397da7a7649e8085de9916c240e8166"</ETag>
+ <Size>1234567</Size>
+ <Owner>
+ <ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID>
+ </Owner>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+</ListBucketResult>
Modified: libcloud/trunk/test/storage/test_cloudfiles.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/test_cloudfiles.py?rev=1139754&r1=1139753&r2=1139754&view=diff
==============================================================================
--- libcloud/trunk/test/storage/test_cloudfiles.py (original)
+++ libcloud/trunk/test/storage/test_cloudfiles.py Sun Jun 26 10:24:39 2011
@@ -90,6 +90,18 @@ class CloudFilesTests(unittest.TestCase)
self.assertEqual(obj.size, 1160520)
self.assertEqual(obj.container.name, 'test_container')
+ def test_list_container_objects_iterator(self):
+ CloudFilesMockHttp.type = 'ITERATOR'
+ container = Container(
+ name='test_container', extra={}, driver=self.driver)
+ objects = self.driver.list_container_objects(container=container)
+ self.assertEqual(len(objects), 5)
+
+ obj = [o for o in objects if o.name == 'foo-test-1'][0]
+ self.assertEqual(obj.hash, '16265549b5bda64ecdaa5156de4c97cc')
+ self.assertEqual(obj.size, 1160520)
+ self.assertEqual(obj.container.name, 'test_container')
+
def test_get_container(self):
container = self.driver.get_container(container_name='test_container')
self.assertEqual(container.name, 'test_container')
@@ -485,8 +497,12 @@ class CloudFilesMockHttp(StorageMockHttp
headers = copy.deepcopy(self.base_headers)
if method == 'GET':
# list_container_objects
- body = self.fixtures.load('list_container_objects.json')
- status_code = httplib.OK
+ if url.find('marker') == -1:
+ body = self.fixtures.load('list_container_objects.json')
+ status_code = httplib.OK
+ else:
+ body = ''
+ status_code = httplib.NO_CONTENT
elif method == 'HEAD':
# get_container
body = self.fixtures.load('list_container_objects_empty.json')
@@ -496,6 +512,22 @@ class CloudFilesMockHttp(StorageMockHttp
})
return (status_code, body, headers, httplib.responses[httplib.OK])
+ def _v1_MossoCloudFS_test_container_ITERATOR(self, method, url, body,
headers):
+ headers = copy.deepcopy(self.base_headers)
+ # list_container_objects
+ if url.find('foo-test-3') != -1:
+ body =
self.fixtures.load('list_container_objects_not_exhausted2.json')
+ status_code = httplib.OK
+ elif url.find('foo-test-5') != -1:
+ body = ''
+ status_code = httplib.NO_CONTENT
+ else:
+ # First request
+ body =
self.fixtures.load('list_container_objects_not_exhausted1.json')
+ status_code = httplib.OK
+
+ return (status_code, body, headers, httplib.responses[httplib.OK])
+
def _v1_MossoCloudFS_test_container_not_found(
self, method, url, body, headers):
# test_get_container_not_found
Modified: libcloud/trunk/test/storage/test_s3.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/storage/test_s3.py?rev=1139754&r1=1139753&r2=1139754&view=diff
==============================================================================
--- libcloud/trunk/test/storage/test_s3.py (original)
+++ libcloud/trunk/test/storage/test_s3.py Sun Jun 26 10:24:39 2011
@@ -105,6 +105,20 @@ class S3Tests(unittest.TestCase):
self.assertEqual(obj.container.name, 'test_container')
self.assertTrue('owner' in obj.meta_data)
+ def test_list_container_objects_iterator_has_more(self):
+ S3MockHttp.type = 'ITERATOR'
+ container = Container(name='test_container', extra={},
+ driver=self.driver)
+ objects = self.driver.list_container_objects(container=container)
+
+ obj = [o for o in objects if o.name == '1.zip'][0]
+ self.assertEqual(obj.hash, '4397da7a7649e8085de9916c240e8166')
+ self.assertEqual(obj.size, 1234567)
+ self.assertEqual(obj.container.name, 'test_container')
+
+ self.assertTrue(obj in objects)
+ self.assertEqual(len(objects), 5)
+
def test_get_container_doesnt_exist(self):
S3MockHttp.type = 'list_containers'
try:
@@ -459,6 +473,19 @@ class S3MockHttp(StorageMockHttp):
self.base_headers,
httplib.responses[httplib.OK])
+ def _test_container_ITERATOR(self, method, url, body, headers):
+ if url.find('3.zip') == -1:
+ # First part of the response (first 3 objects)
+ body =
self.fixtures.load('list_container_objects_not_exhausted1.xml')
+ else:
+ body =
self.fixtures.load('list_container_objects_not_exhausted2.xml')
+
+ return (httplib.OK,
+ body,
+ self.base_headers,
+ httplib.responses[httplib.OK])
+
+
def _test2_test_list_containers(self, method, url, body, headers):
# test_get_object
body = self.fixtures.load('list_containers.xml')
Added: libcloud/trunk/test/test_types.py
URL:
http://svn.apache.org/viewvc/libcloud/trunk/test/test_types.py?rev=1139754&view=auto
==============================================================================
--- libcloud/trunk/test/test_types.py (added)
+++ libcloud/trunk/test/test_types.py Sun Jun 26 10:24:39 2011
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+
+from libcloud.common.types import LazyList
+
+
+class TestLazyList(unittest.TestCase):
+ def setUp(self):
+ super(TestLazyList, self).setUp
+ self._get_more_counter = 0
+
+ def tearDown(self):
+ super(TestLazyList, self).tearDown
+
+ def test_init(self):
+ data = [1, 2, 3, 4, 5]
+ ll = LazyList(get_more=self._get_more_exhausted)
+ ll_list = list(ll)
+ self.assertEqual(ll_list, data)
+
+ def test_iterator(self):
+ data = [1, 2, 3, 4, 5]
+ ll = LazyList(get_more=self._get_more_exhausted)
+ for i, d in enumerate(ll):
+ self.assertEqual(d, data[i])
+
+ def test_empty_list(self):
+ ll = LazyList(get_more=self._get_more_empty)
+
+ self.assertEqual(list(ll), [])
+ self.assertEqual(len(ll), 0)
+ self.assertTrue(10 not in ll)
+
+ def test_iterator_not_exhausted(self):
+ data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+ ll = LazyList(get_more=self._get_more_not_exhausted)
+ number_of_iterations = 0
+ for i, d in enumerate(ll):
+ self.assertEqual(d, data[i])
+ number_of_iterations += 1
+ self.assertEqual(number_of_iterations, 10)
+
+ def test_len(self):
+ ll = LazyList(get_more=self._get_more_not_exhausted)
+ ll = LazyList(get_more=self._get_more_not_exhausted)
+
+ self.assertEqual(len(ll), 10)
+
+ def test_contains(self):
+ ll = LazyList(get_more=self._get_more_not_exhausted)
+
+ self.assertTrue(40 not in ll)
+ self.assertTrue(1 in ll)
+ self.assertTrue(5 in ll)
+ self.assertTrue(10 in ll)
+
+ def test_indexing(self):
+ ll = LazyList(get_more=self._get_more_not_exhausted)
+
+ self.assertEqual(ll[0], 1)
+ self.assertEqual(ll[9], 10)
+ self.assertEqual(ll[-1], 10)
+
+ try:
+ ll[11]
+ except IndexError:
+ pass
+ else:
+ self.fail('Exception was not thrown')
+
+ def test_repr(self):
+ ll1 = LazyList(get_more=self._get_more_empty)
+ ll2 = LazyList(get_more=self._get_more_exhausted)
+ ll3 = LazyList(get_more=self._get_more_not_exhausted)
+
+ self.assertEqual(repr(ll1), '[]')
+ self.assertEqual(repr(ll2), '[1, 2, 3, 4, 5]')
+ self.assertEqual(repr(ll3), '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]')
+
+ def _get_more_empty(self, last_key, value_dict):
+ return [], None, True
+
+ def _get_more_exhausted(self, last_key, value_dict):
+ data = [1, 2, 3, 4, 5]
+ return data, 5, True
+
+ def _get_more_not_exhausted(self, last_key, value_dict):
+ self._get_more_counter += 1
+ if not last_key:
+ data, last_key, exhausted = [1, 2, 3, 4, 5], 5, False
+ else:
+ data, last_key, exhausted = [6, 7, 8, 9, 10], 10, True
+
+ return data, last_key, exhausted
+
+if __name__ == '__main__':
+ sys.exit(unittest.main())
