Author: tomaz
Date: Sat Dec 3 18:12:28 2011
New Revision: 1209961
URL: http://svn.apache.org/viewvc?rev=1209961&view=rev
Log:
Storage tests now pass with Python 3.
Modified:
libcloud/branches/py3k/libcloud/common/base.py
libcloud/branches/py3k/libcloud/storage/base.py
libcloud/branches/py3k/libcloud/storage/drivers/atmos.py
libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py
libcloud/branches/py3k/libcloud/storage/drivers/dummy.py
libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py
libcloud/branches/py3k/libcloud/storage/drivers/s3.py
libcloud/branches/py3k/test/__init__.py
libcloud/branches/py3k/test/storage/test_atmos.py
libcloud/branches/py3k/test/storage/test_base.py
libcloud/branches/py3k/test/storage/test_cloudfiles.py
libcloud/branches/py3k/test/storage/test_google_storage.py
libcloud/branches/py3k/test/storage/test_s3.py
Modified: libcloud/branches/py3k/libcloud/common/base.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/common/base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/common/base.py (original)
+++ libcloud/branches/py3k/libcloud/common/base.py Sat Dec 3 18:12:28 2011
@@ -30,6 +30,8 @@ from libcloud.py3 import urllib
from libcloud.py3 import httplib
from libcloud.py3 import urlparse
from libcloud.py3 import StringIO
+from libcloud.py3 import u
+
from libcloud.common.types import LibcloudError, MalformedResponseError
from libcloud.httplib_ssl import LibcloudHTTPSConnection
Modified: libcloud/branches/py3k/libcloud/storage/base.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/base.py (original)
+++ libcloud/branches/py3k/libcloud/storage/base.py Sat Dec 3 18:12:28 2011
@@ -20,11 +20,14 @@ Provides base classes for working with s
# Backward compatibility for Python 2.5
from __future__ import with_statement
-from libcloud.py3 import httplib
import os.path # pylint: disable-msg=W0404
import hashlib
from os.path import join as pjoin
+from libcloud.py3 import httplib
+from libcloud.py3 import next
+from libcloud.py3 import b
+
from libcloud import utils
from libcloud.common.types import LibcloudError
from libcloud.common.base import ConnectionUserAndKey, BaseDriver
@@ -466,7 +469,7 @@ class StorageDriver(BaseDriver):
stream = utils.read_in_chunks(response, chunk_size)
try:
- data_read = stream.next()
+ data_read = next(stream)
except StopIteration:
# Empty response?
return False
@@ -475,11 +478,11 @@ class StorageDriver(BaseDriver):
with open(file_path, 'wb') as file_handle:
while len(data_read) > 0:
- file_handle.write(data_read)
+ file_handle.write(b(data_read))
bytes_transferred += len(data_read)
try:
- data_read = stream.next()
+ data_read = next(stream)
except StopIteration:
data_read = ''
@@ -505,9 +508,10 @@ class StorageDriver(BaseDriver):
headers = headers or {}
if file_path and not os.path.exists(file_path):
- raise OSError('File %s does not exist' % (file_path))
+ raise OSError('File %s does not exist' % (file_path))
- if iterator is not None and not hasattr(iterator, 'next'):
+ if iterator is not None and not hasattr(iterator, 'next') and not \
+ hasattr(iterator, '__next__'):
raise AttributeError('iterator object must implement next() ' +
'method.')
@@ -584,7 +588,7 @@ class StorageDriver(BaseDriver):
if calculate_hash:
data_hash = self._get_hash_function()
- data_hash.update(data)
+ data_hash.update(b(data))
try:
response.connection.connection.send(data)
@@ -639,7 +643,7 @@ class StorageDriver(BaseDriver):
bytes_transferred = 0
try:
- chunk = generator.next()
+ chunk = next(generator)
except StopIteration:
# Special case when StopIteration is thrown on the first iteration
-
# create a 0-byte long object
@@ -670,10 +674,10 @@ class StorageDriver(BaseDriver):
bytes_transferred += len(chunk)
if calculate_hash:
- data_hash.update(chunk)
+ data_hash.update(b(chunk))
try:
- chunk = generator.next()
+ chunk = next(generator)
except StopIteration:
chunk = ''
Modified: libcloud/branches/py3k/libcloud/storage/drivers/atmos.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/atmos.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/atmos.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/atmos.py Sat Dec 3
18:12:28 2011
@@ -13,15 +13,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import sys
import base64
import hashlib
import hmac
-from libcloud.py3 import httplib
import time
+
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
from libcloud.py3 import urllib
-import urlparse
+from libcloud.py3 import urlparse
+from libcloud.py3 import next
+from libcloud.py3 import b
-from xml.etree import ElementTree
+if PY3:
+ from io import FileIO as file
from libcloud import utils
from libcloud.common.base import ConnectionUserAndKey, XmlResponse
@@ -100,8 +106,8 @@ class AtmosConnection(ConnectionUserAndK
signature.extend([k + ':' + collapse(v) for k, v in xhdrs])
signature = '\n'.join(signature)
key = base64.b64decode(self.key)
- signature = hmac.new(key, signature, hashlib.sha1).digest()
- return base64.b64encode(signature)
+ signature = hmac.new(b(key), b(signature), hashlib.sha1).digest()
+ return base64.b64encode(b(signature))
class AtmosDriver(StorageDriver):
connectionCls = AtmosConnection
@@ -132,7 +138,8 @@ class AtmosDriver(StorageDriver):
path = self._namespace_path(container_name + '/?metadata/system')
try:
result = self.connection.request(path)
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code != 1003:
raise
raise ContainerDoesNotExistError(e, self, container_name)
@@ -146,7 +153,8 @@ class AtmosDriver(StorageDriver):
path = self._namespace_path(container_name + '/')
try:
result = self.connection.request(path, method='POST')
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code != 1016:
raise
raise ContainerAlreadyExistsError(e, self, container_name)
@@ -156,7 +164,8 @@ class AtmosDriver(StorageDriver):
try:
self.connection.request(self._namespace_path(container.name + '/'),
method='DELETE')
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code == 1003:
raise ContainerDoesNotExistError(e, self, container.name)
elif e.code == 1023:
@@ -174,7 +183,8 @@ class AtmosDriver(StorageDriver):
result = self.connection.request(path + '?metadata/user')
user_meta = self._emc_meta(result)
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code != 1003:
raise
raise ObjectDoesNotExistError(e, self, object_name)
@@ -204,7 +214,8 @@ class AtmosDriver(StorageDriver):
try:
self.connection.request(request_path + '?metadata/system')
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code != 1003:
raise
method = 'POST'
@@ -248,7 +259,7 @@ class AtmosDriver(StorageDriver):
generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True)
bytes_transferred = 0
try:
- chunk = generator.next()
+ chunk = next(generator)
except StopIteration:
chunk = ''
@@ -256,7 +267,7 @@ class AtmosDriver(StorageDriver):
while True:
end = bytes_transferred + len(chunk) - 1
- data_hash.update(chunk)
+ data_hash.update(b(chunk))
headers = {
'x-emc-meta': 'md5=' + data_hash.hexdigest(),
}
@@ -267,7 +278,7 @@ class AtmosDriver(StorageDriver):
bytes_transferred += len(chunk)
try:
- chunk = generator.next()
+ chunk = next(generator)
except StopIteration:
break
if len(chunk) == 0:
@@ -327,7 +338,8 @@ class AtmosDriver(StorageDriver):
path = self._namespace_path(obj.container.name + '/' + obj.name)
try:
self.connection.request(path, method='DELETE')
- except AtmosError, e:
+ except AtmosError:
+ e = sys.exc_info()[1]
if e.code != 1003:
raise
raise ObjectDoesNotExistError(e, self, obj.name)
Modified: libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/cloudfiles.py Sat Dec 3
18:12:28 2011
@@ -21,6 +21,12 @@ try:
except ImportError:
import json
+from libcloud.py3 import PY3
+
+if PY3:
+ from io import FileIO as file
+
+
from libcloud.utils import read_in_chunks
from libcloud.common.types import MalformedResponseError, LibcloudError
from libcloud.common.base import Response, RawResponse
Modified: libcloud/branches/py3k/libcloud/storage/drivers/dummy.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/dummy.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/dummy.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/dummy.py Sat Dec 3
18:12:28 2011
@@ -17,6 +17,12 @@ import os.path
import random
import hashlib
+from libcloud.py3 import PY3
+from libcloud.py3 import b
+
+if PY3:
+ from io import FileIO as file
+
from libcloud.common.types import LibcloudError
from libcloud.storage.base import Object, Container, StorageDriver
@@ -61,10 +67,14 @@ class DummyIterator(object):
raise StopIteration
value = self._data[self._current_item]
- self.hash.update(value)
+ self.hash.update(b(value))
self._current_item += 1
return value
+ def __next__(self):
+ return self.next()
+
+
class DummyStorageDriver(StorageDriver):
"""
Dummy Storage driver.
@@ -318,7 +328,7 @@ class DummyStorageDriver(StorageDriver):
... iterator=DummyFileObject(5, 10), extra={})
>>> stream = container.download_object_as_stream(obj)
>>> stream #doctest: +ELLIPSIS
- <closed file ..., mode '<uninitialized file>' at 0x...>
+ <...closed...>
"""
return DummyFileObject()
Modified: libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/google_storage.py Sat Dec
3 18:12:28 2011
@@ -14,8 +14,6 @@
# limitations under the License.
import time
-from libcloud.py3 import httplib
-from libcloud.py3 import urllib
import copy
import base64
import hmac
@@ -23,6 +21,8 @@ import hmac
from hashlib import sha1
from email.utils import formatdate
+from libcloud.py3 import b
+
from libcloud.common.base import ConnectionUserAndKey
from libcloud.storage.drivers.s3 import S3StorageDriver, S3Response
@@ -122,7 +122,7 @@ class GoogleStorageConnection(Connection
string_to_sign = '\n'.join(values_to_sign)
b64_hmac = base64.b64encode(
- hmac.new(secret_key, string_to_sign, digestmod=sha1).digest()
+ hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
)
return b64_hmac
Modified: libcloud/branches/py3k/libcloud/storage/drivers/s3.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/libcloud/storage/drivers/s3.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/libcloud/storage/drivers/s3.py (original)
+++ libcloud/branches/py3k/libcloud/storage/drivers/s3.py Sat Dec 3 18:12:28
2011
@@ -14,8 +14,6 @@
# limitations under the License.
import time
-from libcloud.py3 import httplib
-from libcloud.py3 import urllib
import copy
import base64
import hmac
@@ -23,6 +21,11 @@ import hmac
from hashlib import sha1
from xml.etree.ElementTree import Element, SubElement, tostring
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
+from libcloud.py3 import urllib
+from libcloud.py3 import b
+
from libcloud.utils import fixxpath, findtext, in_development_warning
from libcloud.utils import read_in_chunks
from libcloud.common.types import InvalidCredsError, LibcloudError
@@ -154,7 +157,7 @@ class S3Connection(ConnectionUserAndKey)
string_to_sign = '\n'.join(values_to_sign)
b64_hmac = base64.b64encode(
- hmac.new(secret_key, string_to_sign, digestmod=sha1).digest()
+ hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
)
return b64_hmac
@@ -214,7 +217,12 @@ class S3StorageDriver(StorageDriver):
root = Element('CreateBucketConfiguration')
child = SubElement(root, 'LocationConstraint')
child.text = self.ex_location_name
- data = tostring(root)
+
+ if PY3:
+ encoding = 'unicode'
+ else:
+ encoding = None
+ data = tostring(root, encoding=encoding)
else:
data = ''
Modified: libcloud/branches/py3k/test/__init__.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/__init__.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/__init__.py (original)
+++ libcloud/branches/py3k/test/__init__.py Sat Dec 3 18:12:28 2011
@@ -21,6 +21,8 @@ from cgi import parse_qs
from libcloud.py3 import httplib
from libcloud.py3 import StringIO
from libcloud.py3 import urlparse
+from libcloud.py3 import u
+
XML_HEADERS = {'content-type': 'application/xml'}
@@ -76,7 +78,7 @@ class MockResponse(object):
def __init__(self, status, body, headers=None, reason=None):
self.status = status
- self.body = StringIO(body)
+ self.body = StringIO(u(body))
self.headers = headers or self.headers
self.reason = reason or self.reason
@@ -241,6 +243,9 @@ class MockRawResponse(BaseMockHttpObject
self._current_item += 1
return value
+ def __next__(self):
+ return self.next()
+
def _generate_random_data(self, size):
data = []
current_size = 0
Modified: libcloud/branches/py3k/test/storage/test_atmos.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_atmos.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_atmos.py (original)
+++ libcloud/branches/py3k/test/storage/test_atmos.py Sat Dec 3 18:12:28 2011
@@ -14,13 +14,13 @@
# limitations under the License.
import base64
-import httplib
import os.path
import sys
import unittest
-import urlparse
-from xml.etree import ElementTree
+from libcloud.py3 import httplib
+from libcloud.py3 import urlparse
+from libcloud.py3 import b
import libcloud.utils
@@ -44,7 +44,7 @@ class AtmosTests(unittest.TestCase):
AtmosMockHttp.type = None
AtmosMockHttp.upload_created = False
AtmosMockRawResponse.type = None
- self.driver = AtmosDriver('dummy', base64.b64encode('dummy'))
+ self.driver = AtmosDriver('dummy', base64.b64encode(b('dummy')))
self._remove_test_file()
def tearDown(self):
@@ -342,7 +342,7 @@ class AtmosTests(unittest.TestCase):
def test_signature_algorithm(self):
test_uid = 'fredsmagicuid'
- test_key = base64.b64encode('ssssshhhhhmysecretkey')
+ test_key = base64.b64encode(b('ssssshhhhhmysecretkey'))
test_date = 'Mon, 04 Jul 2011 07:39:19 GMT'
test_values = [
('GET', '/rest/namespace/foo', '', {},
@@ -372,7 +372,7 @@ class AtmosTests(unittest.TestCase):
c.driver = d
headers = c.add_default_headers(headers)
headers['Date'] = headers['x-emc-date'] = test_date
- self.assertEqual(c._calculate_signature({}, headers), expected)
+ self.assertEqual(c._calculate_signature({}, headers), b(expected))
class AtmosMockHttp(StorageMockHttp, unittest.TestCase):
fixtures = StorageFileFixtures('atmos')
Modified: libcloud/branches/py3k/test/storage/test_base.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_base.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_base.py (original)
+++ libcloud/branches/py3k/test/storage/test_base.py Sat Dec 3 18:12:28 2011
@@ -17,9 +17,15 @@ import sys
import unittest
import hashlib
-from StringIO import StringIO
from mock import Mock
+from libcloud.py3 import StringIO
+from libcloud.py3 import PY3
+from libcloud.py3 import b
+
+if PY3:
+ from io import FileIO as file
+
from libcloud.storage.base import StorageDriver
from test import StorageMockHttp # pylint: disable-msg=E0611
@@ -73,7 +79,12 @@ class BaseStorageTests(unittest.TestCase
def test_upload_zero_bytes_long_object_via_stream(self):
iterator = Mock()
- iterator.next.side_effect = StopIteration()
+
+ if PY3:
+ iterator.__next__ = Mock()
+ iterator.__next__.side_effect = StopIteration()
+ else:
+ iterator.next.side_effect = StopIteration()
def mock_send(data):
self.send_called += 1
@@ -88,7 +99,7 @@ class BaseStorageTests(unittest.TestCase
chunked=False, calculate_hash=True)
self.assertTrue(success)
- self.assertEqual(data_hash, hashlib.md5('').hexdigest())
+ self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
self.assertEqual(bytes_transferred, 0)
self.assertEqual(self.send_called, 1)
@@ -99,7 +110,7 @@ class BaseStorageTests(unittest.TestCase
chunked=True, calculate_hash=True)
self.assertTrue(success)
- self.assertEqual(data_hash, hashlib.md5('').hexdigest())
+ self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
self.assertEqual(bytes_transferred, 0)
self.assertEqual(self.send_called, 5)
@@ -116,7 +127,7 @@ class BaseStorageTests(unittest.TestCase
calculate_hash=True)
self.assertTrue(success)
- self.assertEqual(data_hash, hashlib.md5(data).hexdigest())
+ self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
self.assertEqual(bytes_transferred, (len(data)))
self.assertEqual(self.send_called, 1)
Modified: libcloud/branches/py3k/test/storage/test_cloudfiles.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_cloudfiles.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_cloudfiles.py (original)
+++ libcloud/branches/py3k/test/storage/test_cloudfiles.py Sat Dec 3 18:12:28
2011
@@ -17,10 +17,15 @@ import os.path
import sys
import copy
import unittest
-import httplib
import libcloud.utils
+from libcloud.py3 import PY3
+from libcloud.py3 import httplib
+
+if PY3:
+ from io import FileIO as file
+
from libcloud.common.types import LibcloudError, MalformedResponseError
from libcloud.storage.base import Container, Object
from libcloud.storage.types import ContainerAlreadyExistsError
Modified: libcloud/branches/py3k/test/storage/test_google_storage.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_google_storage.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_google_storage.py (original)
+++ libcloud/branches/py3k/test/storage/test_google_storage.py Sat Dec 3
18:12:28 2011
@@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
import sys
-import httplib
import unittest
from libcloud.storage.drivers.google_storage import GoogleStorageDriver
Modified: libcloud/branches/py3k/test/storage/test_s3.py
URL:
http://svn.apache.org/viewvc/libcloud/branches/py3k/test/storage/test_s3.py?rev=1209961&r1=1209960&r2=1209961&view=diff
==============================================================================
--- libcloud/branches/py3k/test/storage/test_s3.py (original)
+++ libcloud/branches/py3k/test/storage/test_s3.py Sat Dec 3 18:12:28 2011
@@ -15,9 +15,10 @@
import os
import sys
-import httplib
import unittest
+from libcloud.py3 import httplib
+
from libcloud.common.types import InvalidCredsError
from libcloud.common.types import LibcloudError
from libcloud.storage.base import Container, Object
@@ -276,7 +277,8 @@ class S3Tests(unittest.TestCase):
self.mock_response_klass.type = 'UNAUTHORIZED'
try:
self.driver.list_containers()
- except InvalidCredsError, e:
+ except InvalidCredsError:
+ e = sys.exc_info()[1]
self.assertEqual(True, isinstance(e, InvalidCredsError))
else:
self.fail('Exception was not thrown')
@@ -514,7 +516,8 @@ class S3Tests(unittest.TestCase):
object_name=object_name,
verify_hash=True,
ex_storage_class='invalid-class')
- except ValueError, e:
+ except ValueError:
+ e = sys.exc_info()[1]
self.assertTrue(str(e).lower().find('invalid storage class') != -1)
else:
self.fail('Exception was not thrown')
