Repository: libcloud Updated Branches: refs/heads/trunk 6f3279c74 -> 3da15748f
http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/loadbalancer/test_slb.py ---------------------------------------------------------------------- diff --git a/libcloud/test/loadbalancer/test_slb.py b/libcloud/test/loadbalancer/test_slb.py new file mode 100644 index 0000000..926aed0 --- /dev/null +++ b/libcloud/test/loadbalancer/test_slb.py @@ -0,0 +1,566 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import unittest + +from libcloud.compute.base import Node +from libcloud.compute.types import NodeState +from libcloud.loadbalancer.base import Member, Algorithm +from libcloud.loadbalancer.drivers.slb import SLBDriver, \ + SLBLoadBalancerHttpListener, SLBLoadBalancerHttpsListener, \ + SLBLoadBalancerTcpListener, SLBLoadBalancerUdpListener +from libcloud.loadbalancer.types import State +from libcloud.test.file_fixtures import LoadBalancerFileFixtures +from libcloud.test import MockHttpTestCase +from libcloud.test.secrets import LB_SLB_PARAMS +from libcloud.utils.py3 import httplib + + +class SLBDriverTestCases(unittest.TestCase): + region = LB_SLB_PARAMS[2] + + def setUp(self): + SLBMockHttp.test = self + SLBDriver.connectionCls.conn_classes = (None, + SLBMockHttp) + SLBMockHttp.type = None + SLBMockHttp.use_param = 'Action' + + self.driver = SLBDriver(*LB_SLB_PARAMS) + + def test_list_protocols(self): + protocols = self.driver.list_protocols() + self.assertEqual(4, len(protocols)) + expected = ['tcp', 'udp', 'http', 'https'] + diff = set(expected) - set(protocols) + self.assertEqual(0, len(diff)) + + def test_list_balancers(self): + balancers = self.driver.list_balancers() + + self.assertEqual(len(balancers), 1) + balancer = balancers[0] + self.assertEqual('15229f88562-cn-hangzhou-dg-a01', balancer.id) + self.assertEqual('abc', balancer.name) + self.assertEqual(State.RUNNING, balancer.state) + self.assertEqual('120.27.186.149', balancer.ip) + self.assertTrue(balancer.port is None) + self.assertEqual(self.driver, balancer.driver) + expected_extra = { + 'create_timestamp': 1452403099000, + 'address_type': 'internet', + 'region_id': 'cn-hangzhou-dg-a01', + 'region_id_alias': 'cn-hangzhou', + 'create_time': '2016-01-10T13:18Z', + 'master_zone_id': 'cn-hangzhou-d', + 'slave_zone_id': 'cn-hangzhou-b', + 'network_type': 'classic' + } + self._validate_extras(expected_extra, balancer.extra) + + def _validate_extras(self, expected, actual): + self.assertTrue(actual is not None) + for key, value in iter(expected.items()): + self.assertTrue(key in actual) + self.assertEqual(value, actual[key], ('extra %(key)s not equal, ' + 'expected: "%(expected)s", ' + 'actual: "%(actual)s"' % + {'key': key, + 'expected': value, + 'actual': actual[key]})) + + def test_list_balancers_with_ids(self): + SLBMockHttp.type = 'list_balancers_ids' + self.balancer_ids = ['id1', 'id2'] + balancers = self.driver.list_balancers( + ex_balancer_ids=self.balancer_ids) + self.assertTrue(balancers is not None) + + def test_list_balancers_with_ex_filters(self): + SLBMockHttp.type = 'list_balancers_filters' + self.ex_filters = {'AddressType': 'internet'} + balancers = self.driver.list_balancers(ex_filters=self.ex_filters) + self.assertTrue(balancers is not None) + + def test_get_balancer(self): + SLBMockHttp.type = 'get_balancer' + balancer = self.driver.get_balancer(balancer_id='tests') + + self.assertEqual(balancer.id, '15229f88562-cn-hangzhou-dg-a01') + self.assertEqual(balancer.name, 'abc') + self.assertEqual(balancer.state, State.RUNNING) + + def test_destroy_balancer(self): + balancer = self.driver.get_balancer(balancer_id='tests') + + self.assertTrue(self.driver.destroy_balancer(balancer)) + + def test_create_balancer(self): + self.name = 'balancer1' + self.port = 80 + self.protocol = 'http' + self.algorithm = Algorithm.WEIGHTED_ROUND_ROBIN + self.extra = { + 'AddressType': 'internet', + 'InternetChargeType': 'paybytraffic', + 'Bandwidth': 1, + 'MasterZoneId': 'cn-hangzhou-d', + 'SlaveZoneId': 'cn-hangzhou-b', + 'StickySession': 'on', + 'HealthCheck': 'on'} + self.members = [Member('node1', None, None)] + + balancer = self.driver.create_balancer(name=self.name, port=self.port, + protocol=self.protocol, + algorithm=self.algorithm, + members=self.members, + **self.extra) + + self.assertEqual(balancer.name, self.name) + self.assertEqual(balancer.port, self.port) + self.assertEqual(balancer.state, State.UNKNOWN) + + def test_create_balancer_no_port_exception(self): + self.assertRaises(AttributeError, self.driver.create_balancer, + None, None, 'http', Algorithm.WEIGHTED_ROUND_ROBIN, + None) + + def test_create_balancer_unsupport_protocol_exception(self): + self.assertRaises(AttributeError, self.driver.create_balancer, + None, 443, 'ssl', Algorithm.WEIGHTED_ROUND_ROBIN, + None) + + def test_create_balancer_multiple_member_ports_exception(self): + members = [Member('m1', '1.2.3.4', 80), + Member('m2', '1.2.3.5', 81)] + self.assertRaises(AttributeError, self.driver.create_balancer, + None, 80, 'http', Algorithm.WEIGHTED_ROUND_ROBIN, + members) + + def test_create_balancer_bandwidth_value_error(self): + self.assertRaises(AttributeError, self.driver.create_balancer, + None, 80, 'http', Algorithm.WEIGHTED_ROUND_ROBIN, + None, Bandwidth='NAN') + + def test_create_balancer_paybybandwidth_without_bandwidth_exception(self): + self.assertRaises(AttributeError, self.driver.create_balancer, + None, 80, 'http', Algorithm.WEIGHTED_ROUND_ROBIN, + None, InternetChargeType='paybybandwidth') + + def test_balancer_list_members(self): + balancer = self.driver.get_balancer(balancer_id='tests') + members = balancer.list_members() + + self.assertEqual(len(members), 1) + self.assertEqual(members[0].balancer, balancer) + self.assertEqual('i-23tshnsdq', members[0].id) + + def test_balancer_list_listeners(self): + balancer = self.driver.get_balancer(balancer_id='tests') + listeners = self.driver.ex_list_listeners(balancer) + + self.assertEqual(1, len(listeners)) + listener = listeners[0] + self.assertEqual('80', listener.port) + + def test_balancer_detach_member(self): + self.balancer = self.driver.get_balancer(balancer_id='tests') + self.member = Member('i-23tshnsdq', None, None) + + self.assertTrue(self.balancer.detach_member(self.member)) + + def test_balancer_attach_compute_node(self): + SLBMockHttp.type = 'attach_compute_node' + self.balancer = self.driver.get_balancer(balancer_id='tests') + self.node = Node(id='node1', name='node-name', + state=NodeState.RUNNING, + public_ips=['1.2.3.4'], private_ips=['4.3.2.1'], + driver=self.driver) + member = self.driver.balancer_attach_compute_node( + self.balancer, self.node) + self.assertEqual(self.node.id, member.id) + self.assertEqual(self.node.public_ips[0], member.ip) + self.assertEqual(self.balancer.port, member.port) + + def test_ex_create_listener(self): + SLBMockHttp.type = 'create_listener' + self.balancer = self.driver.get_balancer(balancer_id='tests') + self.backend_port = 80 + self.protocol = 'http' + self.algorithm = Algorithm.WEIGHTED_ROUND_ROBIN + self.bandwidth = 1 + self.extra = {'StickySession': 'off', 'HealthCheck': 'off'} + self.assertTrue(self.driver.ex_create_listener(self.balancer, + self.backend_port, + self.protocol, + self.algorithm, + self.bandwidth, + **self.extra)) + + def test_ex_create_listener_override_port(self): + SLBMockHttp.type = 'create_listener_override_port' + self.balancer = self.driver.get_balancer(balancer_id='tests') + self.backend_port = 80 + self.protocol = 'http' + self.algorithm = Algorithm.WEIGHTED_ROUND_ROBIN + self.bandwidth = 1 + self.extra = {'StickySession': 'off', + 'HealthCheck': 'off', + 'ListenerPort': 8080} + self.assertTrue(self.driver.ex_create_listener(self.balancer, + self.backend_port, + self.protocol, + self.algorithm, + self.bandwidth, + **self.extra)) + + def test_ex_start_listener(self): + SLBMockHttp.type = 'start_listener' + balancer = self.driver.get_balancer(balancer_id='tests') + self.port = 80 + self.assertTrue(self.driver.ex_start_listener(balancer, self.port)) + + def test_ex_stop_listener(self): + SLBMockHttp.type = 'stop_listener' + balancer = self.driver.get_balancer(balancer_id='tests') + self.port = 80 + self.assertTrue(self.driver.ex_stop_listener(balancer, self.port)) + + def test_ex_upload_certificate(self): + self.name = 'cert1' + self.cert = 'cert-data' + self.key = 'key-data' + certificate = self.driver.ex_upload_certificate(self.name, self.cert, + self.key) + self.assertEqual(self.name, certificate.name) + self.assertEqual('01:DF:AB:CD', certificate.fingerprint) + + def test_ex_list_certificates(self): + certs = self.driver.ex_list_certificates() + self.assertEqual(2, len(certs)) + cert = certs[0] + self.assertEqual('139a00604ad-cn-east-hangzhou-01', cert.id) + self.assertEqual('abe', cert.name) + self.assertEqual('A:B:E', cert.fingerprint) + + def test_ex_list_certificates_ids(self): + SLBMockHttp.type = 'list_certificates_ids' + self.cert_ids = ['cert1', 'cert2'] + certs = self.driver.ex_list_certificates(certificate_ids=self.cert_ids) + self.assertTrue(certs is not None) + + def test_ex_delete_certificate(self): + self.cert_id = 'cert1' + self.assertTrue(self.driver.ex_delete_certificate(self.cert_id)) + + def test_ex_set_certificate_name(self): + self.cert_id = 'cert1' + self.cert_name = 'cert-name' + self.assertTrue(self.driver.ex_set_certificate_name(self.cert_id, + self.cert_name)) + + +class SLBMockHttp(MockHttpTestCase): + fixtures = LoadBalancerFileFixtures('slb') + + def _DescribeLoadBalancers(self, method, url, body, headers): + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _list_balancers_ids_DescribeLoadBalancers(self, method, url, body, + headers): + params = {'LoadBalancerId': ','.join(self.test.balancer_ids)} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _list_balancers_filters_DescribeLoadBalancers(self, method, url, body, + headers): + params = {'AddressType': 'internet'} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _get_balancer_DescribeLoadBalancers(self, method, url, body, headers): + params = {'LoadBalancerId': 'tests'} + self.assertUrlContainsQueryParams(url, params) + return self._DescribeLoadBalancers(method, url, body, headers) + + def _DeleteLoadBalancer(self, method, url, body, headers): + params = {'LoadBalancerId': '15229f88562-cn-hangzhou-dg-a01'} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('delete_load_balancer.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _DescribeLoadBalancerAttribute(self, method, url, body, headers): + body = self.fixtures.load('describe_load_balancer_attribute.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _CreateLoadBalancer(self, method, url, body, headers): + params = {'RegionId': self.test.region, + 'LoadBalancerName': self.test.name} + balancer_keys = [ + 'AddressType', + 'InternetChargeType', + 'Bandwidth', + 'MasterZoneId', + 'SlaveZoneId' + ] + for key in balancer_keys: + params[key] = str(self.test.extra[key]) + + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('create_load_balancer.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _AddBackendServers(self, method, url, body, headers): + _id = self.test.members[0].id + self.assertTrue("ServerId" in url and _id in url) + self.assertTrue("Weight" in url and "100" in url) + body = self.fixtures.load('add_backend_servers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _CreateLoadBalancerHTTPListener(self, method, url, body, headers): + body = self.fixtures.load('create_load_balancer_http_listener.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _StartLoadBalancerListener(self, method, url, body, headers): + body = self.fixtures.load('start_load_balancer_listener.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _RemoveBackendServers(self, method, url, body, headers): + _id = self.test.member.id + servers_json = '["%s"]' % _id + params = {'LoadBalancerId': self.test.balancer.id, + 'BackendServers': servers_json} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('add_backend_servers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _attach_compute_node_DescribeLoadBalancers(self, method, url, body, + headers): + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _attach_compute_node_AddBackendServers(self, method, url, body, + headers): + _id = self.test.node.id + self.assertTrue("ServerId" in url and _id in url) + self.assertTrue("Weight" in url and "100" in url) + body = self.fixtures.load('add_backend_servers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _create_listener_CreateLoadBalancerHTTPListener(self, method, url, + body, headers): + params = {'LoadBalancerId': self.test.balancer.id, + 'RegionId': self.test.region, + 'ListenerPort': str(self.test.balancer.port), + 'BackendServerPort': str(self.test.backend_port), + 'Scheduler': 'wrr', + 'Bandwidth': '1', + 'StickySession': 'off', + 'HealthCheck': 'off'} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('create_load_balancer_http_listener.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _create_listener_DescribeLoadBalancers(self, method, url, body, + headers): + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _create_listener_override_port_CreateLoadBalancerHTTPListener( + self, method, url, body, headers): + params = {'LoadBalancerId': self.test.balancer.id, + 'RegionId': self.test.region, + 'ListenerPort': str(self.test.extra['ListenerPort']), + 'BackendServerPort': str(self.test.backend_port), + 'Scheduler': 'wrr', + 'Bandwidth': '1', + 'StickySession': 'off', + 'HealthCheck': 'off'} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('create_load_balancer_http_listener.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _create_listener_override_port_DescribeLoadBalancers( + self, method, url, body, headers): + body = self.fixtures.load('describe_load_balancers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _start_listener_DescribeLoadBalancers(self, method, url, body, + headers): + return self._DescribeLoadBalancers(method, url, body, headers) + + def _start_listener_StartLoadBalancerListener(self, method, url, body, + headers): + params = {'ListenerPort': str(self.test.port)} + self.assertUrlContainsQueryParams(url, params) + return self._StartLoadBalancerListener(method, url, body, headers) + + def _stop_listener_DescribeLoadBalancers(self, method, url, body, + headers): + return self._DescribeLoadBalancers(method, url, body, headers) + + def _stop_listener_StopLoadBalancerListener(self, method, url, body, + headers): + params = {'ListenerPort': str(self.test.port)} + self.assertUrlContainsQueryParams(url, params) + return self._StartLoadBalancerListener(method, url, body, headers) + + def _UploadServerCertificate(self, method, url, body, headers): + body = self.fixtures.load('upload_server_certificate.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _DescribeServerCertificates(self, method, url, body, headers): + body = self.fixtures.load('describe_server_certificates.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _list_certificates_ids_DescribeServerCertificates(self, method, url, + body, headers): + params = {'RegionId': self.test.region, + 'ServerCertificateId': ','.join(self.test.cert_ids)} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('describe_server_certificates.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _DeleteServerCertificate(self, method, url, body, headers): + params = {'RegionId': self.test.region, + 'ServerCertificateId': self.test.cert_id} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('delete_server_certificate.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _SetServerCertificateName(self, method, url, body, headers): + params = {'RegionId': self.test.region, + 'ServerCertificateId': self.test.cert_id, + 'ServerCertificateName': self.test.cert_name} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('set_server_certificate_name.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + +class AssertDictMixin(object): + def assert_dict_equals(self, expected, actual): + expected_keys = set(expected.keys()) + actual_keys = set(actual.keys()) + self.assertEqual(len(expected_keys), len(actual_keys)) + self.assertEqual(0, len(expected_keys - actual_keys)) + for key in expected: + self.assertEqual(expected[key], actual[key]) + + +class SLBLoadBalancerHttpListenerTestCase(unittest.TestCase, AssertDictMixin): + def setUp(self): + self.listener = SLBLoadBalancerHttpListener.create( + 80, 8080, Algorithm.WEIGHTED_ROUND_ROBIN, 1, + extra={'StickySession': 'on', + 'StickySessionType': 'insert', + 'HealthCheck': 'on'} + ) + + def test_get_required_params(self): + expected = {'Action': 'CreateLoadBalancerHTTPListener', + 'ListenerPort': 80, + 'BackendServerPort': 8080, + 'Scheduler': 'wrr', + 'Bandwidth': 1, + 'StickySession': 'on', + 'HealthCheck': 'on'} + self.assert_dict_equals(expected, + self.listener.get_required_params()) + + def test_get_optional_params(self): + expected = {'StickySessionType': 'insert'} + self.assert_dict_equals(expected, self.listener.get_optional_params()) + + def test_repr(self): + self.assertTrue('SLBLoadBalancerHttpListener' in str(self.listener)) + + +class SLBLoadBalancerHttpsListenerTestCase(unittest.TestCase, AssertDictMixin): + def setUp(self): + self.listener = SLBLoadBalancerHttpsListener.create( + 80, 8080, Algorithm.WEIGHTED_ROUND_ROBIN, 1, + extra={'StickySession': 'on', + 'StickySessionType': 'insert', + 'HealthCheck': 'on', + 'ServerCertificateId': 'fake-cert1'} + ) + + def test_get_required_params(self): + expected = {'Action': 'CreateLoadBalancerHTTPSListener', + 'ListenerPort': 80, + 'BackendServerPort': 8080, + 'Scheduler': 'wrr', + 'Bandwidth': 1, + 'StickySession': 'on', + 'HealthCheck': 'on', + 'ServerCertificateId': 'fake-cert1'} + self.assert_dict_equals(expected, + self.listener.get_required_params()) + + def test_get_optional_params(self): + expected = {'StickySessionType': 'insert'} + self.assert_dict_equals(expected, self.listener.get_optional_params()) + + +class SLBLoadBalancerTcpListenerTestCase(unittest.TestCase, AssertDictMixin): + def setUp(self): + self.listener = SLBLoadBalancerTcpListener.create( + 80, 8080, Algorithm.WEIGHTED_ROUND_ROBIN, 1, + extra={'PersistenceTimeout': 0, + 'HealthCheckDomain': ''} + ) + + def test_get_required_params(self): + expected = {'Action': 'CreateLoadBalancerTCPListener', + 'ListenerPort': 80, + 'BackendServerPort': 8080, + 'Scheduler': 'wrr', + 'Bandwidth': 1} + self.assert_dict_equals(expected, + self.listener.get_required_params()) + + def test_get_optional_params(self): + expected = {'PersistenceTimeout': 0, + 'HealthCheckDomain': ''} + self.assert_dict_equals(expected, self.listener.get_optional_params()) + + +class SLBLoadBalancerUdpListenerTestCase(unittest.TestCase, AssertDictMixin): + def setUp(self): + self.listener = SLBLoadBalancerUdpListener.create( + 80, 8080, Algorithm.WEIGHTED_ROUND_ROBIN, 1, + extra={'PersistenceTimeout': 0} + ) + + def test_get_required_params(self): + expected = {'Action': 'CreateLoadBalancerUDPListener', + 'ListenerPort': 80, + 'BackendServerPort': 8080, + 'Scheduler': 'wrr', + 'Bandwidth': 1} + self.assert_dict_equals(expected, + self.listener.get_required_params()) + + def test_get_optional_params(self): + expected = {'PersistenceTimeout': 0} + self.assert_dict_equals(expected, self.listener.get_optional_params()) + + +if __name__ == "__main__": + sys.exit(unittest.main()) http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/secrets.py-dist ---------------------------------------------------------------------- diff --git a/libcloud/test/secrets.py-dist b/libcloud/test/secrets.py-dist index 216a7d5..879e8d5 100644 --- a/libcloud/test/secrets.py-dist +++ b/libcloud/test/secrets.py-dist @@ -50,9 +50,11 @@ CLOUDFRAMES_PARAMS = ('key', 'secret', False, 'host', 8888) PROFIT_BRICKS_PARAMS = ('user', 'key') VULTR_PARAMS = ('key') PACKET_PARAMS = ('api_key') +ECS_PARAMS = ('access_key', 'access_secret') # Storage STORAGE_S3_PARAMS = ('key', 'secret') +STORAGE_OSS_PARAMS = ('key', 'secret') # Google key = 20 char alphanumeric string starting with GOOG STORAGE_GOOGLE_STORAGE_PARAMS = ('GOOG0123456789ABCXYZ', 'secret') @@ -62,6 +64,7 @@ STORAGE_AZURE_BLOBS_PARAMS = ('account', 'cGFzc3dvcmQ=') # Loadbalancer LB_BRIGHTBOX_PARAMS = ('user', 'key') LB_ELB_PARAMS = ('access_id', 'secret', 'region') +LB_SLB_PARAMS = ('access_id', 'secret', 'region') # DNS DNS_PARAMS_LINODE = ('user', 'key') http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/complete_multipart_upload.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/complete_multipart_upload.xml b/libcloud/test/storage/fixtures/oss/complete_multipart_upload.xml new file mode 100644 index 0000000..ff8da76 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/complete_multipart_upload.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<CompleteMultipartUploadResult> + <Location>http://oss-example.oss-cn-hangzhou.aliyuncs.com/multipart.data</Location> + <Bucket>oss-example</Bucket> + <Key>multipart.data</Key> + <ETag>B864DB6A936D376F9F8D3ED3BBE540DD-3</ETag> +</CompleteMultipartUploadResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p1.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p1.xml b/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p1.xml new file mode 100644 index 0000000..912d947 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p1.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListMultipartUploadsResult> + <Bucket>oss-example</Bucket> + <KeyMarker></KeyMarker> + <UploadIdMarker></UploadIdMarker> + <NextKeyMarker>oss.avi</NextKeyMarker> + <NextUploadIdMarker>0004B99B8E707874FC2D692FA5D77D3F</NextUploadIdMarker> + <Delimiter></Delimiter> + <Prefix></Prefix> + <MaxUploads>2</MaxUploads> + <IsTruncated>true</IsTruncated> + <Upload> + <Key>multipart.data</Key> + <UploadId>0004B999EF518A1FE585B0C9360DC4C8</UploadId> + <Initiated>2012-02-23T04:18:23.000Z</Initiated> + </Upload> + <Upload> + <Key>oss.avi</Key> + <UploadId>0004B99B8E707874FC2D692FA5D77D3F</UploadId> + <Initiated>2012-02-23T06:14:27.000Z</Initiated> + </Upload> +</ListMultipartUploadsResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p2.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p2.xml b/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p2.xml new file mode 100644 index 0000000..9e256ef --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/ex_iterate_multipart_uploads_p2.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListMultipartUploadsResult> + <Bucket>oss-example</Bucket> + <KeyMarker></KeyMarker> + <UploadIdMarker></UploadIdMarker> + <NextKeyMarker></NextKeyMarker> + <NextUploadIdMarker></NextUploadIdMarker> + <Delimiter></Delimiter> + <Prefix></Prefix> + <MaxUploads>2</MaxUploads> + <IsTruncated>false</IsTruncated> + <Upload> + <Key>multipart.data</Key> + <UploadId>0004B999EF5A239BB9138C6227D69F95</UploadId> + <Initiated>2012-02-23T04:18:23.000Z</Initiated> + </Upload> +</ListMultipartUploadsResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/initiate_multipart_upload.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/initiate_multipart_upload.xml b/libcloud/test/storage/fixtures/oss/initiate_multipart_upload.xml new file mode 100644 index 0000000..182c248 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/initiate_multipart_upload.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<InitiateMultipartUploadResult> + <Bucket>multipart_upload</Bucket> + <Key>multipart.data</Key> + <UploadId>0004B9894A22E5B1888A1E29F8236E2D</UploadId> +</InitiateMultipartUploadResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_container_objects.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_container_objects.xml b/libcloud/test/storage/fixtures/oss/list_container_objects.xml new file mode 100644 index 0000000..c301627 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_container_objects.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListBucketResult> + <Name>20150624</Name> + <Prefix></Prefix> + <Marker></Marker> + <MaxKeys>100</MaxKeys> + <Delimiter></Delimiter> + <IsTruncated>false</IsTruncated> + <Contents> + <Key>en/</Key> + <LastModified>2016-01-15T14:43:15.000Z</LastModified> + <ETag>"D41D8CD98F00B204E9800998ECF8427E"</ETag> + <Type>Normal</Type> + <Size>0</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + <DisplayName>1751306716098727</DisplayName> + </Owner> + </Contents> + <Contents> + <Key>test.txt</Key> + <LastModified>2016-01-15T14:42:46.000Z</LastModified> + <ETag>"26AB0DB90D72E28AD0BA1E22EE510510"</ETag> + <Type>Normal</Type> + <Size>2</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + <DisplayName>1751306716098727</DisplayName> + </Owner> + </Contents> +</ListBucketResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_container_objects_chinese.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_container_objects_chinese.xml b/libcloud/test/storage/fixtures/oss/list_container_objects_chinese.xml new file mode 100644 index 0000000..2c4f9fc --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_container_objects_chinese.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListBucketResult> + <Name>test_container</Name> + <Prefix></Prefix> + <Marker></Marker> + <MaxKeys>100</MaxKeys> + <Delimiter></Delimiter> + <IsTruncated>false</IsTruncated> + <Contents> + <Key>WEB\xe6\x8e\xa7\xe5\x88\xb6\xe5\x8f\xb0.odp</Key> + <LastModified>2016-01-15T14:43:06.000Z</LastModified> + <ETag>"281371EA1618CF0E645D6BB90A158276"</ETag> + <Type>Normal</Type> + <Size>1234567</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + </Owner> + </Contents> + <Contents> + <Key>\xe4\xb8\xad\xe6\x96\x87/</Key> + <LastModified>2016-01-15T14:43:34.000Z</LastModified> + <ETag>"D41D8CD98F00B204E9800998ECF8427E"</ETag> + <Type>Normal</Type> + <Size>0</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + <DisplayName>1751306716098727</DisplayName> + </Owner> + </Contents> +</ListBucketResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_container_objects_empty.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_container_objects_empty.xml b/libcloud/test/storage/fixtures/oss/list_container_objects_empty.xml new file mode 100644 index 0000000..463bb71 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_container_objects_empty.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListBucketResult> + <Name>20160116</Name> + <Prefix></Prefix> + <Marker></Marker> + <MaxKeys>100</MaxKeys> + <Delimiter></Delimiter> + <IsTruncated>false</IsTruncated> +</ListBucketResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_container_objects_prefix.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_container_objects_prefix.xml b/libcloud/test/storage/fixtures/oss/list_container_objects_prefix.xml new file mode 100644 index 0000000..38d850a --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_container_objects_prefix.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListBucketResult> + <Name>20150624</Name> + <Prefix>en</Prefix> + <Marker></Marker> + <MaxKeys>100</MaxKeys> + <Delimiter></Delimiter> + <IsTruncated>false</IsTruncated> + <Contents> + <Key>en/</Key> + <LastModified>2016-01-15T14:43:15.000Z</LastModified> + <ETag>"D41D8CD98F00B204E9800998ECF8427E"</ETag> + <Type>Normal</Type> + <Size>0</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + <DisplayName>1751306716098727</DisplayName> + </Owner> + </Contents> + <Contents> + <Key>en/test.txt</Key> + <LastModified>2016-01-15T14:42:46.000Z</LastModified> + <ETag>"26AB0DB90D72E28AD0BA1E22EE510510"</ETag> + <Type>Normal</Type> + <Size>2</Size> + <StorageClass>Standard</StorageClass> + <Owner> + <ID>1751306716098727</ID> + <DisplayName>1751306716098727</DisplayName> + </Owner> + </Contents> +</ListBucketResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_containers.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_containers.xml b/libcloud/test/storage/fixtures/oss/list_containers.xml new file mode 100644 index 0000000..d904ef2 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_containers.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListAllMyBucketsResult> + <Owner> + <ID>ut_test_put_bucket</ID> + <DisplayName>ut_test_put_bucket</DisplayName> + </Owner> + <Buckets> + <Bucket> + <Location>oss-cn-hangzhou-a</Location> + <Name>xz02tphky6fjfiuc0</Name> + <CreationDate>2014-05-15T11:18:32.000Z</CreationDate> + </Bucket> + <Bucket> + <Location>oss-cn-hangzhou-a</Location> + <Name>xz02tphky6fjfiuc1</Name> + <CreationDate>2014-05-15T11:18:32.000Z</CreationDate> + </Bucket> + </Buckets> +</ListAllMyBucketsResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/fixtures/oss/list_containers_empty.xml ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/fixtures/oss/list_containers_empty.xml b/libcloud/test/storage/fixtures/oss/list_containers_empty.xml new file mode 100644 index 0000000..f846f72 --- /dev/null +++ b/libcloud/test/storage/fixtures/oss/list_containers_empty.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ListAllMyBucketsResult> + <Owner> + <ID>af4rf45db0927637c66fb848dfc972b8b5126e1237bde6fe02862b11481fdfd9</ID> + <DisplayName>foobar</DisplayName> + </Owner> + <Buckets> + </Buckets> +</ListAllMyBucketsResult> http://git-wip-us.apache.org/repos/asf/libcloud/blob/e6002e0a/libcloud/test/storage/test_oss.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_oss.py b/libcloud/test/storage/test_oss.py new file mode 100644 index 0000000..e5eb24f --- /dev/null +++ b/libcloud/test/storage/test_oss.py @@ -0,0 +1,881 @@ +# -*- coding=utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import unicode_literals + +import os +import sys +import unittest + +try: + import mock +except ImportError: + from unittest import mock + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + +from libcloud.utils.py3 import b +from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import urlparse +from libcloud.utils.py3 import parse_qs +from libcloud.utils.py3 import PY3 +from libcloud.common.types import InvalidCredsError +from libcloud.common.types import MalformedResponseError +from libcloud.storage.base import Container, Object +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ContainerError +from libcloud.storage.types import ContainerIsNotEmptyError +from libcloud.storage.types import InvalidContainerNameError +from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError +from libcloud.storage.drivers.oss import OSSConnection +from libcloud.storage.drivers.oss import OSSStorageDriver +from libcloud.storage.drivers.oss import CHUNK_SIZE +from libcloud.storage.drivers.dummy import DummyIterator +from libcloud.test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 +from libcloud.test import MockHttpTestCase # pylint: disable-msg=E0611 +from libcloud.test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 +from libcloud.test.secrets import STORAGE_OSS_PARAMS + + +class OSSConnectionTestCase(unittest.TestCase): + def setUp(self): + self.conn = OSSConnection('44CF9590006BF252F707', + 'OtxrzxIsfpFjA7SwPzILwy8Bw21TLhquhboDYROV') + + def test_signature(self): + expected = b('26NBxoKdsyly4EDv6inkoDft/yA=') + headers = { + 'Content-MD5': 'ODBGOERFMDMzQTczRUY3NUE3NzA5QzdFNUYzMDQxNEM=', + 'Content-Type': 'text/html', + 'Expires': 'Thu, 17 Nov 2005 18:49:58 GMT', + 'X-OSS-Meta-Author': 'f...@bar.com', + 'X-OSS-Magic': 'abracadabra', + 'Host': 'oss-example.oss-cn-hangzhou.aliyuncs.com' + } + action = '/oss-example/nelson' + actual = OSSConnection._get_auth_signature('PUT', headers, {}, + headers['Expires'], + self.conn.key, + action, + 'x-oss-') + self.assertEqual(expected, actual) + + +class ObjectTestCase(unittest.TestCase): + def test_object_with_chinese_name(self): + driver = OSSStorageDriver(*STORAGE_OSS_PARAMS) + obj = Object(name='ä¸æ', size=0, hash=None, extra=None, + meta_data=None, container=None, driver=driver) + self.assertTrue(obj.__repr__() is not None) + + +class OSSMockHttp(StorageMockHttp, MockHttpTestCase): + + fixtures = StorageFileFixtures('oss') + base_headers = {} + + def _unauthorized(self, method, url, body, headers): + return (httplib.UNAUTHORIZED, + '', + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_containers_empty(self, method, url, body, headers): + body = self.fixtures.load('list_containers_empty.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_containers(self, method, url, body, headers): + body = self.fixtures.load('list_containers.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_container_objects_empty(self, method, url, body, headers): + body = self.fixtures.load('list_container_objects_empty.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_container_objects(self, method, url, body, headers): + body = self.fixtures.load('list_container_objects.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_container_objects_chinese(self, method, url, body, headers): + body = self.fixtures.load('list_container_objects_chinese.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _list_container_objects_prefix(self, method, url, body, headers): + params = {'prefix': self.test.prefix} + self.assertUrlContainsQueryParams(url, params) + body = self.fixtures.load('list_container_objects_prefix.xml') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _get_container(self, method, url, body, headers): + return self._list_containers(method, url, body, headers) + + def _get_object(self, method, url, body, headers): + return self._list_containers(method, url, body, headers) + + def _notexisted_get_object(self, method, url, body, headers): + return (httplib.NOT_FOUND, + body, + self.base_headers, + httplib.responses[httplib.NOT_FOUND]) + + def _test_get_object(self, method, url, body, headers): + self.base_headers.update( + {'accept-ranges': 'bytes', + 'connection': 'keep-alive', + 'content-length': '0', + 'content-type': 'application/octet-stream', + 'date': 'Sat, 16 Jan 2016 15:38:14 GMT', + 'etag': '"D41D8CD98F00B204E9800998ECF8427E"', + 'last-modified': 'Fri, 15 Jan 2016 14:43:15 GMT', + 'server': 'AliyunOSS', + 'x-oss-object-type': 'Normal', + 'x-oss-request-id': '569A63E6257784731E3D877F', + 'x-oss-meta-rabbits': 'monkeys'}) + + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _invalid_name(self, method, url, body, headers): + # test_create_container_bad_request + return (httplib.BAD_REQUEST, + body, + headers, + httplib.responses[httplib.OK]) + + def _already_exists(self, method, url, body, headers): + # test_create_container_already_existed + return (httplib.CONFLICT, + body, + headers, + httplib.responses[httplib.OK]) + + def _create_container(self, method, url, body, headers): + # test_create_container_success + self.assertEqual('PUT', method) + self.assertEqual('', body) + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _create_container_location(self, method, url, body, headers): + # test_create_container_success + self.assertEqual('PUT', method) + location_constraint = ('<CreateBucketConfiguration>' + '<LocationConstraint>%s</LocationConstraint>' + '</CreateBucketConfiguration>' % + self.test.ex_location) + self.assertEqual(location_constraint, body) + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _delete_container_doesnt_exist(self, method, url, body, headers): + # test_delete_container_doesnt_exist + return (httplib.NOT_FOUND, + body, + headers, + httplib.responses[httplib.OK]) + + def _delete_container_not_empty(self, method, url, body, headers): + # test_delete_container_not_empty + return (httplib.CONFLICT, + body, + headers, + httplib.responses[httplib.OK]) + + def _delete_container(self, method, url, body, headers): + return (httplib.NO_CONTENT, + body, + self.base_headers, + httplib.responses[httplib.NO_CONTENT]) + + def _foo_bar_object_not_found(self, method, url, body, headers): + # test_delete_object_not_found + return (httplib.NOT_FOUND, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_bar_object(self, method, url, body, headers): + # test_delete_object + return (httplib.NO_CONTENT, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_test_stream_data_multipart(self, method, url, body, headers): + headers = {'etag': '"0cc175b9c0f1b6a831c399e269772661"'} + TEST_UPLOAD_ID = '0004B9894A22E5B1888A1E29F8236E2D' + + query_string = urlparse.urlsplit(url).query + query = parse_qs(query_string) + + if not query.get('uploadId', False): + self.fail('Request doesnt contain uploadId query parameter') + + upload_id = query['uploadId'][0] + if upload_id != TEST_UPLOAD_ID: + self.fail('first uploadId doesnt match') + + if method == 'PUT': + # PUT is used for uploading the part. part number is mandatory + if not query.get('partNumber', False): + self.fail('Request is missing partNumber query parameter') + + body = '' + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + elif method == 'DELETE': + # DELETE is done for aborting the upload + body = '' + return (httplib.NO_CONTENT, + body, + headers, + httplib.responses[httplib.NO_CONTENT]) + + else: + commit = ET.fromstring(body) + count = 0 + + for part in commit.findall('Part'): + count += 1 + part_no = part.find('PartNumber').text + etag = part.find('ETag').text + + self.assertEqual(part_no, str(count)) + self.assertEqual(etag, headers['etag']) + + # Make sure that manifest contains at least one part + self.assertTrue(count >= 1) + + body = self.fixtures.load('complete_multipart_upload.xml') + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _list_multipart(self, method, url, body, headers): + query_string = urlparse.urlsplit(url).query + query = parse_qs(query_string) + + if 'key-marker' not in query: + body = self.fixtures.load('ex_iterate_multipart_uploads_p1.xml') + else: + body = self.fixtures.load('ex_iterate_multipart_uploads_p2.xml') + + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + +class OSSMockRawResponse(MockRawResponse, MockHttpTestCase): + + fixtures = StorageFileFixtures('oss') + + def parse_body(self): + if len(self.body) == 0 and not self.parse_zero_length_body: + return self.body + + try: + if PY3: + parser = ET.XMLParser(encoding='utf-8') + body = ET.XML(self.body.encode('utf-8'), parser=parser) + else: + body = ET.XML(self.body) + except: + raise MalformedResponseError("Failed to parse XML", + body=self.body, + driver=self.connection.driver) + return body + + def _foo_bar_object(self, method, url, body, headers): + # test_download_object_success + body = self._generate_random_data(1000) + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_bar_object_invalid_size(self, method, url, body, headers): + # test_upload_object_invalid_file_size + body = '' + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_bar_object_not_found(self, method, url, body, headers): + # test_upload_object_not_found + return (httplib.NOT_FOUND, + body, + headers, + httplib.responses[httplib.NOT_FOUND]) + + def _foo_test_upload_invalid_hash1(self, method, url, body, headers): + body = '' + headers = {} + headers['etag'] = '"foobar"' + # test_upload_object_invalid_hash1 + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_test_upload(self, method, url, body, headers): + # test_upload_object_success + body = '' + headers = {'etag': '"0CC175B9C0F1B6A831C399E269772661"'} + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_test_upload_acl(self, method, url, body, headers): + # test_upload_object_with_acl + body = '' + headers = {'etag': '"0CC175B9C0F1B6A831C399E269772661"'} + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_test_stream_data(self, method, url, body, headers): + # test_upload_object_via_stream + body = '' + headers = {'etag': '"0cc175b9c0f1b6a831c399e269772661"'} + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + + def _foo_test_stream_data_multipart(self, method, url, body, headers): + headers = {} + # POST is done for initiating multipart upload + if method == 'POST': + body = self.fixtures.load('initiate_multipart_upload.xml') + return (httplib.OK, + body, + headers, + httplib.responses[httplib.OK]) + else: + body = '' + return (httplib.BAD_REQUEST, + body, + headers, + httplib.responses[httplib.BAD_REQUEST]) + + +class OSSStorageDriverTestCase(unittest.TestCase): + driver_type = OSSStorageDriver + driver_args = STORAGE_OSS_PARAMS + mock_response_klass = OSSMockHttp + mock_raw_response_klass = OSSMockRawResponse + + @classmethod + def create_driver(self): + return self.driver_type(*self.driver_args) + + def setUp(self): + self.driver_type.connectionCls.conn_classes = ( + None, self.mock_response_klass) + self.driver_type.connectionCls.rawResponseCls = \ + self.mock_raw_response_klass + self.mock_response_klass.type = None + self.mock_response_klass.test = self + self.mock_raw_response_klass.type = None + self.mock_raw_response_klass.test = self + self.driver = self.create_driver() + + def tearDown(self): + self._remove_test_file() + + def _remove_test_file(self): + file_path = os.path.abspath(__file__) + '.temp' + + try: + os.unlink(file_path) + except OSError: + pass + + def test_invalid_credentials(self): + self.mock_response_klass.type = 'unauthorized' + self.assertRaises(InvalidCredsError, self.driver.list_containers) + + def test_list_containers_empty(self): + self.mock_response_klass.type = 'list_containers_empty' + containers = self.driver.list_containers() + self.assertEqual(len(containers), 0) + + def test_list_containers_success(self): + self.mock_response_klass.type = 'list_containers' + containers = self.driver.list_containers() + self.assertEqual(len(containers), 2) + + container = containers[0] + self.assertEqual('xz02tphky6fjfiuc0', container.name) + self.assertTrue('creation_date' in container.extra) + self.assertEqual('2014-05-15T11:18:32.000Z', + container.extra['creation_date']) + self.assertTrue('location' in container.extra) + self.assertEqual('oss-cn-hangzhou-a', container.extra['location']) + self.assertEqual(self.driver, container.driver) + + def test_list_container_objects_empty(self): + self.mock_response_klass.type = 'list_container_objects_empty' + container = Container(name='test_container', extra={}, + driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 0) + + def test_list_container_objects_success(self): + self.mock_response_klass.type = 'list_container_objects' + container = Container(name='test_container', extra={}, + driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 2) + + obj = objects[0] + self.assertEqual(obj.name, 'en/') + self.assertEqual(obj.hash, 'D41D8CD98F00B204E9800998ECF8427E') + self.assertEqual(obj.size, 0) + self.assertEqual(obj.container.name, 'test_container') + self.assertEqual( + obj.extra['last_modified'], '2016-01-15T14:43:15.000Z') + self.assertTrue('owner' in obj.meta_data) + + def test_list_container_objects_with_chinese(self): + self.mock_response_klass.type = 'list_container_objects_chinese' + container = Container(name='test_container', extra={}, + driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 2) + + obj = [o for o in objects + if o.name == 'WEBæ§å¶å°.odp'][0] + self.assertEqual(obj.hash, '281371EA1618CF0E645D6BB90A158276') + self.assertEqual(obj.size, 1234567) + self.assertEqual(obj.container.name, 'test_container') + self.assertEqual( + obj.extra['last_modified'], '2016-01-15T14:43:06.000Z') + self.assertTrue('owner' in obj.meta_data) + + def test_list_container_objects_with_prefix(self): + self.mock_response_klass.type = 'list_container_objects_prefix' + container = Container(name='test_container', extra={}, + driver=self.driver) + self.prefix = 'test_prefix' + objects = self.driver.list_container_objects(container=container, + ex_prefix=self.prefix) + self.assertEqual(len(objects), 2) + + def test_get_container_doesnt_exist(self): + self.mock_response_klass.type = 'get_container' + self.assertRaises(ContainerDoesNotExistError, + self.driver.get_container, + container_name='not-existed') + + def test_get_container_success(self): + self.mock_response_klass.type = 'get_container' + container = self.driver.get_container( + container_name='xz02tphky6fjfiuc0') + self.assertTrue(container.name, 'xz02tphky6fjfiuc0') + + def test_get_object_container_doesnt_exist(self): + self.mock_response_klass.type = 'get_object' + self.assertRaises(ObjectDoesNotExistError, + self.driver.get_object, + container_name='xz02tphky6fjfiuc0', + object_name='notexisted') + + def test_get_object_success(self): + self.mock_response_klass.type = 'get_object' + obj = self.driver.get_object(container_name='xz02tphky6fjfiuc0', + object_name='test') + + self.assertEqual(obj.name, 'test') + self.assertEqual(obj.container.name, 'xz02tphky6fjfiuc0') + self.assertEqual(obj.size, 0) + self.assertEqual(obj.hash, 'D41D8CD98F00B204E9800998ECF8427E') + self.assertEqual(obj.extra['last_modified'], + 'Fri, 15 Jan 2016 14:43:15 GMT') + self.assertEqual(obj.extra['content_type'], 'application/octet-stream') + self.assertEqual(obj.meta_data['rabbits'], 'monkeys') + + def test_create_container_bad_request(self): + # invalid container name, returns a 400 bad request + self.mock_response_klass.type = 'invalid_name' + self.assertRaises(ContainerError, + self.driver.create_container, + container_name='invalid_name') + + def test_create_container_already_exists(self): + # container with this name already exists + self.mock_response_klass.type = 'already_exists' + self.assertRaises(InvalidContainerNameError, + self.driver.create_container, + container_name='new-container') + + def test_create_container_success(self): + # success + self.mock_response_klass.type = 'create_container' + name = 'new_container' + container = self.driver.create_container(container_name=name) + self.assertEqual(container.name, name) + + def test_create_container_with_ex_location(self): + self.mock_response_klass.type = 'create_container_location' + name = 'new_container' + self.ex_location = 'oss-cn-beijing' + container = self.driver.create_container(container_name=name, + ex_location=self.ex_location) + self.assertEqual(container.name, name) + self.assertTrue(container.extra['location'], self.ex_location) + + def test_delete_container_doesnt_exist(self): + container = Container(name='new_container', extra=None, + driver=self.driver) + self.mock_response_klass.type = 'delete_container_doesnt_exist' + self.assertRaises(ContainerDoesNotExistError, + self.driver.delete_container, + container=container) + + def test_delete_container_not_empty(self): + container = Container(name='new_container', extra=None, + driver=self.driver) + self.mock_response_klass.type = 'delete_container_not_empty' + self.assertRaises(ContainerIsNotEmptyError, + self.driver.delete_container, + container=container) + + def test_delete_container_success(self): + self.mock_response_klass.type = 'delete_container' + container = Container(name='new_container', extra=None, + driver=self.driver) + self.assertTrue(self.driver.delete_container(container=container)) + + def test_download_object_success(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver_type) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertTrue(result) + + def test_download_object_invalid_file_size(self): + self.mock_raw_response_klass.type = 'invalid_size' + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver_type) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertFalse(result) + + def test_download_object_not_found(self): + self.mock_raw_response_klass.type = 'not_found' + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver_type) + destination_path = os.path.abspath(__file__) + '.temp' + self.assertRaises(ObjectDoesNotExistError, + self.driver.download_object, + obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + + def test_download_object_as_stream_success(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver_type) + + stream = self.driver.download_object_as_stream(obj=obj, + chunk_size=None) + self.assertTrue(hasattr(stream, '__iter__')) + + def test_upload_object_invalid_hash1(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + self.mock_raw_response_klass.type = 'invalid_hash1' + + old_func = self.driver_type._upload_file + try: + self.driver_type._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_upload' + self.assertRaises(ObjectHashMismatchError, + self.driver.upload_object, + file_path=file_path, + container=container, + object_name=object_name, + verify_hash=True) + finally: + self.driver_type._upload_file = old_func + + def test_upload_object_success(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, '0cc175b9c0f1b6a831c399e269772661', 1000 + + old_func = self.driver_type._upload_file + try: + self.driver_type._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_upload' + extra = {'meta_data': {'some-value': 'foobar'}} + obj = self.driver.upload_object(file_path=file_path, + container=container, + object_name=object_name, + extra=extra, + verify_hash=True) + self.assertEqual(obj.name, 'foo_test_upload') + self.assertEqual(obj.size, 1000) + self.assertTrue('some-value' in obj.meta_data) + finally: + self.driver_type._upload_file = old_func + + def test_upload_object_with_acl(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, '0cc175b9c0f1b6a831c399e269772661', 1000 + + old_func = self.driver_type._upload_file + try: + self.driver_type._upload_file = upload_file + self.mock_raw_response_klass.type = 'acl' + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_upload' + extra = {'acl': 'public-read'} + obj = self.driver.upload_object(file_path=file_path, + container=container, + object_name=object_name, + extra=extra, + verify_hash=True) + self.assertEqual(obj.name, 'foo_test_upload') + self.assertEqual(obj.size, 1000) + self.assertEqual(obj.extra['acl'], 'public-read') + finally: + self.driver_type._upload_file = old_func + + def test_upload_object_with_invalid_acl(self): + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_upload' + extra = {'acl': 'invalid-acl'} + self.assertRaises(AttributeError, + self.driver.upload_object, + file_path=file_path, + container=container, + object_name=object_name, + extra=extra, + verify_hash=True) + + def test_upload_empty_object_via_stream(self): + if self.driver.supports_multipart_upload: + self.mock_raw_response_klass.type = 'multipart' + self.mock_response_klass.type = 'multipart' + else: + self.mock_raw_response_klass.type = None + self.mock_response_klass.type = None + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_stream_data' + iterator = DummyIterator(data=['']) + extra = {'content_type': 'text/plain'} + obj = self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator, + extra=extra) + + self.assertEqual(obj.name, object_name) + self.assertEqual(obj.size, 0) + + def test_upload_small_object_via_stream(self): + if self.driver.supports_multipart_upload: + self.mock_raw_response_klass.type = 'multipart' + self.mock_response_klass.type = 'multipart' + else: + self.mock_raw_response_klass.type = None + self.mock_response_klass.type = None + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_stream_data' + iterator = DummyIterator(data=['2', '3', '5']) + extra = {'content_type': 'text/plain'} + obj = self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator, + extra=extra) + + self.assertEqual(obj.name, object_name) + self.assertEqual(obj.size, 3) + + def test_upload_big_object_via_stream(self): + if self.driver.supports_multipart_upload: + self.mock_raw_response_klass.type = 'multipart' + self.mock_response_klass.type = 'multipart' + else: + self.mock_raw_response_klass.type = None + self.mock_response_klass.type = None + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_stream_data' + iterator = DummyIterator( + data=['2' * CHUNK_SIZE, '3' * CHUNK_SIZE, '5']) + extra = {'content_type': 'text/plain'} + obj = self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator, + extra=extra) + + self.assertEqual(obj.name, object_name) + self.assertEqual(obj.size, CHUNK_SIZE * 2 + 1) + + def test_upload_object_via_stream_abort(self): + if not self.driver.supports_multipart_upload: + return + + self.mock_raw_response_klass.type = 'MULTIPART' + self.mock_response_klass.type = 'MULTIPART' + + def _faulty_iterator(): + for i in range(0, 5): + yield str(i) + raise RuntimeError('Error in fetching data') + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + object_name = 'foo_test_stream_data' + iterator = _faulty_iterator() + extra = {'content_type': 'text/plain'} + + try: + self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator, + extra=extra) + except Exception: + pass + + return + + def test_ex_iterate_multipart_uploads(self): + if not self.driver.supports_multipart_upload: + return + + self.mock_response_klass.type = 'list_multipart' + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + + for upload in self.driver.ex_iterate_multipart_uploads(container, + max_uploads=2): + self.assertTrue(upload.key is not None) + self.assertTrue(upload.id is not None) + self.assertTrue(upload.initiated is not None) + + def test_ex_abort_all_multipart_uploads(self): + if not self.driver.supports_multipart_upload: + return + + self.mock_response_klass.type = 'list_multipart' + + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + + with mock.patch('libcloud.storage.drivers.oss.OSSStorageDriver' + '._abort_multipart', autospec=True) as mock_abort: + self.driver.ex_abort_all_multipart_uploads(container) + + self.assertEqual(3, mock_abort.call_count) + + def test_delete_object_not_found(self): + self.mock_response_klass.type = 'not_found' + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1234, hash=None, extra=None, + meta_data=None, container=container, driver=self.driver) + self.assertRaises(ObjectDoesNotExistError, + self.driver.delete_object, + obj=obj) + + def test_delete_object_success(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1234, hash=None, extra=None, + meta_data=None, container=container, driver=self.driver) + + result = self.driver.delete_object(obj=obj) + self.assertTrue(result) + + +if __name__ == '__main__': + sys.exit(unittest.main())
