merged with drs
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/14a69ee4 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/14a69ee4 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/14a69ee4 Branch: refs/heads/trunk Commit: 14a69ee45c9141329922ef193480eba99ce4adc1 Parents: 067e93e 62d9eb6 Author: mitch <[email protected]> Authored: Fri Oct 26 22:04:46 2018 -0400 Committer: mitch <[email protected]> Committed: Fri Oct 26 22:04:46 2018 -0400 ---------------------------------------------------------------------- .gitignore | 2 + .pylintrc | 1 + .travis.yml | 24 +- CHANGES.rst | 78 +- README.rst | 13 +- contrib/scrap-ec2-sizes.py | 281 + contrib/trigger_rtd_build.py | 9 +- dist/hash-sign.sh | 147 - dist/hash.py | 12 + dist/release.sh | 3 +- dist/sign.sh | 69 + .../images/misc/azure_blobs_account_kind.png | Bin 0 -> 292732 bytes .../misc/azure_blobs_manage_access_keys_1.png | Bin 64259 -> 223140 bytes .../misc/azure_blobs_manage_access_keys_2.png | Bin 26247 -> 388303 bytes docs/_static/images/provider_logos/ntt.png | Bin 0 -> 23017 bytes docs/_static/images/provider_logos/scaleway.png | Bin 0 -> 11527 bytes docs/committer_guide.rst | 10 +- docs/compute/_supported_providers.rst | 3 +- docs/compute/drivers/nttcis.rst | 89 + docs/compute/drivers/openstack.rst | 10 +- docs/compute/drivers/scaleway.rst | 30 + docs/conf.py | 4 +- .../Firewall_Create_Complex_Firewall_Rule.py | 77 + .../nttcis/Firewall_Create_Simple_Rule.py | 28 + .../nttcis/Nodes_Create_mcp2_Customized.py | 27 + .../nttcis/Nodes_Create_mcp2_Uncustomised.py | 57 + docs/examples/compute/nttcis/__init__.py | 0 docs/examples/compute/scaleway/create_node.py | 16 + docs/examples/compute/scaleway/list_nodes.py | 9 + docs/examples/compute/scaleway/list_volumes.py | 12 + .../nttcis/LoadBalancer_create_members.py | 38 + .../nttcis/Load_balancer_add_pool_mon.py | 42 + docs/examples/loadbalancer/nttcis/__init__.py | 0 docs/loadbalancer/drivers/nttcis.rst | 65 + docs/other/changes_in_2_0.rst | 2 +- docs/storage/drivers/azure_blobs.rst | 12 +- libcloud/backup/drivers/nttcis.py | 689 --- libcloud/base.py | 7 + libcloud/common/dimensiondata.py | 2 +- libcloud/common/nttcis.py | 145 +- libcloud/common/openstack.py | 2 + libcloud/common/openstack_identity.py | 14 + libcloud/compute/constants.py | 5431 ++++++++++++++++++ libcloud/compute/drivers/azure.py | 9 +- libcloud/compute/drivers/digitalocean.py | 16 + libcloud/compute/drivers/dimensiondata.py | 14 +- libcloud/compute/drivers/ec2.py | 2124 +------ libcloud/compute/drivers/gce.py | 15 +- libcloud/compute/drivers/nttcis.py | 657 ++- libcloud/compute/drivers/openstack.py | 348 +- libcloud/compute/drivers/scaleway.py | 663 +++ libcloud/compute/providers.py | 2 + libcloud/compute/types.py | 1 + libcloud/drs/__init__.py | 19 + libcloud/drs/base.py | 160 +- libcloud/drs/drivers/__init__.py | 18 + libcloud/drs/drivers/nttcis.py | 42 +- libcloud/drs/providers.py | 9 +- libcloud/drs/types.py | 43 + libcloud/http.py | 1 - libcloud/loadbalancer/base.py | 19 +- libcloud/loadbalancer/drivers/__init__.py | 3 +- libcloud/loadbalancer/drivers/nttcis.py | 86 +- libcloud/storage/drivers/azure_blobs.py | 29 +- libcloud/storage/drivers/dummy.py | 2 - libcloud/storage/drivers/s3.py | 18 +- libcloud/storage/providers.py | 6 +- libcloud/storage/types.py | 1 + libcloud/test/__init__.py | 7 +- libcloud/test/common/test_openstack_identity.py | 18 +- .../fixtures/digitalocean_v2/list_node.json | 112 + .../test/compute/fixtures/nttcis/audit_log.csv | 25 + .../fixtures/nttcis/change_disk_size.xml | 6 + .../fixtures/nttcis/change_disk_speed.xml | 6 + .../change_nic_networkadapter_response.xml | 7 + .../nttcis/datacenter_snapshotWindows.xml | 8 + .../nttcis/deploy_customised_server.xml | 7 + .../fixtures/nttcis/detailed_usage_report.csv | 42 + .../nttcis/exchange_nic_vlans_response.xml | 8 + .../fixtures/nttcis/image_customerImage.xml | 57 + ...age_2ffa36c8_1848_49eb_b4fa_9d908775f68c.xml | 19 + ...age_5234e5c7_01de_4411_8b6e_baeb8d91cf5d.xml | 19 + .../compute/fixtures/nttcis/image_osImage.xml | 40 + ...age_6b4fb0c7_a57b_4f58_b59c_9958f94f971a.xml | 13 + .../nttcis/image_osImage_BAD_REQUEST.xml | 6 + ...age_c14b1a46_2428_44c1_9c1a_b20e6418d08c.xml | 14 + .../fixtures/nttcis/import_image_response.xml | 8 + .../nttcis/infrastructure_datacenter.xml | 188 + .../nttcis/infrastructure_datacenter_NA9.xml | 54 + .../fixtures/nttcis/ip_address_list_create.xml | 9 + .../fixtures/nttcis/ip_address_list_delete.xml | 10 + .../fixtures/nttcis/ip_address_list_edit.xml | 10 + .../fixtures/nttcis/ip_address_lists.xml | 46 + .../nttcis/ip_address_lists_FILTERBYNAME.xml | 14 + .../nttcis/network_addPublicIpBlock.xml | 7 + .../nttcis/network_createFirewallRule.xml | 8 + .../fixtures/nttcis/network_createNatRule.xml | 9 + .../nttcis/network_deleteFirewallRule.xml | 8 + .../fixtures/nttcis/network_deleteNatRule.xml | 8 + .../nttcis/network_deleteNetworkDomain.xml | 8 + .../fixtures/nttcis/network_deleteVlan.xml | 8 + .../nttcis/network_deployNetworkDomain.xml | 8 + .../fixtures/nttcis/network_deployVlan.xml | 9 + .../nttcis/network_editFirewallRule.xml | 8 + .../nttcis/network_editNetworkDomain.xml | 7 + .../fixtures/nttcis/network_editVlan.xml | 7 + .../fixtures/nttcis/network_expandVlan.xml | 8 + .../fixtures/nttcis/network_firewallRule.xml | 35 + ...ule_d0a20f59_77b9_4f28_a63b_e58496b73a6c.xml | 18 + .../compute/fixtures/nttcis/network_natRule.xml | 21 + ...ule_2187a636_7ebb_49a1_a2ff_5d617f496dce.xml | 9 + .../fixtures/nttcis/network_networkDomain.xml | 43 + ...ain_8cdfd607_f429_4df6_9352_162cfc0891be.xml | 9 + .../fixtures/nttcis/network_publicIpBlock.xml | 10 + ...ock_4487241a_f0ca_11e3_9315_d4bed9b167ba.xml | 8 + ...ock_9945dc4a_bdce_11e4_8c14_b8ca3a5d9ef8.xml | 8 + .../nttcis/network_removePublicIpBlock.xml | 8 + .../compute/fixtures/nttcis/network_vlan.xml | 25 + ...lan_0e56433f_d808_4669_821d_812769517ff8.xml | 14 + .../fixtures/nttcis/oec_0_9_myaccount.xml | 26 + .../fixtures/nttcis/port_list_create.xml | 9 + .../fixtures/nttcis/port_list_delete.xml | 10 + .../compute/fixtures/nttcis/port_list_edit.xml | 8 + .../compute/fixtures/nttcis/port_list_get.xml | 15 + .../compute/fixtures/nttcis/port_list_lists.xml | 38 + .../fixtures/nttcis/report_usageMonitoring.xml | 8 + .../fixtures/nttcis/server_GetServer.xml | 42 + .../compute/fixtures/nttcis/server_addDisk.xml | 9 + .../compute/fixtures/nttcis/server_addNic.xml | 9 + .../nttcis/server_antiAffinityRule_list.xml | 42 + .../server_antiAffinityRule_list_PAGINATED.xml | 42 + .../server_changeServerMonitoringPlan.xml | 8 + .../fixtures/nttcis/server_cleanServer.xml | 9 + .../fixtures/nttcis/server_clone_response.xml | 9 + .../nttcis/server_createAntiAffinityRule.xml | 6 + .../server_createAntiAffinityRule_FAIL.xml | 8 + .../nttcis/server_deleteAntiAffinityRule.xml | 7 + .../server_deleteAntiAffinityRule_FAIL.xml | 8 + .../fixtures/nttcis/server_deleteServer.xml | 9 + .../nttcis/server_deleteServer_RESOURCEBUSY.xml | 8 + .../fixtures/nttcis/server_deployServer.xml | 8 + .../nttcis/server_disableServerMonitoring.xml | 8 + .../nttcis/server_editServerMetadata.xml | 6 + .../nttcis/server_enableServerMonitoring.xml | 8 + .../fixtures/nttcis/server_powerOffServer.xml | 8 + .../nttcis/server_powerOffServer_INPROGRESS.xml | 8 + .../fixtures/nttcis/server_rebootServer.xml | 8 + .../nttcis/server_rebootServer_RESOURCEBUSY.xml | 7 + .../nttcis/server_reconfigureServer.xml | 5 + .../fixtures/nttcis/server_removeDisk.xml | 6 + .../fixtures/nttcis/server_removeNic.xml | 9 + .../fixtures/nttcis/server_resetServer.xml | 8 + .../compute/fixtures/nttcis/server_server.xml | 207 + .../fixtures/nttcis/server_server_NA3.xml | 56 + ...ver_e75ead52_692f_4314_8725_c8a4f4d13a87.xml | 30 + .../fixtures/nttcis/server_server_paginated.xml | 58 + .../nttcis/server_server_paginated_empty.xml | 1 + .../fixtures/nttcis/server_shutdownServer.xml | 8 + .../nttcis/server_shutdownServer_INPROGRESS.xml | 8 + .../fixtures/nttcis/server_startServer.xml | 8 + .../nttcis/server_startServer_INPROGRESS.xml | 8 + .../nttcis/server_updateVmwareTools.xml | 8 + .../fixtures/nttcis/summary_usage_report.csv | 13 + .../compute/fixtures/nttcis/tag_applyTags.xml | 6 + .../nttcis/tag_applyTags_BADREQUEST.xml | 6 + .../fixtures/nttcis/tag_createTagKey.xml | 7 + .../nttcis/tag_createTagKey_BADREQUEST.xml | 6 + .../fixtures/nttcis/tag_deleteTagKey.xml | 6 + .../nttcis/tag_deleteTagKey_BADREQUEST.xml | 6 + .../compute/fixtures/nttcis/tag_editTagKey.xml | 6 + .../nttcis/tag_editTagKey_BADREQUEST.xml | 6 + .../compute/fixtures/nttcis/tag_removeTag.xml | 6 + .../nttcis/tag_removeTag_BADREQUEST.xml | 6 + ...Key_5ab77f5f_5aa9_426f_8459_4eab34e03d54.xml | 6 + ...f_5aa9_426f_8459_4eab34e03d54_BADREQUEST.xml | 6 + .../compute/fixtures/nttcis/tag_tagKey_list.xml | 19 + .../fixtures/nttcis/tag_tagKey_list_SINGLE.xml | 8 + .../compute/fixtures/nttcis/tag_tag_list.xml | 36 + .../compute/fixtures/openstack/_v2_0__auth.json | 22 + .../fixtures/openstack_v1.1/_port_v2.json | 32 + .../fixtures/openstack_v1.1/_ports_v2.json | 185 + .../openstack_v1.1/_v2_0__networks.json | 64 + .../openstack_v1.1/_v2_0__networks_POST.json | 33 + .../fixtures/openstack_v1.1/_v2_0__subnets.json | 62 + .../compute/fixtures/scaleway/create_image.json | 21 + .../compute/fixtures/scaleway/create_node.json | 40 + .../fixtures/scaleway/create_volume.json | 13 + .../scaleway/create_volume_snapshot.json | 15 + .../test/compute/fixtures/scaleway/error.json | 1 + .../fixtures/scaleway/error_invalid_image.json | 1 + .../compute/fixtures/scaleway/get_image.json | 21 + .../fixtures/scaleway/list_availability.json | 13 + .../compute/fixtures/scaleway/list_images.json | 42 + .../compute/fixtures/scaleway/list_nodes.json | 74 + .../fixtures/scaleway/list_nodes_empty.json | 3 + .../compute/fixtures/scaleway/list_sizes.json | 76 + .../scaleway/list_volume_snapshots.json | 30 + .../compute/fixtures/scaleway/list_volumes.json | 26 + .../fixtures/scaleway/list_volumes_empty.json | 3 + .../compute/fixtures/scaleway/reboot_node.json | 9 + .../compute/fixtures/scaleway/token_info.json | 14 + .../compute/fixtures/scaleway/user_info.json | 15 + libcloud/test/compute/test_digitalocean_v2.py | 12 + libcloud/test/compute/test_ec2.py | 40 +- libcloud/test/compute/test_nttcis.py | 2898 ++++++++++ libcloud/test/compute/test_openstack.py | 204 +- libcloud/test/compute/test_scaleway.py | 334 ++ .../nttcis/networkDomainVip_addPoolMember.xml | 9 + .../nttcis/networkDomainVip_createNode.xml | 8 + .../nttcis/networkDomainVip_createPool.xml | 9 + .../networkDomainVip_createVirtualListener.xml | 11 + .../networkDomainVip_defaultHealthMonitor.xml | 35 + .../nttcis/networkDomainVip_defaultIrule.xml | 47 + ...tworkDomainVip_defaultPersistenceProfile.xml | 47 + .../nttcis/networkDomainVip_deleteNode.xml | 9 + .../nttcis/networkDomainVip_deletePool.xml | 9 + .../networkDomainVip_deleteVirtualListener.xml | 9 + .../nttcis/networkDomainVip_editNode.xml | 8 + .../nttcis/networkDomainVip_editPool.xml | 9 + .../nttcis/networkDomainVip_editPoolMember.xml | 8 + .../fixtures/nttcis/networkDomainVip_node.xml | 29 + ...ode_34de6ed6_46a4_4dae_a753_2f8d3840c6f9.xml | 13 + .../fixtures/nttcis/networkDomainVip_pool.xml | 37 + .../nttcis/networkDomainVip_poolMember.xml | 29 + ...ber_3dd806a2_c2c8_4c0c_9a4f_5219ea9266c0.xml | 13 + ...ool_4d360b1f_bc2c_4ab7_9884_1f03ba2768f7.xml | 17 + .../networkDomainVip_removePoolMember.xml | 7 + .../nttcis/networkDomainVip_virtualListener.xml | 51 + ...ner_6115469d_a8bb_445b_bb23_d23b5283f2b9.xml | 44 + .../fixtures/nttcis/oec_0_9_myaccount.xml | 26 + libcloud/test/loadbalancer/test_nttcis.py | 647 +++ libcloud/test/secrets.py-dist | 6 +- libcloud/test/storage/test_azure_blobs.py | 4 +- libcloud/test/storage/test_s3.py | 7 +- libcloud/test/test_connection.py | 16 + libcloud/test/test_utils.py | 8 +- libcloud/utils/files.py | 4 +- libcloud/utils/xml.py | 1 - setup.cfg | 4 + setup.py | 8 +- tests/conftest.py | 21 - tests/lib_create_test.py | 240 - tests/lib_edit_test.py | 452 -- tests/lib_list_test.py | 390 -- tests/lib_misc_test.py | 10 - tests/test_lib_list.py | 344 ++ tox.ini | 11 +- 247 files changed, 16170 insertions(+), 4579 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/14a69ee4/libcloud/common/nttcis.py ---------------------------------------------------------------------- diff --cc libcloud/common/nttcis.py index aea3e41,aeb5fa1..fd1976d --- a/libcloud/common/nttcis.py +++ b/libcloud/common/nttcis.py @@@ -13,12 -13,11 +13,13 @@@ # See the License for the specific language governing permissions and # limitations under the License. """ - Dimension Data Common Components + NTTCIS Common Components """ + from base64 import b64encode from time import sleep +from lxml import etree +from io import BytesIO # TODO: use disutils.version when Travis CI fixed the pylint issue with version # from distutils.version import LooseVersion from libcloud.utils.py3 import httplib @@@ -1931,243 -1924,3 +1926,245 @@@ class NttCisNic(object) return ('<NttCisNic: private_ip_v4=%s, vlan=%s,' 'network_adapter_name=%s>' % (self.private_ip_v4, self.vlan, self.network_adapter_name)) ++<<<<<<< HEAD + + +##### Testing new concept below this line + +attrs = {} + + +def processor(mapping, name=None): + mapping = mapping + + map_copy = deepcopy(mapping) + + def add_items(key, value, name=None): + if name in attrs: + print(attrs) + attrs[name].update({key: value}) + elif name is not None: + attrs[name] = value + + else: + attrs.update({key: value}) + if key in map_copy: + del map_copy[key] + elif key in map_copy[name]: + del map_copy[name][key] + if len(map_copy[name]) == 0: + del map_copy[name] + + def handle_map(map, name): + tmp = {} + types = [type(x) for x in map.values()] + if XmlListConfig not in types and XmlDictConfig not in types and dict not in types: + return map + + elif XmlListConfig in types: + result = handle_seq(map, name) + return result + else: + for k, v in map.items(): + if isinstance(v, str): + tmp.update({k: v}) + if isinstance(v, dict): + cls = build_class(k.capitalize(), v) + tmp.update({k: cls}) + elif isinstance(v, XmlDictConfig): + cls = build_class(k.capitalize(), v) + return (k, cls) + return tmp + + def handle_seq(seq, name): + tmp = {} + tmp_list = [] + if isinstance(seq, list): + tmp = [] + for _ in seq: + cls = build_class(name.capitalize(), _) + tmp.append(cls) + return tmp + for k, v in seq.items(): + if isinstance(v, Mapping): + result1 = handle_map(v, k) + elif isinstance(v, MutableSequence): + for _ in v: + if isinstance(_, Mapping): + types = [type(x) for x in _.values()] + if XmlDictConfig in types: + result = handle_map(_, k) + if isinstance(result, tuple): + tmp.update({result[0]: result[1]}) + else: + tmp.update({k: result}) + else: + tmp_list = [build_class(k.capitalize(), i) for i in v] + tmp[k] = tmp_list + print() + elif isinstance(v, str): + tmp.update({k: v}) + return tmp + + def build_class(key, value): + klass = class_factory(key.capitalize(), value) + return klass(value) + + def process(mapping, name): + for k1, v1 in mapping.items(): + if isinstance(v1, Mapping): + types = [type(v) for v in v1.values()] + if MutableSequence not in types and dict not in types: + result = handle_map(v1, k1) + cls = build_class(k1.capitalize(), result) + add_items(k1, cls) + elif XmlListConfig in types: + result = handle_seq(v1, k1) + cls = build_class(list(v1)[0], result) + add_items(k1, cls) + elif dict in types: + result = handle_map(v1, k1) + cls = build_class(k1.capitalize(), result) + add_items(k1, cls, k1) + elif isinstance(v1, list): + tmp = {} + tmp1 = {} + tmp2 = {} + tmp2[k1] = [] + for i, j in enumerate(v1): + if isinstance(j, dict): + key = list(j)[0] + result = handle_map(j, key) + tmp1[k1 + str(i)] = build_class(k1, result) + tmp2[k1].append(tmp1[k1 + str(i)]) + if tmp2: + #cls = build_class(k1.capitalize(), tmp2) + add_items(k1, tmp2[k1], k1) + elif isinstance(v1, str): + add_items(k1, v1) + + + + + if len(map_copy) == 0: + return 1 + #print(attrs) + return process(mapping, name) + + +def class_factory(cls_name, attrs): + + def __init__(self, *args, **kwargs): + for key in attrs: + setattr(self, key, attrs[key]) + + def __iter__(self): + for name in self.__dict__: + yield getattr(self, name) + + def __repr__(self): + values = ', '.join('{}={!r}'.format(*i) for i in zip(self.__dict__, self)) + return '{}({})'.format(self.__class__.__name__, values) + + cls_attrs = dict( + __init__=__init__, + __iter__=__iter__, + __repr__=__repr__) + + return type("NttCis{}".format(cls_name), (object,), cls_attrs) + + +class XmlListConfig(list): + def __init__(self, elem_list): + for element in elem_list: + if element is not None: + # treat like dict + #print(element.attrib, len(element)) + if len(element) >= 0 or element[0].tag != element[1].tag: + self.append(XmlDictConfig(element)) + # treat like list + elif element[0].tag == element[1].tag: + if 'property' in element.tag: + self.append({element.attrib.get('name'): element.attrib.get('value')}) + else: + self.append(element.attrib) + elif element.text: + text = element.text.strip() + if text: + self.append(text) + + +class XmlDictConfig(dict): + + def __init__(self, parent_element): + if parent_element.items(): + if 'property' in parent_element.tag: + self.update({parent_element.attrib.get('name'): parent_element.attrib.get('value')}) + else: + self.update(dict(parent_element.items())) + + for element in parent_element: + if len(element) > 0: + # treat like dict - we assume that if the first two tags + # in a series are different, then they are all different. + if len(element) == 1 or element[0].tag != element[1].tag: + elem_dict = XmlDictConfig(element) + + # treat like list - we assume that if the first two tags + # in a series are the same, then the rest are the same. + else: + # here, we put the list in dictionary; the key is the + # tag name the list elements all share in common, and + # the value is the list itself + elem_dict = {element[0].tag.split('}')[1]: XmlListConfig(element)} + + # if the tag has attributes, add those to the dict + if element.items(): + elem_dict.update(dict(element.items())) + self.update({element.tag.split('}')[1]: elem_dict}) + # this assumes that if you've got an attribute in a tag, + # you won't be having any text. This may or may not be a + # good idea -- time will tell. It works for the way we are + # currently doing XML configuration files... + elif element.items(): + # It is possible to have duplicate element tags. If so, convert to a dict of lists + if element.tag.split('}')[1] in self: + tmp_list = list() + tmp_dict = dict() + if isinstance(self[element.tag.split('}')[1]], list): + tmp_list.append(element.tag.split('}')[1]) + else: + for k, v in self[element.tag.split('}')[1]].items(): + if isinstance(k, XmlListConfig): + tmp_list.append(k) + else: + tmp_dict.update({k: v}) + tmp_list.append(tmp_dict) + tmp_list.append(dict(element.items())) + print() + self[element.tag.split('}')[1]] = tmp_list + else: + self.update({element.tag.split('}')[1]: dict(element.items())}) + # finally, if there are no child tags and no attributes, extract + # the text + else: + self.update({element.tag.split('}')[1]: element.text}) + + +def process_xml(xml): + tree = etree.parse(BytesIO(xml)) + root = tree.getroot() + elem = root.tag.split('}')[1].capitalize() + items = dict(root.items()) + + if 'pageNumber' in items: + converted_xml = XmlListConfig(root) + processor(converted_xml[0]) + else: + converted_xml = XmlDictConfig(root) + processor(converted_xml) + klass = class_factory(elem.capitalize(), attrs) + cls = klass(attrs) + - return cls ++ return cls ++
