support for traffic type and external devices configuration * traffictypes = 'Guest', 'Management', 'Public', 'Storage' - corresponding nic labels are given in the labeldict * External device providers JuniperSrx, NetScaler and F5 load balancing can now be configured
* examples are shown in the configGenerator.py script for basic, advanced and EIP/ELB based zones Conflicts: tools/marvin/marvin/configGenerator.py tools/marvin/marvin/deployDataCenter.py Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/614131a9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/614131a9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/614131a9 Branch: refs/heads/master Commit: 614131a9def4aa4da9085aab3907ffc169f0311f Parents: c91463f Author: Prasanna Santhanam <prasanna.santha...@citrix.com> Authored: Thu Aug 9 14:12:45 2012 +0530 Committer: Prasanna Santhanam <prasanna.santha...@citrix.com> Committed: Thu Aug 9 15:31:28 2012 +0530 ---------------------------------------------------------------------- tools/marvin/marvin/configGenerator.py | 276 ++++++++++++++++++++++++-- tools/marvin/marvin/deployDataCenter.py | 184 +++++++++-------- 2 files changed, 357 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/614131a9/tools/marvin/marvin/configGenerator.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/configGenerator.py b/tools/marvin/marvin/configGenerator.py index ad323ca..ba4b00b 100644 --- a/tools/marvin/marvin/configGenerator.py +++ b/tools/marvin/marvin/configGenerator.py @@ -68,21 +68,22 @@ class zone(): self.vlan = None '''default public network, in advanced mode''' self.ipranges = [] - self.networks = [] + self.physical_networks = [] self.pods = [] self.secondaryStorages = [] - '''enable default virtual router provider''' - vrouter = provider() - vrouter.name = 'VirtualRouter' - self.providers = [vrouter] -class provider(): - def __init__(self): - self.name = None - self.state = None - self.broadcastdomainrange = 'ZONE' - self.zoneid = None - self.servicelist = [] +class traffictype(): + def __init__(self, typ, labeldict=None): + self.typ = typ #Guest/Management/Public + if labeldict: + self.xen = labeldict['xen'] if 'xen' in labeldict.keys() else None + self.kvm = labeldict['kvm'] if 'kvm' in labeldict.keys() else None + self.vmware = labeldict['vmware'] if 'vmware' in labeldict.keys() else None + #{ + # 'xen' : 'cloud-xen', + # 'kvm' : 'cloud-kvm', + # 'vmware' : 'cloud-vmware' + #} class pod(): def __init__(self): @@ -124,7 +125,28 @@ class host(): self.hostmac = None self.hosttags = None self.memory = None - + +class physical_network(): + def __init__(self): + self.name = None + self.tags = [] + self.traffictypes = [] + self.broadcastdomainrange = 'Zone' + self.vlan = None + '''enable default virtual router provider''' + vrouter = provider() + vrouter.name = 'VirtualRouter' + self.providers = [vrouter] + +class provider(): + def __init__(self, name=None): + self.name = name + self.state = None + self.broadcastdomainrange = 'ZONE' + self.zoneid = None + self.servicelist = [] + self.devices = [] + class network(): def __init__(self): self.displaytext = None @@ -156,6 +178,77 @@ class primaryStorage(): class secondaryStorage(): def __init__(self): self.url = None + +class netscaler(): + def __init__(self, hostname=None, username='nsroot', password='nsroot'): + self.hostname = hostname + self.username = username + self.password = password + self.networkdevicetype = 'NetscalerVPXLoadBalancer' + self.publicinterface = '1/1' + self.privateinterface = '1/1' + self.numretries = '2' + self.lbdevicecapacity = '50' + self.lbdevicededicated = 'false' + + def getUrl(self): + return repr(self) + + def __repr__(self): + req = zip(self.__dict__.keys(), self.__dict__.values()) + return self.hostname+"?" + "&".join(["=".join([r[0], r[1]]) \ + for r in req]) + +class srx(): + def __init__(self, hostname=None, username='root', password='admin'): + self.hostname = hostname + self.username = username + self.password = password + self.networkdevicetype = 'JuniperSRXFirewall' + self.publicinterface = '1/1' + self.privateinterface = '1/1' + self.numretries = '2' + self.fwdevicededicated = 'false' + self.timeout = '300' + self.publicnetwork = 'untrusted' + self.privatenetwork = 'trusted' + + def getUrl(self): + return repr(self) + + def __repr__(self): + req = zip(self.__dict__.keys(), self.__dict__.values()) + return self.hostname+"?" + "&".join(["=".join([r[0], r[1]]) \ + for r in req]) + +class bigip(): + def __init__(self, hostname=None, username='root', password='default'): + self.hostname = hostname + self.username = username + self.password = password + self.networkdevicetype = 'F5BigIpLoadBalancer' + self.publicinterface = '1/1' + self.privateinterface = '1/1' + self.numretries = '2' + self.lbdevicededicated = 'false' + self.lbdevicecapacity = '50' + + def getUrl(self): + return repr(self) + + def __repr__(self): + req = zip(self.__dict__.keys(), self.__dict__.values()) + return self.hostname+"?" + "&".join(["=".join([r[0], r[1]]) \ + for r in req]) + +def getDeviceUrl(obj): + req = zip(obj.__dict__.keys(), obj.__dict__.values()) + if obj.hostname: + return "http://" + obj.hostname+"?" + "&".join(["=".join([r[0], r[1]]) \ + for r in req]) + else: + return None + '''sample code to generate setup configuration file''' def describe_setup_in_basic_mode(): @@ -171,6 +264,18 @@ def describe_setup_in_basic_mode(): z.networktype = 'Basic' z.securitygroupenabled = 'True' + #If security groups are reqd + sgprovider = provider() + sgprovider.broadcastdomainrange = 'Pod' + sgprovider.name = 'SecurityGroupProvider' + + pn = physical_network() + pn.name = "test-network" + pn.traffictypes = [traffictype("Guest"), traffictype("Management")] + pn.providers.append(sgprovider) + + z.physical_networks.append(pn) + '''create 10 pods''' for i in range(2): p = pod() @@ -211,7 +316,6 @@ def describe_setup_in_basic_mode(): '''add 2 primary storages''' for m in range(2): primary = primaryStorage() - size=1*1024*1024*1024*1024 primary.name = "primary"+str(l) + str(i) + str(j) + str(m) #primary.url = "nfs://localhost/path%s/size=%d"%(str(l) + str(i) + str(j) + str(m), size) primary.url = "nfs://localhost/path%s"%(str(l) + str(i) + str(j) + str(m)) @@ -265,6 +369,129 @@ def describe_setup_in_basic_mode(): return zs +def describe_setup_in_eip_mode(): + """ + Setting up an EIP/ELB enabled zone with netscaler provider + """ + zs = cloudstackConfiguration() + + for l in range(1): + z = zone() + z.dns1 = "8.8.8.8" + z.dns2 = "4.4.4.4" + z.internaldns1 = "192.168.110.254" + z.internaldns2 = "192.168.110.253" + z.name = "test"+str(l) + z.networktype = 'Basic' + + #If security groups are reqd + sgprovider = provider() + sgprovider.broadcastdomainrange = 'Pod' + sgprovider.name = 'SecurityGroupProvider' + + nsprovider = provider() + nsprovider.name = 'Netscaler' + ns = netscaler() + ns.hostname = '10.147.40.100' + nsprovider.devices.append(ns) + + pn = physical_network() + pn.name = "test-network" + pn.traffictypes = [traffictype("Guest", {"xen": "cloud-guest"}), traffictype("Management"), traffictype("Public", { "xen": "cloud-public"})] + pn.providers.extend([sgprovider, nsprovider]) + z.physical_networks.append(pn) + + '''create 10 pods''' + for i in range(2): + p = pod() + p.name = "test" +str(l) + str(i) + p.gateway = "192.168.%d.1"%i + p.netmask = "255.255.255.0" + p.startip = "192.168.%d.150"%i + p.endip = "192.168.%d.220"%i + + '''add two pod guest ip ranges''' + for j in range(2): + ip = iprange() + ip.gateway = p.gateway + ip.netmask = p.netmask + ip.startip = "192.168.%d.%d"%(i,j*20) + ip.endip = "192.168.%d.%d"%(i,j*20+10) + + p.guestIpRanges.append(ip) + + '''add 10 clusters''' + for j in range(2): + c = cluster() + c.clustername = "test"+str(l)+str(i) + str(j) + c.clustertype = "CloudManaged" + c.hypervisor = "Simulator" + + '''add 10 hosts''' + for k in range(2): + h = host() + h.username = "root" + h.password = "password" + #h.url = "http://Sim/%d%d%d%d/cpucore=1&cpuspeed=8000&memory=%d&localstorage=%d"%(l,i,j,k,memory,localstorage) + h.url = "http://Sim/%d%d%d%d"%(l,i,j,k) + c.hosts.append(h) + + '''add 2 primary storages''' + for m in range(2): + primary = primaryStorage() + primary.name = "primary"+str(l) + str(i) + str(j) + str(m) + #primary.url = "nfs://localhost/path%s/size=%d"%(str(l) + str(i) + str(j) + str(m), size) + primary.url = "nfs://localhost/path%s"%(str(l) + str(i) + str(j) + str(m)) + c.primaryStorages.append(primary) + + p.clusters.append(c) + + z.pods.append(p) + + '''add two secondary''' + for i in range(5): + secondary = secondaryStorage() + secondary.url = "nfs://localhost/path"+str(l) + str(i) + z.secondaryStorages.append(secondary) + + zs.zones.append(z) + + '''Add one mgt server''' + mgt = managementServer() + mgt.mgtSvrIp = "localhost" + zs.mgtSvr.append(mgt) + + '''Add a database''' + db = dbServer() + db.dbSvr = "localhost" + + zs.dbSvr = db + + '''add global configuration''' + global_settings = {'expunge.delay': '60', + 'expunge.interval': '60', + 'expunge.workers': '3', + } + for k,v in global_settings.iteritems(): + cfg = configuration() + cfg.name = k + cfg.value = v + zs.globalConfig.append(cfg) + + ''''add loggers''' + testClientLogger = logger() + testClientLogger.name = "TestClient" + testClientLogger.file = "/tmp/testclient.log" + + testCaseLogger = logger() + testCaseLogger.name = "TestCase" + testCaseLogger.file = "/tmp/testcase.log" + + zs.logger.append(testClientLogger) + zs.logger.append(testCaseLogger) + + return zs + '''sample code to generate setup configuration file''' def describe_setup_in_advanced_mode(): zs = cloudstackConfiguration() @@ -279,6 +506,24 @@ def describe_setup_in_advanced_mode(): z.networktype = 'Advanced' z.guestcidraddress = "10.1.1.0/24" z.vlan = "100-2000" + + pn = physical_network() + pn.name = "test-network" + pn.traffictypes = [traffictype("Guest"), traffictype("Management"), traffictype("Public")] + + vpcprovider = provider('VpcVirtualRouter') + + nsprovider = provider('Netscaler') + nsprovider.devices.append(netscaler(hostname='10.147.40.100')) + + srxprovider = provider('JuniperSRX') + srxprovider.devices.append(srx(hostname='10.147.40.3')) + + f5provider = provider('F5BigIp') + f5provider.devices.append(bigip(hostname='10.147.40.3')) + + pn.providers.extend([vpcprovider, nsprovider, srxprovider, f5provider]) + z.physical_networks.append(pn) '''create 10 pods''' for i in range(2): @@ -310,7 +555,6 @@ def describe_setup_in_advanced_mode(): '''add 2 primary storages''' for m in range(2): primary = primaryStorage() - size=1*1024*1024*1024*1024 primary.name = "primary"+str(l) + str(i) + str(j) + str(m) #primary.url = "nfs://localhost/path%s/size=%d"%(str(l) + str(i) + str(j) + str(m), size) primary.url = "nfs://localhost/path%s"%(str(l) + str(i) + str(j) + str(m)) @@ -406,3 +650,5 @@ if __name__ == "__main__": config = describe_setup_in_basic_mode() generate_setup_config(config, options.output) + + http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/614131a9/tools/marvin/marvin/deployDataCenter.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index 4284d12..68e2350 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -88,7 +88,7 @@ class deployDataCenters(): primarycmd.clusterid = clusterId self.apiClient.createStoragePool(primarycmd) - def createpods(self, pods, zone, zoneId, networkId=None): + def createpods(self, pods, zoneId, networkId=None): if pods is None: return for pod in pods: @@ -140,7 +140,7 @@ class deployDataCenters(): secondarycmd.zoneid = zoneId self.apiClient.addSecondaryStorage(secondarycmd) - def createnetworks(self, networks, zoneId, mode): + def createnetworks(self, networks, zoneId): if networks is None: return for network in networks: @@ -162,13 +162,13 @@ class deployDataCenters(): networkId = networkcmdresponse.id return networkId - def createPhysicalNetwork(self, name, zoneid, vlan=None): + def createPhysicalNetwork(self, net, zoneid): phynet = createPhysicalNetwork.createPhysicalNetworkCmd() phynet.zoneid = zoneid - phynet.name = name - if vlan: - phynet.vlan = vlan - return self.apiClient.createPhysicalNetwork(phynet) + phynet.name = net.name + phynetwrk = self.apiClient.createPhysicalNetwork(phynet) + self.addTrafficTypes(phynetwrk.id, net.traffictypes) + return phynetwrk def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None): upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd() @@ -177,75 +177,86 @@ class deployDataCenters(): if vlan: upnet.vlan = vlan return self.apiClient.updatePhysicalNetwork(upnet) - - def configureProviders(self, phynetwrk, zone): - pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd() - pnetprov.physicalnetworkid = phynetwrk.id - pnetprov.state = "Disabled" - pnetprov.name = "VirtualRouter" - pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov) - - vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() - vrprov.nspid = pnetprovres[0].id - vrprovresponse = self.apiClient.listVirtualRouterElements(vrprov) - vrprovid = vrprovresponse[0].id - - vrconfig = \ - configureVirtualRouterElement.configureVirtualRouterElementCmd() - vrconfig.enabled = "true" - vrconfig.id = vrprovid - vrconfigresponse = \ - self.apiClient.configureVirtualRouterElement(vrconfig) - + + def enableProvider(self, provider_id): upnetprov = \ updateNetworkServiceProvider.updateNetworkServiceProviderCmd() - upnetprov.id = vrprov.nspid + upnetprov.id = provider_id upnetprov.state = "Enabled" - upnetprovresponse = \ self.apiClient.updateNetworkServiceProvider(upnetprov) - if zone.networktype == "Basic" and zone.securitygroupenabled: - sgprovider = configGenerator.provider() - sgprovider.name = "SecurityGroupProvider" - zone.providers.append(sgprovider) - - if zone.networktype == "Advanced": + def configureProviders(self, phynetwrk, providers): + """ + We will enable the virtualrouter elements for all zones. Other providers + like NetScalers, SRX, etc are explicitly added/configured + """ + + for provider in providers: pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd() pnetprov.physicalnetworkid = phynetwrk.id pnetprov.state = "Disabled" - pnetprov.name = "VpcVirtualRouter" + pnetprov.name = provider.name pnetprovres = self.apiClient.listNetworkServiceProviders(pnetprov) - - if pnetprovres and len(pnetprovres) > 0: - vpcvrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() - vpcvrprov.nspid = pnetprovres[0].id - vpcvrprovresponse = self.apiClient.listVirtualRouterElements(vpcvrprov) - vpcvrprovid = vpcvrprovresponse[0].id + + if pnetprovres and len(pnetprovres) > 0: + if provider.name == 'VirtualRouter'\ + or provider.name == 'VpcVirtualRouter': + vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() + vrprov.nspid = pnetprovres[0].id + vrprovresponse = self.apiClient.listVirtualRouterElements(vrprov) + vrprovid = vrprovresponse[0].id - vpcvrconfig = \ - configureVirtualRouterElement.configureVirtualRouterElementCmd() - vpcvrconfig.enabled = "true" - vpcvrconfig.id = vpcvrprovid - vpcvrconfigresponse = \ - self.apiClient.configureVirtualRouterElement(vpcvrconfig) - - upnetprov = \ - updateNetworkServiceProvider.updateNetworkServiceProviderCmd() - upnetprov.id = vpcvrprov.nspid - upnetprov.state = "Enabled" - upnetprovresponse = \ - self.apiClient.updateNetworkServiceProvider(upnetprov) - - def addTrafficTypes(self, physical_network_id, traffictypes=None, \ - network_labels=None): - [self.addTrafficType(physical_network_id, traffictype) for \ - traffictype in traffictypes] - - def addTrafficType(self, physical_network_id, traffictype, \ - network_label=None): + vrconfig = \ + configureVirtualRouterElement.configureVirtualRouterElementCmd() + vrconfig.enabled = "true" + vrconfig.id = vrprovid + self.apiClient.configureVirtualRouterElement(vrconfig) + self.enableProvider(pnetprovres[0].id) + elif provider.name in ['Netscaler', 'JuniperSRX', 'F5BigIp']: + netprov = addNetworkServiceProvider.addNetworkServiceProviderCmd() + netprov.name = provider.name + netprov.physicalnetworkid = phynetwrk.id + result = self.apiClient.addNetworkServiceProvider(netprov) + for device in provider.devices: + if provider.name == 'Netscaler': + dev = addNetscalerLoadBalancer.addNetscalerLoadBalancerCmd() + dev.username = device.username + dev.password = device.password + dev.networkdevicetype = device.networkdevicetype + dev.url = configGenerator.getDeviceUrl(device) + dev.physicalnetworkid = phynetwrk.id + self.apiClient.addNetscalerLoadBalancer(dev) + elif provider.name == 'JuniperSRX': + dev = addSrxFirewall.addSrxFirewallCmd() + dev.username = device.username + dev.password = device.password + dev.networkdevicetype = device.networkdevicetype + dev.url = configGenerator.getDeviceUrl(device) + dev.physicalnetworkid = phynetwrk.id + self.apiClient.addSrxFirewall(dev) + elif provider.name == 'F5BigIp': + dev = addF5LoadBalancer.addF5LoadBalancerCmd() + dev.username = device.username + dev.password = device.password + dev.networkdevicetype = device.networkdevicetype + dev.url = configGenerator.getDeviceUrl(device) + dev.physicalnetworkid = phynetwrk.id + self.apiClient.addF5LoadBalancer(dev) + else: + print "Device %s doesn't match any know provider type"%device + self.enableProvider(result.id) + + def addTrafficTypes(self, physical_network_id, traffictypes): + [self.addTrafficType(physical_network_id, traffic_type) for traffic_type in traffictypes] + + def addTrafficType(self, physical_network_id, traffictype): traffic_type = addTrafficType.addTrafficTypeCmd() traffic_type.physicalnetworkid = physical_network_id - traffic_type.traffictype = traffictype + traffic_type.traffictype = traffictype.typ + if traffictype.labeldict: + traffic_type.kvmnetworklabel = traffictype.labeldict.xen + traffic_type.xennetworklabel = traffictype.labeldict.kvm + traffic_type.vmwarenetworklabel = traffictype.labeldict.vmware return self.apiClient.addTrafficType(traffic_type) def enableZone(self, zoneid, allocation_state="Enabled"): @@ -265,31 +276,21 @@ class deployDataCenters(): createzone.securitygroupenabled = zone.securitygroupenabled createzone.networktype = zone.networktype createzone.guestcidraddress = zone.guestcidraddress - + zoneresponse = self.apiClient.createZone(createzone) zoneId = zoneresponse.id - phynetwrk = self.createPhysicalNetwork(zone.name + "-pnet", \ - zoneId) - - self.configureProviders(phynetwrk, zone) - self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=zone.vlan) + for pnet in zone.physical_networks: + phynetwrk = self.createPhysicalNetwork(pnet, zoneId) + self.configureProviders(phynetwrk, pnet.providers) + self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=zone.vlan) if zone.networktype == "Basic": - self.addTrafficTypes(phynetwrk.id, ["Guest", "Management"]) + listnetworkoffering = listNetworkOfferings.listNetworkOfferingsCmd() - listnetworkoffering = \ - listNetworkOfferings.listNetworkOfferingsCmd() - - if zone.securitygroupenabled: - listnetworkoffering.name = \ - "DefaultSharedNetworkOfferingWithSGService" - else: - # need both name and display text for single result - listnetworkoffering.name = \ - "DefaultSharedNetworkOffering" - listnetworkoffering.displaytext = \ - "Offering for Shared networks" + listnetworkoffering.name = "DefaultSharedNetscalerEIPandELBNetworkOffering" \ + if len(filter(lambda x : x.typ == 'Public', zone.physical_networks.traffictypes)) > 0 \ + else "DefaultSharedNetworkOfferingWithSGService" listnetworkofferingresponse = \ self.apiClient.listNetworkOfferings(listnetworkoffering) @@ -300,19 +301,26 @@ class deployDataCenters(): guestntwrk.zoneid = zoneId guestntwrk.networkofferingid = \ listnetworkofferingresponse[0].id - networkid = self.createnetworks([guestntwrk], zoneId, zone.networktype) - self.createpods(zone.pods, zone, zoneId, networkid) + + networkid = self.createnetworks([guestntwrk], zoneId) + self.createpods(zone.pods, zoneId, networkid) + if self.isEipElbZone(zone): + self.createVlanIpRanges(zone.networktype, zone.ipranges, \ + zoneId) if zone.networktype == "Advanced": - self.createpods(zone.pods, zone, zoneId) - self.addTrafficTypes(phynetwrk.id, ["Guest", "Public", \ - "Management"]) + self.createpods(zone.pods, zoneId) self.createVlanIpRanges(zone.networktype, zone.ipranges, \ zoneId) self.createSecondaryStorages(zone.secondaryStorages, zoneId) self.enableZone(zoneId, "Enabled") return + + def isEipElbZone(self, zone): + if zone.networktype == "Basic" and len(filter(lambda x : x.typ == 'Public', zone.physical_networks.traffictypes)) > 0: + return True + return False def registerApiKey(self): listuser = listUsers.listUsersCmd()