Updated Branches:
refs/heads/master 4495d456e -> 0b617a13f
Fixed Bug: 4899
Added Configuration Support to Marvin.
1. It provides the basic configuration facilities to marvin.
2. User can just add configuration files for his tests, deployment
etc, under one config folder before running their tests.
cs/tools/marvin/marvin/config.
They can remove all hard coded values from code and separate
it out as config at this location.
Either add this to the existing setup.cfg as separate section
or add new configuration.
3. This will thus removes hard coded tests and separate
data from tests.
4. This API is provided as an additional facility under
cloudstackTestClient and users can get the
configuration object as similar to apiclient,dbconnection
etc to drive their test.
5. They just add their configuration for a test,
setup etc,at one single place under configuration dir
and use "getConfigParser" API of cloudstackTestClient
It will give them "configObj".They can either pass their own
config file for parsing to "getConfig" or it will use
default config file @ config/setup.cfg.
6. They will then get the dictionary of parsed
configuration and can use it further to drive their tests or
config drive
7. Test features, can drive their setups thus removing hard coded
values. Configuration default file will be under config and as
setup.cfg.
8. Users can use their own configuration file passed to
"getConfig" API,once configObj is returned.
Signed-off-by: Santhosh Edukulla <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/0b617a13
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/0b617a13
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/0b617a13
Branch: refs/heads/master
Commit: 0b617a13fdf3b8108c2fd940e982b2e58bf9c3e7
Parents: 4495d45
Author: Santhosh Edukulla <[email protected]>
Authored: Tue Oct 29 22:30:26 2013 +0530
Committer: Prasanna Santhanam <[email protected]>
Committed: Thu Oct 31 17:54:13 2013 +0530
----------------------------------------------------------------------
tools/marvin/marvin/cloudstackTestClient.py | 18 ++
tools/marvin/marvin/config/setup.cfg | 228 ++++++++++++++++++++++
tools/marvin/marvin/configGenerator.py | 161 +++++++++++++--
tools/marvin/marvin/integration/lib/utils.py | 2 +-
4 files changed, 388 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0b617a13/tools/marvin/marvin/cloudstackTestClient.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cloudstackTestClient.py
b/tools/marvin/marvin/cloudstackTestClient.py
index be93f35..3e833c7 100644
--- a/tools/marvin/marvin/cloudstackTestClient.py
+++ b/tools/marvin/marvin/cloudstackTestClient.py
@@ -19,6 +19,10 @@ import cloudstackConnection
import asyncJobMgr
import dbConnection
from cloudstackAPI import *
+import random
+import string
+import hashlib
+from configGenerator import ConfigManager
'''
@Desc : CloudStackTestClient is encapsulated class for getting various \
@@ -48,6 +52,17 @@ class cloudstackTestClient(object):
self.createDbConnection(dbSvrDetails.dbSvr, dbSvrDetails.port,
dbSvrDetails.user,
dbSvrDetails.passwd, dbSvrDetails.db)
+ '''
+ Provides the Configuration Object to users through getConfigParser
+ The purpose of this object is to parse the config
+ and provide dictionary of the config so users can
+ use that configuration.Users can later call getConfig
+ on this object and it will return the default parsed
+ config dictionary from default configuration file,
+ they can overwrite it with providing their own
+ configuration file as well.
+ '''
+ self.configObj = ConfigManager()
self.asyncJobMgr = None
self.id = None
self.defaultWorkerThreads = defaultWorkerThreads
@@ -154,6 +169,9 @@ class cloudstackTestClient(object):
def getDbConnection(self):
return self.dbConnection
+ def getConfigParser(self):
+ return self.configObj
+
def getApiClient(self):
self.apiClient.id = self.identifier
return self.apiClient
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0b617a13/tools/marvin/marvin/config/setup.cfg
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/config/setup.cfg
b/tools/marvin/marvin/config/setup.cfg
new file mode 100644
index 0000000..216314f
--- /dev/null
+++ b/tools/marvin/marvin/config/setup.cfg
@@ -0,0 +1,228 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+{
+ "zones": [
+ {
+ "name": "Sandbox-simulator",
+ "guestcidraddress": "10.1.1.0/24",
+ "dns1": "10.147.28.6",
+ "physical_networks": [
+ {
+ "broadcastdomainrange": "Zone",
+ "vlan": "100-200",
+ "name": "Sandbox-pnet",
+ "traffictypes": [
+ {
+ "typ": "Guest"
+ },
+ {
+ "typ": "Management"
+ },
+ {
+ "typ": "Public"
+ }
+ ],
+ "providers": [
+ {
+ "broadcastdomainrange": "ZONE",
+ "name": "VirtualRouter"
+ },
+ {
+ "broadcastdomainrange": "ZONE",
+ "name": "VpcVirtualRouter"
+ },
+ {
+ "broadcastdomainrange": "ZONE",
+ "name": "InternalLbVm"
+ }
+ ],
+ "isolationmethods": [
+ "VLAN"
+ ]
+ }
+ ],
+ "ipranges": [
+ {
+ "startip": "192.168.2.2",
+ "endip": "192.168.2.200",
+ "netmask": "255.255.255.0",
+ "vlan": "50",
+ "gateway": "192.168.2.1"
+ }
+ ],
+ "networktype": "Advanced",
+ "pods": [
+ {
+ "endip": "172.16.15.200",
+ "name": "POD0",
+ "startip": "172.16.15.2",
+ "netmask": "255.255.255.0",
+ "clusters": [
+ {
+ "clustername": "C0",
+ "hypervisor": "simulator",
+ "hosts": [
+ {
+ "username": "root",
+ "url": "http://sim/c0/h0";,
+ "password": "password"
+ },
+ {
+ "username": "root",
+ "url": "http://sim/c0/h1";,
+ "password": "password"
+ }
+ ],
+ "clustertype": "CloudManaged",
+ "primaryStorages": [
+ {
+ "url":
"nfs://10.147.28.6:/export/home/sandbox/primary0",
+ "name": "PS0"
+ },
+ {
+ "url":
"nfs://10.147.28.6:/export/home/sandbox/primary1",
+ "name": "PS1"
+ }
+ ]
+ },
+ {
+ "clustername": "C1",
+ "hypervisor": "simulator",
+ "hosts": [
+ {
+ "username": "root",
+ "url": "http://sim/c1/h0";,
+ "password": "password"
+ }
+ ],
+ "clustertype": "CloudManaged",
+ "primaryStorages": [
+ {
+ "url":
"nfs://10.147.28.6:/export/home/sandbox/primary2",
+ "name": "PS2"
+ }
+ ]
+ }
+ ],
+ "gateway": "172.16.15.1"
+ }
+ ],
+ "internaldns1": "10.147.28.6",
+ "secondaryStorages": [
+ {
+ "url": "nfs://10.147.28.6:/export/home/sandbox/secondary",
+ "provider" : "NFS"
+ }
+ ]
+ }
+ ],
+ "dbSvr": {
+ "dbSvr": "localhost",
+ "passwd": "cloud",
+ "db": "cloud",
+ "port": 3306,
+ "user": "cloud"
+ },
+ "logger": [
+ {
+ "name": "TestClient",
+ "file": "/tmp/testclient.log"
+ },
+ {
+ "name": "TestCase",
+ "file": "/tmp/testcase.log"
+ }
+ ],
+ "globalConfig": [
+ {
+ "name": "network.gc.wait",
+ "value": "60"
+ },
+ {
+ "name": "storage.cleanup.interval",
+ "value": "300"
+ },
+ {
+ "name": "vm.op.wait.interval",
+ "value": "5"
+ },
+ {
+ "name": "default.page.size",
+ "value": "10000"
+ },
+ {
+ "name": "network.gc.interval",
+ "value": "60"
+ },
+ {
+ "name": "instance.name",
+ "value": "QA"
+ },
+ {
+ "name": "workers",
+ "value": "10"
+ },
+ {
+ "name": "account.cleanup.interval",
+ "value": "600"
+ },
+ {
+ "name": "guest.domain.suffix",
+ "value": "sandbox.simulator"
+ },
+ {
+ "name": "expunge.delay",
+ "value": "60"
+ },
+ {
+ "name": "vm.allocation.algorithm",
+ "value": "random"
+ },
+ {
+ "name": "expunge.interval",
+ "value": "60"
+ },
+ {
+ "name": "expunge.workers",
+ "value": "3"
+ },
+ {
+ "name": "check.pod.cidrs",
+ "value": "true"
+ },
+ {
+ "name": "secstorage.allowed.internal.sites",
+ "value": "10.147.28.0/24"
+ },
+ {
+ "name": "direct.agent.load.size",
+ "value": "1000"
+ }
+ ],
+ "mgtSvr": [
+ {
+ "mgtSvrIp": "localhost",
+ "passwd": "password",
+ "user": "root",
+ "port": 8096,
+ "hypervisor": "simulator",
+ "useHttps": "False",
+ "certCAPath": "NA",
+ "certPath": "NA"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0b617a13/tools/marvin/marvin/configGenerator.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/configGenerator.py
b/tools/marvin/marvin/configGenerator.py
index 0cfad30..631e40f 100644
--- a/tools/marvin/marvin/configGenerator.py
+++ b/tools/marvin/marvin/configGenerator.py
@@ -19,6 +19,7 @@ import json
import os
from optparse import OptionParser
import jsonHelper
+from marvin.codes import *
class managementServer(object):
@@ -82,7 +83,7 @@ class zone(object):
self.cacheStorages = []
-class traffictype(object):
+class trafficType(object):
def __init__(self, typ, labeldict=None):
self.typ = typ # Guest/Management/Public
if labeldict:
@@ -153,7 +154,7 @@ class host(object):
self.memory = None
-class physical_network(object):
+class physicalNetwork(object):
def __init__(self):
self.name = None
self.tags = []
@@ -299,6 +300,126 @@ class bigip(object):
for r in req])
+class ConfigManager(object):
+
+ '''
+ @Name: configManager
+ @Desc: 1. It provides the basic configuration facilities to marvin.
+ 2. User can just add configuration files for his tests, deployment
+ etc, under one config folder before running their tests.
+ cs/tools/marvin/marvin/config.
+ They can remove all hard coded values from code and separate
+ it out as config at this location.
+ Either add this to the existing setup.cfg as separate section
+ or add new configuration.
+ 3. This will thus removes hard coded tests and separate
+ data from tests.
+ 4. This API is provided as an additional facility under
+ cloudstackTestClient and users can get the
+ configuration object as similar to apiclient,dbconnection
+ etc to drive their test.
+ 5. They just add their configuration for a test,
+ setup etc,at one single place under configuration dir
+ and use "getConfigParser" API of cloudstackTestClient
+ It will give them "configObj".They can either pass their own
+ config file for parsing to "getConfig" or it will use
+ default config file @ config/setup.cfg.
+ 6. They will then get the dictionary of parsed
+ configuration and can use it further to drive their tests or
+ config drive
+ 7. Test features, can drive their setups thus removing hard coded
+ values. Configuration default file will be under config and as
+ setup.cfg.
+ 8. Users can use their own configuration file passed to
+ "getConfig" API,once configObj is returned.
+ '''
+
+ def __init__(self):
+ self.filePath = "config/config.cfg"
+ self.parsedDict = None
+ if self.__verifyFile(self.filePath) is not False:
+ self.parsedDict = self.__parseConfig(self.filePath)
+
+ def __parseConfig(self, file):
+ '''
+ @Name : __parseConfig
+ @Description: Parses the Input configuration Json file
+ and returns a dictionary from the file.
+ @Input : NA
+ @Output : Returns the parsed dictionary from json file
+ Returns None for invalid input or if parsing failed
+ '''
+ config_dict = None
+ try:
+ configlines = []
+ with open(file, 'r') as fp:
+ for line in fp:
+ if len(line) != 0:
+ ws = line.strip()
+ if ws[0] not in ["#"]:
+ configlines.append(ws)
+ config_dict = json.loads("\n".join(configlines))
+ except Exception, e:
+ #Will replace with log once we have logging done
+ print "\n Exception occurred under __parseConfig", e
+ finally:
+ return config_dict
+
+ def __verifyFile(self, file):
+ '''
+ @Name : __parseConfig
+ @Description: Parses the Input configuration Json file
+ and returns a dictionary from the file.
+ @Input : file NA
+ @Output : True or False based upon file input validity
+ and availability
+ '''
+ if file is None or file == '':
+ return False
+ if os.path.exists(file) is False:
+ return False
+ return True
+
+ def __getSectionData(self, return_dict, section=None):
+ '''
+ @Name: getSectionData
+ @Desc: Gets the Section data of a particular section
+ under parsed dictionary
+ @Input: Parsed Dictionary from configuration file
+ section to be returned from this dict
+ @Output:Section matching inside the parsed data
+ '''
+ if return_dict is not None:
+ inp = return_dict
+ elif self.parsedDict is None:
+ return INVALID_INPUT
+ else:
+ inp = self.parsedDict
+
+ if section is not None:
+ return inp.get(section)
+ else:
+ return inp
+
+ def getConfig(self, file_path=None, section=None):
+ '''
+ @Name: getConfig
+ @Desc : Parses and converts the given configuration file to dictionary
+ @Input : file_path: path where the configuration needs to be passed
+ section: specific section inside the file
+ @Output: INVALID_INPUT: This value is returned if the input
+ is invalid or not able to be parsed
+ Parsed configuration dictionary from json file
+ '''
+ ret = None
+ if file not in [None, '']:
+ if self.__verifyFile(file_path) is False:
+ return INVALID_INPUT
+ else:
+ ret = self.__parseConfig(file_path)
+ return self.__getSectionData(ret, section)
+
+
def getDeviceUrl(obj):
req = zip(obj.__dict__.keys(), obj.__dict__.values())
if obj.hostname:
@@ -309,7 +430,7 @@ def getDeviceUrl(obj):
return None
-def describe_setup_in_basic_mode():
+def descSetupInBasicMode():
'''sample code to generate setup configuration file'''
zs = cloudstackConfiguration()
@@ -328,9 +449,9 @@ def describe_setup_in_basic_mode():
sgprovider.broadcastdomainrange = 'Pod'
sgprovider.name = 'SecurityGroupProvider'
- pn = physical_network()
+ pn = physicalNetwork()
pn.name = "test-network"
- pn.traffictypes = [traffictype("Guest"), traffictype("Management")]
+ pn.traffictypes = [trafficType("Guest"), trafficType("Management")]
pn.providers.append(sgprovider)
z.physical_networks.append(pn)
@@ -428,7 +549,7 @@ def describe_setup_in_basic_mode():
return zs
-def describe_setup_in_eip_mode():
+def descSetupInEipMode():
"""
Setting up an EIP/ELB enabled zone with netscaler provider
"""
@@ -462,12 +583,12 @@ def describe_setup_in_eip_mode():
ns.hostname = '10.147.40.100'
nsprovider.devices.append(ns)
- pn = physical_network()
+ pn = physicalNetwork()
pn.name = "test-network"
- pn.traffictypes = [traffictype("Guest",
+ pn.traffictypes = [trafficType("Guest",
{"xen": "cloud-guest"}),
- traffictype("Management"),
- traffictype("Public", {"xen": "cloud-public"})]
+ trafficType("Management"),
+ trafficType("Public", {"xen": "cloud-public"})]
pn.providers.extend([sgprovider, nsprovider])
z.physical_networks.append(pn)
@@ -563,7 +684,7 @@ def describe_setup_in_eip_mode():
return zs
-def describe_setup_in_advanced_mode():
+def descSetupInAdvancedMode():
'''sample code to generate setup configuration file'''
zs = cloudstackConfiguration()
@@ -578,10 +699,10 @@ def describe_setup_in_advanced_mode():
z.guestcidraddress = "10.1.1.0/24"
z.vlan = "100-2000"
- pn = physical_network()
+ pn = physicalNetwork()
pn.name = "test-network"
- pn.traffictypes = [traffictype("Guest"), traffictype("Management"),
- traffictype("Public")]
+ pn.traffictypes = [trafficType("Guest"), trafficType("Management"),
+ trafficType("Public")]
vpcprovider = provider('VpcVirtualRouter')
@@ -697,7 +818,7 @@ def describe_setup_in_advanced_mode():
'''sample code to generate setup configuration file'''
-def describe_setup_in_advancedsg_mode():
+def descSetupInAdvancedsgMode():
zs = cloudstackConfiguration()
for l in range(1):
@@ -711,9 +832,9 @@ def describe_setup_in_advancedsg_mode():
z.vlan = "100-2000"
z.securitygroupenabled = "true"
- pn = physical_network()
+ pn = physicalNetwork()
pn.name = "test-network"
- pn.traffictypes = [traffictype("Guest"), traffictype("Management")]
+ pn.traffictypes = [trafficType("Guest"), trafficType("Management")]
#If security groups are reqd
sgprovider = provider()
@@ -864,10 +985,10 @@ by default is ./datacenterCfg")
if options.inputfile:
config = getSetupConfig(options.inputfile)
if options.advanced:
- config = describe_setup_in_advanced_mode()
+ config = descSetupInAdvancedMode()
elif options.advancedsg:
- config = describe_setup_in_advancedsg_mode()
+ config = descSetupInAdvancedsgMode()
else:
- config = describe_setup_in_basic_mode()
+ config = descSetupInBasicMode()
generate_setup_config(config, options.output)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0b617a13/tools/marvin/marvin/integration/lib/utils.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/integration/lib/utils.py
b/tools/marvin/marvin/integration/lib/utils.py
index 7d662af..b6e38ec 100644
--- a/tools/marvin/marvin/integration/lib/utils.py
+++ b/tools/marvin/marvin/integration/lib/utils.py
@@ -323,7 +323,7 @@ def is_snapshot_on_nfs(apiclient, dbconn, config, zoneid,
snapshotid):
def validateList(inp):
'''
- @name: validateList
+ @name: validateList
@Description: 1. A utility function to validate
whether the input passed is a list
2. The list is empty or not
