Updated Branches: refs/heads/requestsformarvin [created] 20ce63fcf
WIP refactor for cloudstackConnection Moving to use requests and refactoring to include post data Signed-off-by: Prasanna Santhanam <t...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/20ce63fc Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/20ce63fc Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/20ce63fc Branch: refs/heads/requestsformarvin Commit: 20ce63fcf053f82a9b090f9a33c9fdc9f2897f54 Parents: fad55c5 Author: Prasanna Santhanam <t...@apache.org> Authored: Wed Apr 17 17:06:15 2013 +0530 Committer: Prasanna Santhanam <t...@apache.org> Committed: Wed Apr 17 17:06:15 2013 +0530 ---------------------------------------------------------------------- tools/marvin/marvin/asyncJobMgr.py | 2 +- tools/marvin/marvin/cloudstackConnection.py | 239 +++++++++++----------- tools/marvin/marvin/cloudstackTestClient.py | 3 +- tools/marvin/marvin/codegenerator.py | 4 +- tools/marvin/marvin/deployDataCenter.py | 1 - tools/marvin/marvin/jsonHelper.py | 7 +- 6 files changed, 130 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/asyncJobMgr.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py index 935bebe..40304fa 100644 --- a/tools/marvin/marvin/asyncJobMgr.py +++ b/tools/marvin/marvin/asyncJobMgr.py @@ -54,7 +54,7 @@ class workThread(threading.Thread): try: self.lock.acquire() - result = self.connection.pollAsyncJob(job.jobId, job.responsecls).jobresult + result = self.connection.poll(job.jobId, job.responsecls).jobresult except cloudstackException.cloudstackAPIException, e: result = str(e) finally: http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackConnection.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index 1caeef3..06c846c 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -15,126 +15,133 @@ # specific language governing permissions and limitations # under the License. -import urllib2 +import requests import urllib -import httplib import base64 import hmac import hashlib -import json -import xml.dom.minidom -import types import time -import inspect import cloudstackException from cloudstackAPI import * import jsonHelper +from requests import ConnectionError +from requests import HTTPError +from requests import Timeout +from requests import RequestException class cloudConnection(object): - def __init__(self, mgtSvr, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600, logging=None, protocol='http', path='/client/api'): + def __init__(self, mgtSvr, port=8096, apiKey=None, securityKey=None, asyncTimeout=3600, logging=None, + scheme='http', path='client/api'): self.apiKey = apiKey self.securityKey = securityKey self.mgtSvr = mgtSvr self.port = port self.logging = logging - if protocol != 'http' and protocol != 'https': - raise ValueError("Protocol must be 'http' or 'https'.") - else: - self.protocol=protocol self.path = path - if port == 8096 or (self.apiKey == None and self.securityKey == None): - self.auth = False - else: - self.auth = True self.retries = 5 self.asyncTimeout = asyncTimeout - - def close(self): - try: - self.connection.close() - except: - pass - + self.auth = True + if port == 8096 or \ + (self.apiKey == None and self.securityKey == None): + self.auth = False + if scheme not in ['http', 'https']: + raise RequestException("Protocol must be HTTP") + self.protocol = scheme + self.baseurl = "%s://%s:%d/%s"%(self.protocol, self.mgtSvr, self.port, self.path) + def __copy__(self): - return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout, self.logging, self.protocol, self.path) - - def make_request_with_auth(self, command, requests={}): - requests["command"] = command - requests["apiKey"] = self.apiKey - requests["response"] = "json" - request = zip(requests.keys(), requests.values()) - request.sort(key=lambda x: str.lower(x[0])) - - requestUrl = "&".join(["=".join([r[0], urllib.quote_plus(str(r[1]))]) for r in request]) - hashStr = "&".join(["=".join([str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+", "%20")]) for r in request]) - - sig = urllib.quote_plus(base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip()) - requestUrl += "&signature=%s"%sig + return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout, + self.logging, self.protocol, self.path) - try: - self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr, self.port, self.path, requestUrl)) - if self.logging is not None: - self.logging.debug("sending GET request: %s"%requestUrl) - response = self.connection.read() - if self.logging is not None: - self.logging.info("got response: %s"%response) - except IOError, e: - if hasattr(e, 'reason'): - if self.logging is not None: - self.logging.critical("failed to reach %s because of %s"%(self.mgtSvr, e.reason)) - elif hasattr(e, 'code'): - if self.logging is not None: - self.logging.critical("server returned %d error code"%e.code) - raise e - except httplib.HTTPException, h: - if self.logging is not None: - self.logging.debug("encountered http Exception %s"%h.args) - if self.retries > 0: - self.retries = self.retries - 1 - self.make_request_with_auth(command, requests) - else: - self.retries = 5 - raise h - else: - return response - - def make_request_without_auth(self, command, requests={}): - requests["command"] = command - requests["response"] = "json" - requests = zip(requests.keys(), requests.values()) - requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))]) for request in requests]) - - self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr, self.port, self.path, requestUrl)) - if self.logging is not None: - self.logging.debug("sending GET request without auth: %s"%requestUrl) - response = self.connection.read() - if self.logging is not None: - self.logging.info("got response: %s"%response) - return response - - def pollAsyncJob(self, jobId, response): + def poll(self, jobid, response): + """ + polls the completion of a given jobid + @param jobid: + @param response: + @return: + """ cmd = queryAsyncJobResult.queryAsyncJobResultCmd() - cmd.jobid = jobId + cmd.jobid = jobid timeout = self.asyncTimeout - + while timeout > 0: - asyncResonse = self.make_request(cmd, response, True) - + asyncResonse = self.marvin_request(cmd, response, True) + if asyncResonse.jobstatus == 2: raise cloudstackException.cloudstackAPIException("asyncquery", asyncResonse.jobresult) elif asyncResonse.jobstatus == 1: return asyncResonse - + time.sleep(5) if self.logging is not None: - self.logging.debug("job: %s still processing, will timeout in %ds"%(jobId, timeout)) + self.logging.debug("job: %s still processing, will timeout in %ds"%(jobid, timeout)) timeout = timeout - 5 - - raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobId) - - def make_request(self, cmd, response = None, raw=False): - commandName = cmd.__class__.__name__.replace("Cmd", "") - isAsync = "false" + + raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobid) + + def sign(self, payload): + """ + signs a given request URL when the apiKey and secretKey are known + + @param payload: dict of GET params to be signed + @return: the signature of the payload + """ + params = zip(payload.keys(), map(lambda v: urllib.quote_plus(v), payload.values())) + params.sort(key=lambda k: str.lower(k[0])) + hashStr = "&".join( + ["=".join( + [str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+", "%20")] + ) for r in params] + ) + signature = urllib.quote_plus( + base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip() + ) + return signature + + def request(self, command, auth=True, payload={}, data={}): + """ + Makes requests on the `integration.api.port` + @param command: cloudstack API command name eg: deployVirtualMachineCommand + @param auth: Authentication (apikey,secretKey) => True, else False + @param payload: GET param data composed as a dictionary of key,value pairs + @param data: POST data as a dictionary + @return: + """ + payload["command"] = command + payload["response"] = "json" + + if auth: + payload["apiKey"] = self.apiKey + signature = self.sign(payload) + payload["signature"] = signature + + + try: + if data: + response = requests.get(self.baseurl, params=payload, data=data) + else: + response = requests.get(self.baseurl, params=payload) + except ConnectionError, c: + self.logging.debug("Connection refused. Reason: %s"%(self.baseurl, c)) + raise c + except HTTPError, h: + self.logging.debug("Server returned error code: %s"%h) + raise h + except Timeout, t: + self.logging.debug("Connection timed out with %s"%t) + raise t + except RequestException,r: + self.logging.debug("Error returned by server %s"%r) + raise r + else: + return response + + def sanitize_command(self, cmd): + """ + Removes None values, Validates all required params are present + @param cmd: Cmd object eg: createPhysicalNetwork + @return: + """ requests = {} required = [] for attribute in dir(cmd): @@ -145,11 +152,11 @@ class cloudConnection(object): required = getattr(cmd, attribute) else: requests[attribute] = getattr(cmd, attribute) - + + cmdname = cmd.__class__.__name__.replace("Cmd", "") for requiredPara in required: if requests[requiredPara] is None: - raise cloudstackException.cloudstackAPIException(commandName, "%s is required"%requiredPara) - '''remove none value''' + raise cloudstackException.cloudstackAPIException(cmdname, "%s is required"%requiredPara) for param, value in requests.items(): if value is None: requests.pop(param) @@ -166,28 +173,30 @@ class cloudConnection(object): for k,v in val.iteritems(): requests["%s[%d].%s"%(param,i,k)] = v i = i + 1 - - if self.logging is not None: - self.logging.info("sending command: %s %s"%(commandName, str(requests))) - result = None + return cmdname, isAsync, requests + + def marvin_request(self, cmd, data={}, response_type=None): + """ + Requester for marvin command objects + @param cmd: marvin's command from cloudstackAPI + @param data: any data to be sent in as POST + @param response_type: response type of the command in cmd + @param raw: + @return: + """ + cmdname, isAsync, payload = self.sanitize_command(cmd) + self.logging.info("sending command: %s %s"%(cmdname, str(payload))) if self.auth: - result = self.make_request_with_auth(commandName, requests) + response = self.request(cmdname, auth=True, payload=payload, data=data) else: - result = self.make_request_without_auth(commandName, requests) - - if result is None: - return None - - result = jsonHelper.getResultObj(result, response) - if raw or isAsync == "false": - return result + response = self.request(cmdname, auth=False, payload=payload, data=data) + + self.logging.info("Request: %s Response: %s"%(response.url, response.text)) + response = jsonHelper.getResultObj(response.json, response_type) + + if isAsync == "false": + return response else: - asynJobId = result.jobid - result = self.pollAsyncJob(asynJobId, response) - return result.jobresult - -if __name__ == '__main__': - xml = '<?xml version="1.0" encoding="ISO-8859-1"?><deployVirtualMachineResponse><virtualmachine><id>407</id><name>i-1-407-RS3</name><displayname>i-1-407-RS3</displayname><account>system</account><domainid>1</domainid><domain>ROOT</domain><created>2011-07-30T14:45:19-0700</created><state>Running</state><haenable>false</haenable><zoneid>1</zoneid><zonename>CA1</zonename><hostid>3</hostid><hostname>kvm-50-205</hostname><templateid>4</templateid><templatename>CentOS 5.5(64-bit) no GUI (KVM)</templatename><templatedisplaytext>CentOS 5.5(64-bit) no GUI (KVM)</templatedisplaytext><passwordenabled>false</passwordenabled><serviceofferingid>1</serviceofferingid><serviceofferingname>Small Instance</serviceofferingname><cpunumber>1</cpunumber><cpuspeed>500</cpuspeed><memory>512</memory><guestosid>112</guestosid><rootdeviceid>0</rootdeviceid><rootdevicetype>NetworkFilesystem</rootdevicetype><nic><id>380</id><networkid>203</networkid><netmask>255.255.255.0</netmask><gateway>65.19.181.1</gatew ay><ipaddress>65.19.181.110</ipaddress><isolationuri>vlan://65</isolationuri><broadcasturi>vlan://65</broadcasturi><traffictype>Guest</traffictype><type>Direct</type><isdefault>true</isdefault><macaddress>06:52:da:00:00:08</macaddress></nic><hypervisor>KVM</hypervisor></virtualmachine></deployVirtualMachineResponse>' - conn = cloudConnection(None) - - print conn.paraseReturnXML(xml, deployVirtualMachine.deployVirtualMachineResponse()) + asyncJobId = response.jobid + response = self.poll(asyncJobId, response_type) + return response.jobresult http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackTestClient.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index e4735e4..85552ed 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -111,7 +111,8 @@ class cloudstackTestClient(object): apiKey = registerUserRes.apikey securityKey = registerUserRes.secretkey - newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port, apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging) + newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port, + apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging) self.userApiClient = cloudstackAPIClient.CloudStackAPIClient(newUserConnection) self.userApiClient.connection = newUserConnection return self.userApiClient http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/codegenerator.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py index 0622e5d..b3b2a86 100644 --- a/tools/marvin/marvin/codegenerator.py +++ b/tools/marvin/marvin/codegenerator.py @@ -184,9 +184,9 @@ class codeGenerator: body += "\n" for cmdName in self.cmdsName: - body += self.space + 'def %s(self,command):\n'%cmdName + body += self.space + 'def %s(self, command, postdata={}):\n'%cmdName body += self.space + self.space + 'response = %sResponse()\n'%cmdName - body += self.space + self.space + 'response = self.connection.make_request(command, response)\n' + body += self.space + self.space + 'response = self.connection.marvin_request(command, data=postdata, response_type=response)\n' body += self.space + self.space + 'return response\n' body += '\n' http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/deployDataCenter.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index 2e270a7..d358789 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -407,7 +407,6 @@ class deployDataCenters(): logging=self.testClientLogger) if mgt.apiKey is None: apiKey, securityKey = self.registerApiKey() - self.testClient.close() self.testClient = \ cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, \ apiKey, securityKey, \ http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/jsonHelper.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py index 652cce0..37363bc 100644 --- a/tools/marvin/marvin/jsonHelper.py +++ b/tools/marvin/marvin/jsonHelper.py @@ -19,7 +19,6 @@ import cloudstackException import json import inspect from cloudstackAPI import * -import pdb class jsonLoader: '''The recursive class for building and representing objects with.''' @@ -113,12 +112,8 @@ def finalizeResultObj(result, responseName, responsecls): return result else: return result - - - + def getResultObj(returnObj, responsecls=None): - returnObj = json.loads(returnObj) - if len(returnObj) == 0: return None responseName = filter(lambda a: a!=u'cloudstack-version', returnObj.keys())[0]
