CLOUDSTACK-1037: Make cloudmonkey awesome-er Signed-off-by: Rohit Yadav <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/631b6fd4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/631b6fd4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/631b6fd4 Branch: refs/heads/gslb Commit: 631b6fd46a4371425991127cf19379feea16f869 Parents: 77e3aad Author: Rohit Yadav <[email protected]> Authored: Fri Feb 1 14:34:49 2013 -0800 Committer: Rohit Yadav <[email protected]> Committed: Fri Feb 1 14:34:49 2013 -0800 ---------------------------------------------------------------------- tools/cli/cloudmonkey/__init__.py | 2 +- tools/cli/cloudmonkey/cachegen.py | 97 ----------- tools/cli/cloudmonkey/cachemaker.py | 145 ++++++++++++++++ tools/cli/cloudmonkey/cloudmonkey.py | 256 ++++++++--------------------- tools/cli/cloudmonkey/common.py | 59 ------- tools/cli/cloudmonkey/config.py | 115 +++++++++++++ tools/cli/cloudmonkey/lexer.py | 121 -------------- tools/cli/cloudmonkey/printer.py | 133 +++++++++++++++ tools/cli/cloudmonkey/requester.py | 153 +++++++++++++++++ tools/cli/pom.xml | 4 +- 10 files changed, 615 insertions(+), 470 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/__init__.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/__init__.py b/tools/cli/cloudmonkey/__init__.py index 9674777..e4c4e6d 100644 --- a/tools/cli/cloudmonkey/__init__.py +++ b/tools/cli/cloudmonkey/__init__.py @@ -16,6 +16,6 @@ # under the License. try: - from common import __version__ + from config import __version__ except ImportError, e: print e http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/cachegen.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/cachegen.py b/tools/cli/cloudmonkey/cachegen.py deleted file mode 100644 index 509c0c6..0000000 --- a/tools/cli/cloudmonkey/cachegen.py +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -try: - import re - from marvin.cloudstackAPI import * - from marvin import cloudstackAPI -except ImportError, e: - import sys - print "ImportError", e - sys.exit(1) - -completions = cloudstackAPI.__all__ - - -def get_api_module(api_name, api_class_strs=[]): - try: - api_mod = __import__("marvin.cloudstackAPI.%s" % api_name, - globals(), locals(), api_class_strs, -1) - except ImportError, e: - print "Error: API not found", e - return None - return api_mod - - -def main(): - """ - cachegen.py creates a precached dictionary for all the available verbs in - the predefined grammar of cloudmonkey, it dumps the dictionary in an - importable python module. This way we cheat on the runtime overhead of - completing commands and help docs. This reduces the overall search and - cache_miss (computation) complexity from O(n) to O(1) for any valid cmd. - """ - pattern = re.compile("[A-Z]") - verbs = list(set([x[:pattern.search(x).start()] for x in completions - if pattern.search(x) is not None]).difference(['cloudstack'])) - # datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}} - cache_verbs = {} - for verb in verbs: - completions_found = filter(lambda x: x.startswith(verb), completions) - cache_verbs[verb] = {} - for api_name in completions_found: - api_cmd_str = "%sCmd" % api_name - api_mod = get_api_module(api_name, [api_cmd_str]) - if api_mod is None: - continue - try: - api_cmd = getattr(api_mod, api_cmd_str)() - required = api_cmd.required - doc = api_mod.__doc__ - except AttributeError, e: - print "Error: API attribute %s not found!" % e - params = filter(lambda x: '__' not in x and 'required' not in x, - dir(api_cmd)) - if len(required) > 0: - doc += "\nRequired args: %s" % " ".join(required) - doc += "\nArgs: %s" % " ".join(params) - api_name_lower = api_name.replace(verb, '').lower() - cache_verbs[verb][api_name_lower] = [api_name, params, doc, - required] - f = open("precache.py", "w") - f.write("""# Auto-generated code by cachegen.py -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License.""") - f.write("\nprecached_verbs = %s" % cache_verbs) - f.close() - -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/cachemaker.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/cachemaker.py b/tools/cli/cloudmonkey/cachemaker.py new file mode 100644 index 0000000..264fa60 --- /dev/null +++ b/tools/cli/cloudmonkey/cachemaker.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + import json + import os + import re + + from requester import monkeyrequest +except ImportError, e: + import sys + print "ImportError", e + sys.exit(1) + + +def getvalue(dictionary, key): + if key in dictionary: + return dictionary[key] + else: + return None + + +def csv_str_as_list(string): + if string is not None: + return filter(lambda x: x.strip() != '', string.split(',')) + else: + return [] + + +def cachegen_from_file(json_file): + f = open(json_file, 'r') + data = f.read() + f.close() + try: + apis = json.loads(data) + except ValueError, e: + print "Error processing json in cachegen()", e + return cachegen(apis) + + +def cachegen(apis): + pattern = re.compile("[A-Z]") + responsekey = filter(lambda x: 'response' in x, apis.keys()) + + if len(responsekey) == 0: + print "[cachegen] Invalid dictionary, has no response" + return None + if len(responsekey) != 1: + print "[cachegen] Multiple responsekeys, chosing first one" + + responsekey = responsekey[0] + verbs = set() + cache = {} + cache['count'] = getvalue(apis[responsekey], 'count') + + for api in getvalue(apis[responsekey], 'api'): + name = getvalue(api, 'name') + response = getvalue(api, 'response') + + idx = pattern.search(name).start() + verb = name[:idx] + subject = name[idx:] + + apidict = {} + apidict['name'] = name + apidict['description'] = getvalue(api, 'description') + apidict['isasync'] = getvalue(api, 'isasync') + apidict['related'] = csv_str_as_list(getvalue(api, 'related')) + + required = [] + apiparams = [] + for param in getvalue(api, 'params'): + apiparam = {} + apiparam['name'] = getvalue(param, 'name') + apiparam['description'] = getvalue(param, 'description') + apiparam['required'] = (getvalue(param, 'required') is True) + apiparam['length'] = int(getvalue(param, 'length')) + apiparam['type'] = getvalue(param, 'type') + apiparam['related'] = csv_str_as_list(getvalue(param, 'related')) + if apiparam['required']: + required.append(apiparam['name']) + apiparams.append(apiparam) + + apidict['requiredparams'] = required + apidict['params'] = apiparams + apidict['response'] = getvalue(api, 'response') + cache[verb] = {subject: apidict} + verbs.add(verb) + + cache['verbs'] = list(verbs) + return cache + + +def main(json_file): + """ + cachegen.py creates a precache datastore of all available apis of + CloudStack and dumps the precache dictionary in an + importable python module. This way we cheat on the runtime overhead of + completing commands and help docs. This reduces the overall search and + cache_miss (computation) complexity from O(n) to O(1) for any valid cmd. + """ + f = open("precache.py", "w") + f.write("""# -*- coding: utf-8 -*- +# Auto-generated code by cachegen.py +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License.""") + f.write("\nprecache = %s" % cachegen_from_file(json_file)) + f.close() + +if __name__ == "__main__": + json_file = 'listapis.json' + if os.path.exists(json_file): + main(json_file) + else: + pass + #print "[ERROR] cli:cachegen is unable to locate %s" % json_file http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/cloudmonkey.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/cloudmonkey.py b/tools/cli/cloudmonkey/cloudmonkey.py index 646ad40..49f3ee9 100644 --- a/tools/cli/cloudmonkey/cloudmonkey.py +++ b/tools/cli/cloudmonkey/cloudmonkey.py @@ -20,7 +20,6 @@ try: import atexit import cmd - import codecs import json import logging import os @@ -28,18 +27,16 @@ try: import re import shlex import sys - import time - import types - from ConfigParser import ConfigParser, SafeConfigParser from urllib2 import HTTPError, URLError from httplib import BadStatusLine - from prettytable import PrettyTable - from common import __version__, config_dir, config_file, config_fields - from common import precached_verbs - from lexer import monkeyprint + from config import __version__, config_file + from config import precached_verbs, read_config, write_config + from printer import monkeyprint + from requester import monkeyrequest + from prettytable import PrettyTable from marvin.cloudstackConnection import cloudConnection from marvin.cloudstackException import cloudstackAPIException from marvin.cloudstackAPI import * @@ -70,8 +67,7 @@ class CloudMonkeyShell(cmd.Cmd, object): intro = ("â Apache CloudStack ðµ cloudmonkey " + __version__ + ". Type help or ? to list commands.\n") ruler = "=" - config_dir = config_dir - config_file = config_file + apicache = {} # datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}} cache_verbs = precached_verbs config_options = [] @@ -79,31 +75,8 @@ class CloudMonkeyShell(cmd.Cmd, object): def __init__(self, pname, verbs): self.program_name = pname self.verbs = verbs - global config_fields - first_time = False - if not os.path.exists(self.config_dir): - os.makedirs(self.config_dir) - if os.path.exists(self.config_file): - config = self.read_config() - else: - first_time = True - config = self.write_config(first_time) - - for section in config_fields.keys(): - for key in config_fields[section].keys(): - try: - self.config_options.append(key) - setattr(self, key, config.get(section, key)) - except Exception: - print "Please fix `%s` in %s" % (key, self.config_file) - sys.exit() - - if first_time: - print "Welcome! Using `set` configure the necessary settings:" - print " ".join(sorted(self.config_options)) - print "Config file:", self.config_file - print "For debugging, tail -f", self.log_file, "\n" + self.config_options = read_config(self.get_attr, self.set_attr) self.prompt = self.prompt.strip() + " " # Cosmetic fix for prompt logging.basicConfig(filename=self.log_file, @@ -111,40 +84,20 @@ class CloudMonkeyShell(cmd.Cmd, object): logger.debug("Loaded config fields:\n%s" % map(lambda x: "%s=%s" % (x, getattr(self, x)), self.config_options)) - cmd.Cmd.__init__(self) - if not os.path.exists(self.config_file): - config = self.write_config() try: if os.path.exists(self.history_file): readline.read_history_file(self.history_file) atexit.register(readline.write_history_file, self.history_file) except IOError: - print("Error: history support") + monkeyprint("Error: history support") - def read_config(self): - config = ConfigParser() - try: - with open(self.config_file, 'r') as cfg: - config.readfp(cfg) - except IOError, e: - self.print_shell("Error: config_file not found", e) - return config - - def write_config(self, first_time=False): - global config_fields - config = ConfigParser() - for section in config_fields.keys(): - config.add_section(section) - for key in config_fields[section].keys(): - if first_time: - config.set(section, key, config_fields[section][key]) - else: - config.set(section, key, getattr(self, key)) - with open(self.config_file, 'w') as cfg: - config.write(cfg) - return config + def get_attr(self, field): + return getattr(self, field) + + def set_attr(self, field, value): + return setattr(self, field, value) def emptyline(self): pass @@ -158,20 +111,8 @@ class CloudMonkeyShell(cmd.Cmd, object): except KeyboardInterrupt: print("^C") - def print_shell(self, *args): - output = "" - try: - for arg in args: - arg = str(arg) - if isinstance(type(args), types.NoneType): - continue - output += arg - if self.color == 'true': - monkeyprint(output) - else: - print output - except Exception, e: - self.print_shell("Error: " + e) + def monkeyprint(self, *args): + monkeyprint((self.color == 'true'), *args) def print_result(self, result, result_filter=None): if result is None or len(result) == 0: @@ -179,7 +120,7 @@ class CloudMonkeyShell(cmd.Cmd, object): def printer_helper(printer, toprow): if printer: - self.print_shell(printer) + self.monkeyprint(printer) return PrettyTable(toprow) def print_result_tabular(result, result_filter=None): @@ -200,16 +141,16 @@ class CloudMonkeyShell(cmd.Cmd, object): if printer and row: printer.add_row(row) if printer: - self.print_shell(printer) + self.monkeyprint(printer) def print_result_as_dict(result, result_filter=None): for key in sorted(result.keys(), key=lambda x: x not in ['id', 'count', 'name'] and x): if not (isinstance(result[key], list) or isinstance(result[key], dict)): - self.print_shell("%s = %s" % (key, result[key])) + self.monkeyprint("%s = %s" % (key, result[key])) else: - self.print_shell(key + ":") + self.monkeyprint(key + ":") self.print_result(result[key], result_filter) def print_result_as_list(result, result_filter=None): @@ -220,7 +161,7 @@ class CloudMonkeyShell(cmd.Cmd, object): break self.print_result(node) if len(result) > 1: - self.print_shell(self.ruler * 80) + self.monkeyprint(self.ruler * 80) if isinstance(result, dict): print_result_as_dict(result, result_filter) @@ -229,92 +170,18 @@ class CloudMonkeyShell(cmd.Cmd, object): elif isinstance(result, str): print result elif not (str(result) is None): - self.print_shell(result) - - def make_request(self, command, requests={}, isAsync=False): - conn = cloudConnection(self.host, port=int(self.port), - apiKey=self.apikey, securityKey=self.secretkey, - asyncTimeout=self.timeout, logging=logger, - protocol=self.protocol, path=self.path) - response = None - logger.debug("====START Request====") - logger.debug("Requesting command=%s, args=%s" % (command, requests)) - try: - response = conn.make_request_with_auth(command, requests) - except cloudstackAPIException, e: - self.print_shell("API Error:", e) - except HTTPError, e: - self.print_shell(e) - except (URLError, BadStatusLine), e: - self.print_shell("Connection Error:", e) - logger.debug("====END Request====\n") - - def process_json(response): - try: - response = json.loads(str(response)) - except ValueError, e: - pass - return response - - response = process_json(response) - if response is None: - return - - isAsync = isAsync and (self.asyncblock == "true") - responsekey = filter(lambda x: 'response' in x, response.keys())[0] - if isAsync and 'jobid' in response[responsekey]: - jobId = response[responsekey]['jobid'] - command = "queryAsyncJobResult" - requests = {'jobid': jobId} - timeout = int(self.timeout) - pollperiod = 3 - progress = 1 - while timeout > 0: - print '\r' + '.' * progress, - sys.stdout.flush() - response = process_json(conn.make_request_with_auth(command, - requests)) - responsekeys = filter(lambda x: 'response' in x, - response.keys()) - if len(responsekeys) < 1: - continue - result = response[responsekeys[0]] - jobstatus = result['jobstatus'] - if jobstatus == 2: - jobresult = result["jobresult"] - self.print_shell("\rAsync query failed for jobid", - jobId, "\nError", jobresult["errorcode"], - jobresult["errortext"]) - return - elif jobstatus == 1: - print '\r', - return response - time.sleep(pollperiod) - timeout = timeout - pollperiod - progress += 1 - logger.debug("job: %s to timeout in %ds" % (jobId, timeout)) - self.print_shell("Error:", "Async query timeout for jobid", jobId) - + self.monkeyprint(result) + + def make_request(self, command, args={}, isasync=False): + response, error = monkeyrequest(command, args, isasync, + self.asyncblock, logger, + self.host, self.port, + self.apikey, self.secretkey, + self.timeout, self.protocol, self.path) + if error is not None: + self.monkeyprint(error) return response - def get_api_module(self, api_name, api_class_strs=[]): - try: - api_mod = __import__("marvin.cloudstackAPI.%s" % api_name, - globals(), locals(), api_class_strs, -1) - except ImportError, e: - self.print_shell("Error: API not found", e) - return None - return api_mod - - def pipe_runner(self, args): - if args.find(' |') > -1: - pname = self.program_name - if '.py' in pname: - pname = "python " + pname - self.do_shell("%s %s" % (pname, args)) - return True - return False - def default(self, args): if self.pipe_runner(args): return @@ -340,31 +207,20 @@ class CloudMonkeyShell(cmd.Cmd, object): map(lambda x: x.strip(), args_dict.pop('filter').split(','))) - api_cmd_str = "%sCmd" % api_name - api_mod = self.get_api_module(api_name, [api_cmd_str]) - if api_mod is None: - return - - try: - api_cmd = getattr(api_mod, api_cmd_str) - except AttributeError, e: - self.print_shell("Error: API attribute %s not found!" % e) - return - for attribute in args_dict.keys(): setattr(api_cmd, attribute, args_dict[attribute]) - command = api_cmd() - missing_args = filter(lambda x: x not in args_dict.keys(), - command.required) + #command = api_cmd() + #missing_args = filter(lambda x: x not in args_dict.keys(), + # command.required) - if len(missing_args) > 0: - self.print_shell("Missing arguments: ", ' '.join(missing_args)) - return + #if len(missing_args) > 0: + # self.monkeyprint("Missing arguments: ", ' '.join(missing_args)) + # return isAsync = False - if "isAsync" in dir(command): - isAsync = (command.isAsync == "true") + #if "isAsync" in dir(command): + # isAsync = (command.isAsync == "true") result = self.make_request(api_name, args_dict, isAsync) if result is None: @@ -375,7 +231,7 @@ class CloudMonkeyShell(cmd.Cmd, object): self.print_result(result[responsekey], field_filter) print except Exception as e: - self.print_shell("ð Error on parsing and printing", e) + self.monkeyprint("ð Error on parsing and printing", e) def completedefault(self, text, line, begidx, endidx): partitions = line.partition(" ") @@ -403,6 +259,17 @@ class CloudMonkeyShell(cmd.Cmd, object): autocompletions.append("filter=") return [s for s in autocompletions if s.startswith(search_string)] + def do_sync(self, args): + """ + Asks cloudmonkey to discovery and sync apis available on user specified + CloudStack host server which has the API discovery plugin, on failure + it rollbacks last datastore or api precached datastore. + """ + response = self.make_request("listApis") + f = open('test.json', "w") + f.write(json.dumps(response)) + f.close() + def do_api(self, args): """ Make raw api calls. Syntax: api <apiName> <args>=<values>. @@ -413,7 +280,7 @@ class CloudMonkeyShell(cmd.Cmd, object): if len(args) > 0: return self.default(args) else: - self.print_shell("Please use a valid syntax") + self.monkeyprint("Please use a valid syntax") def complete_api(self, text, line, begidx, endidx): mline = line.partition(" ")[2] @@ -435,7 +302,7 @@ class CloudMonkeyShell(cmd.Cmd, object): key, value = (args[0], args[2]) setattr(self, key, value) # keys and attributes should have same names self.prompt = self.prompt.strip() + " " # prompt fix - self.write_config() + write_config(self.get_attr) def complete_set(self, text, line, begidx, endidx): mline = line.partition(" ")[2] @@ -443,6 +310,15 @@ class CloudMonkeyShell(cmd.Cmd, object): return [s[offs:] for s in self.config_options if s.startswith(mline)] + def pipe_runner(self, args): + if args.find(' |') > -1: + pname = self.program_name + if '.py' in pname: + pname = "python " + pname + self.do_shell("%s %s" % (pname, args)) + return True + return False + def do_shell(self, args): """ Execute shell commands using shell <command> or !<command> @@ -474,9 +350,9 @@ class CloudMonkeyShell(cmd.Cmd, object): subject = fields[2].partition(" ")[0] if subject in self.cache_verbs[verb]: - self.print_shell(self.cache_verbs[verb][subject][2]) + self.monkeyprint(self.cache_verbs[verb][subject][2]) else: - self.print_shell("Error: no such api (%s) on %s" % + self.monkeyprint("Error: no such api (%s) on %s" % (subject, verb)) def complete_help(self, text, line, begidx, endidx): @@ -500,7 +376,7 @@ class CloudMonkeyShell(cmd.Cmd, object): """ Quit CloudMonkey CLI """ - self.print_shell("Bye!") + self.monkeyprint("Bye!") return self.do_EOF(args) def do_EOF(self, args): @@ -526,10 +402,10 @@ def main(): helpdoc = res[2] args = args_partition[2] except KeyError, e: - self.print_shell("Error: invalid %s api arg" % verb, e) + self.monkeyprint("Error: invalid %s api arg" % verb, e) return if ' --help' in args or ' -h' in args: - self.print_shell(helpdoc) + self.monkeyprint(helpdoc) return self.default("%s %s" % (cmd, args)) return grammar_closure http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/common.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/common.py b/tools/cli/cloudmonkey/common.py deleted file mode 100644 index 05767a5..0000000 --- a/tools/cli/cloudmonkey/common.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Use following rules for versioning: -# <cloudstack version>-<cli increment, starts from 0> -__version__ = "4.1.0-0" - -try: - from os.path import expanduser - import os - from precache import precached_verbs -except ImportError, e: - precached_verbs = {} - -param_type = ['boolean', 'date', 'float', 'integer', 'short', 'list', - 'long', 'object', 'map', 'string', 'tzdate', 'uuid'] - -config_dir = expanduser('~/.cloudmonkey') -config_file = expanduser(config_dir + '/config') - -# cloudmonkey config fields -config_fields = {'core': {}, 'ui': {}, 'server': {}, 'user': {}} - -# core -config_fields['core']['cache_file'] = expanduser(config_dir + '/cache') -config_fields['core']['history_file'] = expanduser(config_dir + '/history') -config_fields['core']['log_file'] = expanduser(config_dir + '/log') - -# ui -config_fields['ui']['color'] = 'true' -config_fields['ui']['prompt'] = '> ' -config_fields['ui']['tabularize'] = 'false' - -# server -config_fields['server']['asyncblock'] = 'true' -config_fields['server']['host'] = 'localhost' -config_fields['server']['path'] = '/client/api' -config_fields['server']['port'] = '8080' -config_fields['server']['protocol'] = 'http' -config_fields['server']['timeout'] = '3600' - -# user -config_fields['user']['apikey'] = '' -config_fields['user']['secretkey'] = '' http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/config.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/config.py b/tools/cli/cloudmonkey/config.py new file mode 100644 index 0000000..b6de641 --- /dev/null +++ b/tools/cli/cloudmonkey/config.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Use following rules for versioning: +# <cloudstack version>-<cli increment, starts from 0> +__version__ = "4.1.0-0" + +try: + import os + import sys + + from ConfigParser import ConfigParser, SafeConfigParser + from os.path import expanduser + from precache import precached_verbs +except ImportError, e: + precached_verbs = {} + +param_type = ['boolean', 'date', 'float', 'integer', 'short', 'list', + 'long', 'object', 'map', 'string', 'tzdate', 'uuid'] + +iterable_type = ['set', 'list', 'object'] + +config_dir = expanduser('~/.cloudmonkey') +config_file = expanduser(config_dir + '/config') + +# cloudmonkey config fields +config_fields = {'core': {}, 'ui': {}, 'server': {}, 'user': {}} + +# core +config_fields['core']['cache_file'] = expanduser(config_dir + '/cache') +config_fields['core']['history_file'] = expanduser(config_dir + '/history') +config_fields['core']['log_file'] = expanduser(config_dir + '/log') + +# ui +config_fields['ui']['color'] = 'true' +config_fields['ui']['prompt'] = '> ' +config_fields['ui']['tabularize'] = 'false' + +# server +config_fields['server']['asyncblock'] = 'true' +config_fields['server']['host'] = 'localhost' +config_fields['server']['path'] = '/client/api' +config_fields['server']['port'] = '8080' +config_fields['server']['protocol'] = 'http' +config_fields['server']['timeout'] = '3600' + +# user +config_fields['user']['apikey'] = '' +config_fields['user']['secretkey'] = '' + + +def write_config(get_attr, first_time=False): + global config_fields, config_file + config = ConfigParser() + for section in config_fields.keys(): + config.add_section(section) + for key in config_fields[section].keys(): + if first_time: + config.set(section, key, config_fields[section][key]) + else: + config.set(section, key, get_attr(key)) + with open(config_file, 'w') as cfg: + config.write(cfg) + return config + + +def read_config(get_attr, set_attr): + global config_fields, config_dir, config_file + if not os.path.exists(config_dir): + os.makedirs(config_dir) + + config_options = reduce(lambda x, y: x + y, map(lambda x: + config_fields[x].keys(), config_fields.keys())) + + if os.path.exists(config_file): + config = ConfigParser() + try: + with open(config_file, 'r') as cfg: + config.readfp(cfg) + except IOError, e: + print "Error: config_file not found", e + else: + config = write_config(get_attr, True) + print "Welcome! Using `set` configure the necessary settings:" + print " ".join(sorted(config_options)) + print "Config file:", config_file + + missing_keys = [] + for section in config_fields.keys(): + for key in config_fields[section].keys(): + try: + set_attr(key, config.get(section, key)) + except Exception: + missing_keys.appned(key) + + if len(missing_keys) > 0: + print "Please fix `%s` in %s" % (key, config_file) + sys.exit() + + return config_options http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/lexer.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/lexer.py b/tools/cli/cloudmonkey/lexer.py deleted file mode 100644 index 373c9f2..0000000 --- a/tools/cli/cloudmonkey/lexer.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -try: - from pygments import highlight - from pygments.console import ansiformat - from pygments.formatter import Formatter - from pygments.formatters import Terminal256Formatter - from pygments.lexer import bygroups, include, RegexLexer - from pygments.token import * - - import sys -except ImportError, e: - print e - - -MONKEY_COLORS = { - Token: '', - Whitespace: 'reset', - Text: 'reset', - - Name: 'green', - Operator: 'teal', - Operator.Word: 'lightgray', - String: 'purple', - - Keyword: '_red_', - Error: 'red', - Literal: 'yellow', - Number: 'blue', -} - - -def get_colorscheme(): - return MONKEY_COLORS - - -class MonkeyLexer(RegexLexer): - keywords = ['[a-z]*id', '^[a-z A-Z]*:'] - attributes = ['[Tt]rue', '[Ff]alse'] - params = ['[a-z]*[Nn]ame', 'type', '[Ss]tate'] - - uuid_rgx = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' - date_rgx = r'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}-[0-9]{4}' - - def makelistre(lis): - return r'(' + r'|'.join(lis) + r')' - - tokens = { - 'root': [ - (r' ', Whitespace), - (date_rgx, Number), - (uuid_rgx, Literal), - (r'(?:\b\d+\b(?:-\b\d+|%)?)', Number), - (r'^[-=]*\n', Operator.Word), - (r'Error', Error), - (makelistre(keywords), Keyword), - (makelistre(attributes), Literal), - (makelistre(params) + r'( = )(.*)', bygroups(Name, Operator, - String)), - (makelistre(params), Name), - (r'(^[a-zA-Z]* )(=)', bygroups(Name, Operator)), - (r'\S+', Text), - ] - } - - def analyse_text(text): - npos = text.find('\n') - if npos < 3: - return False - return text[0] == '[' and text[npos - 1] == ']' - - -class MonkeyFormatter(Formatter): - def __init__(self, **options): - Formatter.__init__(self, **options) - self.colorscheme = get_colorscheme() - - def format(self, tokensource, outfile): - self.encoding = outfile.encoding - return Formatter.format(self, tokensource, outfile) - - def format_unencoded(self, tokensource, outfile): - for ttype, value in tokensource: - color = self.colorscheme.get(ttype) - while color is None: - ttype = ttype[:-1] - color = self.colorscheme.get(ttype) - if color: - spl = value.split('\n') - for line in spl[:-1]: - if line: - outfile.write(ansiformat(color, line)) - outfile.write('\n') - if spl[-1]: - outfile.write(ansiformat(color, spl[-1])) - else: - outfile.write(value) - - -def monkeyprint(text): - fmter = MonkeyFormatter() - lexer = MonkeyLexer() - lexer.encoding = 'utf-8' - fmter.encoding = 'utf-8' - highlight(text, lexer, fmter, sys.stdout) http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/printer.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/printer.py b/tools/cli/cloudmonkey/printer.py new file mode 100644 index 0000000..40f8473 --- /dev/null +++ b/tools/cli/cloudmonkey/printer.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + from pygments import highlight + from pygments.console import ansiformat + from pygments.formatter import Formatter + from pygments.formatters import Terminal256Formatter + from pygments.lexer import bygroups, include, RegexLexer + from pygments.token import * + + import sys + import types +except ImportError, e: + print e + + +MONKEY_COLORS = { + Token: '', + Whitespace: 'reset', + Text: 'reset', + + Name: 'green', + Operator: 'teal', + Operator.Word: 'lightgray', + String: 'purple', + + Keyword: '_red_', + Error: 'red', + Literal: 'yellow', + Number: 'blue', +} + + +def get_colorscheme(): + return MONKEY_COLORS + + +class MonkeyLexer(RegexLexer): + keywords = ['[a-z]*id', '^[a-z A-Z]*:'] + attributes = ['[Tt]rue', '[Ff]alse'] + params = ['[a-z]*[Nn]ame', 'type', '[Ss]tate'] + + uuid_rgx = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + date_rgx = r'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}-[0-9]{4}' + + def makelistre(lis): + return r'(' + r'|'.join(lis) + r')' + + tokens = { + 'root': [ + (r' ', Whitespace), + (date_rgx, Number), + (uuid_rgx, Literal), + (r'(?:\b\d+\b(?:-\b\d+|%)?)', Number), + (r'^[-=]*\n', Operator.Word), + (r'Error', Error), + (makelistre(keywords), Keyword), + (makelistre(attributes), Literal), + (makelistre(params) + r'( = )(.*)', bygroups(Name, Operator, + String)), + (makelistre(params), Name), + (r'(^[a-zA-Z]* )(=)', bygroups(Name, Operator)), + (r'\S+', Text), + ] + } + + def analyse_text(text): + npos = text.find('\n') + if npos < 3: + return False + return text[0] == '[' and text[npos - 1] == ']' + + +class MonkeyFormatter(Formatter): + def __init__(self, **options): + Formatter.__init__(self, **options) + self.colorscheme = get_colorscheme() + + def format(self, tokensource, outfile): + return Formatter.format(self, tokensource, outfile) + + def format_unencoded(self, tokensource, outfile): + for ttype, value in tokensource: + color = self.colorscheme.get(ttype) + while color is None: + ttype = ttype[:-1] + color = self.colorscheme.get(ttype) + if color: + spl = value.split('\n') + for line in spl[:-1]: + if line: + outfile.write(ansiformat(color, line)) + outfile.write('\n') + if spl[-1]: + outfile.write(ansiformat(color, spl[-1])) + else: + outfile.write(value) + + +def monkeyprint(color=True, *args): + fmter = MonkeyFormatter() + lexer = MonkeyLexer() + lexer.encoding = 'utf-8' + fmter.encoding = 'utf-8' + output = "" + try: + for arg in args: + if isinstance(type(arg), types.NoneType): + continue + output += str(arg) + except Exception, e: + print e + + if color: + highlight(output, lexer, fmter, sys.stdout) + else: + print output http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/cloudmonkey/requester.py ---------------------------------------------------------------------- diff --git a/tools/cli/cloudmonkey/requester.py b/tools/cli/cloudmonkey/requester.py new file mode 100644 index 0000000..5c4cd1e --- /dev/null +++ b/tools/cli/cloudmonkey/requester.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +try: + import base64 + import hashlib + import hmac + import httplib + import json + import os + import pdb + import re + import shlex + import sys + import time + import types + import urllib + import urllib2 + +except ImportError, e: + print "Import error in %s : %s" % (__name__, e) + import sys + sys.exit() + + +def logger_debug(logger, message): + if logger is not None: + logger.debug(message) + + +def make_request(command, args, logger, host, port, + apikey, secretkey, protocol, path): + response = None + error = None + + if protocol != 'http' and protocol != 'https': + error = "Protocol must be 'http' or 'https'" + return None, error + + if args is None: + args = {} + + args["command"] = command + args["apiKey"] = apikey + args["response"] = "json" + request = zip(args.keys(), args.values()) + request.sort(key=lambda x: str.lower(x[0])) + + request_url = "&".join(["=".join([r[0], urllib.quote_plus(str(r[1]))]) + for r in request]) + hashStr = "&".join(["=".join([str.lower(r[0]), + str.lower(urllib.quote_plus(str(r[1]))).replace("+", + "%20")]) for r in request]) + + sig = urllib.quote_plus(base64.encodestring(hmac.new(secretkey, hashStr, + hashlib.sha1).digest()).strip()) + request_url += "&signature=%s" % sig + request_url = "%s://%s:%s%s?%s" % (protocol, host, port, path, request_url) + + try: + logger_debug(logger, "Request sent: %s" % request_url) + connection = urllib2.urlopen(request_url) + response = connection.read() + except Exception, e: + error = str(e) + + logger_debug(logger, "Response received: %s" % response) + if error is not None: + logger_debug(logger, error) + + return response, error + + +def monkeyrequest(command, args, isasync, asyncblock, logger, host, port, + apikey, secretkey, timeout, protocol, path): + + response = None + error = None + logger_debug(logger, "======== START Request ========") + logger_debug(logger, "Requesting command=%s, args=%s" % (command, args)) + response, error = make_request(command, args, logger, host, port, + apikey, secretkey, protocol, path) + logger_debug(logger, "======== END Request ========\n") + + if error is not None: + return response, error + + def process_json(response): + try: + response = json.loads(str(response)) + except ValueError, e: + error = "Error processing json response, %s" % e + logger_debug(logger, "Error processing json", e) + return response + + response = process_json(response) + if response is None: + return response, error + + isasync = isasync and (asyncblock == "true") + responsekey = filter(lambda x: 'response' in x, response.keys())[0] + + if isasync and 'jobid' in response[responsekey]: + jobid = response[responsekey]['jobid'] + command = "queryAsyncJobResult" + request = {'jobid': jobid} + timeout = int(timeout) + pollperiod = 3 + progress = 1 + while timeout > 0: + print '\r' + '.' * progress, + time.sleep(pollperiod) + timeout = timeout - pollperiod + progress += 1 + logger_debug(logger, "Job %s to timeout in %ds" % (jobid, timeout)) + sys.stdout.flush() + response, error = monkeyrequest(command, request, isasync, + asyncblock, logger, + host, port, apikey, secretkey, + timeout, protocol, path) + response = process_json(response) + responsekeys = filter(lambda x: 'response' in x, response.keys()) + if len(responsekeys) < 1: + continue + result = response[responsekeys[0]] + jobstatus = result['jobstatus'] + if jobstatus == 2: + jobresult = result["jobresult"] + error = "\rAsync job %s failed\nError %s, %s" % (jobid, + jobresult["errorcode"], jobresult["errortext"]) + return response, error + elif jobstatus == 1: + print '\r', + return response, error + error = "Error: Async query timeout occurred for jobid %s" % jobid + + return response, error http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/631b6fd4/tools/cli/pom.xml ---------------------------------------------------------------------- diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml index aba5ec3..582cc57 100644 --- a/tools/cli/pom.xml +++ b/tools/cli/pom.xml @@ -72,7 +72,7 @@ </configuration> </execution> <execution> - <id>cachegen</id> + <id>cachemaker</id> <phase>compile</phase> <goals> <goal>exec</goal> @@ -81,7 +81,7 @@ <workingDirectory>${basedir}/cloudmonkey</workingDirectory> <executable>python</executable> <arguments> - <argument>cachegen.py</argument> + <argument>cachemaker.py</argument> </arguments> </configuration> </execution>
