Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-48-aria-cli c1f8eb6a9 -> 1cbd81b3b
more review fixes Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/1cbd81b3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/1cbd81b3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/1cbd81b3 Branch: refs/heads/ARIA-48-aria-cli Commit: 1cbd81b3bb950dd589435af4bce6f0c1c1fc6411 Parents: c1f8eb6 Author: Ran Ziv <[email protected]> Authored: Wed Apr 19 13:19:26 2017 +0300 Committer: Ran Ziv <[email protected]> Committed: Wed Apr 19 13:19:26 2017 +0300 ---------------------------------------------------------------------- aria/cli/commands/executions.py | 7 +- aria/cli/commands/node_templates.py | 6 +- aria/cli/commands/nodes.py | 7 +- aria/cli/commands/plugins.py | 50 ++--------- aria/cli/commands/service_templates.py | 38 ++++----- aria/cli/commands/services.py | 5 +- aria/cli/commands/workflows.py | 6 +- aria/cli/constants.py | 18 ---- aria/cli/core/aria.py | 10 +-- aria/cli/defaults.py | 20 +++++ aria/cli/helptexts.py | 2 +- aria/cli/logger.py | 16 ++-- aria/cli/service_template_utils.py | 39 +++------ aria/cli/table.py | 90 +++++++++++++------- aria/cli/utils.py | 88 ++++--------------- aria/modeling/service_common.py | 10 +-- aria/orchestrator/exceptions.py | 7 ++ aria/orchestrator/plugin.py | 23 +++++ aria/orchestrator/workflow_runner.py | 2 +- aria/orchestrator/workflows/executor/celery.py | 4 +- aria/orchestrator/workflows/executor/dry.py | 3 +- aria/orchestrator/workflows/executor/process.py | 3 +- aria/orchestrator/workflows/executor/thread.py | 3 +- aria/utils/http.py | 62 ++++++++++++++ 24 files changed, 258 insertions(+), 261 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/executions.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py index cd12ead..adec56b 100644 --- a/aria/cli/commands/executions.py +++ b/aria/cli/commands/executions.py @@ -51,7 +51,7 @@ def show(execution_id, model_storage, logger): logger.info('Showing execution {0}'.format(execution_id)) execution = model_storage.execution.get(execution_id) - print_data(EXECUTION_COLUMNS, execution.to_dict(), 'Execution:', max_width=50) + print_data(EXECUTION_COLUMNS, execution, 'Execution:', col_max_width=50) # print execution parameters logger.info('Execution Inputs:') @@ -63,7 +63,6 @@ def show(execution_id, model_storage, logger): logger.info('\t{0}: \t{1}'.format(input_name, input_value)) else: logger.info('\tNo inputs') - logger.info('') @executions.command(name='list', @@ -93,9 +92,9 @@ def list(service_name, logger.info('Listing all executions...') filters = {} - executions_list = [e.to_dict() for e in model_storage.execution.list( + executions_list = model_storage.execution.list( filters=filters, - sort=utils.storage_sort_param(sort_by, descending))] + sort=utils.storage_sort_param(sort_by, descending)).items print_data(EXECUTION_COLUMNS, executions_list, 'Executions:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/node_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/node_templates.py b/aria/cli/commands/node_templates.py index c79e125..f0ca2c1 100644 --- a/aria/cli/commands/node_templates.py +++ b/aria/cli/commands/node_templates.py @@ -44,7 +44,7 @@ def show(node_template_id, model_storage, logger): logger.info('Showing node template {0}'.format(node_template_id)) node_template = model_storage.node_template.get(node_template_id) - print_data(NODE_TEMPLATE_COLUMNS, node_template.to_dict(), 'Node template:', max_width=50) + print_data(NODE_TEMPLATE_COLUMNS, node_template, 'Node template:', col_max_width=50) # print node template properties logger.info('Node template properties:') @@ -86,8 +86,8 @@ def list(service_template_name, sort_by, descending, model_storage, logger): logger.info('Listing all node templates...') filters = {} - node_templates_list = [nt.to_dict() for nt in model_storage.node_template.list( + node_templates_list = model_storage.node_template.list( filters=filters, - sort=utils.storage_sort_param(sort_by, descending))] + sort=utils.storage_sort_param(sort_by, descending)).items print_data(NODE_TEMPLATE_COLUMNS, node_templates_list, 'Node templates:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/nodes.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/nodes.py b/aria/cli/commands/nodes.py index b1f2acc..0a00478 100644 --- a/aria/cli/commands/nodes.py +++ b/aria/cli/commands/nodes.py @@ -43,7 +43,7 @@ def show(node_id, model_storage, logger): logger.info('Showing node {0}'.format(node_id)) node = model_storage.node.get(node_id) - print_data(NODE_COLUMNS, node.to_dict(), 'Node:', 50) + print_data(NODE_COLUMNS, node, 'Node:', 50) # print node attributes logger.info('Node attributes:') @@ -52,7 +52,6 @@ def show(node_id, model_storage, logger): logger.info('\t{0}: {1}'.format(prop_name, prop_value)) else: logger.info('\tNo attributes') - logger.info('') @nodes.command(name='list', @@ -81,8 +80,8 @@ def list(service_name, logger.info('Listing all nodes...') filters = {} - nodes_list = [node.to_dict() for node in model_storage.node.list( + nodes_list = model_storage.node.list( filters=filters, - sort=utils.storage_sort_param(sort_by, descending))] + sort=utils.storage_sort_param(sort_by, descending)).items print_data(NODE_COLUMNS, nodes_list, 'Nodes:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/plugins.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/plugins.py b/aria/cli/commands/plugins.py index 41a272e..22552d6 100644 --- a/aria/cli/commands/plugins.py +++ b/aria/cli/commands/plugins.py @@ -13,11 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import zipfile - from .. import utils from ..core import aria -from ..exceptions import AriaCliError from ..table import print_data @@ -37,52 +34,21 @@ def plugins(): short_help='Validate a plugin') @aria.argument('plugin-path') @aria.options.verbose() [email protected]_plugin_manager @aria.pass_logger -def validate(plugin_path, logger): - """Validate a plugin +def validate(plugin_path, plugin_manager, logger): + """Validate a plugin archive - This will try to validate the plugin's archive is not corrupted. - A valid plugin is a wagon (http://github.com/cloudify-cosomo/wagon) + A valid plugin is a wagon (http://github.com/cloudify-cosmo/wagon) in the zip format (suffix may also be .wgn). `PLUGIN_PATH` is the path to wagon archive to validate. """ logger.info('Validating plugin {0}...'.format(plugin_path)) - - if not zipfile.is_zipfile(plugin_path): - raise AriaCliError( - 'Archive {0} is of an unsupported type. Only ' - 'zip/wgn is allowed'.format(plugin_path)) - with zipfile.ZipFile(plugin_path, 'r') as zip_file: - infos = zip_file.infolist() - try: - package_name = infos[0].filename[:infos[0].filename.index('/')] - package_json_path = "{0}/{1}".format(package_name, 'package.json') - zip_file.getinfo(package_json_path) - except (KeyError, ValueError, IndexError): - raise AriaCliError( - 'Failed to validate plugin {0} ' - '(package.json was not found in archive)'.format(plugin_path)) - + plugin_manager.validate_plugin(plugin_path) logger.info('Plugin validated successfully') -# @plugins.command(name='delete', -# short_help='Delete a plugin') -# @aria.argument('plugin-id') -# @aria.options.verbose() -# @aria.pass_model_storage -# @aria.pass_logger -# def delete(plugin_id, model_storage, logger): -# """Delete a plugin -# -# `PLUGIN_ID` is the id of the plugin to delete. -# """ -# logger.info('Deleting plugin {0}...'.format(plugin_id)) -# model_storage.plugin.delete(plugin_id=plugin_id) -# logger.info('Plugin deleted') - - @plugins.command(name='install', short_help='Install a plugin') @aria.argument('plugin-path') @@ -114,7 +80,7 @@ def show(plugin_id, model_storage, logger): """ logger.info('Showing plugin {0}...'.format(plugin_id)) plugin = model_storage.plugin.get(plugin_id) - print_data(PLUGIN_COLUMNS, plugin.to_dict(), 'Plugin:') + print_data(PLUGIN_COLUMNS, plugin, 'Plugin:') @plugins.command(name='list', @@ -128,6 +94,6 @@ def list(sort_by, descending, model_storage, logger): """List all plugins on the manager """ logger.info('Listing all plugins...') - plugins_list = [p.to_dict() for p in model_storage.plugin.list( - sort=utils.storage_sort_param(sort_by, descending))] + plugins_list = model_storage.plugin.list( + sort=utils.storage_sort_param(sort_by, descending)).items print_data(PLUGIN_COLUMNS, plugins_list, 'Plugins:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/service_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py index 2ef37c0..189ba51 100644 --- a/aria/cli/commands/service_templates.py +++ b/aria/cli/commands/service_templates.py @@ -18,16 +18,16 @@ import os from .. import csar from .. import service_template_utils +from .. import table from .. import utils from ..core import aria -from ..table import print_data from ...core import Core from ...storage import exceptions as storage_exceptions -DESCRIPTION_LIMIT = 20 +DESCRIPTION_FIELD_LENGTH_LIMIT = 20 SERVICE_TEMPLATE_COLUMNS = \ - ['id', 'name', 'main_file_name', 'created_at', 'updated_at'] + ['id', 'name', 'description', 'main_file_name', 'created_at', 'updated_at'] @aria.group(name='service-templates') @@ -51,11 +51,14 @@ def show(service_template_name, model_storage, logger): """ logger.info('Showing service template {0}...'.format(service_template_name)) service_template = model_storage.service_template.get_by_name(service_template_name) - services = [d.to_dict() for d in service_template.services] service_template_dict = service_template.to_dict() - service_template_dict['#services'] = len(services) + service_template_dict['#services'] = len(service_template.services) + + column_formatters = \ + dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT)) columns = SERVICE_TEMPLATE_COLUMNS + ['#services'] - print_data(columns, service_template_dict, 'Service-template:', max_width=50) + table.print_data(columns, service_template_dict, 'Service-template:', + column_formatters=column_formatters, col_max_width=50) if service_template_dict['description'] is not None: logger.info('Description:') @@ -63,8 +66,8 @@ def show(service_template_name, model_storage, logger): os.linesep)) logger.info('Existing services:') - logger.info('{0}{1}'.format([s['name'] for s in services], - os.linesep)) + for service in service_template.services: + logger.info('\t{0}'.format(service.name)) @service_templates.command(name='list', @@ -77,20 +80,15 @@ def show(service_template_name, model_storage, logger): def list(sort_by, descending, model_storage, logger): """List all service templates """ - def trim_description(service_template): - if service_template['description'] is not None: - if len(service_template['description']) >= DESCRIPTION_LIMIT: - service_template['description'] = '{0}..'.format( - service_template['description'][:DESCRIPTION_LIMIT - 2]) - else: - service_template['description'] = '' - return service_template logger.info('Listing all service templates...') - service_templates_list = [trim_description(b.to_dict()) for b in - model_storage.service_template.list( - sort=utils.storage_sort_param(sort_by, descending))] - print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:') + service_templates_list = model_storage.service_template.list( + sort=utils.storage_sort_param(sort_by, descending)).items + + column_formatters = \ + dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT)) + table.print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:', + column_formatters=column_formatters) @service_templates.command(name='store', http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/services.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py index afa5e42..e1569a6 100644 --- a/aria/cli/commands/services.py +++ b/aria/cli/commands/services.py @@ -64,9 +64,9 @@ def list(service_template_name, logger.info('Listing all services...') filters = {} - services_list = [d.to_dict() for d in model_storage.service.list( + services_list = model_storage.service.list( sort=utils.storage_sort_param(sort_by=sort_by, descending=descending), - filters=filters)] + filters=filters).items print_data(SERVICE_COLUMNS, services_list, 'Services:') @@ -177,4 +177,3 @@ def inputs(service_name, model_storage, logger): logger.info(inputs_string.getvalue()) else: logger.info('\tNo inputs') - logger.info('') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/workflows.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/workflows.py b/aria/cli/commands/workflows.py index d380fac..5bd23b7 100644 --- a/aria/cli/commands/workflows.py +++ b/aria/cli/commands/workflows.py @@ -52,7 +52,7 @@ def show(workflow_name, service_name, model_storage, logger): 'service_template_name': service.service_template_name, 'service_name': service.name } - print_data(WORKFLOW_COLUMNS, workflow.to_dict(), 'Workflows:', defaults=defaults) + print_data(WORKFLOW_COLUMNS, workflow, 'Workflows:', defaults=defaults) # print workflow inputs required_inputs = dict() @@ -78,7 +78,6 @@ def show(workflow_name, service_name, model_storage, logger): else: logger.info('\t\t{0}: \t{1}'.format(input_name, input.value)) - logger.info('') @workflows.command(name='list', @@ -92,8 +91,7 @@ def list(service_name, model_storage, logger): """ logger.info('Listing workflows for service {0}...'.format(service_name)) service = model_storage.service.get_by_name(service_name) - workflows_list = [wf.to_dict() for wf in - sorted(service.workflows.values(), key=lambda w: w.name)] + workflows_list = sorted(service.workflows.values(), key=lambda w: w.name) defaults = { 'service_template_name': service.service_template_name, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/constants.py ---------------------------------------------------------------------- diff --git a/aria/cli/constants.py b/aria/cli/constants.py deleted file mode 100644 index c68fb5e..0000000 --- a/aria/cli/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -DEFAULT_SERVICE_TEMPLATE_FILENAME = 'service_template.yaml' -HELP_TEXT_COLUMN_BUFFER = 5 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/core/aria.py ---------------------------------------------------------------------- diff --git a/aria/cli/core/aria.py b/aria/cli/core/aria.py index fb5a81b..fe6dc4b 100644 --- a/aria/cli/core/aria.py +++ b/aria/cli/core/aria.py @@ -27,9 +27,9 @@ from ..env import ( env, logger ) +from .. import defaults from .. import helptexts from ..inputs import inputs_to_dict -from ..constants import DEFAULT_SERVICE_TEMPLATE_FILENAME from ...utils.exceptions import get_exception_as_string from ... import __version__ @@ -316,13 +316,13 @@ class Options(object): '--descending', required=False, is_flag=True, - default=False, + default=defaults.SORT_DESCENDING_DEFAULT, help=helptexts.DESCENDING) self.service_template_filename = click.option( '-n', '--service-template-filename', - default=DEFAULT_SERVICE_TEMPLATE_FILENAME, + default=defaults.SERVICE_TEMPLATE_FILENAME_DEFAULT, help=helptexts.SERVICE_TEMPLATE_FILENAME) @staticmethod @@ -354,7 +354,7 @@ class Options(object): help=help) @staticmethod - def task_max_attempts(default=30): + def task_max_attempts(default=defaults.TASK_MAX_ATTEMPTS_DEFAULT): return click.option( '--task-max-attempts', type=int, @@ -370,7 +370,7 @@ class Options(object): help=helptexts.SORT_BY) @staticmethod - def task_retry_interval(default=30): + def task_retry_interval(default=defaults.TASK_RETRY_INTERVAL_DEFAULT): return click.option( '--task-retry-interval', type=int, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/defaults.py ---------------------------------------------------------------------- diff --git a/aria/cli/defaults.py b/aria/cli/defaults.py new file mode 100644 index 0000000..6befd25 --- /dev/null +++ b/aria/cli/defaults.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +SERVICE_TEMPLATE_FILENAME_DEFAULT = 'service_template.yaml' +TASK_MAX_ATTEMPTS_DEFAULT = 30 +TASK_RETRY_INTERVAL_DEFAULT = 30 +SORT_DESCENDING_DEFAULT = False http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/helptexts.py ---------------------------------------------------------------------- diff --git a/aria/cli/helptexts.py b/aria/cli/helptexts.py index 6e31f47..1a3f6c0 100644 --- a/aria/cli/helptexts.py +++ b/aria/cli/helptexts.py @@ -29,7 +29,7 @@ EXECUTION_ID = "The unique identifier for the execution" SERVICE_TEMPLATE_PATH = "The path to the application's service template file" SERVICE_TEMPLATE_FILENAME = ( "The name of the archive's main service template file. " - "This is only relevant if uploading a non-csar archive") + "This is only relevant if uploading a (non-csar) archive") INPUTS_PARAMS_USAGE = ( '(Can be provided as wildcard based paths ' '(*.yaml, /my_inputs/, etc..) to YAML files, a JSON string or as ' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/logger.py ---------------------------------------------------------------------- diff --git a/aria/cli/logger.py b/aria/cli/logger.py index 2f012d9..1ffa918 100644 --- a/aria/cli/logger.py +++ b/aria/cli/logger.py @@ -25,7 +25,7 @@ MEDIUM_VERBOSE = 2 LOW_VERBOSE = 1 NO_VERBOSE = 0 -DEFAULT_LOGGER_CONFIG = { +LOGGER_CONFIG_TEMPLATE = { "version": 1, "formatters": { "file": { @@ -57,7 +57,7 @@ class Logging(object): def __init__(self, config): self._log_file = None self._verbosity_level = NO_VERBOSE - self._all_loggers = [] + self._all_loggers_names = [] self._configure_loggers(config) self._lgr = logging.getLogger('aria.cli.main') @@ -73,21 +73,21 @@ class Logging(object): def verbosity_level(self): return self._verbosity_level - def is_high_verbose_level(self): - return self.verbosity_level == HIGH_VERBOSE - @verbosity_level.setter def verbosity_level(self, level): self._verbosity_level = level if self.is_high_verbose_level(): - for logger_name in self._all_loggers: + for logger_name in self._all_loggers_names: logging.getLogger(logger_name).setLevel(logging.DEBUG) + def is_high_verbose_level(self): + return self.verbosity_level == HIGH_VERBOSE + def _configure_loggers(self, config): loggers_config = config.logging.loggers logfile = config.logging.filename - logger_dict = copy.deepcopy(DEFAULT_LOGGER_CONFIG) + logger_dict = copy.deepcopy(LOGGER_CONFIG_TEMPLATE) if logfile: # set filename on file handler logger_dict['handlers']['file']['filename'] = logfile @@ -102,6 +102,7 @@ class Logging(object): loggers = {} for logger_name in loggers_config: loggers[logger_name] = dict(handlers=list(logger_dict['handlers'].keys())) + self._all_loggers_names.append(logger_name) logger_dict['loggers'] = loggers # set level for all loggers @@ -109,6 +110,5 @@ class Logging(object): log = logging.getLogger(logger_name) level = logging._levelNames[logging_level.upper()] log.setLevel(level) - self._all_loggers.append(logger_name) dictconfig.dictConfig(logger_dict) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/service_template_utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/service_template_utils.py b/aria/cli/service_template_utils.py index 0300449..382cce1 100644 --- a/aria/cli/service_template_utils.py +++ b/aria/cli/service_template_utils.py @@ -26,28 +26,23 @@ def get(source, service_template_filename): """Get a source and return a path to the main service template file The behavior based on then source argument content is: - - + - local yaml file: return the file - local archive: extract it locally and return path service template file - - local yaml file: return the file - URL: - - return it (download=False) - - download and get service template from downloaded file (download=True) + - download and get service template from downloaded archive - github repo: - - map it to a URL and return it (download=False) - - download and get service template from downloaded file (download=True) + - download and get service template from downloaded archive Supported archive types are: csar, zip, tar, tar.gz and tar.bz2 :param source: Path/URL/github repo to archive/service-template file :type source: str - :param service_template_filename: Path to service template (if source is an archive file) + :param service_template_filename: Path to service template (if source is an archive [but + not a csar archive - with csars archives, this is read from the metadata file]) :type service_template_filename: str - :param download: Download service template file if source is URL/github repo - :type download: bool - :return: Path to file (if archive/service-template file passed) or url + :return: Path to main service template file :rtype: str - """ if urlparse(source).scheme: downloaded_file = utils.download_file(source) @@ -115,26 +110,12 @@ def _map_to_github_url(source): return url -# def generate_id(service_template_path, -# service_template_filename=DEFAULT_SERVICE_TEMPLATE_FILENAME): -# """The name of the service template will be the name of the folder. -# If service_template_filename is provided, it will be appended to the folder. -# """ -# service_template_id = os.path.split(os.path.dirname(os.path.abspath( -# service_template_path)))[-1] -# if service_template_filename != DEFAULT_SERVICE_TEMPLATE_FILENAME: -# filename, _ = os.path.splitext(os.path.basename(service_template_filename)) -# service_template_id = (service_template_id + '.' + filename) -# return service_template_id.replace('_', '-') - - def _is_archive(source): return archive_utils.is_archive(source) or csar.is_csar_archive(source) def _extract_csar_archive(archive): - if csar.is_csar_archive(archive): - reader = csar.read(source=archive) - main_service_template_file_name = os.path.basename(reader.entry_definitions) - return os.path.join(reader.destination, - main_service_template_file_name) + reader = csar.read(source=archive) + main_service_template_file_name = os.path.basename(reader.entry_definitions) + return os.path.join(reader.destination, + main_service_template_file_name) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/table.py ---------------------------------------------------------------------- diff --git a/aria/cli/table.py b/aria/cli/table.py index 36dcbea..11d791e 100644 --- a/aria/cli/table.py +++ b/aria/cli/table.py @@ -21,7 +21,25 @@ from prettytable import PrettyTable from .env import logger -def generate(cols, data, defaults=None): +def print_data(columns, items, header_text, + column_formatters=None, col_max_width=None, defaults=None): + if items is None: + items = [] + elif not isinstance(items, list): + items = [items] + + pretty_table = _generate(columns, data=items, column_formatters=column_formatters, + defaults=defaults) + if col_max_width: + pretty_table.max_width = col_max_width + _log(header_text, pretty_table) + + +def _log(title, table): + logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, table)) + + +def _generate(cols, data, column_formatters=None, defaults=None): """ Return a new PrettyTable instance representing the list. @@ -32,10 +50,16 @@ def generate(cols, data, defaults=None): for example: ['id','name'] - data - An iterable of dictionaries, each dictionary must - have key's corresponding to the cols items. + data - An iterable of dictionaries or objects, each element must + have keys or attributes corresponding to the cols items. + + for example: [{'id':'123', 'name':'Pete'}] - for example: [{'id':'123', 'name':'Pete'] + column_formatters - A dictionary from a column name to a function that may manipulate + the values printed for this column. + (See below for a few built-in formatter examples) + + for example: {'created_at': timestamp_formatter} defaults - A dictionary specifying default values for key's that don't exist in the data itself. @@ -45,25 +69,26 @@ def generate(cols, data, defaults=None): """ def get_values_per_column(column, row_data): - if column in row_data: - if row_data[column] and isinstance(row_data[column], basestring): - try: - datetime.strptime(row_data[column][:10], '%Y-%m-%d') - row_data[column] = \ - row_data[column].replace('T', ' ').replace('Z', ' ') - except ValueError: - # not a timestamp - pass - elif row_data[column] and isinstance(row_data[column], list): - row_data[column] = ','.join(row_data[column]) - elif not row_data[column]: - # if it's empty list, don't print [] - row_data[column] = '' - return row_data[column] + if hasattr(row_data, column) or (isinstance(row_data, dict) and column in row_data): + val = row_data[column] if isinstance(row_data, dict) else getattr(row_data, column) + + if val and isinstance(val, list): + val = [str(element) for element in val] + val = ','.join(val) + elif val is None or isinstance(val, list): + # don't print `[]` or `None` (but do print `0`, `False`, etc.) + val = '' + + if column in column_formatters: + # calling the user's column formatter to manipulate the value + val = column_formatters[column](val) + + return val else: return defaults[column] - pretty_table = PrettyTable([col for col in cols]) + column_formatters = column_formatters or dict() + pretty_table = PrettyTable(list(cols)) for datum in data: values_row = [] @@ -74,17 +99,18 @@ def generate(cols, data, defaults=None): return pretty_table -def log(title, table): - logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, table)) - +def timestamp_formatter(value): + try: + datetime.strptime(value[:10], '%Y-%m-%d') + return value.replace('T', ' ').replace('Z', ' ') + except ValueError: + # not a timestamp + return value -def print_data(columns, items, header_text, max_width=None, defaults=None): - if items is None: - items = [] - elif not isinstance(items, list): - items = [items] - pretty_table = generate(columns, data=items, defaults=defaults) - if max_width: - pretty_table.max_width = max_width - log(header_text, pretty_table) +def trim_formatter_generator(max_length): + def trim_formatter(value): + if len(value) >= max_length: + value = '{0}..'.format(value[:max_length - 2]) + return value + return trim_formatter http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/utils.py b/aria/cli/utils.py index 3cc68c9..852f24d 100644 --- a/aria/cli/utils.py +++ b/aria/cli/utils.py @@ -15,41 +15,19 @@ import os import sys -import string -import random -import tempfile from StringIO import StringIO from backports.shutil_get_terminal_size import get_terminal_size -import requests from .env import logger from .exceptions import AriaCliError - - -def dump_to_file(collection, file_path): - with open(file_path, 'a') as f: - f.write(os.linesep.join(collection)) - f.write(os.linesep) - - -def is_virtual_env(): - return hasattr(sys, 'real_prefix') +from ..utils import http def storage_sort_param(sort_by, descending): return {sort_by: 'desc' if descending else 'asc'} -def generate_random_string(size=6, - chars=string.ascii_uppercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) - - -def generate_suffixed_id(id): - return '{0}_{1}'.format(id, generate_random_string()) - - def get_parameter_templates_as_string(parameter_templates): params_string = StringIO() @@ -64,43 +42,28 @@ def get_parameter_templates_as_string(parameter_templates): return params_string.getvalue() -def download_file(url, destination=None): - """Download file. - - :param url: Location of the file to download - :type url: str - :param destination: - Location where the file should be saved (autogenerated by default) - :type destination: str | None - :returns: Location where the file was saved - :rtype: str - +def check_overriding_storage_exceptions(e, model_class, name): """ - chunk_size = 1024 - - if not destination: - file_descriptor, destination = tempfile.mkstemp() - os.close(file_descriptor) - logger.info('Downloading {0} to {1}...'.format(url, destination)) - - try: - response = requests.get(url, stream=True) - except requests.exceptions.RequestException as ex: - raise AriaCliError( - 'Failed to download {0}. ({1})'.format(url, str(ex))) + This method checks whether the storage exception is a known type where we'd like to override + the exception message; If so, it raises a new error. Otherwise it simply returns. + """ + assert isinstance(e, BaseException) + if 'UNIQUE constraint failed' in e.message: + new_message = \ + 'Could not store {model_class} `{name}`{linesep}' \ + 'There already a exists a {model_class} with the same name' \ + .format(model_class=model_class, name=name, linesep=os.linesep) + trace = sys.exc_info()[2] + raise type(e), type(e)(new_message), trace # pylint: disable=raising-non-exception - final_url = response.url - if final_url != url: - logger.debug('Redirected to {0}'.format(final_url)) +def download_file(url): + progress_bar = generate_progress_handler(url, 'Downloading') try: - with open(destination, 'wb') as destination_file: - for chunk in response.iter_content(chunk_size): - destination_file.write(chunk) - except IOError as ex: + destination = http.download_file(url, logger=logger, progress_handler=progress_bar) + except Exception as e: raise AriaCliError( - 'Failed to download {0}. ({1})'.format(url, str(ex))) - + 'Failed to download {0}. ({1})'.format(url, str(e))) return destination @@ -150,18 +113,3 @@ def generate_progress_handler(file_path, action='', max_bar_length=80): sys.stdout.write(os.linesep) return print_progress - - -def check_overriding_storage_exceptions(e, model_class, name): - """ - This method checks whether the storage exception is a known type where we'd like to override - the exception message; If so, it raises a new error. Otherwise it simply returns. - """ - assert isinstance(e, BaseException) - if 'UNIQUE constraint failed' in e.message: - new_message = \ - 'Could not store {model_class} `{name}`{linesep}' \ - 'There already a exists a {model_class} with the same name' \ - .format(model_class=model_class, name=name, linesep=os.linesep) - trace = sys.exc_info()[2] - raise type(e), type(e)(new_message), trace # pylint: disable=raising-non-exception http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/modeling/service_common.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py index e7fda29..1188f34 100644 --- a/aria/modeling/service_common.py +++ b/aria/modeling/service_common.py @@ -87,14 +87,8 @@ class ParameterBase(TemplateModelMixin): if self.description: console.puts(context.style.meta(self.description)) - @staticmethod - def unwrap_dict(parameters_dict): - """ - Takes a parameters dict and simplifies it into key-value dict - :param parameters_dict: a parameter-name to parameter dict - :return: a parameter-name to parameter value dict - """ - return dict((k, v.value) for k, v in parameters_dict.iteritems()) + def unwrap(self): + return self.name, self.value @classmethod def wrap(cls, name, value, description=None): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/exceptions.py b/aria/orchestrator/exceptions.py index fd3b66d..8d3dcc6 100644 --- a/aria/orchestrator/exceptions.py +++ b/aria/orchestrator/exceptions.py @@ -25,6 +25,13 @@ class OrchestratorError(AriaError): pass +class InvalidPluginError(AriaError): + """ + Raised when an invalid plugin is validated unsuccessfully + """ + pass + + class PluginAlreadyExistsError(AriaError): """ Raised when a plugin with the same package name and package version already exists http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/plugin.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py index d526e9c..b79d7fc 100644 --- a/aria/orchestrator/plugin.py +++ b/aria/orchestrator/plugin.py @@ -17,6 +17,7 @@ import os import tempfile import subprocess import sys +import zipfile from datetime import datetime import wagon @@ -69,6 +70,28 @@ class PluginManager(object): self._plugins_dir, '{0}-{1}'.format(plugin.package_name, plugin.package_version)) + @staticmethod + def validate_plugin(source): + """ + validate a plugin archive. + A valid plugin is a wagon (http://github.com/cloudify-cosmo/wagon) + in the zip format (suffix may also be .wgn). + """ + if not zipfile.is_zipfile(source): + raise exceptions.InvalidPluginError( + 'Archive {0} is of an unsupported type. Only ' + 'zip/wgn is allowed'.format(source)) + with zipfile.ZipFile(source, 'r') as zip_file: + infos = zip_file.infolist() + try: + package_name = infos[0].filename[:infos[0].filename.index('/')] + package_json_path = "{0}/{1}".format(package_name, 'package.json') + zip_file.getinfo(package_json_path) + except (KeyError, ValueError, IndexError): + raise exceptions.InvalidPluginError( + 'Failed to validate plugin {0} ' + '(package.json was not found in archive)'.format(source)) + def _install_wagon(self, source, prefix): pip_freeze_output = self._pip_freeze() file_descriptor, constraint_path = tempfile.mkstemp(prefix='constraint-', suffix='.txt') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflow_runner.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflow_runner.py b/aria/orchestrator/workflow_runner.py index 0051e8e..8779f06 100644 --- a/aria/orchestrator/workflow_runner.py +++ b/aria/orchestrator/workflow_runner.py @@ -80,7 +80,7 @@ class WorkflowRunner(object): task_retry_interval=task_retry_interval) # transforming the execution inputs to dict, to pass them to the workflow function - execution_inputs_dict = models.Parameter.unwrap_dict(self.execution.inputs) + execution_inputs_dict = dict(inp.unwrap() for inp in self.execution.inputs.values()) self._tasks_graph = workflow_fn(ctx=workflow_context, **execution_inputs_dict) executor = executor or ProcessExecutor(plugin_manager=plugin_manager) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/celery.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/celery.py b/aria/orchestrator/workflows/executor/celery.py index 3c98197..7bd9b7c 100644 --- a/aria/orchestrator/workflows/executor/celery.py +++ b/aria/orchestrator/workflows/executor/celery.py @@ -22,8 +22,6 @@ import Queue from aria.orchestrator.workflows.executor import BaseExecutor -from ....modeling.models import Parameter - class CeleryExecutor(BaseExecutor): """ @@ -46,7 +44,7 @@ class CeleryExecutor(BaseExecutor): def execute(self, task): self._tasks[task.id] = task - inputs = Parameter.unwrap_dict(task.inputs.iteritems()) + inputs = dict(inp.unwrap() for inp in task.inputs.values()) inputs['ctx'] = task.context self._results[task.id] = self._app.send_task( task.operation_mapping, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/dry.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/dry.py b/aria/orchestrator/workflows/executor/dry.py index b14f5d7..d894b25 100644 --- a/aria/orchestrator/workflows/executor/dry.py +++ b/aria/orchestrator/workflows/executor/dry.py @@ -20,7 +20,6 @@ Dry executor from datetime import datetime from .base import BaseExecutor -from ....modeling.models import Parameter class DryExecutor(BaseExecutor): @@ -38,7 +37,7 @@ class DryExecutor(BaseExecutor): actor_type = type(task.actor).__name__.lower() implementation = '{0} > '.format(task.plugin) if task.plugin else '' implementation += task.implementation - inputs = Parameter.unwrap_dict(task.inputs) + inputs = dict(inp.unwrap() for inp in task.inputs.values()) task.context.logger.info( 'Executing {actor_type} {task.actor.name} operation {task.interface_name} ' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/process.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py index 3c2b5fe..851d78e 100644 --- a/aria/orchestrator/workflows/executor/process.py +++ b/aria/orchestrator/workflows/executor/process.py @@ -48,7 +48,6 @@ from aria.utils import exceptions from aria.orchestrator.workflows.executor import base from aria.storage import instrumentation from aria.modeling import types as modeling_types -from aria.modeling.models import Parameter _IS_WIN = os.name == 'nt' @@ -149,7 +148,7 @@ class ProcessExecutor(base.BaseExecutor): return { 'task_id': task.id, 'implementation': task.implementation, - 'operation_inputs': Parameter.unwrap_dict(task.inputs), + 'operation_inputs': dict(inp.unwrap() for inp in task.inputs.values()), 'port': self._server_port, 'context': task.context.serialization_dict, } http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/thread.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/thread.py b/aria/orchestrator/workflows/executor/thread.py index 8b443cc..f422592 100644 --- a/aria/orchestrator/workflows/executor/thread.py +++ b/aria/orchestrator/workflows/executor/thread.py @@ -23,7 +23,6 @@ import threading from aria.utils import imports from .base import BaseExecutor -from ....modeling.models import Parameter class ThreadExecutor(BaseExecutor): @@ -60,7 +59,7 @@ class ThreadExecutor(BaseExecutor): self._task_started(task) try: task_func = imports.load_attribute(task.implementation) - inputs = Parameter.unwrap_dict(task.inputs) + inputs = dict(inp.unwrap() for inp in task.inputs.values()) task_func(ctx=task.context, **inputs) self._task_succeeded(task) except BaseException as e: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/utils/http.py ---------------------------------------------------------------------- diff --git a/aria/utils/http.py b/aria/utils/http.py new file mode 100644 index 0000000..7bdfd79 --- /dev/null +++ b/aria/utils/http.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import requests + + +def download_file(url, destination=None, logger=None, progress_handler=None): + """Download file. + + May raise IOError as well as requests.exceptions.RequestException + :param url: Location of the file to download + :type url: str + :param destination: + Location where the file should be saved (autogenerated by default) + :type destination: str | None + :returns: Location where the file was saved + :rtype: str + + """ + chunk_size = 1024 + + if not destination: + file_descriptor, destination = tempfile.mkstemp() + os.close(file_descriptor) + if logger: + logger.info('Downloading {0} to {1}...'.format(url, destination)) + + response = requests.get(url, stream=True) + final_url = response.url + if final_url != url and logger: + logger.debug('Redirected to {0}'.format(final_url)) + + read_bytes = 0 + total_size = int(response.headers['Content-Length']) \ + if 'Content-Length' in response.headers else None + try: + with open(destination, 'wb') as destination_file: + for chunk in response.iter_content(chunk_size): + destination_file.write(chunk) + if total_size and progress_handler: + # Only showing progress bar if we have the total content length + read_bytes += chunk_size + progress_handler(read_bytes, total_size) + finally: + response.close() + + return destination
