http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/commands/plugins.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/plugins.py b/aria/cli/commands/plugins.py new file mode 100644 index 0000000..9e7d449 --- /dev/null +++ b/aria/cli/commands/plugins.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import zipfile + +from ..table import print_data +from ..cli import aria +from ..exceptions import AriaCliError +from ..utils import storage_sort_param + + +PLUGIN_COLUMNS = ['id', 'package_name', 'package_version', 'supported_platform', + 'distribution', 'distribution_release', 'uploaded_at'] + + +@aria.group(name='plugins') +@aria.options.verbose() +def plugins(): + """Handle plugins + """ + pass + + +@plugins.command(name='validate', + short_help='Validate a plugin') +@aria.argument('plugin-path') +@aria.options.verbose() +@aria.pass_logger +def validate(plugin_path, logger): + """Validate a plugin + + This will try to validate the plugin's archive is not corrupted. + A valid plugin is a wagon (http://github.com/cloudify-cosomo/wagon) + in the zip format (suffix may also be .wgn). + + `PLUGIN_PATH` is the path to wagon archive to validate. + """ + logger.info('Validating plugin {0}...'.format(plugin_path)) + + if not zipfile.is_zipfile(plugin_path): + raise AriaCliError( + 'Archive {0} is of an unsupported type. Only ' + 'zip/wgn is allowed'.format(plugin_path)) + with zipfile.ZipFile(plugin_path, 'r') as zip_file: + infos = zip_file.infolist() + try: + package_name = infos[0].filename[:infos[0].filename.index('/')] + package_json_path = "{0}/{1}".format(package_name, 'package.json') + zip_file.getinfo(package_json_path) + except (KeyError, ValueError, IndexError): + raise AriaCliError( + 'Failed to validate plugin {0} ' + '(package.json was not found in archive)'.format(plugin_path)) + + logger.info('Plugin validated successfully') + + +# @plugins.command(name='delete', +# short_help='Delete a plugin') +# @aria.argument('plugin-id') +# @aria.options.verbose() +# @aria.pass_model_storage +# @aria.pass_logger +# def delete(plugin_id, model_storage, logger): +# """Delete a plugin +# +# `PLUGIN_ID` is the id of the plugin to delete. +# """ +# logger.info('Deleting plugin {0}...'.format(plugin_id)) +# model_storage.plugin.delete(plugin_id=plugin_id) +# logger.info('Plugin deleted') + + +@plugins.command(name='install', + short_help='Install a plugin') +@aria.argument('plugin-path') +@aria.options.verbose() +@aria.pass_context +@aria.pass_plugin_manager +@aria.pass_logger +def install(ctx, plugin_path, plugin_manager, logger): + """Install a plugin + + `PLUGIN_PATH` is the path to wagon archive to install. + """ + ctx.invoke(validate, plugin_path=plugin_path) + logger.info('Installing plugin {0}...'.format(plugin_path)) + plugin = plugin_manager.install(plugin_path) + logger.info("Plugin installed. The plugin's id is {0}".format(plugin.id)) + + +@plugins.command(name='show', + short_help='show plugin information') +@aria.argument('plugin-id') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def show(plugin_id, model_storage, logger): + """Show information for a specific plugin + + `PLUGIN_ID` is the id of the plugin to show information on. + """ + logger.info('Showing plugin {0}...'.format(plugin_id)) + plugin = model_storage.plugin.get(plugin_id) + print_data(PLUGIN_COLUMNS, plugin.to_dict(), 'Plugin:') + + +@plugins.command(name='list', + short_help='List plugins') +@aria.options.sort_by('uploaded_at') +@aria.options.descending +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def list(sort_by, descending, model_storage, logger): + """List all plugins on the manager + """ + logger.info('Listing all plugins...') + plugins_list = [p.to_dict() for p in model_storage.plugin.list( + sort=storage_sort_param(sort_by, descending))] + print_data(PLUGIN_COLUMNS, plugins_list, 'Plugins:')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/commands/service_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py new file mode 100644 index 0000000..8e0e91c --- /dev/null +++ b/aria/cli/commands/service_templates.py @@ -0,0 +1,220 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +from .. import utils +from .. import csar +from .. import service_template_utils +from ..cli import aria +from ..table import print_data +from ..exceptions import AriaCliError +from ..utils import handle_storage_exception +from ...core import Core +from ...exceptions import AriaException +from ...storage import exceptions as storage_exceptions + + +DESCRIPTION_LIMIT = 20 +SERVICE_TEMPLATE_COLUMNS = \ + ['id', 'name', 'main_file_name', 'created_at', 'updated_at'] + + +@aria.group(name='service-templates') +@aria.options.verbose() +def service_templates(): + """Handle service templates on the manager + """ + pass + + +@service_templates.command(name='show', + short_help='Show service template information') +@aria.argument('service-template-name') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def show(service_template_name, model_storage, logger): + """Show information for a specific service templates + + `SERVICE_TEMPLATE_NAME` is the name of the service template to show information on. + """ + logger.info('Showing service template {0}...'.format(service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + services = [d.to_dict() for d in service_template.services] + service_template_dict = service_template.to_dict() + service_template_dict['#services'] = len(services) + columns = SERVICE_TEMPLATE_COLUMNS + ['#services'] + print_data(columns, service_template_dict, 'Service-template:', max_width=50) + + if service_template_dict['description'] is not None: + logger.info('Description:') + logger.info('{0}{1}'.format(service_template_dict['description'].encode('UTF-8') or '', + os.linesep)) + + logger.info('Existing services:') + logger.info('{0}{1}'.format([s['name'] for s in services], + os.linesep)) + + +@service_templates.command(name='list', + short_help='List service templates') +@aria.options.sort_by() +@aria.options.descending +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def list(sort_by, descending, model_storage, logger): + """List all service templates + """ + def trim_description(service_template): + if service_template['description'] is not None: + if len(service_template['description']) >= DESCRIPTION_LIMIT: + service_template['description'] = '{0}..'.format( + service_template['description'][:DESCRIPTION_LIMIT - 2]) + else: + service_template['description'] = '' + return service_template + + logger.info('Listing all service templates...') + service_templates_list = [trim_description(b.to_dict()) for b in + model_storage.service_template.list( + sort=utils.storage_sort_param(sort_by, descending))] + print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:') + + +@service_templates.command(name='store', + short_help='Store a service template') +@aria.argument('service-template-path') +@aria.argument('service-template-name') +@aria.options.service_template_filename +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_resource_storage +@aria.pass_plugin_manager +@aria.pass_logger +def store(service_template_path, service_template_name, service_template_filename, + model_storage, resource_storage, plugin_manager, logger): + """Store a service template + + `SERVICE_TEMPLATE_PATH` is the path of the service template to store. + + `SERVICE_TEMPLATE_NAME` is the name of the service template to store. + """ + logger.info('Storing service template {0}...'.format(service_template_name)) + + service_template_path = service_template_utils.get(service_template_path, + service_template_filename) + core = Core(model_storage, resource_storage, plugin_manager) + try: + core.create_service_template(service_template_path, + os.path.dirname(service_template_path), + service_template_name) + except storage_exceptions.StorageError as e: + handle_storage_exception(e, 'service template', service_template_name) + logger.info('Service template {0} stored'.format(service_template_name)) + + +@service_templates.command(name='delete', + short_help='Delete a service template') +@aria.argument('service-template-name') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_resource_storage +@aria.pass_plugin_manager +@aria.pass_logger +def delete(service_template_name, model_storage, resource_storage, plugin_manager, logger): + """Delete a service template + `SERVICE_TEMPLATE_NAME` is the name of the service template to delete. + """ + logger.info('Deleting service template {0}...'.format(service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + core = Core(model_storage, resource_storage, plugin_manager) + try: + core.delete_service_template(service_template.id) + except storage_exceptions.NotFoundError: + raise AriaCliError() + logger.info('Service template {0} deleted'.format(service_template_name)) + + +@service_templates.command(name='inputs', + short_help='Show service template inputs') +@aria.argument('service-template-name') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def inputs(service_template_name, model_storage, logger): + """Show inputs for a specific service template + + `SERVICE_TEMPLATE_NAME` is the name of the service template to show inputs for. + """ + logger.info('Showing inputs for service template {0}...'.format(service_template_name)) + print_service_template_inputs(model_storage, service_template_name, logger) + + +@service_templates.command(name='validate', + short_help='Validate a service template') +@aria.argument('service-template') +@aria.options.service_template_filename +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_resource_storage +@aria.pass_plugin_manager +@aria.pass_logger +def validate(service_template, service_template_filename, + model_storage, resource_storage, plugin_manager, logger): + """Validate a service template + + `SERVICE_TEMPLATE` is the path or url of the service template or archive to validate. + """ + logger.info('Validating service template: {0}'.format(service_template)) + service_template_path = service_template_utils.get(service_template, service_template_filename) + core = Core(model_storage, resource_storage, plugin_manager) + + try: + core.validate_service_template(service_template_path) + except AriaException as e: + # TODO: gather errors from parser and dump them via CLI? + raise AriaCliError(str(e)) + + logger.info('Service template validated successfully') + + +@service_templates.command(name='create-archive', + short_help='Create a csar archive') +@aria.argument('service-template-path') +@aria.argument('destination') +@aria.options.verbose() +@aria.pass_logger +def create_archive(service_template_path, destination, logger): + """Create a csar archive + + `service_template_path` is the path of the service template to create the archive from + `destination` is the path of the output csar archive + """ + logger.info('Creating a csar archive') + csar.write(os.path.dirname(service_template_path), service_template_path, destination, logger) + logger.info('Csar archive created at {0}'.format(destination)) + + +def print_service_template_inputs(model_storage, service_template_name, logger): + service_template = model_storage.service_template.get_by_name(service_template_name) + + logger.info('Service template inputs:') + if service_template.inputs: + logger.info(utils.get_parameter_templates_as_string(service_template.inputs)) + else: + logger.info('\tNo inputs') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/commands/services.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py new file mode 100644 index 0000000..b785006 --- /dev/null +++ b/aria/cli/commands/services.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from StringIO import StringIO + +from . import service_templates +from ..cli import aria, helptexts +from ..exceptions import AriaCliError +from ..table import print_data +from ..utils import storage_sort_param, handle_storage_exception +from ...core import Core +from ...exceptions import AriaException +from ...storage import exceptions as storage_exceptions + + +SERVICE_COLUMNS = ['id', 'name', 'service_template_name', 'created_at', 'updated_at'] + + +@aria.group(name='services') +@aria.options.verbose() +def services(): + """Handle services + """ + pass + + +@services.command(name='list', short_help='List services') +@aria.options.service_template_name() +@aria.options.sort_by() +@aria.options.descending +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def list(service_template_name, + sort_by, + descending, + model_storage, + logger): + """List services + + If `--service-template-name` is provided, list services for that service template. + Otherwise, list services for all service templates. + """ + if service_template_name: + logger.info('Listing services for service template {0}...'.format( + service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + filters = dict(service_template=service_template) + else: + logger.info('Listing all services...') + filters = {} + + services_list = [d.to_dict() for d in model_storage.service.list( + sort=storage_sort_param(sort_by=sort_by, descending=descending), + filters=filters)] + print_data(SERVICE_COLUMNS, services_list, 'Services:') + + +@services.command(name='create', + short_help='Create a services') +@aria.argument('service-name', required=False) +@aria.options.service_template_name(required=True) +@aria.options.inputs +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_resource_storage +@aria.pass_plugin_manager +@aria.pass_logger +def create(service_template_name, + service_name, + inputs, # pylint: disable=redefined-outer-name + model_storage, + resource_storage, + plugin_manager, + logger): + """Create a service + + `SERVICE_NAME` is the name of the service you'd like to create. + + """ + logger.info('Creating new service from service template {0}...'.format( + service_template_name)) + + try: + core = Core(model_storage, resource_storage, plugin_manager) + service_template = model_storage.service_template.get_by_name(service_template_name) + service = core.create_service(service_template.id, inputs, service_name) + except storage_exceptions.StorageError as e: + handle_storage_exception(e, 'service', service_name) + except AriaException as e: + logger.info(str(e)) + service_templates.print_service_template_inputs(model_storage, service_template_name, + logger) + raise AriaCliError(str(e)) + logger.info("Service created. The service's name is {0}".format(service.name)) + + +@services.command(name='delete', + short_help='Delete a service') +@aria.argument('service-name') +@aria.options.force(help=helptexts.IGNORE_RUNNING_NODES) +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_resource_storage +@aria.pass_plugin_manager +@aria.pass_logger +def delete(service_name, force, model_storage, resource_storage, plugin_manager, logger): + """Delete a service + + `SERVICE_NAME` is the name of the service to delete. + """ + logger.info('Deleting service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + core = Core(model_storage, resource_storage, plugin_manager) + core.delete_service(service.id, force=force) + logger.info('Service {0} deleted'.format(service_name)) + + +@services.command(name='outputs', + short_help='Show service outputs') +@aria.argument('service-name') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def outputs(service_name, model_storage, logger): + """Show outputs for a specific service + + `SERVICE_NAME` is the name of the service to print outputs for. + """ + logger.info('Showing outputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + #TODO fix this section.. + outputs_def = service.outputs + response = model_storage.service.outputs.get(service_name) + outputs_ = StringIO() + for output_name, output in response.outputs.iteritems(): + outputs_.write(' - "{0}":{1}'.format(output_name, os.linesep)) + description = outputs_def[output_name].get('description', '') + outputs_.write(' Description: {0}{1}'.format(description, + os.linesep)) + outputs_.write(' Value: {0}{1}'.format(output, os.linesep)) + logger.info(outputs_.getvalue()) + + +@services.command(name='inputs', + short_help='Show service inputs') +@aria.argument('service-name') +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def inputs(service_name, model_storage, logger): + """Show inputs for a specific service + + `SERVICE_NAME` is the id of the service to print inputs for. + """ + logger.info('Showing inputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + if service.inputs: + inputs_ = StringIO() + for input_name, input in service.inputs.iteritems(): + inputs_.write(' - "{0}":{1}'.format(input_name, os.linesep)) + inputs_.write(' Value: {0}{1}'.format(input.value, os.linesep)) + logger.info(inputs_.getvalue()) + else: + logger.info('\tNo inputs') + logger.info('') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/commands/workflows.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/workflows.py b/aria/cli/commands/workflows.py new file mode 100644 index 0000000..72dea5b --- /dev/null +++ b/aria/cli/commands/workflows.py @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..table import print_data +from ..cli import aria +from ..exceptions import AriaCliError + +WORKFLOW_COLUMNS = ['name', 'service_template_name', 'service_name'] + + +@aria.group(name='workflows') +def workflows(): + """Handle service workflows + """ + pass + + +@workflows.command(name='show', + short_help='Show workflow information') +@aria.argument('workflow-name') +@aria.options.service_name(required=True) +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def show(workflow_name, service_name, model_storage, logger): + """Show information for a specific workflow of a specific service + + `WORKFLOW_NAME` is the name of the workflow to get information on. + """ + logger.info('Retrieving workflow {0} for service {1}'.format( + workflow_name, service_name)) + service = model_storage.service.get_by_name(service_name) + workflow = next((wf for wf in service.workflows.values() if + wf.name == workflow_name), None) + if not workflow: + raise AriaCliError( + 'Workflow {0} not found for service {1}'.format(workflow_name, service_name)) + + defaults = { + 'service_template_name': service.service_template_name, + 'service_name': service.name + } + print_data(WORKFLOW_COLUMNS, workflow.to_dict(), 'Workflows:', defaults=defaults) + + # print workflow inputs + required_inputs = dict() + optional_inputs = dict() + for input_name, input in workflow.inputs.iteritems(): + inputs_group = optional_inputs if input.value is not None else required_inputs + inputs_group[input_name] = input + + logger.info('Workflow Inputs:') + logger.info('\tMandatory Inputs:') + for input_name, input in required_inputs.iteritems(): + if input.description is not None: + logger.info('\t\t{0}\t({1})'.format(input_name, + input.description)) + else: + logger.info('\t\t{0}'.format(input_name)) + + logger.info('\tOptional Inputs:') + for input_name, input in optional_inputs.iteritems(): + if input.description is not None: + logger.info('\t\t{0}: \t{1}\t({2})'.format( + input_name, input.value, input.description)) + else: + logger.info('\t\t{0}: \t{1}'.format(input_name, + input.value)) + logger.info('') + + +@workflows.command(name='list', + short_help='List workflows for a service') +@aria.options.service_name(required=True) +@aria.options.verbose() +@aria.pass_model_storage +@aria.pass_logger +def list(service_name, model_storage, logger): + """List all workflows of a specific service + """ + logger.info('Listing workflows for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + workflows_list = [wf.to_dict() for wf in + sorted(service.workflows.values(), key=lambda w: w.name)] + + defaults = { + 'service_template_name': service.service_template_name, + 'service_name': service.name + } + print_data(WORKFLOW_COLUMNS, workflows_list, 'Workflows:', defaults=defaults) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/config.py ---------------------------------------------------------------------- diff --git a/aria/cli/config.py b/aria/cli/config.py deleted file mode 100644 index d82886d..0000000 --- a/aria/cli/config.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -CLI configuration -""" - -import os -import logging -from getpass import getuser -from tempfile import gettempdir - -from yaml import safe_load - -from .storage import config_file_path - -# path to a file where cli logs will be saved. -logging_filename = os.path.join(gettempdir(), 'aria_cli_{0}.log'.format(getuser())) -# loggers log level to show -logger_level = logging.INFO -# loggers log level to show -colors = True - -import_resolver = None - - -def load_configurations(): - """ - Dynamically load attributes into the config module from the ``config.yaml`` defined in the user - configuration directory - """ - config_path = config_file_path() - with open(config_path) as config_file: - globals().update(safe_load(config_file) or {}) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/config/__init__.py ---------------------------------------------------------------------- diff --git a/aria/cli/config/__init__.py b/aria/cli/config/__init__.py new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/aria/cli/config/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/config/config.py ---------------------------------------------------------------------- diff --git a/aria/cli/config/config.py b/aria/cli/config/config.py new file mode 100644 index 0000000..7d76830 --- /dev/null +++ b/aria/cli/config/config.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import yaml +import pkg_resources + +from jinja2.environment import Template + + +class CliConfig(object): + + def __init__(self, config_path): + with open(config_path) as f: + self._config = yaml.safe_load(f.read()) + + @classmethod + def create_config(cls, workdir): + config_path = os.path.join(workdir, 'config.yaml') + if not os.path.isfile(config_path): + config_template = pkg_resources.resource_string( + __package__, + 'config_template.yaml') + + default_values = { + 'log_path': os.path.join(workdir, 'cli.log'), + 'enable_colors': True + } + + template = Template(config_template) + rendered = template.render(**default_values) + with open(config_path, 'w') as f: + f.write(rendered) + f.write(os.linesep) + + return cls(config_path) + + @property + def colors(self): + return self._config.get('colors', False) + + @property + def logging(self): + return self.Logging(self._config.get('logging')) + + class Logging(object): + + def __init__(self, logging): + self._logging = logging or {} + + @property + def filename(self): + return self._logging.get('filename') + + @property + def loggers(self): + return self._logging.get('loggers', {}) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/config/config_template.yaml ---------------------------------------------------------------------- diff --git a/aria/cli/config/config_template.yaml b/aria/cli/config/config_template.yaml new file mode 100644 index 0000000..13f2cf9 --- /dev/null +++ b/aria/cli/config/config_template.yaml @@ -0,0 +1,12 @@ +colors: {{ enable_colors }} + +logging: + + # path to a file where cli logs will be saved. + filename: {{ log_path }} + + # configuring level per logger + loggers: + + # main logger of the cli. provides basic descriptions for executed operations. + aria.cli.main: info http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/constants.py ---------------------------------------------------------------------- diff --git a/aria/cli/constants.py b/aria/cli/constants.py new file mode 100644 index 0000000..c68fb5e --- /dev/null +++ b/aria/cli/constants.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +DEFAULT_SERVICE_TEMPLATE_FILENAME = 'service_template.yaml' +HELP_TEXT_COLUMN_BUFFER = 5 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/csar.py ---------------------------------------------------------------------- diff --git a/aria/cli/csar.py b/aria/cli/csar.py index b185f46..5bc35ac 100644 --- a/aria/cli/csar.py +++ b/aria/cli/csar.py @@ -14,12 +14,13 @@ # limitations under the License. import os +import logging import pprint import tempfile import zipfile import requests -from ruamel import yaml # @UnresolvedImport +from ruamel import yaml META_FILE = 'TOSCA-Metadata/TOSCA.meta' @@ -135,7 +136,7 @@ class _CSARReader(object): self.logger.debug('Attempting to parse CSAR metadata YAML') with open(csar_metafile) as f: self.metadata.update(yaml.load(f)) - self.logger.debug('CSAR metadata:\n{0}'.format(pprint.pformat(self.metadata))) + self.logger.debug('CSAR metadata:{0}{1}'.format(os.linesep, pprint.pformat(self.metadata))) def _validate(self): def validate_key(key, expected=None): @@ -167,5 +168,11 @@ class _CSARReader(object): f.write(chunk) -def read(source, destination, logger): +def read(source, destination=None, logger=None): + destination = destination or tempfile.mkdtemp() + logger = logger or logging.getLogger('dummy') return _CSARReader(source=source, destination=destination, logger=logger) + + +def is_csar_archive(source): + return source.endswith('.csar') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/dry.py ---------------------------------------------------------------------- diff --git a/aria/cli/dry.py b/aria/cli/dry.py deleted file mode 100644 index fc6c0c5..0000000 --- a/aria/cli/dry.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from threading import RLock - -from ..modeling import models -from ..orchestrator.decorators import operation -from ..utils.collections import OrderedDict -from ..utils.console import puts, Colored -from ..utils.formatting import safe_repr - - -_TERMINAL_LOCK = RLock() - - -def convert_to_dry(service): - """ - Converts all operations on the service (on workflows, node interfaces, and relationship - interfaces) to run dryly. - """ - - for workflow in service.workflows.itervalues(): - convert_operation_to_dry(workflow) - - for node in service.nodes.itervalues(): - for interface in node.interfaces.itervalues(): - for oper in interface.operations.itervalues(): - convert_operation_to_dry(oper) - for relationship in node.outbound_relationships: - for interface in relationship.interfaces.itervalues(): - for oper in interface.operations.itervalues(): - convert_operation_to_dry(oper) - - for group in service.groups.itervalues(): - for interface in group.interfaces.itervalues(): - for oper in interface.operations.itervalues(): - convert_operation_to_dry(oper) - - -def convert_operation_to_dry(oper): - """ - Converts a single :class:`Operation` to run dryly. - """ - - plugin = oper.plugin.name \ - if oper.plugin is not None else None - if oper.inputs is None: - oper.inputs = OrderedDict() - oper.inputs['_implementation'] = models.Parameter(name='_implementation', - type_name='string', - value=oper.implementation) - oper.inputs['_plugin'] = models.Parameter(name='_plugin', - type_name='string', - value=plugin) - oper.implementation = '{0}.{1}'.format(__name__, 'dry_operation') - oper.plugin_specification = None - - -@operation -def dry_operation(ctx, _plugin, _implementation, **kwargs): - """ - The dry operation simply prints out information about the operation to the console. - """ - - with _TERMINAL_LOCK: - print ctx.name - if hasattr(ctx, 'relationship'): - puts('> Relationship: {0} -> {1}'.format( - Colored.red(ctx.relationship.source_node.name), - Colored.red(ctx.relationship.target_node.name))) - else: - puts('> Node: {0}'.format(Colored.red(ctx.node.name))) - puts(' Operation: {0}'.format(Colored.green(ctx.name))) - _dump_implementation(_plugin, _implementation) - - -def _dump_implementation(plugin, implementation): - if plugin: - puts(' Plugin: {0}'.format(Colored.magenta(plugin, bold=True))) - if implementation: - puts(' Implementation: {0}'.format(Colored.magenta(safe_repr(implementation)))) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/env.py ---------------------------------------------------------------------- diff --git a/aria/cli/env.py b/aria/cli/env.py new file mode 100644 index 0000000..7fe656f --- /dev/null +++ b/aria/cli/env.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import json +import pkgutil + +from .config import config +from .logger import Logging +from .. import (application_model_storage, application_resource_storage) +from ..orchestrator.plugin import PluginManager +from ..storage.sql_mapi import SQLAlchemyModelAPI +from ..storage.filesystem_rapi import FileSystemResourceAPI + + +ARIA_DEFAULT_WORKDIR_NAME = '.aria' + + +class _Environment(object): + + def __init__(self, workdir): + + self._workdir = workdir + self._init_workdir() + + self._config = config.CliConfig.create_config(workdir) + self._logging = Logging(self._config) + + self._model_storage_dir = os.path.join(workdir, 'models') + self._resource_storage_dir = os.path.join(workdir, 'resources') + self._plugins_dir = os.path.join(workdir, 'plugins') + + # initialized lazily + self._model_storage = None + self._resource_storage = None + self._plugin_manager = None + + @property + def workdir(self): + return self._workdir + + @property + def config(self): + return self._config + + @property + def logging(self): + return self._logging + + @property + def model_storage(self): + if not self._model_storage: + self._model_storage = self._init_sqlite_model_storage() + return self._model_storage + + @property + def resource_storage(self): + if not self._resource_storage: + self._resource_storage = self._init_fs_resource_storage() + return self._resource_storage + + @property + def plugin_manager(self): + if not self._plugin_manager: + self._plugin_manager = self._init_plugin_manager() + return self._plugin_manager + + @staticmethod + def get_version_data(): + data = pkgutil.get_data(__package__, 'VERSION') + return json.loads(data) + + def _init_workdir(self): + if not os.path.exists(self._workdir): + os.makedirs(self._workdir) + + def _init_sqlite_model_storage(self): + if not os.path.exists(self._model_storage_dir): + os.makedirs(self._model_storage_dir) + + initiator_kwargs = dict(base_dir=self._model_storage_dir) + return application_model_storage( + SQLAlchemyModelAPI, + initiator_kwargs=initiator_kwargs) + + def _init_fs_resource_storage(self): + if not os.path.exists(self._resource_storage_dir): + os.makedirs(self._resource_storage_dir) + + fs_kwargs = dict(directory=self._resource_storage_dir) + return application_resource_storage( + FileSystemResourceAPI, + api_kwargs=fs_kwargs) + + def _init_plugin_manager(self): + if not os.path.exists(self._plugins_dir): + os.makedirs(self._plugins_dir) + + return PluginManager(self.model_storage, self._plugins_dir) + + +env = _Environment(os.path.join( + os.environ.get('ARIA_WORKDIR', os.path.expanduser('~')), ARIA_DEFAULT_WORKDIR_NAME)) + +logger = env.logging.logger http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/cli/exceptions.py b/aria/cli/exceptions.py index 6897731..89cfacd 100644 --- a/aria/cli/exceptions.py +++ b/aria/cli/exceptions.py @@ -13,59 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -CLI various exception classes -""" +from ..exceptions import AriaError -class AriaCliError(Exception): - """ - General CLI Exception class - """ - pass - - -class AriaCliFormatInputsError(AriaCliError): - """ - Raised when provided inputs are malformed. - """ - - def __init__(self, message, inputs): - self.inputs = inputs - super(AriaCliFormatInputsError, self).__init__(message) - - def user_message(self): - """ - Describes the format error in detail. - """ - return ( - 'Invalid input format: {0}, ' - 'the expected format is: ' - 'key1=value1;key2=value2'.format(self.inputs)) - -class AriaCliYAMLInputsError(AriaCliError): - """ - Raised when an invalid yaml file is provided - """ +class AriaCliError(AriaError): pass - - -class AriaCliInvalidInputsError(AriaCliFormatInputsError): - """ - Raised when provided inputs are invalid. - """ - - def user_message(self): - """ - Describes the error in detail. - """ - return ( - 'Invalid input: {0}. input must represent a dictionary.\n' - 'Valid values can be one of:\n' - '- a path to a YAML file\n' - '- a path to a directory containing YAML files\n' - '- a single quoted wildcard based path (e.g. "*-inputs.yaml")\n' - '- a string formatted as JSON\n' - '- a string formatted as key1=value1;key2=value2'.format(self.inputs) - ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/inputs.py ---------------------------------------------------------------------- diff --git a/aria/cli/inputs.py b/aria/cli/inputs.py new file mode 100644 index 0000000..78db846 --- /dev/null +++ b/aria/cli/inputs.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import glob +import yaml + +from .env import logger +from.exceptions import AriaCliError + + +def inputs_to_dict(resources): + """Returns a dictionary of inputs + + `resources` can be: + - A list of files. + - A single file + - A directory containing multiple input files + - A key1=value1;key2=value2 pairs string. + - A string formatted as JSON/YAML. + - Wildcard based string (e.g. *-inputs.yaml) + """ + if not resources: + return dict() + + parsed_dict = {} + + for resource in resources: + logger.debug('Processing inputs source: {0}'.format(resource)) + # Workflow parameters always pass an empty dictionary. We ignore it + if isinstance(resource, basestring): + try: + parsed_dict.update(_parse_single_input(resource)) + except AriaCliError: + raise AriaCliError( + "Invalid input: {0}. It must represent a dictionary. " + "Valid values can be one of:{1} " + "- A path to a YAML file{1} " + "- A path to a directory containing YAML files{1} " + "- A single quoted wildcard based path " + "(e.g. '*-inputs.yaml'){1} " + "- A string formatted as JSON/YAML{1} " + "- A string formatted as key1=value1;key2=value2".format( + resource, os.linesep)) + return parsed_dict + + +def _parse_single_input(resource): + try: + # parse resource as string representation of a dictionary + return _plain_string_to_dict(resource) + except AriaCliError: + input_files = glob.glob(resource) + parsed_dict = dict() + if os.path.isdir(resource): + for input_file in os.listdir(resource): + parsed_dict.update( + _parse_yaml_path(os.path.join(resource, input_file))) + elif input_files: + for input_file in input_files: + parsed_dict.update(_parse_yaml_path(input_file)) + else: + parsed_dict.update(_parse_yaml_path(resource)) + return parsed_dict + + +def _parse_yaml_path(resource): + + try: + # if resource is a path - parse as a yaml file + if os.path.isfile(resource): + with open(resource) as f: + content = yaml.load(f.read()) + else: + # parse resource content as yaml + content = yaml.load(resource) + except yaml.error.YAMLError as e: + raise AriaCliError("'{0}' is not a valid YAML. {1}".format( + resource, str(e))) + + # Emtpy files return None + content = content or dict() + if not isinstance(content, dict): + raise AriaCliError() + + return content + + +def _plain_string_to_dict(input_string): + input_string = input_string.strip() + input_dict = {} + mapped_inputs = input_string.split(';') + for mapped_input in mapped_inputs: + mapped_input = mapped_input.strip() + if not mapped_input: + continue + split_mapping = mapped_input.split('=') + try: + key = split_mapping[0].strip() + value = split_mapping[1].strip() + except IndexError: + raise AriaCliError( + "Invalid input format: {0}, the expected format is: " + "key1=value1;key2=value2".format(input_string)) + input_dict[key] = value + return input_dict http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/logger.py ---------------------------------------------------------------------- diff --git a/aria/cli/logger.py b/aria/cli/logger.py new file mode 100644 index 0000000..2f012d9 --- /dev/null +++ b/aria/cli/logger.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import copy +import logging +from logutils import dictconfig + + +HIGH_VERBOSE = 3 +MEDIUM_VERBOSE = 2 +LOW_VERBOSE = 1 +NO_VERBOSE = 0 + +DEFAULT_LOGGER_CONFIG = { + "version": 1, + "formatters": { + "file": { + "format": "%(asctime)s [%(levelname)s] %(message)s" + }, + "console": { + "format": "%(message)s" + } + }, + "handlers": { + "file": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "file", + "maxBytes": "5000000", + "backupCount": "20" + }, + "console": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "console" + } + }, + "disable_existing_loggers": False +} + + +class Logging(object): + + def __init__(self, config): + self._log_file = None + self._verbosity_level = NO_VERBOSE + self._all_loggers = [] + self._configure_loggers(config) + self._lgr = logging.getLogger('aria.cli.main') + + @property + def logger(self): + return self._lgr + + @property + def log_file(self): + return self._log_file + + @property + def verbosity_level(self): + return self._verbosity_level + + def is_high_verbose_level(self): + return self.verbosity_level == HIGH_VERBOSE + + @verbosity_level.setter + def verbosity_level(self, level): + self._verbosity_level = level + if self.is_high_verbose_level(): + for logger_name in self._all_loggers: + logging.getLogger(logger_name).setLevel(logging.DEBUG) + + def _configure_loggers(self, config): + loggers_config = config.logging.loggers + logfile = config.logging.filename + + logger_dict = copy.deepcopy(DEFAULT_LOGGER_CONFIG) + if logfile: + # set filename on file handler + logger_dict['handlers']['file']['filename'] = logfile + logfile_dir = os.path.dirname(logfile) + if not os.path.exists(logfile_dir): + os.makedirs(logfile_dir) + self._log_file = logfile + else: + del logger_dict['handlers']['file'] + + # add handlers to all loggers + loggers = {} + for logger_name in loggers_config: + loggers[logger_name] = dict(handlers=list(logger_dict['handlers'].keys())) + logger_dict['loggers'] = loggers + + # set level for all loggers + for logger_name, logging_level in loggers_config.iteritems(): + log = logging.getLogger(logger_name) + level = logging._levelNames[logging_level.upper()] + log.setLevel(level) + self._all_loggers.append(logger_name) + + dictconfig.dictConfig(logger_dict) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/main.py ---------------------------------------------------------------------- diff --git a/aria/cli/main.py b/aria/cli/main.py new file mode 100644 index 0000000..d06ad8a --- /dev/null +++ b/aria/cli/main.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aria import install_aria_extensions +from aria.cli import commands +from aria.cli.cli import aria + + +@aria.group(name='aria') +@aria.options.verbose() +@aria.options.version +def _aria(): + """ARIA's Command Line Interface + + To activate bash-completion. Run: `eval "$(_ARIA_COMPLETE=source aria)"` + + ARIA's working directory resides by default in ~/.aria. To change it, set + the environment variable `ARIA_WORKDIR` to something else (e.g. /tmp/). + """ + aria.set_cli_except_hook() + + +def _register_commands(): + """ + Register the CLI's commands. + """ + + _aria.add_command(commands.service_templates.service_templates) + _aria.add_command(commands.node_templates.node_templates) + _aria.add_command(commands.services.services) + _aria.add_command(commands.nodes.nodes) + _aria.add_command(commands.workflows.workflows) + _aria.add_command(commands.executions.executions) + _aria.add_command(commands.plugins.plugins) + _aria.add_command(commands.logs.logs) + + +_register_commands() + + +def main(): + install_aria_extensions() + _aria() + + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/service_template_utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/service_template_utils.py b/aria/cli/service_template_utils.py new file mode 100644 index 0000000..0300449 --- /dev/null +++ b/aria/cli/service_template_utils.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from urlparse import urlparse + +from . import csar +from . import utils +from .exceptions import AriaCliError +from ..utils import archive as archive_utils + + +def get(source, service_template_filename): + """Get a source and return a path to the main service template file + + The behavior based on then source argument content is: + - + - local archive: + extract it locally and return path service template file + - local yaml file: return the file + - URL: + - return it (download=False) + - download and get service template from downloaded file (download=True) + - github repo: + - map it to a URL and return it (download=False) + - download and get service template from downloaded file (download=True) + + Supported archive types are: csar, zip, tar, tar.gz and tar.bz2 + + :param source: Path/URL/github repo to archive/service-template file + :type source: str + :param service_template_filename: Path to service template (if source is an archive file) + :type service_template_filename: str + :param download: Download service template file if source is URL/github repo + :type download: bool + :return: Path to file (if archive/service-template file passed) or url + :rtype: str + + """ + if urlparse(source).scheme: + downloaded_file = utils.download_file(source) + return _get_service_template_file_from_archive( + downloaded_file, service_template_filename) + elif os.path.isfile(source): + if _is_archive(source): + return _get_service_template_file_from_archive(source, service_template_filename) + else: + # Maybe check if yaml. + return source + elif len(source.split('/')) == 2: + url = _map_to_github_url(source) + downloaded_file = utils.download_file(url) + return _get_service_template_file_from_archive( + downloaded_file, service_template_filename) + else: + raise AriaCliError( + 'You must provide either a path to a local file, a remote URL ' + 'or a GitHub `organization/repository[:tag/branch]`') + + +def _get_service_template_file_from_archive(archive, service_template_filename): + """Extract archive to temporary location and get path to service template file. + + :param archive: Path to archive file + :type archive: str + :param service_template_filename: Path to service template file relative to archive + :type service_template_filename: str + :return: Absolute path to service template file + :rtype: str + + """ + if csar.is_csar_archive(archive): + service_template_file = _extract_csar_archive(archive) + else: + extract_directory = archive_utils.extract_archive(archive) + service_template_dir = os.path.join( + extract_directory, + os.listdir(extract_directory)[0], + ) + service_template_file = os.path.join(service_template_dir, service_template_filename) + + if not os.path.isfile(service_template_file): + raise AriaCliError( + 'Could not find `{0}`. Please provide the name of the main ' + 'service template file by using the `-n/--service-template-filename` flag' + .format(service_template_filename)) + return service_template_file + + +def _map_to_github_url(source): + """Returns a path to a downloaded github archive. + + :param source: github repo in the format of `org/repo[:tag/branch]`. + :type source: str + :return: URL to the archive file for the given repo in github + :rtype: str + + """ + source_parts = source.split(':', 1) + repo = source_parts[0] + tag = source_parts[1] if len(source_parts) == 2 else 'master' + url = 'https://github.com/{0}/archive/{1}.tar.gz'.format(repo, tag) + return url + + +# def generate_id(service_template_path, +# service_template_filename=DEFAULT_SERVICE_TEMPLATE_FILENAME): +# """The name of the service template will be the name of the folder. +# If service_template_filename is provided, it will be appended to the folder. +# """ +# service_template_id = os.path.split(os.path.dirname(os.path.abspath( +# service_template_path)))[-1] +# if service_template_filename != DEFAULT_SERVICE_TEMPLATE_FILENAME: +# filename, _ = os.path.splitext(os.path.basename(service_template_filename)) +# service_template_id = (service_template_id + '.' + filename) +# return service_template_id.replace('_', '-') + + +def _is_archive(source): + return archive_utils.is_archive(source) or csar.is_csar_archive(source) + + +def _extract_csar_archive(archive): + if csar.is_csar_archive(archive): + reader = csar.read(source=archive) + main_service_template_file_name = os.path.basename(reader.entry_definitions) + return os.path.join(reader.destination, + main_service_template_file_name) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/storage.py ---------------------------------------------------------------------- diff --git a/aria/cli/storage.py b/aria/cli/storage.py deleted file mode 100644 index fa1518b..0000000 --- a/aria/cli/storage.py +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Filesystem related CLI storage location and configuration -""" - -import os -import getpass -from shutil import rmtree - -work_space_directory = '.aria' -storage_directory_name = 'local-storage' - - -def user_space(user_name=getpass.getuser()): - """ - Base work directory - """ - user_path = '~{0}'.format(user_name) - real_path = os.path.expanduser(user_path) - if os.path.exists(real_path): - return os.path.join(real_path, work_space_directory) - return os.path.join(os.getcwd(), work_space_directory) - - -def local_storage(user_name=getpass.getuser()): - """ - Base storage directory - """ - return os.path.join(user_space(user_name), storage_directory_name) - - -def local_model_storage(): - """ - Model storage directory - """ - return os.path.join(local_storage(), 'models') - - -def local_resource_storage(): - """ - Resource storage directory - """ - return os.path.join(local_storage(), 'resources') - - -def config_file_path(): - """ - Configuration file path - """ - path = os.path.join(user_space(), 'config.yaml') - if not os.path.exists(path): - open(path, 'w').close() - return path - - -def create_user_space(user_name=getpass.getuser(), override=False): - """ - Creates the base work directory - """ - path = user_space(user_name) - if os.path.exists(path): - if override: - rmtree(path, ignore_errors=True) - else: - raise IOError('user space {0} already exists'.format(path)) - os.mkdir(path) - return path - - -def create_local_storage(user_name=getpass.getuser(), override=False): - """ - Creates the base storage directory - """ - path = local_storage(user_name) - if os.path.exists(path): - if override: - rmtree(path, ignore_errors=True) - else: - raise IOError('local storage {0} already exists'.format(path)) - os.mkdir(path) - return path http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/table.py ---------------------------------------------------------------------- diff --git a/aria/cli/table.py b/aria/cli/table.py new file mode 100644 index 0000000..36dcbea --- /dev/null +++ b/aria/cli/table.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +from prettytable import PrettyTable + +from .env import logger + + +def generate(cols, data, defaults=None): + """ + Return a new PrettyTable instance representing the list. + + Arguments: + + cols - An iterable of strings that specify what + are the columns of the table. + + for example: ['id','name'] + + data - An iterable of dictionaries, each dictionary must + have key's corresponding to the cols items. + + for example: [{'id':'123', 'name':'Pete'] + + defaults - A dictionary specifying default values for + key's that don't exist in the data itself. + + for example: {'serviceId':'123'} will set the + serviceId value for all rows to '123'. + + """ + def get_values_per_column(column, row_data): + if column in row_data: + if row_data[column] and isinstance(row_data[column], basestring): + try: + datetime.strptime(row_data[column][:10], '%Y-%m-%d') + row_data[column] = \ + row_data[column].replace('T', ' ').replace('Z', ' ') + except ValueError: + # not a timestamp + pass + elif row_data[column] and isinstance(row_data[column], list): + row_data[column] = ','.join(row_data[column]) + elif not row_data[column]: + # if it's empty list, don't print [] + row_data[column] = '' + return row_data[column] + else: + return defaults[column] + + pretty_table = PrettyTable([col for col in cols]) + + for datum in data: + values_row = [] + for col in cols: + values_row.append(get_values_per_column(col, datum)) + pretty_table.add_row(values_row) + + return pretty_table + + +def log(title, table): + logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, table)) + + +def print_data(columns, items, header_text, max_width=None, defaults=None): + if items is None: + items = [] + elif not isinstance(items, list): + items = [items] + + pretty_table = generate(columns, data=items, defaults=defaults) + if max_width: + pretty_table.max_width = max_width + log(header_text, pretty_table) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/cli/utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/utils.py b/aria/cli/utils.py new file mode 100644 index 0000000..fad1b07 --- /dev/null +++ b/aria/cli/utils.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import string +import random +import tempfile +from StringIO import StringIO + +from backports.shutil_get_terminal_size import get_terminal_size +import requests + +from .env import logger +from .exceptions import AriaCliError + + +def dump_to_file(collection, file_path): + with open(file_path, 'a') as f: + f.write(os.linesep.join(collection)) + f.write(os.linesep) + + +def is_virtual_env(): + return hasattr(sys, 'real_prefix') + + +def storage_sort_param(sort_by, descending): + return {sort_by: 'desc' if descending else 'asc'} + + +def generate_random_string(size=6, + chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + +def generate_suffixed_id(id): + return '{0}_{1}'.format(id, generate_random_string()) + + +def get_parameter_templates_as_string(parameter_templates): + params_string = StringIO() + + for param_name, param_template in parameter_templates.iteritems(): + params_string.write('\t{0}:{1}'.format(param_name, os.linesep)) + param_dict = param_template.to_dict() + del param_dict['id'] # not interested in printing the id + for k, v in param_dict.iteritems(): + params_string.write('\t\t{0}: {1}{2}'.format(k, v, os.linesep)) + + params_string.write(os.linesep) + return params_string.getvalue() + + +def download_file(url, destination=None): + """Download file. + + :param url: Location of the file to download + :type url: str + :param destination: + Location where the file should be saved (autogenerated by default) + :type destination: str | None + :returns: Location where the file was saved + :rtype: str + + """ + chunk_size = 1024 + + if not destination: + file_descriptor, destination = tempfile.mkstemp() + os.close(file_descriptor) + logger.info('Downloading {0} to {1}...'.format(url, destination)) + + try: + response = requests.get(url, stream=True) + except requests.exceptions.RequestException as ex: + raise AriaCliError( + 'Failed to download {0}. ({1})'.format(url, str(ex))) + + final_url = response.url + if final_url != url: + logger.debug('Redirected to {0}'.format(final_url)) + + try: + with open(destination, 'wb') as destination_file: + for chunk in response.iter_content(chunk_size): + destination_file.write(chunk) + except IOError as ex: + raise AriaCliError( + 'Failed to download {0}. ({1})'.format(url, str(ex))) + + return destination + + +def generate_progress_handler(file_path, action='', max_bar_length=80): + """Returns a function that prints a progress bar in the terminal + + :param file_path: The name of the file being transferred + :param action: Uploading/Downloading + :param max_bar_length: Maximum allowed length of the bar. Default: 80 + :return: The configured print_progress function + """ + # We want to limit the maximum line length to 80, but allow for a smaller + # terminal size. We also include the action string, and some extra chars + terminal_width = get_terminal_size().columns + + # This takes care of the case where there is no terminal (e.g. unittest) + terminal_width = terminal_width or max_bar_length + bar_length = min(max_bar_length, terminal_width) - len(action) - 12 + + # Shorten the file name if it's too long + file_name = os.path.basename(file_path) + if len(file_name) > (bar_length / 4) + 3: + file_name = file_name[:bar_length / 4] + '...' + + bar_length -= len(file_name) + + def print_progress(read_bytes, total_bytes): + """Print upload/download progress on a single line + + Call this function in a loop to create a progress bar in the terminal + + :param read_bytes: Number of bytes already processed + :param total_bytes: Total number of bytes in the file + """ + + filled_length = min(bar_length, int(round(bar_length * read_bytes / + float(total_bytes)))) + percents = min(100.00, round( + 100.00 * (read_bytes / float(total_bytes)), 2)) + bar = '#' * filled_length + '-' * (bar_length - filled_length) # pylint: disable=blacklisted-name + + # The \r caret makes sure the cursor moves back to the beginning of + # the line + sys.stdout.write('\r{0} {1} |{2}| {3}%'.format( + action, file_name, bar, percents)) + if read_bytes >= total_bytes: + sys.stdout.write(os.linesep) + + return print_progress + + +def handle_storage_exception(e, model_class, name): + if 'UNIQUE constraint failed' in e.message: + msg = 'Could not store {model_class} `{name}`{linesep}' \ + 'There already a exists a {model_class} with the same name' \ + .format(model_class=model_class, name=name, linesep=os.linesep) + raise AriaCliError(msg) + raise AriaCliError() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/core.py ---------------------------------------------------------------------- diff --git a/aria/core.py b/aria/core.py new file mode 100644 index 0000000..0be53c6 --- /dev/null +++ b/aria/core.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import exceptions +from .parser import consumption +from .parser.loading.location import UriLocation +from .storage import exceptions as storage_exceptions + + +class Core(object): + + def __init__(self, + model_storage, + resource_storage, + plugin_manager): + self._model_storage = model_storage + self._resource_storage = resource_storage + self._plugin_manager = plugin_manager + + @property + def model_storage(self): + return self._model_storage + + @property + def resource_storage(self): + return self._resource_storage + + @property + def plugin_manager(self): + return self._plugin_manager + + def validate_service_template(self, service_template_path): + self._parse_service_template(service_template_path) + + def create_service_template(self, service_template_path, service_template_dir, + service_template_name): + context = self._parse_service_template(service_template_path) + service_template = context.modeling.template + service_template.name = service_template_name + self.model_storage.service_template.put(service_template) + self.resource_storage.service_template.upload( + entry_id=str(service_template.id), source=service_template_dir) + + def delete_service_template(self, service_template_id): + service_template = self.model_storage.service_template.get(service_template_id) + if service_template.services: + raise exceptions.DependentServicesError( + "Can't delete service template {0} - Service template has existing services") + + self.model_storage.service_template.delete(service_template) + self.resource_storage.service_template.delete(entry_id=str(service_template.id)) + + def create_service(self, service_template_id, inputs, service_name=None): + + service_template = self.model_storage.service_template.get(service_template_id) + + # creating an empty ConsumptionContext, initiating a threadlocal context + consumption.ConsumptionContext() + # setting no autoflush for the duration of instantiation - this helps avoid dependency + # constraints as they're being set up + with self.model_storage._all_api_kwargs['session'].no_autoflush: + service = service_template.instantiate(None, inputs) + + # If the user didn't enter a name for this service, we'll want to auto generate it. + # But how will we ensure a unique but simple name? We'll append the services' unique id + # to the service_templates name. Since this service is not in the storage yet, we'll put it + # there, and pull out its id. + self.model_storage.service.put(service) + service.name = service_name or '{0}_{1}'.format(service_template.name, service.id) + try: + self.model_storage.service.update(service) + except storage_exceptions.StorageError: + self.model_storage.service.delete(service) + raise + return service + + def delete_service(self, service_id, force=False): + service = self.model_storage.service.get(service_id) + + active_executions = [e for e in service.executions if e.is_active()] + if active_executions: + raise exceptions.DependentActiveExecutionsError( + "Can't delete service {0} - there is an active execution for this service. " + "Active execution id: {1}".format(service.name, active_executions[0].id)) + + if not force: + available_nodes = [str(n.id) for n in service.nodes.values() if n.is_available()] + if available_nodes: + raise exceptions.DependentAvailableNodesError( + "Can't delete service {0} - there are available nodes for this service. " + "Available node ids: {1}".format(service.name, ', '.join(available_nodes))) + + self.model_storage.service.delete(service) + + @staticmethod + def _parse_service_template(service_template_path): + context = consumption.ConsumptionContext() + context.presentation.location = UriLocation(service_template_path) + consumption.ConsumerChain( + context, + ( + consumption.Read, + consumption.Validate, + consumption.ServiceTemplate + )).consume() + if context.validation.dump_issues(): + raise exceptions.ParsingError('Failed to parse service template') + return context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/exceptions.py b/aria/exceptions.py index a180ce1..bdf9f78 100644 --- a/aria/exceptions.py +++ b/aria/exceptions.py @@ -44,3 +44,28 @@ class AriaException(Exception): # Make sure it's our traceback cause_traceback = traceback self.cause_traceback = cause_traceback + + +class DependentServicesError(AriaError): + """ + Raised when attempting to delete a service template which has existing services + """ + pass + + +class DependentActiveExecutionsError(AriaError): + """ + Raised when attempting to delete a service which has active executions + """ + pass + + +class DependentAvailableNodesError(AriaError): + """ + Raised when attempting to delete a service which has available nodes + """ + pass + + +class ParsingError(AriaError): + pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/logger.py ---------------------------------------------------------------------- diff --git a/aria/logger.py b/aria/logger.py index e3039f5..dd54264 100644 --- a/aria/logger.py +++ b/aria/logger.py @@ -19,8 +19,20 @@ Logging related mixins and functions import logging from logging import handlers as logging_handlers +# NullHandler doesn't exist in < 27. this workaround is from +# http://docs.python.org/release/2.6/library/logging.html#configuring-logging-for-a-library +try: + from logging import NullHandler # pylint: disable=unused-import +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass from datetime import datetime + +TASK_LOGGER_NAME = 'aria.executions.task' + + _base_logger = logging.getLogger('aria') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/modeling/__init__.py ---------------------------------------------------------------------- diff --git a/aria/modeling/__init__.py b/aria/modeling/__init__.py index 4dfc39d..4ac79e7 100644 --- a/aria/modeling/__init__.py +++ b/aria/modeling/__init__.py @@ -19,6 +19,7 @@ from . import ( mixins, types, models, + utils, service_template as _service_template_bases, service_instance as _service_instance_bases, service_changes as _service_changes_bases, @@ -45,4 +46,5 @@ __all__ = ( 'types', 'models', 'model_bases', + 'utils' ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/modeling/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/modeling/exceptions.py b/aria/modeling/exceptions.py index 6931c78..8225f37 100644 --- a/aria/modeling/exceptions.py +++ b/aria/modeling/exceptions.py @@ -32,3 +32,21 @@ class CannotEvaluateFunctionException(ModelingException): """ ARIA modeling exception: cannot evaluate the function at this time. """ + + +class MissingRequiredInputsException(ModelingException): + """ + ARIA modeling exception: Required inputs have been omitted + """ + + +class InputsOfWrongTypeException(ModelingException): + """ + ARIA modeling exception: Inputs of the wrong types have been provided + """ + + +class UndeclaredInputsException(ModelingException): + """ + ARIA modeling exception: Undeclared inputs have been provided + """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/modeling/models.py ---------------------------------------------------------------------- diff --git a/aria/modeling/models.py b/aria/modeling/models.py index 170efb2..584b877 100644 --- a/aria/modeling/models.py +++ b/aria/modeling/models.py @@ -16,6 +16,10 @@ # pylint: disable=abstract-method from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ( + Column, + Text +) from . import ( service_template, @@ -26,7 +30,6 @@ from . import ( mixins, ) - aria_declarative_base = declarative_base(cls=mixins.ModelIDMixin) @@ -84,7 +87,7 @@ __all__ = ( # region service template models class ServiceTemplate(aria_declarative_base, service_template.ServiceTemplateBase): - pass + name = Column(Text, index=True, unique=True) class NodeTemplate(aria_declarative_base, service_template.NodeTemplateBase): @@ -140,7 +143,7 @@ class PluginSpecification(aria_declarative_base, service_template.PluginSpecific # region service instance models class Service(aria_declarative_base, service_instance.ServiceBase): - pass + name = Column(Text, index=True, unique=True) class Node(aria_declarative_base, service_instance.NodeBase): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/aaf66420/aria/modeling/orchestration.py ---------------------------------------------------------------------- diff --git a/aria/modeling/orchestration.py b/aria/modeling/orchestration.py index b32a8a1..a2f041b 100644 --- a/aria/modeling/orchestration.py +++ b/aria/modeling/orchestration.py @@ -39,7 +39,6 @@ from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.declarative import declared_attr from ..orchestrator.exceptions import (TaskAbortException, TaskRetryException) -from .types import Dict from .mixins import ModelMixin from . import ( relationship, @@ -55,9 +54,7 @@ class ExecutionBase(ModelMixin): __tablename__ = 'execution' __private_fields__ = ['service_fk', - 'service_name', - 'service_template', - 'service_template_name'] + 'service_template'] TERMINATED = 'terminated' FAILED = 'failed' @@ -97,17 +94,14 @@ class ExecutionBase(ModelMixin): ended_at = Column(DateTime, nullable=True, index=True) error = Column(Text, nullable=True) is_system_workflow = Column(Boolean, nullable=False, default=False) - parameters = Column(Dict) status = Column(Enum(*STATES, name='execution_status'), default=PENDING) workflow_name = Column(Text) - @property def has_ended(self): return self.status in self.END_STATES - @property def is_active(self): - return not self.has_ended + return not self.has_ended() @declared_attr def logs(cls): @@ -121,6 +115,10 @@ class ExecutionBase(ModelMixin): def tasks(cls): return relationship.one_to_many(cls, 'task') + @declared_attr + def inputs(cls): + return relationship.many_to_many(cls, 'parameter', prefix='inputs', dict_key='name') + # region foreign keys @declared_attr @@ -264,10 +262,7 @@ class TaskBase(ModelMixin): __private_fields__ = ['node_fk', 'relationship_fk', 'plugin_fk', - 'execution_fk' - 'node_name', - 'relationship_name', - 'execution_name'] + 'execution_fk'] PENDING = 'pending' RETRYING = 'retrying' @@ -322,11 +317,9 @@ class TaskBase(ModelMixin): ended_at = Column(DateTime, default=None) retry_count = Column(Integer, default=0) - @property def has_ended(self): return self.status in (self.SUCCESS, self.FAILED) - @property def is_waiting(self): return self.status in (self.PENDING, self.RETRYING)