http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/commands/service_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py new file mode 100644 index 0000000..b855529 --- /dev/null +++ b/aria/cli/commands/service_templates.py @@ -0,0 +1,207 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import json + +from .. import utils +from .. import csar +from ..cli import aria +from .. import service_template_utils +from ..table import print_data +from ..exceptions import AriaCliError +from ...core import Core +from ...exceptions import AriaException + + +DESCRIPTION_LIMIT = 20 +SERVICE_TEMPLATE_COLUMNS = \ + ['id', 'name', 'main_file_name', 'created_at', 'updated_at'] + + [email protected](name='service-templates') [email protected]() +def service_templates(): + """Handle service templates on the manager + """ + pass + + +@service_templates.command(name='show', + short_help='Show service template information') [email protected]('service-template-id') [email protected]() [email protected]_model_storage [email protected]_logger +def show(service_template_id, model_storage, logger): + """Show information for a specific service templates + + `SERVICE_TEMPLATE_ID` is the id of the service template to show information on. + """ + logger.info('Showing service template {0}...'.format(service_template_id)) + service_template = model_storage.service_template.get(service_template_id) + services = [d.to_dict() for d in service_template.services] + service_template_dict = service_template.to_dict() + service_template_dict['#services'] = len(services) + columns = SERVICE_TEMPLATE_COLUMNS + ['#services'] + print_data(columns, service_template_dict, 'Service-template:', max_width=50) + + logger.info('Description:') + logger.info('{0}\n'.format(service_template_dict['description'].encode('UTF-8') or '')) + + logger.info('Existing services:') + logger.info('{0}\n'.format(json.dumps([d['name'] for d in services]))) + + +@service_templates.command(name='list', + short_help='List service templates') [email protected]_by() [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(sort_by, descending, model_storage, logger): + """List all service templates + """ + def trim_description(service_template): + if service_template['description'] is not None: + if len(service_template['description']) >= DESCRIPTION_LIMIT: + service_template['description'] = '{0}..'.format( + service_template['description'][:DESCRIPTION_LIMIT - 2]) + else: + service_template['description'] = '' + return service_template + + logger.info('Listing all service templates...') + service_templates = [trim_description(b.to_dict()) for b in model_storage.service_template.list( + sort=utils.storage_sort_param(sort_by, descending))] + print_data(SERVICE_TEMPLATE_COLUMNS, service_templates, 'Service templates:') + + +@service_templates.command(name='store', + short_help='Store a service template') [email protected]('service-template-path') [email protected]('service-template-name') [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def store(service_template_path, service_template_name, model_storage, resource_storage, + plugin_manager, logger): + """Store a service template + + `SERVICE_TEMPLATE_PATH` is the path of the service template to store. + + `SERVICE_TEMPLATE_NAME` is the name of the service template to store. + """ + logger.info('Storing service template {0}...'.format(service_template_name)) + + service_template_path = service_template_utils.get(service_template_path) + core = Core(model_storage, resource_storage, plugin_manager) + core.create_service_template(service_template_path, + os.path.dirname(service_template_path), + service_template_name) + + logger.info('Service template stored') + + +@service_templates.command(name='delete', + short_help='Delete a service template') [email protected]('service-template-id') [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def delete(service_template_id, model_storage, resource_storage, plugin_manager, logger): + """Delete a service template + `SERVICE_TEMPLATE_ID` is the id of the service template to delete. + """ + logger.info('Deleting service template {0}...'.format(service_template_id)) + core = Core(model_storage, resource_storage, plugin_manager) + core.delete_service_template(service_template_id) + logger.info('Service template {0} deleted'.format(service_template_id)) + + +@service_templates.command(name='inputs', + short_help='Show service template inputs') [email protected]('service-template-name') [email protected]() [email protected]_model_storage [email protected]_logger +def inputs(service_template_name, model_storage, logger): + """Show inputs for a specific service template + + `SERVICE_TEMPLATE_NAME` is the name of the service template to show inputs for. + """ + logger.info('Showing inputs for service template {0}...'.format(service_template_name)) + print_service_template_inputs(model_storage, service_template_name) + + +@service_templates.command(name='validate', + short_help='Validate a service template') [email protected]('service-template') [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def validate_service_template(service_template, model_storage, resource_storage, plugin_manager, + logger): + """Validate a service template + + `SERVICE_TEMPLATE` is the path or url of the service template or archive to validate. + """ + logger.info('Validating service template: {0}'.format(service_template)) + service_template_path = service_template_utils.get(service_template) + core = Core(model_storage, resource_storage, plugin_manager) + + try: + core.validate_service_template(service_template_path) + except AriaException as e: + # TODO: gather errors from parser and dump them via CLI? + raise AriaCliError(str(e)) + + logger.info('Service template validated successfully') + + +@service_templates.command(name='create-archive', + short_help='Create a csar archive') [email protected]('service-template-path') [email protected]('destination') [email protected]() [email protected]_logger +def create_archive(service_template_path, destination, logger): + """Create a csar archive + + `service_template_path` is the path of the service template to create the archive from + `destination` is the path of the output csar archive + """ + logger.info('Creating a csar archive') + csar.write(os.path.dirname(service_template_path), service_template_path, destination, logger) + logger.info('Csar archive created at {0}'.format(destination)) + + [email protected]_logger +def print_service_template_inputs(model_storage, service_template_name, logger): + service_template = model_storage.service_template.get_by_name(service_template_name) + + logger.info('Service template inputs:') + if service_template.inputs: + logger.info(utils.get_parameter_templates_as_string(service_template.inputs)) + else: + logger.info('\tNo inputs')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/commands/services.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py new file mode 100644 index 0000000..ce1139b --- /dev/null +++ b/aria/cli/commands/services.py @@ -0,0 +1,175 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from StringIO import StringIO + +from . import service_templates +from ..cli import aria, helptexts +from ..exceptions import AriaCliError +from ..table import print_data +from ..utils import storage_sort_param +from ...core import Core +from ...exceptions import AriaException + + +SERVICE_COLUMNS = ['id', 'name', 'service_template_name', 'created_at', 'updated_at'] + + [email protected](name='services') [email protected]() +def services(): + """Handle services + """ + pass + + [email protected](name='list', short_help='List services') [email protected]_template_id() [email protected]_by() [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_template_id, + sort_by, + descending, + model_storage, + logger): + """List services + + If `--service-template-id` is provided, list services for that service template. + Otherwise, list services for all service templates. + """ + if service_template_id: + logger.info('Listing services for service template {0}...'.format( + service_template_id)) + service_template = model_storage.service_template.get(service_template_id) + filters = dict(service_template=service_template) + else: + logger.info('Listing all service...') + filters = {} + + services = [d.to_dict() for d in model_storage.service.list( + sort=storage_sort_param(sort_by=sort_by, descending=descending), + filters=filters)] + print_data(SERVICE_COLUMNS, services, 'Services:') + + [email protected](name='create', + short_help='Create a services') [email protected]('service-name', required=False) [email protected]_template_name(required=True) [email protected] [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def create(service_template_name, + service_name, + inputs, + model_storage, + resource_storage, + plugin_manager, + logger): + """Create a service + + `SERVICE_NAME` is the name of the service you'd like to create. + + """ + logger.info('Creating new service from service template {0}...'.format( + service_template_name)) + + try: + core = Core(model_storage, resource_storage, plugin_manager) + service = core.create_service(service_template_name, inputs, service_name) + except AriaException as e: + logger.info(str(e)) + service_templates.print_service_template_inputs(model_storage, service_template_name) + raise AriaCliError(str(e)) + + logger.info("Service created. The service's name is {0}".format(service.name)) + + [email protected](name='delete', + short_help='Delete a service') [email protected]('service-name') [email protected](help=helptexts.IGNORE_RUNNING_NODES) [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def delete(service_name, force, model_storage, resource_storage, plugin_manager, logger): + """Delete a service + + `SERVICE_NAME` is the name of the service to delete. + """ + logger.info('Deleting service {0}...'.format(service_name)) + core = Core(model_storage, resource_storage, plugin_manager) + core.delete_service(service_name, force=force) + logger.info('Service {0} deleted'.format(service_name)) + + [email protected](name='outputs', + short_help='Show service outputs') [email protected]('service-name') [email protected]() [email protected]_model_storage [email protected]_logger +def outputs(service_name, model_storage, logger): + """Show outputs for a specific service + + `SERVICE_NAME` is the name of the service to print outputs for. + """ + logger.info('Showing outputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + #TODO fix this section.. + outputs_def = service.outputs + response = model_storage.service.outputs.get(service_name) + outputs_ = StringIO() + for output_name, output in response.outputs.iteritems(): + outputs_.write(' - "{0}":{1}'.format(output_name, os.linesep)) + description = outputs_def[output_name].get('description', '') + outputs_.write(' Description: {0}{1}'.format(description, + os.linesep)) + outputs_.write(' Value: {0}{1}'.format(output, os.linesep)) + logger.info(outputs_.getvalue()) + + [email protected](name='inputs', + short_help='Show service inputs') [email protected]('service-name') [email protected]() [email protected]_model_storage [email protected]_logger +def inputs(service_name, model_storage, logger): + """Show inputs for a specific service + + `SERVICE_NAME` is the id of the service to print inputs for. + """ + logger.info('Showing inputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + if service.inputs: + inputs_ = StringIO() + for input_name, input in service.inputs.iteritems(): + inputs_.write(' - "{0}":{1}'.format(input_name, os.linesep)) + inputs_.write(' Value: {0}{1}'.format(input.value, os.linesep)) + logger.info(inputs_.getvalue()) + else: + logger.info('\tNo inputs') + logger.info('') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/commands/workflows.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/workflows.py b/aria/cli/commands/workflows.py new file mode 100644 index 0000000..2180168 --- /dev/null +++ b/aria/cli/commands/workflows.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..table import print_data +from .. import utils +from ..cli import aria +from ..exceptions import AriaCliError +from ...storage.exceptions import StorageError + +WORKFLOW_COLUMNS = ['name', 'service_template_name', 'service_name'] + + [email protected](name='workflows') +def workflows(): + """Handle service workflows + """ + pass + + [email protected](name='show', + short_help='Show workflow information') [email protected]('workflow-name') [email protected]_name(required=True) [email protected]() [email protected]_model_storage [email protected]_logger +def show(workflow_name, service_name, model_storage, logger): + """Show information for a specific workflow of a specific service + + `WORKFLOW_NAME` is the name of the workflow to get information on. + """ + try: + logger.info('Retrieving workflow {0} for service {1}'.format( + workflow_name, service_name)) + service = model_storage.service.get(service_name) + workflow = next((wf for wf in service.workflows if + wf.name == workflow_name), None) + if not workflow: + raise AriaCliError( + 'Workflow {0} not found for service {1}'.format(workflow_name, service_name)) + except StorageError: + raise AriaCliError('service {0} not found'.format(service_name)) + + defaults = { + 'service_template_name': service.service_template_name, + 'service_name': service.name + } + print_data(WORKFLOW_COLUMNS, workflow, 'Workflows:', defaults=defaults) + + # print workflow parameters + mandatory_params = dict() + optional_params = dict() + for param_name, param in workflow.parameters.iteritems(): + params_group = optional_params if 'default' in param else \ + mandatory_params + params_group[param_name] = param + + logger.info('Workflow Parameters:') + logger.info('\tMandatory Parameters:') + for param_name, param in mandatory_params.iteritems(): + if 'description' in param: + logger.info('\t\t{0}\t({1})'.format(param_name, + param['description'])) + else: + logger.info('\t\t{0}'.format(param_name)) + + logger.info('\tOptional Parameters:') + for param_name, param in optional_params.iteritems(): + if 'description' in param: + logger.info('\t\t{0}: \t{1}\t({2})'.format( + param_name, param['default'], param['description'])) + else: + logger.info('\t\t{0}: \t{1}'.format(param_name, + param['default'])) + logger.info('') + + [email protected](name='list', + short_help='List workflows for a service') [email protected]_name(required=True) [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_name, model_storage, logger): + """List all workflows of a specific service + """ + logger.info('Listing workflows for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + workflows = [wf.to_dict() for wf in sorted(service.workflows.values(), key=lambda w: w.name)] + + defaults = { + 'service_template_name': service.service_template_name, + 'service_name': service.name + } + print_data(WORKFLOW_COLUMNS, workflows, 'Workflows:', defaults=defaults) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/config.py ---------------------------------------------------------------------- diff --git a/aria/cli/config.py b/aria/cli/config.py deleted file mode 100644 index d82886d..0000000 --- a/aria/cli/config.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -CLI configuration -""" - -import os -import logging -from getpass import getuser -from tempfile import gettempdir - -from yaml import safe_load - -from .storage import config_file_path - -# path to a file where cli logs will be saved. -logging_filename = os.path.join(gettempdir(), 'aria_cli_{0}.log'.format(getuser())) -# loggers log level to show -logger_level = logging.INFO -# loggers log level to show -colors = True - -import_resolver = None - - -def load_configurations(): - """ - Dynamically load attributes into the config module from the ``config.yaml`` defined in the user - configuration directory - """ - config_path = config_file_path() - with open(config_path) as config_file: - globals().update(safe_load(config_file) or {}) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/config/__init__.py ---------------------------------------------------------------------- diff --git a/aria/cli/config/__init__.py b/aria/cli/config/__init__.py new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/aria/cli/config/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/config/config.py ---------------------------------------------------------------------- diff --git a/aria/cli/config/config.py b/aria/cli/config/config.py new file mode 100644 index 0000000..7d76830 --- /dev/null +++ b/aria/cli/config/config.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import yaml +import pkg_resources + +from jinja2.environment import Template + + +class CliConfig(object): + + def __init__(self, config_path): + with open(config_path) as f: + self._config = yaml.safe_load(f.read()) + + @classmethod + def create_config(cls, workdir): + config_path = os.path.join(workdir, 'config.yaml') + if not os.path.isfile(config_path): + config_template = pkg_resources.resource_string( + __package__, + 'config_template.yaml') + + default_values = { + 'log_path': os.path.join(workdir, 'cli.log'), + 'enable_colors': True + } + + template = Template(config_template) + rendered = template.render(**default_values) + with open(config_path, 'w') as f: + f.write(rendered) + f.write(os.linesep) + + return cls(config_path) + + @property + def colors(self): + return self._config.get('colors', False) + + @property + def logging(self): + return self.Logging(self._config.get('logging')) + + class Logging(object): + + def __init__(self, logging): + self._logging = logging or {} + + @property + def filename(self): + return self._logging.get('filename') + + @property + def loggers(self): + return self._logging.get('loggers', {}) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/config/config_template.yaml ---------------------------------------------------------------------- diff --git a/aria/cli/config/config_template.yaml b/aria/cli/config/config_template.yaml new file mode 100644 index 0000000..13f2cf9 --- /dev/null +++ b/aria/cli/config/config_template.yaml @@ -0,0 +1,12 @@ +colors: {{ enable_colors }} + +logging: + + # path to a file where cli logs will be saved. + filename: {{ log_path }} + + # configuring level per logger + loggers: + + # main logger of the cli. provides basic descriptions for executed operations. + aria.cli.main: info http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/constants.py ---------------------------------------------------------------------- diff --git a/aria/cli/constants.py b/aria/cli/constants.py new file mode 100644 index 0000000..67c094d --- /dev/null +++ b/aria/cli/constants.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +SAMPLE_SERVICE_TEMPLATE_FILENAME = 'service_template.yaml' +HELP_TEXT_COLUMN_BUFFER = 5 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/csar.py ---------------------------------------------------------------------- diff --git a/aria/cli/csar.py b/aria/cli/csar.py index b185f46..5ab581b 100644 --- a/aria/cli/csar.py +++ b/aria/cli/csar.py @@ -14,12 +14,13 @@ # limitations under the License. import os +import logging import pprint import tempfile import zipfile import requests -from ruamel import yaml # @UnresolvedImport +from ruamel import yaml META_FILE = 'TOSCA-Metadata/TOSCA.meta' @@ -167,5 +168,11 @@ class _CSARReader(object): f.write(chunk) -def read(source, destination, logger): +def read(source, destination=None, logger=None): + destination = destination or tempfile.mkdtemp() + logger = logger or logging.getLogger('dummy') return _CSARReader(source=source, destination=destination, logger=logger) + + +def is_csar_archive(source): + return source.endswith('.csar') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/dry.py ---------------------------------------------------------------------- diff --git a/aria/cli/dry.py b/aria/cli/dry.py deleted file mode 100644 index 098638f..0000000 --- a/aria/cli/dry.py +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from threading import RLock - -from ..modeling import models -from ..orchestrator.decorators import operation -from ..utils.collections import OrderedDict -from ..utils.console import puts, Colored -from ..utils.formatting import safe_repr - - -_TERMINAL_LOCK = RLock() - - -def convert_to_dry(service): - """ - Converts all operations on the service (on workflows, node interfaces, and relationship - interfaces) to run dryly. - """ - - for workflow in service.workflows.itervalues(): - convert_operation_to_dry(workflow) - - for node in service.nodes.itervalues(): - for interface in node.interfaces.itervalues(): - for oper in interface.operations.itervalues(): - convert_operation_to_dry(oper) - for relationship in node.outbound_relationships: - for interface in relationship.interfaces.itervalues(): - for oper in interface.operations.itervalues(): - convert_operation_to_dry(oper) - - -def convert_operation_to_dry(oper): - """ - Converts a single :class:`Operation` to run dryly. - """ - - plugin = oper.plugin_specification.name \ - if oper.plugin_specification is not None else None - if oper.inputs is None: - oper.inputs = OrderedDict() - oper.inputs['_implementation'] = models.Parameter(name='_implementation', - type_name='string', - value=oper.implementation) - oper.inputs['_plugin'] = models.Parameter(name='_plugin', - type_name='string', - value=plugin) - oper.implementation = '{0}.{1}'.format(__name__, 'dry_operation') - oper.plugin_specification = None - - -@operation -def dry_operation(ctx, _plugin, _implementation, **kwargs): - """ - The dry operation simply prints out information about the operation to the console. - """ - - with _TERMINAL_LOCK: - print ctx.name - if hasattr(ctx, 'relationship'): - puts('> Relationship: {0} -> {1}'.format( - Colored.red(ctx.relationship.source_node.name), - Colored.red(ctx.relationship.target_node.name))) - else: - puts('> Node: {0}'.format(Colored.red(ctx.node.name))) - puts(' Operation: {0}'.format(Colored.green(ctx.name))) - _dump_implementation(_plugin, _implementation) - - -def _dump_implementation(plugin, implementation): - if plugin: - puts(' Plugin: {0}'.format(Colored.magenta(plugin, bold=True))) - if implementation: - puts(' Implementation: {0}'.format(Colored.magenta(safe_repr(implementation)))) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/env.py ---------------------------------------------------------------------- diff --git a/aria/cli/env.py b/aria/cli/env.py new file mode 100644 index 0000000..5d34141 --- /dev/null +++ b/aria/cli/env.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import json +import pkgutil + +from .config import config +from .logger import Logging +from .. import (application_model_storage, application_resource_storage) +from ..orchestrator.plugin import PluginManager +from ..storage.sql_mapi import SQLAlchemyModelAPI +from ..storage.filesystem_rapi import FileSystemResourceAPI + + +ARIA_DEFAULT_WORKDIR_NAME = '.aria' + + +class Environment(object): + + def __init__(self, workdir): + + self._workdir = workdir + self._init_workdir() + + self._config = config.CliConfig.create_config(workdir) + self._logging = Logging(self._config) + + self._model_storage_dir = os.path.join(workdir, 'models') + self._resource_storage_dir = os.path.join(workdir, 'resources') + self._plugins_dir = os.path.join(workdir, 'plugins') + + # initialized lazily + self._model_storage = None + self._resource_storage = None + self._plugin_manager = None + + @property + def workdir(self): + return self._workdir + + @property + def config(self): + return self._config + + @property + def logging(self): + return self._logging + + @property + def model_storage(self): + if not self._model_storage: + self._model_storage = self._init_sqlite_model_storage() + return self._model_storage + + @property + def resource_storage(self): + if not self._resource_storage: + self._resource_storage = self._init_fs_resource_storage() + return self._resource_storage + + @property + def plugin_manager(self): + if not self._plugin_manager: + self._plugin_manager = self._init_plugin_manager() + return self._plugin_manager + + @staticmethod + def get_version_data(): + data = pkgutil.get_data(__package__, 'VERSION') + return json.loads(data) + + def _init_workdir(self): + if not os.path.exists(self._workdir): + os.makedirs(self._workdir) + + def _init_sqlite_model_storage(self): + if not os.path.exists(self._model_storage_dir): + os.makedirs(self._model_storage_dir) + + initiator_kwargs = dict(base_dir=self._model_storage_dir) + return application_model_storage( + SQLAlchemyModelAPI, + initiator_kwargs=initiator_kwargs) + + def _init_fs_resource_storage(self): + if not os.path.exists(self._resource_storage_dir): + os.makedirs(self._resource_storage_dir) + + fs_kwargs = dict(directory=self._resource_storage_dir) + return application_resource_storage( + FileSystemResourceAPI, + api_kwargs=fs_kwargs) + + def _init_plugin_manager(self): + if not os.path.exists(self._plugins_dir): + os.makedirs(self._plugins_dir) + + return PluginManager(self._model_storage, self._plugins_dir) + + +env = Environment(os.path.join( + os.environ.get('ARIA_WORKDIR', os.path.expanduser('~')), ARIA_DEFAULT_WORKDIR_NAME)) + +logger = env.logging.logger http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/cli/exceptions.py b/aria/cli/exceptions.py index 6897731..89cfacd 100644 --- a/aria/cli/exceptions.py +++ b/aria/cli/exceptions.py @@ -13,59 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -CLI various exception classes -""" +from ..exceptions import AriaError -class AriaCliError(Exception): - """ - General CLI Exception class - """ - pass - - -class AriaCliFormatInputsError(AriaCliError): - """ - Raised when provided inputs are malformed. - """ - - def __init__(self, message, inputs): - self.inputs = inputs - super(AriaCliFormatInputsError, self).__init__(message) - - def user_message(self): - """ - Describes the format error in detail. - """ - return ( - 'Invalid input format: {0}, ' - 'the expected format is: ' - 'key1=value1;key2=value2'.format(self.inputs)) - -class AriaCliYAMLInputsError(AriaCliError): - """ - Raised when an invalid yaml file is provided - """ +class AriaCliError(AriaError): pass - - -class AriaCliInvalidInputsError(AriaCliFormatInputsError): - """ - Raised when provided inputs are invalid. - """ - - def user_message(self): - """ - Describes the error in detail. - """ - return ( - 'Invalid input: {0}. input must represent a dictionary.\n' - 'Valid values can be one of:\n' - '- a path to a YAML file\n' - '- a path to a directory containing YAML files\n' - '- a single quoted wildcard based path (e.g. "*-inputs.yaml")\n' - '- a string formatted as JSON\n' - '- a string formatted as key1=value1;key2=value2'.format(self.inputs) - ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/inputs.py ---------------------------------------------------------------------- diff --git a/aria/cli/inputs.py b/aria/cli/inputs.py new file mode 100644 index 0000000..2077b67 --- /dev/null +++ b/aria/cli/inputs.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import glob +import yaml + +from .env import logger +from.exceptions import AriaCliError + + +def inputs_to_dict(resources): + """Returns a dictionary of inputs + + `resources` can be: + - A list of files. + - A single file + - A directory containing multiple input files + - A key1=value1;key2=value2 pairs string. + - A string formatted as JSON/YAML. + - Wildcard based string (e.g. *-inputs.yaml) + """ + if not resources: + return dict() + + parsed_dict = {} + + for resource in resources: + logger.debug('Processing inputs source: {0}'.format(resource)) + # Workflow parameters always pass an empty dictionary. We ignore it + if isinstance(resource, basestring): + try: + parsed_dict.update(_parse_single_input(resource)) + except AriaCliError: + raise AriaCliError( + "Invalid input: {0}. It must represent a dictionary. " + "Valid values can be one of:\n " + "- A path to a YAML file\n " + "- A path to a directory containing YAML files\n " + "- A single quoted wildcard based path " + "(e.g. '*-inputs.yaml')\n " + "- A string formatted as JSON/YAML\n " + "- A string formatted as key1=value1;key2=value2".format( + resource)) + return parsed_dict + + +def _parse_single_input(resource): + try: + # parse resource as string representation of a dictionary + return plain_string_to_dict(resource) + except AriaCliError: + input_files = glob.glob(resource) + parsed_dict = dict() + if os.path.isdir(resource): + for input_file in os.listdir(resource): + parsed_dict.update( + _parse_yaml_path(os.path.join(resource, input_file))) + elif input_files: + for input_file in input_files: + parsed_dict.update(_parse_yaml_path(input_file)) + else: + parsed_dict.update(_parse_yaml_path(resource)) + return parsed_dict + + +def _parse_yaml_path(resource): + + try: + # if resource is a path - parse as a yaml file + if os.path.isfile(resource): + with open(resource) as f: + content = yaml.load(f.read()) + else: + # parse resource content as yaml + content = yaml.load(resource) + except yaml.error.YAMLError as e: + raise AriaCliError("'{0}' is not a valid YAML. {1}".format( + resource, str(e))) + + # Emtpy files return None + content = content or dict() + if not isinstance(content, dict): + raise AriaCliError() + + return content + + +def plain_string_to_dict(input_string): + input_string = input_string.strip() + input_dict = {} + mapped_inputs = input_string.split(';') + for mapped_input in mapped_inputs: + mapped_input = mapped_input.strip() + if not mapped_input: + continue + split_mapping = mapped_input.split('=') + try: + key = split_mapping[0].strip() + value = split_mapping[1].strip() + except IndexError: + raise AriaCliError( + "Invalid input format: {0}, the expected format is: " + "key1=value1;key2=value2".format(input_string)) + input_dict[key] = value + return input_dict http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/logger.py ---------------------------------------------------------------------- diff --git a/aria/cli/logger.py b/aria/cli/logger.py new file mode 100644 index 0000000..289dbd3 --- /dev/null +++ b/aria/cli/logger.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import copy +import logging +import logging.config + + +HIGH_VERBOSE = 3 +MEDIUM_VERBOSE = 2 +LOW_VERBOSE = 1 +NO_VERBOSE = 0 + +DEFAULT_LOGGER_CONFIG = { + "version": 1, + "formatters": { + "file": { + "format": "%(asctime)s [%(levelname)s] %(message)s" + }, + "console": { + "format": "%(message)s" + } + }, + "handlers": { + "file": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "file", + "maxBytes": "5000000", + "backupCount": "20" + }, + "console": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "console" + } + } +} + + +class Logging(object): + + def __init__(self, config): + self._log_file = None + self._verbosity_level = NO_VERBOSE + self._all_loggers = [] + self._configure_loggers(config) + self._lgr = logging.getLogger('aria.cli.main') + + @property + def logger(self): + return self._lgr + + @property + def log_file(self): + return self._log_file + + @property + def verbosity_level(self): + return self._verbosity_level + + def is_high_verbose_level(self): + return self.verbosity_level == HIGH_VERBOSE + + @verbosity_level.setter + def verbosity_level(self, level): + self._verbosity_level = level + if self.is_high_verbose_level(): + for logger_name in self._all_loggers: + logging.getLogger(logger_name).setLevel(logging.DEBUG) + + def _configure_loggers(self, config): + loggers_config = config.logging.loggers + logfile = config.logging.filename + + logger_dict = copy.deepcopy(DEFAULT_LOGGER_CONFIG) + if logfile: + # set filename on file handler + logger_dict['handlers']['file']['filename'] = logfile + logfile_dir = os.path.dirname(logfile) + if not os.path.exists(logfile_dir): + os.makedirs(logfile_dir) + self._log_file = logfile + else: + del logger_dict['handlers']['file'] + + # add handlers to all loggers + loggers = {} + for logger_name in loggers_config: + loggers[logger_name] = dict(handlers=list(logger_dict['handlers'].keys())) + logger_dict['loggers'] = loggers + + # set level for all loggers + for logger_name, logging_level in loggers_config.iteritems(): + log = logging.getLogger(logger_name) + level = logging._levelNames[logging_level.upper()] + log.setLevel(level) + self._all_loggers.append(logger_name) + + logging.config.dictConfig(logger_dict) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/main.py ---------------------------------------------------------------------- diff --git a/aria/cli/main.py b/aria/cli/main.py new file mode 100644 index 0000000..4544e40 --- /dev/null +++ b/aria/cli/main.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#TODO handle +if __name__ == '__main__' and __package__ is None: + import aria.cli + __package__ = 'aria.cli' + +# from . import env +from . import logger +from .cli import aria +from .commands import service_templates +from .commands import node_templates +from .commands import services +from .commands import nodes +from .commands import workflows +from .commands import executions +from .commands import plugins +from .commands import logs +from .. import install_aria_extensions + + [email protected](name='aria') [email protected]() [email protected] +def _aria(): + """ARIA's Command Line Interface + + To activate bash-completion. Run: `eval "$(_ARIA_COMPLETE=source aria)"` + + ARIA's working directory resides by default in ~/.aria. To change it, set + the environment variable `ARIA_WORKDIR` to something else (e.g. /tmp/). + """ + aria.set_cli_except_hook() + + +def _register_commands(): + """ + Register the CLI's commands. + """ + + _aria.add_command(service_templates.service_templates) + _aria.add_command(node_templates.node_templates) + _aria.add_command(services.services) + _aria.add_command(nodes.nodes) + _aria.add_command(workflows.workflows) + _aria.add_command(executions.executions) + _aria.add_command(plugins.plugins) + _aria.add_command(logs.logs) + + +_register_commands() + + +def main(): + install_aria_extensions() + _aria() + + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/service_template_utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/service_template_utils.py b/aria/cli/service_template_utils.py new file mode 100644 index 0000000..4ef4ff1 --- /dev/null +++ b/aria/cli/service_template_utils.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from urlparse import urlparse + +from . import csar +from . import utils +from .exceptions import AriaCliError +from .constants import SAMPLE_SERVICE_TEMPLATE_FILENAME +from ..utils import archive as archive_utils + + +def get(source, service_template_filename=SAMPLE_SERVICE_TEMPLATE_FILENAME): + """Get a source and return a path to the main service template file + + The behavior based on then source argument content is: + - + - local archive: + extract it locally and return path service template file + - local yaml file: return the file + - URL: + - return it (download=False) + - download and get service template from downloaded file (download=True) + - github repo: + - map it to a URL and return it (download=False) + - download and get service template from downloaded file (download=True) + + Supported archive types are: csar, zip, tar, tar.gz and tar.bz2 + + :param source: Path/URL/github repo to archive/service-template file + :type source: str + :param service_template_filename: Path to service template (if source is an archive file) + :type service_template_filename: str + :param download: Download service template file if source is URL/github repo + :type download: bool + :return: Path to file (if archive/service-template file passed) or url + :rtype: str + + """ + if urlparse(source).scheme: + downloaded_file = utils.download_file(source) + return _get_service_template_file_from_archive( + downloaded_file, service_template_filename) + elif os.path.isfile(source): + if _is_archive(source): + return _get_service_template_file_from_archive(source, service_template_filename) + else: + # Maybe check if yaml. + return source + elif len(source.split('/')) == 2: + url = _map_to_github_url(source) + downloaded_file = utils.download_file(url) + return _get_service_template_file_from_archive( + downloaded_file, service_template_filename) + else: + raise AriaCliError( + 'You must provide either a path to a local file, a remote URL ' + 'or a GitHub `organization/repository[:tag/branch]`') + + +def _get_service_template_file_from_archive(archive, service_template_filename): + """Extract archive to temporary location and get path to service template file. + + :param archive: Path to archive file + :type archive: str + :param service_template_filename: Path to service template file relative to archive + :type service_template_filename: str + :return: Absolute path to service template file + :rtype: str + + """ + if csar.is_csar_archive(archive): + service_template_file = _extract_csar_archive(archive) + else: + extract_directory = archive_utils.extract_archive(archive)(archive) + service_template_dir = os.path.join( + extract_directory, + os.listdir(extract_directory)[0], + ) + service_template_file = os.path.join(service_template_dir, service_template_filename) + + if not os.path.isfile(service_template_file): + raise AriaCliError( + 'Could not find `{0}`. Please provide the name of the main ' + 'service template file by using the `-n/--service-template-filename` flag' + .format(service_template_filename)) + return service_template_file + + +def _map_to_github_url(source): + """Returns a path to a downloaded github archive. + + :param source: github repo in the format of `org/repo[:tag/branch]`. + :type source: str + :return: URL to the archive file for the given repo in github + :rtype: str + + """ + source_parts = source.split(':', 1) + repo = source_parts[0] + tag = source_parts[1] if len(source_parts) == 2 else 'master' + url = 'https://github.com/{0}/archive/{1}.tar.gz'.format(repo, tag) + return url + + +def generate_id(service_template_path, service_template_filename=SAMPLE_SERVICE_TEMPLATE_FILENAME): + """The name of the service template will be the name of the folder. + If service_template_filename is provided, it will be appended to the folder. + """ + service_template_id = os.path.split(os.path.dirname(os.path.abspath( + service_template_path)))[-1] + if not service_template_filename == SAMPLE_SERVICE_TEMPLATE_FILENAME: + filename, _ = os.path.splitext(os.path.basename(service_template_filename)) + service_template_id = (service_template_id + '.' + filename) + return service_template_id.replace('_', '-') + + +def _is_archive(source): + return archive_utils.is_archive(source) or csar.is_csar_archive(source) + + +def _extract_csar_archive(archive): + if csar.is_csar_archive(archive): + reader = csar.read(source=archive) + main_service_template_file_name = os.path.basename(reader.entry_definitions) + return os.path.join(reader.destination, + main_service_template_file_name) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/storage.py ---------------------------------------------------------------------- diff --git a/aria/cli/storage.py b/aria/cli/storage.py deleted file mode 100644 index fa1518b..0000000 --- a/aria/cli/storage.py +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Filesystem related CLI storage location and configuration -""" - -import os -import getpass -from shutil import rmtree - -work_space_directory = '.aria' -storage_directory_name = 'local-storage' - - -def user_space(user_name=getpass.getuser()): - """ - Base work directory - """ - user_path = '~{0}'.format(user_name) - real_path = os.path.expanduser(user_path) - if os.path.exists(real_path): - return os.path.join(real_path, work_space_directory) - return os.path.join(os.getcwd(), work_space_directory) - - -def local_storage(user_name=getpass.getuser()): - """ - Base storage directory - """ - return os.path.join(user_space(user_name), storage_directory_name) - - -def local_model_storage(): - """ - Model storage directory - """ - return os.path.join(local_storage(), 'models') - - -def local_resource_storage(): - """ - Resource storage directory - """ - return os.path.join(local_storage(), 'resources') - - -def config_file_path(): - """ - Configuration file path - """ - path = os.path.join(user_space(), 'config.yaml') - if not os.path.exists(path): - open(path, 'w').close() - return path - - -def create_user_space(user_name=getpass.getuser(), override=False): - """ - Creates the base work directory - """ - path = user_space(user_name) - if os.path.exists(path): - if override: - rmtree(path, ignore_errors=True) - else: - raise IOError('user space {0} already exists'.format(path)) - os.mkdir(path) - return path - - -def create_local_storage(user_name=getpass.getuser(), override=False): - """ - Creates the base storage directory - """ - path = local_storage(user_name) - if os.path.exists(path): - if override: - rmtree(path, ignore_errors=True) - else: - raise IOError('local storage {0} already exists'.format(path)) - os.mkdir(path) - return path http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/table.py ---------------------------------------------------------------------- diff --git a/aria/cli/table.py b/aria/cli/table.py new file mode 100644 index 0000000..9c195f5 --- /dev/null +++ b/aria/cli/table.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +from .env import logger + +from prettytable import PrettyTable + + +def generate(cols, data, defaults=None): + """ + Return a new PrettyTable instance representing the list. + + Arguments: + + cols - An iterable of strings that specify what + are the columns of the table. + + for example: ['id','name'] + + data - An iterable of dictionaries, each dictionary must + have key's corresponding to the cols items. + + for example: [{'id':'123', 'name':'Pete'] + + defaults - A dictionary specifying default values for + key's that don't exist in the data itself. + + for example: {'serviceId':'123'} will set the + serviceId value for all rows to '123'. + + """ + def get_values_per_column(column, row_data): + if column in row_data: + if row_data[column] and isinstance(row_data[column], basestring): + try: + datetime.strptime(row_data[column][:10], '%Y-%m-%d') + row_data[column] = \ + row_data[column].replace('T', ' ').replace('Z', ' ') + except ValueError: + # not a timestamp + pass + elif row_data[column] and isinstance(row_data[column], list): + row_data[column] = ','.join(row_data[column]) + elif not row_data[column]: + # if it's empty list, don't print [] + row_data[column] = '' + return row_data[column] + else: + return defaults[column] + + pt = PrettyTable([col for col in cols]) + + for d in data: + values_row = [] + for c in cols: + values_row.append(get_values_per_column(c, d)) + pt.add_row(values_row) + + return pt + + +def log(title, tb): + logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, tb)) + + +def print_data(columns, items, header_text, max_width=None, defaults=None): + if items is None: + items = [] + elif not isinstance(items, list): + items = [items] + + pt = generate(columns, data=items, defaults=defaults) + if max_width: + pt.max_width = max_width + log(header_text, pt) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/cli/utils.py ---------------------------------------------------------------------- diff --git a/aria/cli/utils.py b/aria/cli/utils.py new file mode 100644 index 0000000..3b68729 --- /dev/null +++ b/aria/cli/utils.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import string +import random +import tempfile +from StringIO import StringIO + +from backports.shutil_get_terminal_size import get_terminal_size +import requests + +from .env import logger +from .exceptions import AriaCliError + + +def dump_to_file(collection, file_path): + with open(file_path, 'a') as f: + f.write(os.linesep.join(collection)) + f.write(os.linesep) + + +def is_virtual_env(): + return hasattr(sys, 'real_prefix') + + +def storage_sort_param(sort_by, descending): + return {sort_by: 'desc' if descending else 'asc'} + + +def generate_random_string(size=6, + chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + +def generate_suffixed_id(id): + return '{0}_{1}'.format(id, generate_random_string()) + + +def get_parameter_templates_as_string(parameter_templates): + params_string = StringIO() + + for param_name, param_template in parameter_templates.iteritems(): + params_string.write('\t{0}:{1}'.format(param_name, os.linesep)) + param_dict = param_template.to_dict() + del param_dict['id'] # not interested in printing the id + for k, v in param_dict.iteritems(): + params_string.write('\t\t{0}: {1}{2}'.format(k, v, os.linesep)) + + params_string.write(os.linesep) + return params_string.getvalue() + + +def download_file(url, destination=None): + """Download file. + + :param url: Location of the file to download + :type url: str + :param destination: + Location where the file should be saved (autogenerated by default) + :type destination: str | None + :returns: Location where the file was saved + :rtype: str + + """ + CHUNK_SIZE = 1024 + + if not destination: + fd, destination = tempfile.mkstemp() + os.close(fd) + logger.info('Downloading {0} to {1}...'.format(url, destination)) + + try: + response = requests.get(url, stream=True) + except requests.exceptions.RequestException as ex: + raise AriaCliError( + 'Failed to download {0}. ({1})'.format(url, str(ex))) + + final_url = response.url + if final_url != url: + logger.debug('Redirected to {0}'.format(final_url)) + + try: + with open(destination, 'wb') as destination_file: + for chunk in response.iter_content(CHUNK_SIZE): + destination_file.write(chunk) + except IOError as ex: + raise AriaCliError( + 'Failed to download {0}. ({1})'.format(url, str(ex))) + + return destination + + +def generate_progress_handler(file_path, action='', max_bar_length=80): + """Returns a function that prints a progress bar in the terminal + + :param file_path: The name of the file being transferred + :param action: Uploading/Downloading + :param max_bar_length: Maximum allowed length of the bar. Default: 80 + :return: The configured print_progress function + """ + # We want to limit the maximum line length to 80, but allow for a smaller + # terminal size. We also include the action string, and some extra chars + terminal_width = get_terminal_size().columns + + # This takes care of the case where there is no terminal (e.g. unittest) + terminal_width = terminal_width or max_bar_length + bar_length = min(max_bar_length, terminal_width) - len(action) - 12 + + # Shorten the file name if it's too long + file_name = os.path.basename(file_path) + if len(file_name) > (bar_length / 4) + 3: + file_name = file_name[:bar_length / 4] + '...' + + bar_length -= len(file_name) + + def print_progress(read_bytes, total_bytes): + """Print upload/download progress on a single line + + Call this function in a loop to create a progress bar in the terminal + + :param read_bytes: Number of bytes already processed + :param total_bytes: Total number of bytes in the file + """ + + filled_length = min(bar_length, int(round(bar_length * read_bytes / + float(total_bytes)))) + percents = min(100.00, round( + 100.00 * (read_bytes / float(total_bytes)), 2)) + bar = '#' * filled_length + '-' * (bar_length - filled_length) + + # The \r caret makes sure the cursor moves back to the beginning of + # the line + sys.stdout.write('\r{0} {1} |{2}| {3}%'.format( + action, file_name, bar, percents)) + if read_bytes >= total_bytes: + sys.stdout.write('\n') + + return print_progress http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/core.py ---------------------------------------------------------------------- diff --git a/aria/core.py b/aria/core.py new file mode 100644 index 0000000..96a967f --- /dev/null +++ b/aria/core.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import exceptions +from .modeling import models +from .modeling import utils as modeling_utils +from .parser.consumption import ( + ConsumptionContext, + ConsumerChain, + Read, + Validate, + ServiceTemplate) +from .parser.loading.location import UriLocation + + +class Core(object): + + def __init__(self, + model_storage, + resource_storage, + plugin_manager): + self._model_storage = model_storage + self._resource_storage = resource_storage + self._plugin_manager = plugin_manager + + @property + def model_storage(self): + return self._model_storage + + @property + def resource_storage(self): + return self._resource_storage + + @property + def plugin_manager(self): + return self._plugin_manager + + def validate_service_template(self, service_template_path): + self._parse_service_template(service_template_path) + + def create_service_template(self, service_template_path, service_template_dir, + service_template_name): + context = self._parse_service_template(service_template_path) + service_template = context.modeling.template + service_template.name = service_template_name + self.model_storage.service_template.put(service_template) + self.resource_storage.service_template.upload( + entry_id=str(service_template.id), source=service_template_dir) + + def delete_service_template(self, service_template_id): + service_template = self.model_storage.service_template.get(service_template_id) + if service_template.services.all(): + raise exceptions.DependentServicesError( + "Can't delete service template {0} - Service template has existing services") + + self.model_storage.service_template.delete(service_template) + self.resource_storage.service_template.delete(entry_id=str(service_template.id)) + + def create_service(self, service_template_name, inputs, service_name=None): + service_template = self.model_storage.service_template.get_by_name(service_template_name) + + # creating an empty ConsumptionContext, initiating a threadlocal context + ConsumptionContext() + with self.model_storage._all_api_kwargs['session'].no_autoflush: + service = service_template.instantiate(None) + + template_inputs = service_template.inputs + service.inputs = modeling_utils.create_inputs(inputs, template_inputs) + # TODO: now that we have inputs, we should scan properties and inputs and evaluate functions + + # first put the service model so it could have an id, as fallback for setting its name + self.model_storage.service.put(service) + service.name = service_name or '{0}_{1}'.format(service_template_name, service.id) + self.model_storage.service.update(service) + return service + + def delete_service(self, service_name, force=False): + service = self.model_storage.service.get_by_name(service_name) + + active_executions = [e for e in service.executions + if e.status not in models.Execution.ACTIVE_STATES] + if active_executions: + raise exceptions.DependentActiveExecutionsError( + "Can't delete service {0} - there is an active execution for this service. " + "Active execution id: {1}".format(service_name, active_executions[0].id)) + + if not force: + available_nodes = [n for n in service.nodes.values() + if n.state not in ('deleted', 'errored')] + if available_nodes: + raise exceptions.DependentAvailableNodesError( + "Can't delete service {0} - there are available nodes for this service. " + "Available node ids: {1}".format(service_name, available_nodes)) + + self.model_storage.service.delete(service) + + @staticmethod + def _parse_service_template(service_template_path): + context = ConsumptionContext() + context.presentation.location = UriLocation(service_template_path) + ConsumerChain(context, (Read, Validate, ServiceTemplate)).consume() + if context.validation.dump_issues(): + raise exceptions.ParsingError('Failed to parse service template') + return context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/exceptions.py b/aria/exceptions.py index a180ce1..72adda5 100644 --- a/aria/exceptions.py +++ b/aria/exceptions.py @@ -44,3 +44,19 @@ class AriaException(Exception): # Make sure it's our traceback cause_traceback = traceback self.cause_traceback = cause_traceback + + +class DependentServicesError(AriaError): + pass + + +class DependentActiveExecutionsError(AriaError): + pass + + +class DependentAvailableNodesError(AriaError): + pass + + +class ParsingError(AriaError): + pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/__init__.py ---------------------------------------------------------------------- diff --git a/aria/modeling/__init__.py b/aria/modeling/__init__.py index 4dfc39d..4ac79e7 100644 --- a/aria/modeling/__init__.py +++ b/aria/modeling/__init__.py @@ -19,6 +19,7 @@ from . import ( mixins, types, models, + utils, service_template as _service_template_bases, service_instance as _service_instance_bases, service_changes as _service_changes_bases, @@ -45,4 +46,5 @@ __all__ = ( 'types', 'models', 'model_bases', + 'utils' ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/modeling/exceptions.py b/aria/modeling/exceptions.py index 6931c78..f699560 100644 --- a/aria/modeling/exceptions.py +++ b/aria/modeling/exceptions.py @@ -32,3 +32,21 @@ class CannotEvaluateFunctionException(ModelingException): """ ARIA modeling exception: cannot evaluate the function at this time. """ + + +class MissingRequiredInputsException(ModelingException): + """ + ARIA modeling exception: Required inputs have been omitted + """ + + +class InputOfWrongTypeException(ModelingException): + """ + ARIA modeling exception: Inputs of the wrong types have been provided + """ + + +class UndeclaredInputsException(ModelingException): + """ + ARIA modeling exception: Undeclared inputs have been provided + """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/orchestration.py ---------------------------------------------------------------------- diff --git a/aria/modeling/orchestration.py b/aria/modeling/orchestration.py index f0bd4b2..15abde4 100644 --- a/aria/modeling/orchestration.py +++ b/aria/modeling/orchestration.py @@ -55,9 +55,7 @@ class ExecutionBase(ModelMixin): __tablename__ = 'execution' __private_fields__ = ['service_fk', - 'service_name', - 'service_template', - 'service_template_name'] + 'service_template'] TERMINATED = 'terminated' FAILED = 'failed' @@ -97,7 +95,6 @@ class ExecutionBase(ModelMixin): ended_at = Column(DateTime, nullable=True, index=True) error = Column(Text, nullable=True) is_system_workflow = Column(Boolean, nullable=False, default=False) - parameters = Column(Dict) status = Column(Enum(*STATES, name='execution_status'), default=PENDING) workflow_name = Column(Text) @@ -121,6 +118,10 @@ class ExecutionBase(ModelMixin): def tasks(cls): return relationship.one_to_many(cls, 'task') + @declared_attr + def inputs(cls): + return relationship.many_to_many(cls, 'parameter', prefix='inputs', dict_key='name') + # region foreign keys @declared_attr @@ -227,10 +228,7 @@ class TaskBase(ModelMixin): __private_fields__ = ['node_fk', 'relationship_fk', 'plugin_fk', - 'execution_fk', - 'node_name', - 'relationship_name', - 'execution_name'] + 'execution_fk'] PENDING = 'pending' RETRYING = 'retrying' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/service_changes.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_changes.py b/aria/modeling/service_changes.py index b1a75a2..1974424 100644 --- a/aria/modeling/service_changes.py +++ b/aria/modeling/service_changes.py @@ -45,9 +45,7 @@ class ServiceUpdateBase(ModelMixin): __tablename__ = 'service_update' __private_fields__ = ['service_fk', - 'execution_fk', - 'execution_name', - 'service_name'] + 'execution_fk'] created_at = Column(DateTime, nullable=False, index=True) service_plan = Column(Dict, nullable=False) @@ -125,8 +123,7 @@ class ServiceUpdateStepBase(ModelMixin): __tablename__ = 'service_update_step' - __private_fields__ = ['service_update_fk', - 'service_update_name'] + __private_fields__ = ['service_update_fk'] _action_types = namedtuple('ACTION_TYPES', 'ADD, REMOVE, MODIFY') ACTION_TYPES = _action_types(ADD='add', REMOVE='remove', MODIFY='modify') @@ -222,8 +219,7 @@ class ServiceModificationBase(ModelMixin): __tablename__ = 'service_modification' - __private_fields__ = ['service_fk', - 'service_name'] + __private_fields__ = ['service_fk'] STARTED = 'started' FINISHED = 'finished' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/service_instance.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_instance.py b/aria/modeling/service_instance.py index d15aa7e..e6d2fe2 100644 --- a/aria/modeling/service_instance.py +++ b/aria/modeling/service_instance.py @@ -86,8 +86,7 @@ class ServiceBase(InstanceModelMixin): __tablename__ = 'service' __private_fields__ = ['substitution_fk', - 'service_template_fk', - 'service_template_name'] + 'service_template_fk'] # region foreign keys @@ -369,8 +368,7 @@ class NodeBase(InstanceModelMixin): __private_fields__ = ['type_fk', 'host_fk', 'service_fk', - 'node_template_fk', - 'service_name'] + 'node_template_fk'] INITIAL = 'initial' CREATING = 'creating' @@ -450,6 +448,11 @@ class NodeBase(InstanceModelMixin): """Required for use by SQLAlchemy queries""" return association_proxy('service', 'name') + @declared_attr + def node_template_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('node_template', 'name') + # endregion # region one_to_one relationships @@ -1157,9 +1160,7 @@ class RelationshipBase(InstanceModelMixin): 'target_node_fk', 'target_capability_fk', 'requirement_template_fk', - 'relationship_template_fk', - 'source_node_name', - 'target_node_name'] + 'relationship_template_fk'] # region foreign keys http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/service_template.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_template.py b/aria/modeling/service_template.py index 8355521..86cf81a 100644 --- a/aria/modeling/service_template.py +++ b/aria/modeling/service_template.py @@ -287,7 +287,6 @@ class ServiceTemplateBase(TemplateModelMixin): updated_at=now, description=deepcopy_with_locators(self.description), service_template=self) - #service.name = '{0}_{1}'.format(self.name, service.id) context.modeling.instance = service @@ -309,12 +308,6 @@ class ServiceTemplateBase(TemplateModelMixin): utils.instantiate_dict(self, service.inputs, self.inputs) utils.instantiate_dict(self, service.outputs, self.outputs) - for name, the_input in context.modeling.inputs.iteritems(): - if name not in service.inputs: - context.validation.report('input "{0}" is not supported'.format(name)) - else: - service.inputs[name].value = the_input - return service def validate(self): @@ -438,8 +431,7 @@ class NodeTemplateBase(TemplateModelMixin): __tablename__ = 'node_template' __private_fields__ = ['type_fk', - 'service_template_fk', - 'service_template_name'] + 'service_template_fk'] # region foreign_keys @@ -462,6 +454,11 @@ class NodeTemplateBase(TemplateModelMixin): """Required for use by SQLAlchemy queries""" return association_proxy('service_template', 'name') + @declared_attr + def type_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('type', 'name') + # endregion # region one_to_one relationships @@ -548,6 +545,7 @@ class NodeTemplateBase(TemplateModelMixin): type=self.type, description=deepcopy_with_locators(self.description), state=models.Node.INITIAL, + runtime_properties={}, node_template=self) utils.instantiate_dict(node, node.properties, self.properties) utils.instantiate_dict(node, node.interfaces, self.interface_templates) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9852f278/aria/modeling/utils.py ---------------------------------------------------------------------- diff --git a/aria/modeling/utils.py b/aria/modeling/utils.py index 0b4015c..f172a50 100644 --- a/aria/modeling/utils.py +++ b/aria/modeling/utils.py @@ -13,12 +13,95 @@ # See the License for the specific language governing permissions and # limitations under the License. +from json import JSONEncoder +from StringIO import StringIO + +from . import exceptions from ..parser.consumption import ConsumptionContext from ..parser.exceptions import InvalidValueError from ..parser.presentation import Value from ..utils.collections import OrderedDict from ..utils.console import puts -from .exceptions import CannotEvaluateFunctionException +from ..utils.type import validate_value_type + + +class ModelJSONEncoder(JSONEncoder): + def default(self, o): + from .mixins import ModelMixin + if isinstance(o, ModelMixin): + if hasattr(o, 'value'): + dict_to_return = o.to_dict(fields=('value',)) + return dict_to_return['value'] + else: + return o.to_dict() + else: + return JSONEncoder.default(self, o) + + +def create_inputs(inputs, template_inputs): + """ + :param inputs: key-value dict + :param template_inputs: parameter name to parameter object dict + :return: dict of parameter name to Parameter models + """ + merged_inputs = _merge_and_validate_inputs(inputs, template_inputs) + + from . import models + input_models = [] + for input_name, input_val in merged_inputs.iteritems(): + parameter = models.Parameter( + name=input_name, + type_name=template_inputs[input_name].type_name, + description=template_inputs[input_name].description, + value=input_val) + input_models.append(parameter) + + return {input.name: input for input in input_models} + + +def _merge_and_validate_inputs(inputs, template_inputs): + """ + :param inputs: key-value dict + :param template_inputs: parameter name to parameter object dict + :return: + """ + merged_inputs = inputs.copy() + + missing_inputs = [] + wrong_type_inputs = {} + for input_name, input_template in template_inputs.iteritems(): + if input_name not in inputs: + if input_template.value is not None: + merged_inputs[input_name] = input_template.value # apply default value + else: + missing_inputs.append(input_name) + else: + # Validate input type + try: + validate_value_type(inputs[input_name], input_template.type_name) + except ValueError: + wrong_type_inputs[input_name] = input_template.type_name + + if missing_inputs: + raise exceptions.MissingRequiredInputsException( + 'Required inputs {0} have not been specified - expected inputs: {1}' + .format(missing_inputs, template_inputs.keys())) + + if wrong_type_inputs: + error_message = StringIO() + for param_name, param_type in wrong_type_inputs.iteritems(): + error_message.write('Input "{0}" must be of type {1}\n'. + format(param_name, param_type)) + raise exceptions.InputOfWrongTypeException(error_message.getvalue()) + + undeclared_inputs = [input_name for input_name in inputs.keys() + if input_name not in template_inputs] + if undeclared_inputs: + raise exceptions.UndeclaredInputsException( + 'Undeclared inputs have been specified: {0}; Expected inputs: {1}' + .format(undeclared_inputs, template_inputs.keys())) + + return merged_inputs def coerce_value(container, value, report_issues=False): @@ -35,7 +118,7 @@ def coerce_value(container, value, report_issues=False): try: value = value._evaluate(context, container) value = coerce_value(container, value, report_issues) - except CannotEvaluateFunctionException: + except exceptions.CannotEvaluateFunctionException: pass except InvalidValueError as e: if report_issues:
