ARIA-48 Revamped ARIA CLI This is a large commit which revolves mostly around creating the new CLI, but is also tying ARIA's various components together for real for the first time - allowing a complete run of the "hello-world" example and more.
This commit introduces a few other important modules: - aria/core.py - used for managing service-templates and services. - aria/orchestator/workflow_runner.py - used for managing a workflow execution on a service. - aria/orchestrator/dry.py - a "dry executor", used for dry-executing workflows and printing the tasks that would run. Other fixes that were required for the successful usage of ARIA end-to-end have also been introduced in this commit, but there have been too many to list; Review the commit for more info. Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/8e5a1ec2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/8e5a1ec2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/8e5a1ec2 Branch: refs/heads/ARIA-139-attributes Commit: 8e5a1ec2fb8d072dc9725be700fce3c570d51de3 Parents: a7e7826 Author: Ran Ziv <[email protected]> Authored: Tue Mar 28 12:17:46 2017 +0300 Committer: Ran Ziv <[email protected]> Committed: Wed Apr 19 16:36:32 2017 +0300 ---------------------------------------------------------------------- aria/.pylintrc | 2 +- aria/__init__.py | 2 +- aria/cli/args_parser.py | 269 --------- aria/cli/cli.py | 113 ---- aria/cli/commands.py | 546 ------------------- aria/cli/commands/__init__.py | 26 + aria/cli/commands/executions.py | 172 ++++++ aria/cli/commands/logs.py | 65 +++ aria/cli/commands/node_templates.py | 93 ++++ aria/cli/commands/nodes.py | 87 +++ aria/cli/commands/plugins.py | 99 ++++ aria/cli/commands/reset.py | 40 ++ aria/cli/commands/service_templates.py | 208 +++++++ aria/cli/commands/services.py | 179 ++++++ aria/cli/commands/workflows.py | 100 ++++ aria/cli/config.py | 46 -- aria/cli/config/__init__.py | 14 + aria/cli/config/config.py | 73 +++ aria/cli/config/config_template.yaml | 12 + aria/cli/core/__init__.py | 14 + aria/cli/core/aria.py | 429 +++++++++++++++ aria/cli/csar.py | 13 +- aria/cli/defaults.py | 20 + aria/cli/dry.py | 93 ---- aria/cli/env.py | 124 +++++ aria/cli/exceptions.py | 54 +- aria/cli/helptexts.py | 49 ++ aria/cli/inputs.py | 118 ++++ aria/cli/logger.py | 114 ++++ aria/cli/main.py | 58 ++ aria/cli/service_template_utils.py | 121 ++++ aria/cli/storage.py | 95 ---- aria/cli/table.py | 116 ++++ aria/cli/utils.py | 115 ++++ aria/core.py | 124 +++++ aria/exceptions.py | 29 + aria/logger.py | 12 + aria/modeling/__init__.py | 2 + aria/modeling/exceptions.py | 25 + aria/modeling/models.py | 9 +- aria/modeling/orchestration.py | 21 +- aria/modeling/service_changes.py | 10 +- aria/modeling/service_common.py | 15 +- aria/modeling/service_instance.py | 16 +- aria/modeling/service_template.py | 73 ++- aria/modeling/utils.py | 92 +++- aria/orchestrator/context/common.py | 43 +- aria/orchestrator/context/operation.py | 2 - aria/orchestrator/context/workflow.py | 20 +- aria/orchestrator/exceptions.py | 28 + .../execution_plugin/ctx_proxy/server.py | 3 +- .../execution_plugin/instantiation.py | 2 +- aria/orchestrator/plugin.py | 27 +- aria/orchestrator/runner.py | 101 ---- aria/orchestrator/workflow_runner.py | 161 ++++++ aria/orchestrator/workflows/api/task.py | 96 ++-- aria/orchestrator/workflows/builtin/__init__.py | 1 + .../workflows/builtin/execute_operation.py | 16 +- aria/orchestrator/workflows/builtin/utils.py | 82 ++- aria/orchestrator/workflows/core/engine.py | 6 +- aria/orchestrator/workflows/core/task.py | 2 - aria/orchestrator/workflows/exceptions.py | 10 +- aria/orchestrator/workflows/executor/celery.py | 2 +- aria/orchestrator/workflows/executor/dry.py | 51 ++ aria/orchestrator/workflows/executor/process.py | 2 +- aria/orchestrator/workflows/executor/thread.py | 3 +- aria/parser/consumption/__init__.py | 20 +- aria/parser/consumption/modeling.py | 3 +- aria/storage/core.py | 6 +- aria/storage/exceptions.py | 4 + aria/storage/instrumentation.py | 7 +- aria/storage/sql_mapi.py | 30 +- aria/utils/application.py | 294 ---------- aria/utils/archive.py | 63 +++ aria/utils/exceptions.py | 11 + aria/utils/file.py | 13 + aria/utils/formatting.py | 28 + aria/utils/http.py | 62 +++ aria/utils/threading.py | 24 + aria/utils/type.py | 61 +++ .../use-cases/block-storage-1/inputs.yaml | 3 + .../use-cases/block-storage-2/inputs.yaml | 3 + .../use-cases/block-storage-3/inputs.yaml | 2 + .../use-cases/block-storage-4/inputs.yaml | 2 + .../use-cases/block-storage-5/inputs.yaml | 3 + .../use-cases/block-storage-6/inputs.yaml | 3 + .../use-cases/compute-1/inputs.yaml | 1 + .../use-cases/multi-tier-1/inputs.yaml | 1 + .../use-cases/network-1/inputs.yaml | 1 + .../use-cases/network-2/inputs.yaml | 1 + .../use-cases/network-3/inputs.yaml | 1 + .../use-cases/object-storage-1/inputs.yaml | 1 + .../use-cases/software-component-1/inputs.yaml | 1 + .../simple_v1_0/modeling/__init__.py | 3 +- requirements.in | 9 + requirements.txt | 22 +- setup.py | 2 +- tests/.pylintrc | 2 +- tests/cli/__init__.py | 14 + tests/cli/base_test.py | 77 +++ tests/cli/runner.py | 27 + tests/cli/test_node_templates.py | 133 +++++ tests/cli/test_nodes.py | 101 ++++ tests/cli/test_service_templates.py | 246 +++++++++ tests/cli/test_services.py | 205 +++++++ tests/cli/utils.py | 101 ++++ tests/conftest.py | 14 +- tests/end2end/test_orchestrator.py | 63 --- tests/end2end/test_tosca_simple_v1_0.py | 112 ---- tests/fixtures.py | 70 +++ tests/mock/context.py | 7 +- tests/mock/models.py | 135 ++++- tests/mock/topology.py | 8 +- tests/mock/workflow.py | 26 + tests/modeling/test_models.py | 17 +- tests/orchestrator/context/test_operation.py | 45 +- .../context/test_resource_render.py | 12 +- tests/orchestrator/context/test_serialize.py | 13 +- tests/orchestrator/context/test_toolbelt.py | 11 +- tests/orchestrator/context/test_workflow.py | 10 +- .../orchestrator/execution_plugin/test_local.py | 15 +- tests/orchestrator/execution_plugin/test_ssh.py | 48 +- tests/orchestrator/test_runner.py | 74 --- tests/orchestrator/test_workflow_runner.py | 292 ++++++++++ tests/orchestrator/workflows/api/test_task.py | 18 +- .../orchestrator/workflows/core/test_engine.py | 10 +- .../test_task_graph_into_exececution_graph.py | 112 ---- .../test_task_graph_into_execution_graph.py | 112 ++++ .../workflows/executor/test_process_executor.py | 34 +- ...process_executor_concurrent_modifications.py | 3 +- .../executor/test_process_executor_extension.py | 3 +- .../test_process_executor_tracked_changes.py | 6 +- tests/parser/service_templates.py | 6 +- tests/parser/test_tosca_simple_v1_0.py | 112 ++++ tests/utils/test_plugin.py | 29 +- tests/utils/test_threading.py | 33 ++ tox.ini | 2 +- 137 files changed, 5368 insertions(+), 2440 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/.pylintrc ---------------------------------------------------------------------- diff --git a/aria/.pylintrc b/aria/.pylintrc index 7222605..7da8c56 100644 --- a/aria/.pylintrc +++ b/aria/.pylintrc @@ -77,7 +77,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,logging-format-interpolation,import-error,redefined-variable-type,broad-except,protected-access,global-statement,no-member +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,logging-format-interpolation,import-error,redefined-variable-type,broad-except,protected-access,global-statement,no-member,u nused-argument [REPORTS] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index b9251d5..df75b1e 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -84,6 +84,6 @@ def application_resource_storage(api, api_kwargs=None, initiator=None, initiator return storage.ResourceStorage(api_cls=api, api_kwargs=api_kwargs, - items=['blueprint', 'deployment', 'plugin'], + items=['service_template', 'service', 'plugin'], initiator=initiator, initiator_kwargs=initiator_kwargs) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/args_parser.py ---------------------------------------------------------------------- diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py deleted file mode 100644 index 81ee513..0000000 --- a/aria/cli/args_parser.py +++ /dev/null @@ -1,269 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Argument parsing configuration and functions -""" - -import argparse -from functools import partial - -from ..utils.argparse import ArgumentParser - -NO_VERBOSE = 0 - - -class SmartFormatter(argparse.HelpFormatter): - """ - TODO: what is this? - """ - def _split_lines(self, text, width): - if text.startswith('R|'): - return text[2:].splitlines() - return super(SmartFormatter, self)._split_lines(text, width) - - -def sub_parser_decorator(func=None, **parser_settings): - """ - Decorated for sub_parser argument definitions - """ - if not func: - return partial(sub_parser_decorator, **parser_settings) - - def _wrapper(parser): - sub_parser = parser.add_parser(**parser_settings) - sub_parser.add_argument( - '-v', '--verbose', - dest='verbosity', - action='count', - default=NO_VERBOSE, - help='Set verbosity level (can be passed multiple times)') - func(sub_parser) - return sub_parser - return _wrapper - - -def config_parser(parser=None): - """ - Top level argparse configuration - """ - parser = parser or ArgumentParser( - prog='ARIA', - description="ARIA's Command Line Interface", - formatter_class=SmartFormatter) - parser.add_argument('-v', '--version', action='version') - sub_parser = parser.add_subparsers(title='Commands', dest='command') - add_init_parser(sub_parser) - add_execute_parser(sub_parser) - add_parse_parser(sub_parser) - add_workflow_parser(sub_parser) - add_spec_parser(sub_parser) - add_csar_create_parser(sub_parser) - add_csar_open_parser(sub_parser) - add_csar_validate_parser(sub_parser) - return parser - - -@sub_parser_decorator( - name='parse', - help='Parse a blueprint', - formatter_class=SmartFormatter) -def add_parse_parser(parse): - """ - ``parse`` command parser configuration - """ - parse.add_argument( - 'uri', - help='URI or file path to service template') - parse.add_argument( - 'consumer', - nargs='?', - default='validate', - help='"validate" (default), "presentation", "template", "types", "instance", or consumer ' - 'class name (full class path or short name)') - parse.add_argument( - '--loader-source', - default='aria.parser.loading.DefaultLoaderSource', - help='loader source class for the parser') - parse.add_argument( - '--reader-source', - default='aria.parser.reading.DefaultReaderSource', - help='reader source class for the parser') - parse.add_argument( - '--presenter-source', - default='aria.parser.presentation.DefaultPresenterSource', - help='presenter source class for the parser') - parse.add_argument( - '--presenter', - help='force use of this presenter class in parser') - parse.add_argument( - '--prefix', nargs='*', - help='prefixes for imports') - parse.add_flag_argument( - 'debug', - help_true='print debug info', - help_false='don\'t print debug info') - parse.add_flag_argument( - 'cached-methods', - help_true='enable cached methods', - help_false='disable cached methods', - default=True) - - -@sub_parser_decorator( - name='workflow', - help='Run a workflow on a blueprint', - formatter_class=SmartFormatter) -def add_workflow_parser(workflow): - """ - ``workflow`` command parser configuration - """ - workflow.add_argument( - 'uri', - help='URI or file path to service template') - workflow.add_argument( - '-w', '--workflow', - default='install', - help='The workflow name') - workflow.add_flag_argument( - 'dry', - default=True, - help_true='dry run', - help_false='wet run') - - -@sub_parser_decorator( - name='init', - help='Initialize environment', - formatter_class=SmartFormatter) -def add_init_parser(init): - """ - ``init`` command parser configuration - """ - init.add_argument( - '-d', '--deployment-id', - required=True, - help='A unique ID for the deployment') - init.add_argument( - '-p', '--blueprint-path', - dest='blueprint_path', - required=True, - help='The path to the desired blueprint') - init.add_argument( - '-i', '--inputs', - dest='input', - action='append', - help='R|Inputs for the local workflow creation \n' - '(Can be provided as wildcard based paths (*.yaml, etc..) to YAML files, \n' - 'a JSON string or as "key1=value1;key2=value2"). \n' - 'This argument can be used multiple times') - init.add_argument( - '-b', '--blueprint-id', - dest='blueprint_id', - required=True, - help='The blueprint ID' - ) - - -@sub_parser_decorator( - name='execute', - help='Execute a workflow', - formatter_class=SmartFormatter) -def add_execute_parser(execute): - """ - ``execute`` command parser configuration - """ - execute.add_argument( - '-d', '--deployment-id', - required=True, - help='A unique ID for the deployment') - execute.add_argument( - '-w', '--workflow', - dest='workflow_id', - help='The workflow to execute') - execute.add_argument( - '-p', '--parameters', - dest='parameters', - action='append', - help='R|Parameters for the workflow execution\n' - '(Can be provided as wildcard based paths (*.yaml, etc..) to YAML files,\n' - 'a JSON string or as "key1=value1;key2=value2").\n' - 'This argument can be used multiple times.') - execute.add_argument( - '--task-retries', - dest='task_retries', - type=int, - help='How many times should a task be retried in case of failure') - execute.add_argument( - '--task-retry-interval', - dest='task_retry_interval', - default=1, - type=int, - help='How many seconds to wait before each task is retried') - - -@sub_parser_decorator( - name='csar-create', - help='Create a CSAR file from a TOSCA service template directory', - formatter_class=SmartFormatter) -def add_csar_create_parser(parse): - parse.add_argument( - 'source', - help='Service template directory') - parse.add_argument( - 'entry', - help='Entry definition file relative to service template directory') - parse.add_argument( - '-d', '--destination', - help='Output CSAR zip destination', - required=True) - - -@sub_parser_decorator( - name='csar-open', - help='Extracts a CSAR file to a TOSCA service template directory', - formatter_class=SmartFormatter) -def add_csar_open_parser(parse): - parse.add_argument( - 'source', - help='CSAR file location') - parse.add_argument( - '-d', '--destination', - help='Output directory to extract the CSAR into', - required=True) - - -@sub_parser_decorator( - name='csar-validate', - help='Validates a CSAR file', - formatter_class=SmartFormatter) -def add_csar_validate_parser(parse): - parse.add_argument( - 'source', - help='CSAR file location') - - -@sub_parser_decorator( - name='spec', - help='Specification tool', - formatter_class=SmartFormatter) -def add_spec_parser(spec): - """ - ``spec`` command parser configuration - """ - spec.add_argument( - '--csv', - action='store_true', - help='output as CSV') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/cli.py ---------------------------------------------------------------------- diff --git a/aria/cli/cli.py b/aria/cli/cli.py deleted file mode 100644 index 8d014b3..0000000 --- a/aria/cli/cli.py +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -CLI Entry point -""" - -import os -import logging -import tempfile - -from .. import install_aria_extensions -from ..logger import ( - create_logger, - create_console_log_handler, - create_file_log_handler, - LoggerMixin, -) -from ..utils.exceptions import print_exception -from .args_parser import config_parser -from .commands import ( - ParseCommand, - WorkflowCommand, - InitCommand, - ExecuteCommand, - CSARCreateCommand, - CSAROpenCommand, - CSARValidateCommand, - SpecCommand, -) - -__version__ = '0.1.0' - - -class AriaCli(LoggerMixin): - """ - Context manager based class that enables proper top level error handling - """ - - def __init__(self, *args, **kwargs): - super(AriaCli, self).__init__(*args, **kwargs) - self.commands = { - 'parse': ParseCommand.with_logger(base_logger=self.logger), - 'workflow': WorkflowCommand.with_logger(base_logger=self.logger), - 'init': InitCommand.with_logger(base_logger=self.logger), - 'execute': ExecuteCommand.with_logger(base_logger=self.logger), - 'csar-create': CSARCreateCommand.with_logger(base_logger=self.logger), - 'csar-open': CSAROpenCommand.with_logger(base_logger=self.logger), - 'csar-validate': CSARValidateCommand.with_logger(base_logger=self.logger), - 'spec': SpecCommand.with_logger(base_logger=self.logger), - } - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """ - Here we will handle errors - :param exc_type: - :param exc_val: - :param exc_tb: - :return: - """ - # todo: error handling - # todo: cleanup if needed - # TODO: user message if needed - pass - - def run(self): - """ - Parses user arguments and run the appropriate command - """ - parser = config_parser() - args, unknown_args = parser.parse_known_args() - - command_handler = self.commands[args.command] - self.logger.debug('Running command: {args.command} handler: {0}'.format( - command_handler, args=args)) - try: - command_handler(args, unknown_args) - except Exception as e: - print_exception(e) - - -def main(): - """ - CLI entry point - """ - install_aria_extensions() - create_logger( - handlers=[ - create_console_log_handler(), - create_file_log_handler(file_path=os.path.join(tempfile.gettempdir(), 'aria_cli.log')), - ], - level=logging.INFO) - with AriaCli() as aria: - aria.run() - - -if __name__ == '__main__': - main() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands.py b/aria/cli/commands.py deleted file mode 100644 index ee329e7..0000000 --- a/aria/cli/commands.py +++ /dev/null @@ -1,546 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -CLI various commands implementation -""" - -import json -import os -import sys -import csv -import shutil -import tempfile -from glob import glob -from importlib import import_module - -from ruamel import yaml # @UnresolvedImport - -from .. import extension -from ..logger import LoggerMixin -from ..parser import iter_specifications -from ..parser.consumption import ( - ConsumptionContext, - ConsumerChain, - Read, - Validate, - ServiceTemplate, - Types, - Inputs, - ServiceInstance -) -from ..parser.loading import LiteralLocation, UriLocation -from ..utils.application import StorageManager -from ..utils.caching import cachedmethod -from ..utils.console import (puts, Colored, indent) -from ..utils.imports import (import_fullname, import_modules) -from ..utils.collections import OrderedDict -from ..orchestrator import WORKFLOW_DECORATOR_RESERVED_ARGUMENTS -from ..orchestrator.runner import Runner -from ..orchestrator.workflows.builtin import BUILTIN_WORKFLOWS -from .dry import convert_to_dry - -from .exceptions import ( - AriaCliFormatInputsError, - AriaCliYAMLInputsError, - AriaCliInvalidInputsError -) -from . import csar - - -class BaseCommand(LoggerMixin): - """ - Base class for CLI commands. - """ - - def __repr__(self): - return 'AriaCli({cls.__name__})'.format(cls=self.__class__) - - def __call__(self, args_namespace, unknown_args): - """ - __call__ method is called when running command - :param args_namespace: - """ - pass - - def parse_inputs(self, inputs): - """ - Returns a dictionary of inputs `resources` can be: - - A list of files. - - A single file - - A directory containing multiple input files - - A key1=value1;key2=value2 pairs string. - - Wildcard based string (e.g. *-inputs.yaml) - """ - - parsed_dict = {} - - def _format_to_dict(input_string): - self.logger.info('Processing inputs source: {0}'.format(input_string)) - try: - input_string = input_string.strip() - try: - parsed_dict.update(json.loads(input_string)) - except BaseException: - parsed_dict.update((i.split('=') - for i in input_string.split(';') - if i)) - except Exception as exc: - raise AriaCliFormatInputsError(str(exc), inputs=input_string) - - def _handle_inputs_source(input_path): - self.logger.info('Processing inputs source: {0}'.format(input_path)) - try: - with open(input_path) as input_file: - content = yaml.safe_load(input_file) - except yaml.YAMLError as exc: - raise AriaCliYAMLInputsError( - '"{0}" is not a valid YAML. {1}'.format(input_path, str(exc))) - if isinstance(content, dict): - parsed_dict.update(content) - return - if content is None: - return - raise AriaCliInvalidInputsError('Invalid inputs', inputs=input_path) - - for input_string in inputs if isinstance(inputs, list) else [inputs]: - if os.path.isdir(input_string): - for input_file in os.listdir(input_string): - _handle_inputs_source(os.path.join(input_string, input_file)) - continue - input_files = glob(input_string) - if input_files: - for input_file in input_files: - _handle_inputs_source(input_file) - continue - _format_to_dict(input_string) - return parsed_dict - - -class ParseCommand(BaseCommand): - """ - :code:`parse` command. - - Given a blueprint, emits information in human-readable, JSON, or YAML format from various phases - of the ARIA parser. - """ - - def __call__(self, args_namespace, unknown_args): - super(ParseCommand, self).__call__(args_namespace, unknown_args) - - if args_namespace.prefix: - for prefix in args_namespace.prefix: - extension.parser.uri_loader_prefix().append(prefix) - - cachedmethod.ENABLED = args_namespace.cached_methods - - context = ParseCommand.create_context_from_namespace(args_namespace) - context.args = unknown_args - - consumer = ConsumerChain(context, (Read, Validate)) - - consumer_class_name = args_namespace.consumer - dumper = None - if consumer_class_name == 'validate': - dumper = None - elif consumer_class_name == 'presentation': - dumper = consumer.consumers[0] - elif consumer_class_name == 'template': - consumer.append(ServiceTemplate) - elif consumer_class_name == 'types': - consumer.append(ServiceTemplate, Types) - elif consumer_class_name == 'instance': - consumer.append(ServiceTemplate, Inputs, ServiceInstance) - else: - consumer.append(ServiceTemplate, Inputs, ServiceInstance) - consumer.append(import_fullname(consumer_class_name)) - - if dumper is None: - # Default to last consumer - dumper = consumer.consumers[-1] - - consumer.consume() - - if not context.validation.dump_issues(): - dumper.dump() - exit(1) - - @staticmethod - def create_context_from_namespace(namespace, **kwargs): - args = vars(namespace).copy() - args.update(kwargs) - return ParseCommand.create_context(**args) - - @staticmethod - def create_context(uri, - loader_source, - reader_source, - presenter_source, - presenter, - debug, - **kwargs): - context = ConsumptionContext() - context.loading.loader_source = import_fullname(loader_source)() - context.reading.reader_source = import_fullname(reader_source)() - context.presentation.location = UriLocation(uri) if isinstance(uri, basestring) else uri - context.presentation.presenter_source = import_fullname(presenter_source)() - context.presentation.presenter_class = import_fullname(presenter) - context.presentation.print_exceptions = debug - return context - - -class WorkflowCommand(BaseCommand): - """ - :code:`workflow` command. - """ - - WORKFLOW_POLICY_INTERNAL_PROPERTIES = ('implementation', 'dependencies') - - def __call__(self, args_namespace, unknown_args): - super(WorkflowCommand, self).__call__(args_namespace, unknown_args) - - context = self._parse(args_namespace.uri) - workflow_fn, inputs = self._get_workflow(context, args_namespace.workflow) - self._dry = args_namespace.dry - self._run(context, args_namespace.workflow, workflow_fn, inputs) - - def _parse(self, uri): - # Parse - context = ConsumptionContext() - context.presentation.location = UriLocation(uri) - consumer = ConsumerChain(context, (Read, Validate, ServiceTemplate, Inputs, - ServiceInstance)) - consumer.consume() - - if context.validation.dump_issues(): - exit(1) - - return context - - def _get_workflow(self, context, workflow_name): - if workflow_name in BUILTIN_WORKFLOWS: - workflow_fn = import_fullname('aria.orchestrator.workflows.builtin.{0}'.format( - workflow_name)) - inputs = {} - else: - workflow = context.modeling.instance.policies.get(workflow_name) - if workflow is None: - raise AttributeError('workflow policy does not exist: "{0}"'.format(workflow_name)) - if workflow.type.role != 'workflow': - raise AttributeError('policy is not a workflow: "{0}"'.format(workflow_name)) - - sys.path.append(os.path.dirname(str(context.presentation.location))) - - workflow_fn = import_fullname(workflow.properties['implementation'].value) - - for k in workflow.properties: - if k in WORKFLOW_DECORATOR_RESERVED_ARGUMENTS: - raise AttributeError('workflow policy "{0}" defines a reserved property: "{1}"' - .format(workflow_name, k)) - - inputs = OrderedDict([ - (k, v.value) for k, v in workflow.properties.iteritems() - if k not in WorkflowCommand.WORKFLOW_POLICY_INTERNAL_PROPERTIES - ]) - - return workflow_fn, inputs - - def _run(self, context, workflow_name, workflow_fn, inputs): - # Storage - def _initialize_storage(model_storage): - if self._dry: - convert_to_dry(context.modeling.instance) - context.modeling.store(model_storage) - - # Create runner - runner = Runner(workflow_name, workflow_fn, inputs, _initialize_storage, - lambda: context.modeling.instance.id) - - # Run - runner.run() - - -class InitCommand(BaseCommand): - """ - :code:`init` command. - - Broken. Currently maintained for reference. - """ - - _IN_VIRTUAL_ENV = hasattr(sys, 'real_prefix') - - def __call__(self, args_namespace, unknown_args): - super(InitCommand, self).__call__(args_namespace, unknown_args) - self._workspace_setup() - inputs = self.parse_inputs(args_namespace.input) if args_namespace.input else None - plan, deployment_plan = self._parse_blueprint(args_namespace.blueprint_path, inputs) - self._create_storage( - blueprint_plan=plan, - blueprint_path=args_namespace.blueprint_path, - deployment_plan=deployment_plan, - blueprint_id=args_namespace.blueprint_id, - deployment_id=args_namespace.deployment_id, - main_file_name=os.path.basename(args_namespace.blueprint_path)) - self.logger.info('Initiated {0}'.format(args_namespace.blueprint_path)) - self.logger.info( - 'If you make changes to the blueprint, ' - 'run `aria local init -p {0}` command again to apply them'.format( - args_namespace.blueprint_path)) - - def _workspace_setup(self): - try: - create_user_space() - self.logger.debug( - 'created user space path in: {0}'.format(user_space())) - except IOError: - self.logger.debug( - 'user space path already exist - {0}'.format(user_space())) - try: - create_local_storage() - self.logger.debug( - 'created local storage path in: {0}'.format(local_storage())) - except IOError: - self.logger.debug( - 'local storage path already exist - {0}'.format(local_storage())) - return local_storage() - - def _parse_blueprint(self, blueprint_path, inputs=None): - # TODO - pass - - @staticmethod - def _create_storage( - blueprint_path, - blueprint_plan, - deployment_plan, - blueprint_id, - deployment_id, - main_file_name=None): - resource_storage = application_resource_storage( - FileSystemResourceDriver(local_resource_storage())) - model_storage = application_model_storage( - FileSystemModelDriver(local_model_storage())) - resource_storage.setup() - model_storage.setup() - storage_manager = StorageManager( - model_storage=model_storage, - resource_storage=resource_storage, - blueprint_path=blueprint_path, - blueprint_id=blueprint_id, - blueprint_plan=blueprint_plan, - deployment_id=deployment_id, - deployment_plan=deployment_plan - ) - storage_manager.create_blueprint_storage( - blueprint_path, - main_file_name=main_file_name - ) - storage_manager.create_nodes_storage() - storage_manager.create_deployment_storage() - storage_manager.create_node_instances_storage() - - -class ExecuteCommand(BaseCommand): - """ - :code:`execute` command. - - Broken. Currently maintained for reference. - """ - - def __call__(self, args_namespace, unknown_args): - super(ExecuteCommand, self).__call__(args_namespace, unknown_args) - parameters = (self.parse_inputs(args_namespace.parameters) - if args_namespace.parameters else {}) - resource_storage = application_resource_storage( - FileSystemResourceDriver(local_resource_storage())) - model_storage = application_model_storage( - FileSystemModelDriver(local_model_storage())) - deployment = model_storage.service_instance.get(args_namespace.deployment_id) - - try: - workflow = deployment.workflows[args_namespace.workflow_id] - except KeyError: - raise ValueError( - '{0} workflow does not exist. existing workflows are: {1}'.format( - args_namespace.workflow_id, - deployment.workflows.keys())) - - workflow_parameters = self._merge_and_validate_execution_parameters( - workflow, - args_namespace.workflow_id, - parameters - ) - workflow_context = WorkflowContext( - name=args_namespace.workflow_id, - model_storage=model_storage, - resource_storage=resource_storage, - deployment_id=args_namespace.deployment_id, - workflow_id=args_namespace.workflow_id, - parameters=workflow_parameters, - ) - workflow_function = self._load_workflow_handler(workflow['operation']) - tasks_graph = workflow_function(workflow_context, **workflow_context.parameters) - executor = ProcessExecutor() - workflow_engine = Engine(executor=executor, - workflow_context=workflow_context, - tasks_graph=tasks_graph) - workflow_engine.execute() - executor.close() - - @staticmethod - def _merge_and_validate_execution_parameters( - workflow, - workflow_name, - execution_parameters): - merged_parameters = {} - workflow_parameters = workflow.get('parameters', {}) - missing_mandatory_parameters = set() - - for name, param in workflow_parameters.iteritems(): - if 'default' not in param: - if name not in execution_parameters: - missing_mandatory_parameters.add(name) - continue - merged_parameters[name] = execution_parameters[name] - continue - merged_parameters[name] = (execution_parameters[name] if name in execution_parameters - else param['default']) - - if missing_mandatory_parameters: - raise ValueError( - 'Workflow "{0}" must be provided with the following ' - 'parameters to execute: {1}'.format( - workflow_name, ','.join(missing_mandatory_parameters))) - - custom_parameters = dict( - (k, v) for (k, v) in execution_parameters.iteritems() - if k not in workflow_parameters) - - if custom_parameters: - raise ValueError( - 'Workflow "{0}" does not have the following parameters declared: {1}. ' - 'Remove these parameters'.format( - workflow_name, ','.join(custom_parameters.keys()))) - - return merged_parameters - - @staticmethod - def _load_workflow_handler(handler_path): - module_name, spec_handler_name = handler_path.rsplit('.', 1) - try: - module = import_module(module_name) - return getattr(module, spec_handler_name) - except ImportError: - # TODO: exception handler - raise - except AttributeError: - # TODO: exception handler - raise - - -class BaseCSARCommand(BaseCommand): - @staticmethod - def _parse_and_dump(reader): - context = ConsumptionContext() - context.loading.prefixes += [os.path.join(reader.destination, 'definitions')] - context.presentation.location = LiteralLocation(reader.entry_definitions_yaml) - chain = ConsumerChain(context, (Read, Validate, Model, Instance)) - chain.consume() - if context.validation.dump_issues(): - raise RuntimeError('Validation failed') - dumper = chain.consumers[-1] - dumper.dump() - - def _read(self, source, destination): - reader = csar.read( - source=source, - destination=destination, - logger=self.logger) - self.logger.info( - 'Path: {r.destination}\n' - 'TOSCA meta file version: {r.meta_file_version}\n' - 'CSAR Version: {r.csar_version}\n' - 'Created By: {r.created_by}\n' - 'Entry definitions: {r.entry_definitions}' - .format(r=reader)) - self._parse_and_dump(reader) - - def _validate(self, source): - workdir = tempfile.mkdtemp() - try: - self._read( - source=source, - destination=workdir) - finally: - shutil.rmtree(workdir, ignore_errors=True) - - -class CSARCreateCommand(BaseCSARCommand): - def __call__(self, args_namespace, unknown_args): - super(CSARCreateCommand, self).__call__(args_namespace, unknown_args) - csar.write( - source=args_namespace.source, - entry=args_namespace.entry, - destination=args_namespace.destination, - logger=self.logger) - self._validate(args_namespace.destination) - - -class CSAROpenCommand(BaseCSARCommand): - def __call__(self, args_namespace, unknown_args): - super(CSAROpenCommand, self).__call__(args_namespace, unknown_args) - self._read( - source=args_namespace.source, - destination=args_namespace.destination) - - -class CSARValidateCommand(BaseCSARCommand): - def __call__(self, args_namespace, unknown_args): - super(CSARValidateCommand, self).__call__(args_namespace, unknown_args) - self._validate(args_namespace.source) - - -class SpecCommand(BaseCommand): - """ - :code:`spec` command. - - Emits all uses of :code:`@dsl_specification` in the codebase, in human-readable or CSV format. - """ - - def __call__(self, args_namespace, unknown_args): - super(SpecCommand, self).__call__(args_namespace, unknown_args) - - # Make sure that all @dsl_specification decorators are processed - for pkg in extension.parser.specification_package(): - import_modules(pkg) - - # TODO: scan YAML documents as well - - if args_namespace.csv: - writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - writer.writerow(('Specification', 'Section', 'Code', 'URL')) - for spec, sections in iter_specifications(): - for section, details in sections: - writer.writerow((spec, section, details['code'], details['url'])) - - else: - for spec, sections in iter_specifications(): - puts(Colored.cyan(spec)) - with indent(2): - for section, details in sections: - puts(Colored.blue(section)) - with indent(2): - for k, v in details.iteritems(): - puts('%s: %s' % (Colored.magenta(k), v)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/__init__.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/__init__.py b/aria/cli/commands/__init__.py new file mode 100644 index 0000000..a01a029 --- /dev/null +++ b/aria/cli/commands/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import ( + executions, + logs, + node_templates, + nodes, + plugins, + reset, + service_templates, + services, + workflows +) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/executions.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py new file mode 100644 index 0000000..e100f0d --- /dev/null +++ b/aria/cli/commands/executions.py @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .. import helptexts +from .. import table +from .. import utils +from ..core import aria +from ...modeling.models import Execution +from ...orchestrator.workflow_runner import WorkflowRunner +from ...orchestrator.workflows.executor.dry import DryExecutor +from ...utils import formatting +from ...utils import threading + +EXECUTION_COLUMNS = ['id', 'workflow_name', 'status', 'service_name', + 'created_at', 'error'] + + [email protected](name='executions') [email protected]() +def executions(): + """Handle workflow executions + """ + pass + + [email protected](name='show', + short_help='Show execution information') [email protected]('execution-id') [email protected]() [email protected]_model_storage [email protected]_logger +def show(execution_id, model_storage, logger): + """Show information for a specific execution + + `EXECUTION_ID` is the execution to get information on. + """ + logger.info('Showing execution {0}'.format(execution_id)) + execution = model_storage.execution.get(execution_id) + + table.print_data(EXECUTION_COLUMNS, execution, 'Execution:', col_max_width=50) + + # print execution parameters + logger.info('Execution Inputs:') + if execution.inputs: + #TODO check this section, havent tested it + execution_inputs = [ei.to_dict() for ei in execution.inputs] + for input_name, input_value in formatting.decode_dict( + execution_inputs).iteritems(): + logger.info('\t{0}: \t{1}'.format(input_name, input_value)) + else: + logger.info('\tNo inputs') + + [email protected](name='list', + short_help='List service executions') [email protected]_name(required=False) [email protected]_by() [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_name, + sort_by, + descending, + model_storage, + logger): + """List executions + + If `SERVICE_NAME` is provided, list executions for that service. + Otherwise, list executions for all services. + """ + if service_name: + logger.info('Listing executions for service {0}...'.format( + service_name)) + service = model_storage.service.get_by_name(service_name) + filters = dict(service=service) + else: + logger.info('Listing all executions...') + filters = {} + + executions_list = model_storage.execution.list( + filters=filters, + sort=utils.storage_sort_param(sort_by, descending)).items + + table.print_data(EXECUTION_COLUMNS, executions_list, 'Executions:') + + [email protected](name='start', + short_help='Execute a workflow') [email protected]('workflow-name') [email protected]_name(required=True) [email protected](help=helptexts.EXECUTION_INPUTS) [email protected]_execution [email protected]_max_attempts() [email protected]_retry_interval() [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def start(workflow_name, + service_name, + inputs, + dry, + task_max_attempts, + task_retry_interval, + model_storage, + resource_storage, + plugin_manager, + logger): + """Execute a workflow + + `WORKFLOW_NAME` is the name of the workflow to execute (e.g. `uninstall`) + """ + service = model_storage.service.get_by_name(service_name) + executor = DryExecutor() if dry else None # use WorkflowRunner's default executor + + workflow_runner = \ + WorkflowRunner(workflow_name, service.id, inputs, + model_storage, resource_storage, plugin_manager, + executor, task_max_attempts, task_retry_interval) + + execution_thread_name = '{0}_{1}'.format(service_name, workflow_name) + execution_thread = threading.ExceptionThread(target=workflow_runner.execute, + name=execution_thread_name) + execution_thread.daemon = True # allows force-cancel to exit immediately + + logger.info('Starting {0}execution. Press Ctrl+C cancel'.format('dry ' if dry else '')) + execution_thread.start() + try: + while execution_thread.is_alive(): + # using join without a timeout blocks and ignores KeyboardInterrupt + execution_thread.join(1) + except KeyboardInterrupt: + _cancel_execution(workflow_runner, execution_thread, logger) + + # raise any errors from the execution thread (note these are not workflow execution errors) + execution_thread.raise_error_if_exists() + + execution = workflow_runner.execution + logger.info('Execution has ended with "{0}" status'.format(execution.status)) + if execution.status == Execution.FAILED and execution.error: + logger.info('Execution error:{0}{1}'.format(os.linesep, execution.error)) + + if dry: + # remove traces of the dry execution (including tasks, logs, inputs..) + model_storage.execution.delete(execution) + + +def _cancel_execution(workflow_runner, execution_thread, logger): + logger.info('Cancelling execution. Press Ctrl+C again to force-cancel') + try: + workflow_runner.cancel() + while execution_thread.is_alive(): + execution_thread.join(1) + except KeyboardInterrupt: + logger.info('Force-cancelling execution') + # TODO handle execution (update status etc.) and exit process http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/logs.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/logs.py b/aria/cli/commands/logs.py new file mode 100644 index 0000000..6c83347 --- /dev/null +++ b/aria/cli/commands/logs.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import utils +from ..core import aria + + [email protected](name='logs') [email protected]() +def logs(): + """Show logs from workflow executions + """ + pass + + [email protected](name='list', + short_help='List execution logs') [email protected]('execution-id') [email protected]() [email protected]_model_storage [email protected]_logger +def list(execution_id, + model_storage, + logger): + """Display logs for an execution + """ + logger.info('Listing logs for execution id {0}'.format(execution_id)) + logs_list = model_storage.log.list(filters=dict(execution_fk=execution_id), + sort=utils.storage_sort_param('created_at', False)) + # TODO: print logs nicely + if logs_list: + for log in logs_list: + logger.info(log) + else: + logger.info('\tNo logs') + + [email protected](name='delete', + short_help='Delete execution logs') [email protected]('execution-id') [email protected]() [email protected]_model_storage [email protected]_logger +def delete(execution_id, model_storage, logger): + """Delete logs of an execution + + `EXECUTION_ID` is the execution logs to delete. + """ + logger.info('Deleting logs for execution id {0}'.format(execution_id)) + logs_list = model_storage.log.list(filters=dict(execution_fk=execution_id)) + for log in logs_list: + model_storage.log.delete(log) + logger.info('Deleted logs for execution id {0}'.format(execution_id)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/node_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/node_templates.py b/aria/cli/commands/node_templates.py new file mode 100644 index 0000000..50c755e --- /dev/null +++ b/aria/cli/commands/node_templates.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import table +from .. import utils +from ..core import aria + + +NODE_TEMPLATE_COLUMNS = ['id', 'name', 'description', 'service_template_name', 'type_name'] + + [email protected](name='node-templates') [email protected]() +def node_templates(): + """Handle a service template's node templates + """ + pass + + +@node_templates.command(name='show', + short_help='Show node information') [email protected]('node-template-id') +# @aria.options.service_template_name(required=True) [email protected]() [email protected]_model_storage [email protected]_logger +def show(node_template_id, model_storage, logger): + """Show information for a specific node of a specific service template + + `NODE_TEMPLATE_ID` is the node id to get information on. + """ + logger.info('Showing node template {0}'.format(node_template_id)) + node_template = model_storage.node_template.get(node_template_id) + + table.print_data(NODE_TEMPLATE_COLUMNS, node_template, 'Node template:', col_max_width=50) + + # print node template properties + logger.info('Node template properties:') + if node_template.properties: + logger.info(utils.get_parameter_templates_as_string(node_template.properties)) + else: + logger.info('\tNo properties') + + # print node IDs + nodes = node_template.nodes + logger.info('Nodes:') + if nodes: + for node in nodes: + logger.info('\t{0}'.format(node.name)) + else: + logger.info('\tNo nodes') + + +@node_templates.command(name='list', + short_help='List node templates for a service template') [email protected]_template_name() [email protected]_by('service_template_name') [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_template_name, sort_by, descending, model_storage, logger): + """List node templates + + If `SERVICE_TEMPLATE_NAME` is provided, list nodes for that service template. + Otherwise, list node templates for all service templates. + """ + if service_template_name: + logger.info('Listing node templates for service template {0}...'.format( + service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + filters = dict(service_template=service_template) + else: + logger.info('Listing all node templates...') + filters = {} + + node_templates_list = model_storage.node_template.list( + filters=filters, + sort=utils.storage_sort_param(sort_by, descending)) + + table.print_data(NODE_TEMPLATE_COLUMNS, node_templates_list, 'Node templates:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/nodes.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/nodes.py b/aria/cli/commands/nodes.py new file mode 100644 index 0000000..e43493f --- /dev/null +++ b/aria/cli/commands/nodes.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import table +from .. import utils +from ..core import aria + + +NODE_COLUMNS = ['id', 'name', 'service_name', 'node_template_name', 'state'] + + [email protected](name='nodes') [email protected]() +def nodes(): + """Handle a service's nodes + """ + pass + + [email protected](name='show', + short_help='Show node information') [email protected]('node_id') [email protected]() [email protected]_model_storage [email protected]_logger +def show(node_id, model_storage, logger): + """Showing information for a specific node + + `NODE_ID` is the id of the node to get information on. + """ + logger.info('Showing node {0}'.format(node_id)) + node = model_storage.node.get(node_id) + + table.print_data(NODE_COLUMNS, node, 'Node:', col_max_width=50) + + # print node attributes + logger.info('Node attributes:') + if node.runtime_properties: + for prop_name, prop_value in node.runtime_properties.iteritems(): + logger.info('\t{0}: {1}'.format(prop_name, prop_value)) + else: + logger.info('\tNo attributes') + + [email protected](name='list', + short_help='List node for a service') [email protected]_name(required=False) [email protected]_by('service_name') [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_name, + sort_by, + descending, + model_storage, + logger): + """List nodes + + If `SERVICE_NAME` is provided, list nodes for that service. + Otherwise, list nodes for all services. + """ + if service_name: + logger.info('Listing nodes for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + filters = dict(service=service) + else: + logger.info('Listing all nodes...') + filters = {} + + nodes_list = model_storage.node.list( + filters=filters, + sort=utils.storage_sort_param(sort_by, descending)) + + table.print_data(NODE_COLUMNS, nodes_list, 'Nodes:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/plugins.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/plugins.py b/aria/cli/commands/plugins.py new file mode 100644 index 0000000..670288e --- /dev/null +++ b/aria/cli/commands/plugins.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import table +from .. import utils +from ..core import aria + + +PLUGIN_COLUMNS = ['id', 'package_name', 'package_version', 'supported_platform', + 'distribution', 'distribution_release', 'uploaded_at'] + + [email protected](name='plugins') [email protected]() +def plugins(): + """Handle plugins + """ + pass + + [email protected](name='validate', + short_help='Validate a plugin') [email protected]('plugin-path') [email protected]() [email protected]_plugin_manager [email protected]_logger +def validate(plugin_path, plugin_manager, logger): + """Validate a plugin archive + + A valid plugin is a wagon (http://github.com/cloudify-cosmo/wagon) + in the zip format (suffix may also be .wgn). + + `PLUGIN_PATH` is the path to wagon archive to validate. + """ + logger.info('Validating plugin {0}...'.format(plugin_path)) + plugin_manager.validate_plugin(plugin_path) + logger.info('Plugin validated successfully') + + [email protected](name='install', + short_help='Install a plugin') [email protected]('plugin-path') [email protected]() [email protected]_context [email protected]_plugin_manager [email protected]_logger +def install(ctx, plugin_path, plugin_manager, logger): + """Install a plugin + + `PLUGIN_PATH` is the path to wagon archive to install. + """ + ctx.invoke(validate, plugin_path=plugin_path) + logger.info('Installing plugin {0}...'.format(plugin_path)) + plugin = plugin_manager.install(plugin_path) + logger.info("Plugin installed. The plugin's id is {0}".format(plugin.id)) + + [email protected](name='show', + short_help='show plugin information') [email protected]('plugin-id') [email protected]() [email protected]_model_storage [email protected]_logger +def show(plugin_id, model_storage, logger): + """Show information for a specific plugin + + `PLUGIN_ID` is the id of the plugin to show information on. + """ + logger.info('Showing plugin {0}...'.format(plugin_id)) + plugin = model_storage.plugin.get(plugin_id) + table.print_data(PLUGIN_COLUMNS, plugin, 'Plugin:') + + [email protected](name='list', + short_help='List plugins') [email protected]_by('uploaded_at') [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(sort_by, descending, model_storage, logger): + """List all plugins on the manager + """ + logger.info('Listing all plugins...') + plugins_list = model_storage.plugin.list( + sort=utils.storage_sort_param(sort_by, descending)).items + table.print_data(PLUGIN_COLUMNS, plugins_list, 'Plugins:') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/reset.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/reset.py b/aria/cli/commands/reset.py new file mode 100644 index 0000000..1fe0714 --- /dev/null +++ b/aria/cli/commands/reset.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import helptexts +from ..core import aria +from ..env import env +from ..exceptions import AriaCliError + + [email protected](name='reset', + short_help="Reset ARIA's working directory") [email protected](help=helptexts.FORCE_RESET) [email protected]_config [email protected]_logger [email protected]() +def reset(force, reset_config, logger): + """ + Reset ARIA working directory + Resetting the working directory will result in the deletion of all state in ARIA; The user + configuration will remain intact, unless the `reset_config` flag has been set as well, in + which case the entire ARIA working directory shall be removed. + """ + if not force: + raise AriaCliError("To reset the ARIA's working directory, you must also provide the force" + " flag ('-f'/'--force').") + + env.reset(reset_config=reset_config) + logger.info("ARIA's working directory has been reset") http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/service_templates.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py new file mode 100644 index 0000000..97367c2 --- /dev/null +++ b/aria/cli/commands/service_templates.py @@ -0,0 +1,208 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +from .. import csar +from .. import service_template_utils +from .. import table +from .. import utils +from ..core import aria +from ...core import Core +from ...storage import exceptions as storage_exceptions + + +DESCRIPTION_FIELD_LENGTH_LIMIT = 20 +SERVICE_TEMPLATE_COLUMNS = \ + ['id', 'name', 'description', 'main_file_name', 'created_at', 'updated_at'] + + [email protected](name='service-templates') [email protected]() +def service_templates(): + """Handle service templates on the manager + """ + pass + + +@service_templates.command(name='show', + short_help='Show service template information') [email protected]('service-template-name') [email protected]() [email protected]_model_storage [email protected]_logger +def show(service_template_name, model_storage, logger): + """Show information for a specific service templates + + `SERVICE_TEMPLATE_NAME` is the name of the service template to show information on. + """ + logger.info('Showing service template {0}...'.format(service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + service_template_dict = service_template.to_dict() + service_template_dict['#services'] = len(service_template.services) + + column_formatters = \ + dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT)) + columns = SERVICE_TEMPLATE_COLUMNS + ['#services'] + table.print_data(columns, service_template_dict, 'Service-template:', + column_formatters=column_formatters, col_max_width=50) + + if service_template_dict['description'] is not None: + logger.info('Description:') + logger.info('{0}{1}'.format(service_template_dict['description'].encode('UTF-8') or '', + os.linesep)) + + if service_template.services: + logger.info('Existing services:') + for service in service_template.services: + logger.info('\t{0}'.format(service.name)) + + +@service_templates.command(name='list', + short_help='List service templates') [email protected]_by() [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(sort_by, descending, model_storage, logger): + """List all service templates + """ + + logger.info('Listing all service templates...') + service_templates_list = model_storage.service_template.list( + sort=utils.storage_sort_param(sort_by, descending)) + + column_formatters = \ + dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT)) + table.print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:', + column_formatters=column_formatters) + + +@service_templates.command(name='store', + short_help='Store a service template') [email protected]('service-template-path') [email protected]('service-template-name') [email protected]_template_filename [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def store(service_template_path, service_template_name, service_template_filename, + model_storage, resource_storage, plugin_manager, logger): + """Store a service template + + `SERVICE_TEMPLATE_PATH` is the path of the service template to store. + + `SERVICE_TEMPLATE_NAME` is the name of the service template to store. + """ + logger.info('Storing service template {0}...'.format(service_template_name)) + + service_template_path = service_template_utils.get(service_template_path, + service_template_filename) + core = Core(model_storage, resource_storage, plugin_manager) + try: + core.create_service_template(service_template_path, + os.path.dirname(service_template_path), + service_template_name) + except storage_exceptions.StorageError as e: + utils.check_overriding_storage_exceptions(e, 'service template', service_template_name) + raise + logger.info('Service template {0} stored'.format(service_template_name)) + + +@service_templates.command(name='delete', + short_help='Delete a service template') [email protected]('service-template-name') [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def delete(service_template_name, model_storage, resource_storage, plugin_manager, logger): + """Delete a service template + `SERVICE_TEMPLATE_NAME` is the name of the service template to delete. + """ + logger.info('Deleting service template {0}...'.format(service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + core = Core(model_storage, resource_storage, plugin_manager) + core.delete_service_template(service_template.id) + logger.info('Service template {0} deleted'.format(service_template_name)) + + +@service_templates.command(name='inputs', + short_help='Show service template inputs') [email protected]('service-template-name') [email protected]() [email protected]_model_storage [email protected]_logger +def inputs(service_template_name, model_storage, logger): + """Show inputs for a specific service template + + `SERVICE_TEMPLATE_NAME` is the name of the service template to show inputs for. + """ + logger.info('Showing inputs for service template {0}...'.format(service_template_name)) + print_service_template_inputs(model_storage, service_template_name, logger) + + +@service_templates.command(name='validate', + short_help='Validate a service template') [email protected]('service-template') [email protected]_template_filename [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def validate(service_template, service_template_filename, + model_storage, resource_storage, plugin_manager, logger): + """Validate a service template + + `SERVICE_TEMPLATE` is the path or url of the service template or archive to validate. + """ + logger.info('Validating service template: {0}'.format(service_template)) + service_template_path = service_template_utils.get(service_template, service_template_filename) + core = Core(model_storage, resource_storage, plugin_manager) + core.validate_service_template(service_template_path) + logger.info('Service template validated successfully') + + +@service_templates.command(name='create-archive', + short_help='Create a csar archive') [email protected]('service-template-path') [email protected]('destination') [email protected]() [email protected]_logger +def create_archive(service_template_path, destination, logger): + """Create a csar archive + + `service_template_path` is the path of the service template to create the archive from + `destination` is the path of the output csar archive + """ + logger.info('Creating a csar archive') + csar.write(os.path.dirname(service_template_path), service_template_path, destination, logger) + logger.info('Csar archive created at {0}'.format(destination)) + + +def print_service_template_inputs(model_storage, service_template_name, logger): + service_template = model_storage.service_template.get_by_name(service_template_name) + + logger.info('Service template inputs:') + if service_template.inputs: + logger.info(utils.get_parameter_templates_as_string(service_template.inputs)) + else: + logger.info('\tNo inputs') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8e5a1ec2/aria/cli/commands/services.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py new file mode 100644 index 0000000..50b530a --- /dev/null +++ b/aria/cli/commands/services.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from StringIO import StringIO + +from . import service_templates +from .. import helptexts +from .. import table +from .. import utils +from ..core import aria +from ...core import Core +from ...modeling import exceptions as modeling_exceptions +from ...storage import exceptions as storage_exceptions + + +SERVICE_COLUMNS = ['id', 'name', 'service_template_name', 'created_at', 'updated_at'] + + [email protected](name='services') [email protected]() +def services(): + """Handle services + """ + pass + + [email protected](name='list', short_help='List services') [email protected]_template_name() [email protected]_by() [email protected] [email protected]() [email protected]_model_storage [email protected]_logger +def list(service_template_name, + sort_by, + descending, + model_storage, + logger): + """List services + + If `--service-template-name` is provided, list services for that service template. + Otherwise, list services for all service templates. + """ + if service_template_name: + logger.info('Listing services for service template {0}...'.format( + service_template_name)) + service_template = model_storage.service_template.get_by_name(service_template_name) + filters = dict(service_template=service_template) + else: + logger.info('Listing all services...') + filters = {} + + services_list = model_storage.service.list( + sort=utils.storage_sort_param(sort_by=sort_by, descending=descending), + filters=filters) + table.print_data(SERVICE_COLUMNS, services_list, 'Services:') + + [email protected](name='create', + short_help='Create a services') [email protected]('service-name', required=False) [email protected]_template_name(required=True) [email protected](help=helptexts.SERVICE_INPUTS) [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def create(service_template_name, + service_name, + inputs, # pylint: disable=redefined-outer-name + model_storage, + resource_storage, + plugin_manager, + logger): + """Create a service + + `SERVICE_NAME` is the name of the service you'd like to create. + + """ + logger.info('Creating new service from service template {0}...'.format( + service_template_name)) + core = Core(model_storage, resource_storage, plugin_manager) + service_template = model_storage.service_template.get_by_name(service_template_name) + + try: + service = core.create_service(service_template.id, inputs, service_name) + except storage_exceptions.StorageError as e: + utils.check_overriding_storage_exceptions(e, 'service', service_name) + raise + except modeling_exceptions.InputsException: + service_templates.print_service_template_inputs(model_storage, service_template_name, + logger) + raise + logger.info("Service created. The service's name is {0}".format(service.name)) + + [email protected](name='delete', + short_help='Delete a service') [email protected]('service-name') [email protected](help=helptexts.IGNORE_AVAILABLE_NODES) [email protected]() [email protected]_model_storage [email protected]_resource_storage [email protected]_plugin_manager [email protected]_logger +def delete(service_name, force, model_storage, resource_storage, plugin_manager, logger): + """Delete a service + + `SERVICE_NAME` is the name of the service to delete. + """ + logger.info('Deleting service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + core = Core(model_storage, resource_storage, plugin_manager) + core.delete_service(service.id, force=force) + logger.info('Service {0} deleted'.format(service_name)) + + [email protected](name='outputs', + short_help='Show service outputs') [email protected]('service-name') [email protected]() [email protected]_model_storage [email protected]_logger +def outputs(service_name, model_storage, logger): + """Show outputs for a specific service + + `SERVICE_NAME` is the name of the service to print outputs for. + """ + logger.info('Showing outputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + #TODO fix this section.. + outputs_def = service.outputs + response = model_storage.service.outputs.get(service_name) + outputs_ = StringIO() + for output_name, output in response.outputs.iteritems(): + outputs_.write(' - "{0}":{1}'.format(output_name, os.linesep)) + description = outputs_def[output_name].get('description', '') + outputs_.write(' Description: {0}{1}'.format(description, + os.linesep)) + outputs_.write(' Value: {0}{1}'.format(output, os.linesep)) + logger.info(outputs_.getvalue()) + + [email protected](name='inputs', + short_help='Show service inputs') [email protected]('service-name') [email protected]() [email protected]_model_storage [email protected]_logger +def inputs(service_name, model_storage, logger): + """Show inputs for a specific service + + `SERVICE_NAME` is the id of the service to print inputs for. + """ + logger.info('Showing inputs for service {0}...'.format(service_name)) + service = model_storage.service.get_by_name(service_name) + if service.inputs: + inputs_string = StringIO() + for input_name, input_ in service.inputs.iteritems(): + inputs_string.write(' - "{0}":{1}'.format(input_name, os.linesep)) + inputs_string.write(' Value: {0}{1}'.format(input_.value, os.linesep)) + logger.info(inputs_string.getvalue()) + else: + logger.info('\tNo inputs')
