ARIA-105 Integrate parser and orchestrator models
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/9841ca4a Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/9841ca4a Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/9841ca4a Branch: refs/heads/ARIA-105-integrate-modeling Commit: 9841ca4ae8df4353a75250ce57adfaaacec3aa88 Parents: 95177d0 Author: Tal Liron <[email protected]> Authored: Fri Feb 17 16:00:40 2017 -0600 Committer: Tal Liron <[email protected]> Committed: Tue Mar 21 12:36:49 2017 -0500 ---------------------------------------------------------------------- aria/VERSION.py | 4 +- aria/__init__.py | 44 +- aria/cli/args_parser.py | 11 +- aria/cli/commands.py | 59 +- aria/cli/dry.py | 88 + aria/exceptions.py | 6 +- aria/logger.py | 2 +- aria/modeling/__init__.py | 48 + aria/modeling/exceptions.py | 34 + aria/modeling/functions.py | 32 + aria/modeling/mixins.py | 142 ++ aria/modeling/models.py | 286 +++ aria/modeling/orchestration.py | 351 ++++ aria/modeling/relationship.py | 402 +++++ aria/modeling/service_changes.py | 228 +++ aria/modeling/service_common.py | 277 +++ aria/modeling/service_instance.py | 1564 ++++++++++++++++ aria/modeling/service_template.py | 1701 ++++++++++++++++++ aria/modeling/types.py | 304 ++++ aria/modeling/utils.py | 121 ++ aria/orchestrator/__init__.py | 2 +- aria/orchestrator/context/common.py | 31 +- aria/orchestrator/context/operation.py | 12 +- aria/orchestrator/context/workflow.py | 12 +- aria/orchestrator/decorators.py | 6 +- aria/orchestrator/runner.py | 15 +- aria/orchestrator/workflows/api/task.py | 286 ++- aria/orchestrator/workflows/api/task_graph.py | 4 +- .../workflows/builtin/execute_operation.py | 59 +- aria/orchestrator/workflows/builtin/heal.py | 188 +- aria/orchestrator/workflows/builtin/utils.py | 42 +- .../orchestrator/workflows/builtin/workflows.py | 85 +- aria/orchestrator/workflows/core/engine.py | 16 +- .../workflows/core/events_handler.py | 2 +- aria/orchestrator/workflows/core/task.py | 49 +- aria/orchestrator/workflows/events_logging.py | 2 +- aria/orchestrator/workflows/exceptions.py | 13 +- aria/orchestrator/workflows/executor/celery.py | 2 +- aria/orchestrator/workflows/executor/process.py | 7 +- aria/orchestrator/workflows/executor/thread.py | 3 +- aria/parser/consumption/__init__.py | 6 +- aria/parser/consumption/modeling.py | 95 +- aria/parser/consumption/style.py | 2 +- aria/parser/modeling/__init__.py | 50 +- aria/parser/modeling/context.py | 90 +- aria/parser/modeling/elements.py | 128 -- aria/parser/modeling/exceptions.py | 22 - aria/parser/modeling/instance_elements.py | 1041 ----------- aria/parser/modeling/model_elements.py | 1221 ------------- aria/parser/modeling/storage.py | 186 -- aria/parser/modeling/types.py | 146 -- aria/parser/modeling/utils.py | 146 -- aria/parser/reading/__init__.py | 4 +- aria/parser/reading/locator.py | 37 +- aria/storage/__init__.py | 10 +- aria/storage/core.py | 10 +- aria/storage/instrumentation.py | 9 +- aria/storage/modeling/__init__.py | 35 - aria/storage/modeling/elements.py | 106 -- aria/storage/modeling/instance_elements.py | 1288 ------------- aria/storage/modeling/model.py | 223 --- aria/storage/modeling/orchestrator_elements.py | 497 ----- aria/storage/modeling/structure.py | 320 ---- aria/storage/modeling/template_elements.py | 1387 -------------- aria/storage/modeling/type.py | 302 ---- aria/storage/modeling/utils.py | 139 -- aria/storage_initializer.py | 134 -- aria/utils/exceptions.py | 44 +- aria/utils/formatting.py | 19 +- aria/utils/uuid.py | 66 + docs/requirements.txt | 4 +- .../simple_v1_0/functions.py | 9 +- .../simple_v1_0/modeling/__init__.py | 526 ++++-- .../simple_v1_0/presenter.py | 6 +- tests/end2end/test_orchestrator.py | 18 +- tests/end2end/test_tosca_simple_v1_0.py | 6 +- tests/mock/context.py | 4 +- tests/mock/models.py | 253 +-- tests/mock/operations.py | 46 +- tests/mock/topology.py | 88 +- tests/modeling/__init__.py | 34 + tests/modeling/test_mixins.py | 219 +++ tests/modeling/test_model_storage.py | 102 ++ tests/modeling/test_models.py | 837 +++++++++ tests/orchestrator/context/__init__.py | 4 - tests/orchestrator/context/test_operation.py | 203 ++- .../context/test_resource_render.py | 6 +- tests/orchestrator/context/test_serialize.py | 26 +- tests/orchestrator/context/test_toolbelt.py | 61 +- tests/orchestrator/context/test_workflow.py | 14 +- .../execution_plugin/test_common.py | 4 +- .../orchestrator/execution_plugin/test_local.py | 23 +- tests/orchestrator/execution_plugin/test_ssh.py | 35 +- tests/orchestrator/test_runner.py | 7 +- tests/orchestrator/workflows/api/test_task.py | 189 +- .../workflows/builtin/test_execute_operation.py | 27 +- .../orchestrator/workflows/builtin/test_heal.py | 20 +- .../orchestrator/workflows/core/test_engine.py | 28 +- tests/orchestrator/workflows/core/test_task.py | 84 +- .../test_task_graph_into_exececution_graph.py | 27 +- .../workflows/executor/test_executor.py | 13 +- .../workflows/executor/test_process_executor.py | 6 +- ...process_executor_concurrent_modifications.py | 23 +- .../executor/test_process_executor_extension.py | 29 +- .../test_process_executor_tracked_changes.py | 30 +- tests/parser/utils.py | 14 +- .../tosca-simple-1.0/node-cellar/workflows.py | 17 +- tests/storage/__init__.py | 18 +- tests/storage/test_instrumentation.py | 53 +- tests/storage/test_model_storage.py | 103 -- tests/storage/test_models.py | 875 --------- tests/storage/test_resource_storage.py | 113 +- tests/storage/test_structures.py | 218 --- 113 files changed, 8704 insertions(+), 10021 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/VERSION.py ---------------------------------------------------------------------- diff --git a/aria/VERSION.py b/aria/VERSION.py index 7e11072..9ce332c 100644 --- a/aria/VERSION.py +++ b/aria/VERSION.py @@ -14,8 +14,8 @@ # limitations under the License. """ -Aria Version module: - * version: Aria Package version +ARIA Version module: + * version: ARIA Package version """ version = '0.1.0' # pylint: disable=C0103 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 6b10501..b9251d5 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. """ -Aria top level package +ARIA top level package """ import sys @@ -27,6 +27,7 @@ from . import ( utils, parser, storage, + modeling, orchestrator, cli ) @@ -69,48 +70,9 @@ def application_model_storage(api, api_kwargs=None, initiator=None, initiator_kw """ Initiate model storage """ - models_to_register = [ - storage.modeling.model.Parameter, - - storage.modeling.model.MappingTemplate, - storage.modeling.model.SubstitutionTemplate, - storage.modeling.model.ServiceTemplate, - storage.modeling.model.NodeTemplate, - storage.modeling.model.GroupTemplate, - storage.modeling.model.InterfaceTemplate, - storage.modeling.model.OperationTemplate, - storage.modeling.model.ArtifactTemplate, - storage.modeling.model.PolicyTemplate, - storage.modeling.model.GroupPolicyTemplate, - storage.modeling.model.GroupPolicyTriggerTemplate, - storage.modeling.model.RequirementTemplate, - storage.modeling.model.CapabilityTemplate, - - storage.modeling.model.Mapping, - storage.modeling.model.Substitution, - storage.modeling.model.ServiceInstance, - storage.modeling.model.Node, - storage.modeling.model.Group, - storage.modeling.model.Interface, - storage.modeling.model.Operation, - storage.modeling.model.Capability, - storage.modeling.model.Artifact, - storage.modeling.model.Policy, - storage.modeling.model.GroupPolicy, - storage.modeling.model.GroupPolicyTrigger, - storage.modeling.model.Relationship, - - storage.modeling.model.Execution, - storage.modeling.model.ServiceInstanceUpdate, - storage.modeling.model.ServiceInstanceUpdateStep, - storage.modeling.model.ServiceInstanceModification, - storage.modeling.model.Plugin, - storage.modeling.model.Task, - storage.modeling.model.Log - ] return storage.ModelStorage(api_cls=api, api_kwargs=api_kwargs, - items=models_to_register, + items=modeling.models.models_to_register, initiator=initiator, initiator_kwargs=initiator_kwargs or {}) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/cli/args_parser.py ---------------------------------------------------------------------- diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py index 50fec39..81ee513 100644 --- a/aria/cli/args_parser.py +++ b/aria/cli/args_parser.py @@ -91,7 +91,7 @@ def add_parse_parser(parse): 'consumer', nargs='?', default='validate', - help='"validate" (default), "presentation", "model", "types", "instance", or consumer ' + help='"validate" (default), "presentation", "template", "types", "instance", or consumer ' 'class name (full class path or short name)') parse.add_argument( '--loader-source', @@ -137,10 +137,11 @@ def add_workflow_parser(workflow): '-w', '--workflow', default='install', help='The workflow name') - workflow.add_argument( - '-i', '--service-instance-id', - required=False, - help='A unique ID for the service instance') + workflow.add_flag_argument( + 'dry', + default=True, + help_true='dry run', + help_false='wet run') @sub_parser_decorator( http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/cli/commands.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 91d748f..1eef61d 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -36,13 +36,12 @@ from ..parser.consumption import ( ConsumerChain, Read, Validate, - Model, + ServiceTemplate, Types, Inputs, - Instance + ServiceInstance ) from ..parser.loading import LiteralLocation, UriLocation -from ..parser.modeling.storage import initialize_storage from ..utils.application import StorageManager from ..utils.caching import cachedmethod from ..utils.console import (puts, Colored, indent) @@ -51,6 +50,7 @@ from ..utils.collections import OrderedDict from ..orchestrator import WORKFLOW_DECORATOR_RESERVED_ARGUMENTS from ..orchestrator.runner import Runner from ..orchestrator.workflows.builtin import BUILTIN_WORKFLOWS +from .dry import convert_to_dry from .exceptions import ( AriaCliFormatInputsError, @@ -157,14 +157,14 @@ class ParseCommand(BaseCommand): dumper = None elif consumer_class_name == 'presentation': dumper = consumer.consumers[0] - elif consumer_class_name == 'model': - consumer.append(Model) + elif consumer_class_name == 'template': + consumer.append(ServiceTemplate) elif consumer_class_name == 'types': - consumer.append(Model, Types) + consumer.append(ServiceTemplate, Types) elif consumer_class_name == 'instance': - consumer.append(Model, Inputs, Instance) + consumer.append(ServiceTemplate, Inputs, ServiceInstance) else: - consumer.append(Model, Inputs, Instance) + consumer.append(ServiceTemplate, Inputs, ServiceInstance) consumer.append(import_fullname(consumer_class_name)) if dumper is None: @@ -211,16 +211,17 @@ class WorkflowCommand(BaseCommand): def __call__(self, args_namespace, unknown_args): super(WorkflowCommand, self).__call__(args_namespace, unknown_args) - service_instance_id = args_namespace.service_instance_id or 1 context = self._parse(args_namespace.uri) workflow_fn, inputs = self._get_workflow(context, args_namespace.workflow) - self._run(context, args_namespace.workflow, workflow_fn, inputs, service_instance_id) + self._dry = args_namespace.dry + self._run(context, args_namespace.workflow, workflow_fn, inputs) def _parse(self, uri): # Parse context = ConsumptionContext() context.presentation.location = UriLocation(uri) - consumer = ConsumerChain(context, (Read, Validate, Model, Inputs, Instance)) + consumer = ConsumerChain(context, (Read, Validate, ServiceTemplate, Inputs, + ServiceInstance)) consumer.consume() if context.validation.dump_issues(): @@ -230,43 +231,45 @@ class WorkflowCommand(BaseCommand): def _get_workflow(self, context, workflow_name): if workflow_name in BUILTIN_WORKFLOWS: - workflow_fn = import_fullname('aria.orchestrator.workflows.builtin.%s' % workflow_name) + workflow_fn = import_fullname('aria.orchestrator.workflows.builtin.{0}'.format( + workflow_name)) inputs = {} else: + workflow = context.modeling.instance.policies.get(workflow_name) + if workflow is None: + raise AttributeError('workflow policy does not exist: "{0}"'.format(workflow_name)) + if workflow.type.role != 'workflow': + raise AttributeError('policy is not a workflow: "{0}"'.format(workflow_name)) + try: - policy = context.modeling.instance.policies[workflow_name] - except KeyError: - raise AttributeError('workflow policy does not exist: "%s"' % workflow_name) - if context.modeling.policy_types.get_role(policy.type_name) != 'workflow': - raise AttributeError('policy is not a workflow: "%s"' % workflow_name) - - try: - sys.path.append(policy.properties['implementation'].value) + sys.path.append(workflow.properties['implementation'].value) except KeyError: pass - workflow_fn = import_fullname(policy.properties['function'].value) + workflow_fn = import_fullname(workflow.properties['function'].value) - for k in policy.properties: + for k in workflow.properties: if k in WORKFLOW_DECORATOR_RESERVED_ARGUMENTS: - raise AttributeError('workflow policy "%s" defines a reserved property: "%s"' % - (workflow_name, k)) + raise AttributeError('workflow policy "{0}" defines a reserved property: "{1}"' + .format(workflow_name, k)) inputs = OrderedDict([ - (k, v.value) for k, v in policy.properties.iteritems() + (k, v.value) for k, v in workflow.properties.iteritems() if k not in WorkflowCommand.WORKFLOW_POLICY_INTERNAL_PROPERTIES ]) return workflow_fn, inputs - def _run(self, context, workflow_name, workflow_fn, inputs, service_instance_id): + def _run(self, context, workflow_name, workflow_fn, inputs): # Storage def _initialize_storage(model_storage): - initialize_storage(context, model_storage, service_instance_id) + if self._dry: + convert_to_dry(context.modeling.instance) + context.modeling.store(model_storage) # Create runner runner = Runner(workflow_name, workflow_fn, inputs, _initialize_storage, - service_instance_id) + lambda: context.modeling.instance.id) # Run runner.run() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/cli/dry.py ---------------------------------------------------------------------- diff --git a/aria/cli/dry.py b/aria/cli/dry.py new file mode 100644 index 0000000..82faf42 --- /dev/null +++ b/aria/cli/dry.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import RLock + +from ..modeling import models +from ..orchestrator.decorators import operation +from ..utils.collections import OrderedDict +from ..utils.console import puts, Colored +from ..utils.formatting import safe_repr + + +_TERMINAL_LOCK = RLock() + + +def convert_to_dry(service): + """ + Converts all operations on the service (on workflows, node interfaces, and relationship + interfaces) to run dryly. + """ + + for workflow in service.workflows: + convert_operation_to_dry(workflow) + + for node in service.nodes.itervalues(): + for interface in node.interfaces.itervalues(): + for oper in interface.operations.itervalues(): + convert_operation_to_dry(oper) + for relationship in node.outbound_relationships: + for interface in relationship.interfaces.itervalues(): + for oper in interface.operations.itervalues(): + convert_operation_to_dry(oper) + + +def convert_operation_to_dry(oper): + """ + Converts a single :class:`Operation` to run dryly. + """ + + plugin = oper.plugin_specification.name \ + if oper.plugin_specification is not None else None + if oper.inputs is None: + oper.inputs = OrderedDict() + oper.inputs['_implementation'] = models.Parameter(name='_implementation', + type_name='string', + value=oper.implementation) + oper.inputs['_plugin'] = models.Parameter(name='_plugin', + type_name='string', + value=plugin) + oper.implementation = '{0}.{1}'.format(__name__, 'dry_operation') + oper.plugin_specification = None + + +@operation +def dry_operation(ctx, _plugin, _implementation, **kwargs): + """ + The dry operation simply prints out information about the operation to the console. + """ + + with _TERMINAL_LOCK: + print ctx.name + if hasattr(ctx, 'relationship'): + puts('> Relationship: {0} -> {1}'.format( + Colored.red(ctx.relationship.source_node.name), + Colored.red(ctx.relationship.target_node.name))) + else: + puts('> Node: {0}'.format(Colored.red(ctx.node.name))) + puts(' Operation: {0}'.format(Colored.green(ctx.name))) + _dump_implementation(_plugin, _implementation) + + +def _dump_implementation(plugin, implementation): + if plugin: + puts(' Plugin: {0}'.format(Colored.magenta(plugin, bold=True))) + if implementation: + puts(' Implementation: {0}'.format(Colored.magenta(safe_repr(implementation)))) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/exceptions.py b/aria/exceptions.py index 28f8be9..a180ce1 100644 --- a/aria/exceptions.py +++ b/aria/exceptions.py @@ -14,8 +14,8 @@ # limitations under the License. """ -Aria exceptions module -Every sub-package in Aria has a module with its exceptions. +ARIA exceptions module +Every sub-package in ARIA has a module with its exceptions. aria.exceptions module conveniently collects all these exceptions for easier imports. """ @@ -43,4 +43,4 @@ class AriaException(Exception): if cause == e: # Make sure it's our traceback cause_traceback = traceback - self.cause_tb = cause_traceback + self.cause_traceback = cause_traceback http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/logger.py ---------------------------------------------------------------------- diff --git a/aria/logger.py b/aria/logger.py index 42e3679..e3039f5 100644 --- a/aria/logger.py +++ b/aria/logger.py @@ -167,7 +167,7 @@ class _SQLAlchemyHandler(logging.Handler): task_fk=record.task_id, actor=record.prefix, level=record.levelname, - msg=record.msg, + msg=str(record.msg), created_at=created_at, ) self._session.add(log) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/__init__.py ---------------------------------------------------------------------- diff --git a/aria/modeling/__init__.py b/aria/modeling/__init__.py new file mode 100644 index 0000000..4dfc39d --- /dev/null +++ b/aria/modeling/__init__.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple + +from . import ( + mixins, + types, + models, + service_template as _service_template_bases, + service_instance as _service_instance_bases, + service_changes as _service_changes_bases, + service_common as _service_common_bases, + orchestration as _orchestration_bases +) + + +_ModelBasesCls = namedtuple('ModelBase', 'service_template,' + 'service_instance,' + 'service_changes,' + 'service_common,' + 'orchestration') + +model_bases = _ModelBasesCls(service_template=_service_template_bases, + service_instance=_service_instance_bases, + service_changes=_service_changes_bases, + service_common=_service_common_bases, + orchestration=_orchestration_bases) + + +__all__ = ( + 'mixins', + 'types', + 'models', + 'model_bases', +) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/modeling/exceptions.py b/aria/modeling/exceptions.py new file mode 100644 index 0000000..6931c78 --- /dev/null +++ b/aria/modeling/exceptions.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..exceptions import AriaException + + +class ModelingException(AriaException): + """ + ARIA modeling exception. + """ + + +class ValueFormatException(ModelingException): + """ + ARIA modeling exception: the value is in the wrong format. + """ + + +class CannotEvaluateFunctionException(ModelingException): + """ + ARIA modeling exception: cannot evaluate the function at this time. + """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/functions.py ---------------------------------------------------------------------- diff --git a/aria/modeling/functions.py b/aria/modeling/functions.py new file mode 100644 index 0000000..02f4454 --- /dev/null +++ b/aria/modeling/functions.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class Function(object): + """ + An intrinsic function. + + Serves as a placeholder for a value that should eventually be derived by calling the function. + """ + + @property + def as_raw(self): + raise NotImplementedError + + def _evaluate(self, context, container): + raise NotImplementedError + + def __deepcopy__(self, memo): + # Circumvent cloning in order to maintain our state + return self http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/mixins.py ---------------------------------------------------------------------- diff --git a/aria/modeling/mixins.py b/aria/modeling/mixins.py new file mode 100644 index 0000000..e6db5a3 --- /dev/null +++ b/aria/modeling/mixins.py @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +classes: + * ModelMixin - abstract model implementation. + * ModelIDMixin - abstract model implementation with IDs. +""" + +from sqlalchemy.ext import associationproxy +from sqlalchemy import ( + Column, + Integer, + Text +) + +from . import utils + + +class ModelMixin(object): + + @utils.classproperty + def __modelname__(cls): # pylint: disable=no-self-argument + return getattr(cls, '__mapiname__', cls.__tablename__) + + @classmethod + def id_column_name(cls): + raise NotImplementedError + + @classmethod + def name_column_name(cls): + raise NotImplementedError + + def to_dict(self, fields=None, suppress_error=False): + """ + Return a dict representation of the model + + :param suppress_error: If set to True, sets ``None`` to attributes that it's unable to + retrieve (e.g., if a relationship wasn't established yet, and so it's + impossible to access a property through it) + """ + + res = dict() + fields = fields or self.fields() + for field in fields: + try: + field_value = getattr(self, field) + except AttributeError: + if suppress_error: + field_value = None + else: + raise + if isinstance(field_value, list): + field_value = list(field_value) + elif isinstance(field_value, dict): + field_value = dict(field_value) + elif isinstance(field_value, ModelMixin): + field_value = field_value.to_dict() + res[field] = field_value + + return res + + @classmethod + def fields(cls): + """ + Return the list of field names for this table + + Mostly for backwards compatibility in the code (that uses ``fields``) + """ + + fields = set(cls._iter_association_proxies()) + fields.update(cls.__table__.columns.keys()) + return fields - set(getattr(cls, '__private_fields__', [])) + + @classmethod + def _iter_association_proxies(cls): + for col, value in vars(cls).items(): + if isinstance(value, associationproxy.AssociationProxy): + yield col + + def __repr__(self): + return '<{cls} id=`{id}`>'.format( + cls=self.__class__.__name__, + id=getattr(self, self.name_column_name())) + + +class ModelIDMixin(object): + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(Text, index=True) + + @classmethod + def id_column_name(cls): + return 'id' + + @classmethod + def name_column_name(cls): + return 'name' + + +class InstanceModelMixin(ModelMixin): + """ + Mixin for :class:`ServiceInstance` models. + + All models support validation, diagnostic dumping, and representation as + raw data (which can be translated into JSON or YAML) via ``as_raw``. + """ + + @property + def as_raw(self): + raise NotImplementedError + + def validate(self): + pass + + def coerce_values(self, container, report_issues): + pass + + def dump(self): + pass + + +class TemplateModelMixin(InstanceModelMixin): + """ + Mixin for :class:`ServiceTemplate` models. + + All model models can be instantiated into :class:`ServiceInstance` models. + """ + + def instantiate(self, container): + raise NotImplementedError http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/models.py ---------------------------------------------------------------------- diff --git a/aria/modeling/models.py b/aria/modeling/models.py new file mode 100644 index 0000000..a01783b --- /dev/null +++ b/aria/modeling/models.py @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=abstract-method + +from sqlalchemy.ext.declarative import declarative_base + +from . import ( + service_template, + service_instance, + service_changes, + service_common, + orchestration, + mixins, +) + + +aria_declarative_base = declarative_base(cls=mixins.ModelIDMixin) + + +# See also models_to_register at the bottom of this file +__all__ = ( + 'aria_declarative_base', + 'models_to_register', + + # Service template models + 'ServiceTemplate', + 'NodeTemplate', + 'GroupTemplate', + 'PolicyTemplate', + 'SubstitutionTemplate', + 'SubstitutionTemplateMapping', + 'RequirementTemplate', + 'RelationshipTemplate', + 'CapabilityTemplate', + 'InterfaceTemplate', + 'OperationTemplate', + 'ArtifactTemplate', + + # Service instance models + 'Service', + 'Node', + 'Group', + 'Policy', + 'Substitution', + 'SubstitutionMapping', + 'Relationship', + 'Capability', + 'Interface', + 'Operation', + 'Artifact', + + # Service changes models + 'ServiceUpdate', + 'ServiceUpdateStep', + 'ServiceModification', + + # Common service models + 'Parameter', + 'Type', + 'Metadata', + 'PluginSpecification', + + # Orchestration models + 'Execution', + 'Plugin', + 'Task', + 'Log' +) + + +# region service template models + +class ServiceTemplate(aria_declarative_base, service_template.ServiceTemplateBase): + pass + + +class NodeTemplate(aria_declarative_base, service_template.NodeTemplateBase): + pass + + +class GroupTemplate(aria_declarative_base, service_template.GroupTemplateBase): + pass + + +class PolicyTemplate(aria_declarative_base, service_template.PolicyTemplateBase): + pass + + +class SubstitutionTemplate(aria_declarative_base, service_template.SubstitutionTemplateBase): + pass + + +class SubstitutionTemplateMapping(aria_declarative_base, + service_template.SubstitutionTemplateMappingBase): + pass + + +class RequirementTemplate(aria_declarative_base, service_template.RequirementTemplateBase): + pass + + +class RelationshipTemplate(aria_declarative_base, service_template.RelationshipTemplateBase): + pass + + +class CapabilityTemplate(aria_declarative_base, service_template.CapabilityTemplateBase): + pass + + +class InterfaceTemplate(aria_declarative_base, service_template.InterfaceTemplateBase): + pass + + +class OperationTemplate(aria_declarative_base, service_template.OperationTemplateBase): + pass + + +class ArtifactTemplate(aria_declarative_base, service_template.ArtifactTemplateBase): + pass + +# endregion + + +# region service instance models + +class Service(aria_declarative_base, service_instance.ServiceBase): + pass + + +class Node(aria_declarative_base, service_instance.NodeBase): + pass + + +class Group(aria_declarative_base, service_instance.GroupBase): + pass + + +class Policy(aria_declarative_base, service_instance.PolicyBase): + pass + + +class Substitution(aria_declarative_base, service_instance.SubstitutionBase): + pass + + +class SubstitutionMapping(aria_declarative_base, service_instance.SubstitutionMappingBase): + pass + + +class Relationship(aria_declarative_base, service_instance.RelationshipBase): + pass + + +class Capability(aria_declarative_base, service_instance.CapabilityBase): + pass + + +class Interface(aria_declarative_base, service_instance.InterfaceBase): + pass + + +class Operation(aria_declarative_base, service_instance.OperationBase): + pass + + +class Artifact(aria_declarative_base, service_instance.ArtifactBase): + pass + +# endregion + + +# region service changes models + +class ServiceUpdate(aria_declarative_base, service_changes.ServiceUpdateBase): + pass + + +class ServiceUpdateStep(aria_declarative_base, service_changes.ServiceUpdateStepBase): + pass + + +class ServiceModification(aria_declarative_base, service_changes.ServiceModificationBase): + pass + +# endregion + + +# region common service models + +class Parameter(aria_declarative_base, service_common.ParameterBase): + pass + + +class Type(aria_declarative_base, service_common.TypeBase): + pass + + +class Metadata(aria_declarative_base, service_common.MetadataBase): + pass + + +class PluginSpecification(aria_declarative_base, service_common.PluginSpecificationBase): + pass + +# endregion + + +# region orchestration models + +class Execution(aria_declarative_base, orchestration.ExecutionBase): + pass + + +class Plugin(aria_declarative_base, orchestration.PluginBase): + pass + + +class Task(aria_declarative_base, orchestration.TaskBase): + pass + + +class Log(aria_declarative_base, orchestration.LogBase): + pass + +# endregion + + +# See also __all__ at the top of this file +models_to_register = [ + # Service template models + ServiceTemplate, + NodeTemplate, + GroupTemplate, + PolicyTemplate, + SubstitutionTemplate, + SubstitutionTemplateMapping, + RequirementTemplate, + RelationshipTemplate, + CapabilityTemplate, + InterfaceTemplate, + OperationTemplate, + ArtifactTemplate, + + # Service instance models + Service, + Node, + Group, + Policy, + SubstitutionMapping, + Substitution, + Relationship, + Capability, + Interface, + Operation, + Artifact, + + # Service changes models + ServiceUpdate, + ServiceUpdateStep, + ServiceModification, + + # Common service models + Parameter, + Type, + Metadata, + PluginSpecification, + + # Orchestration models + Execution, + Plugin, + Task, + Log +] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/orchestration.py ---------------------------------------------------------------------- diff --git a/aria/modeling/orchestration.py b/aria/modeling/orchestration.py new file mode 100644 index 0000000..0277756 --- /dev/null +++ b/aria/modeling/orchestration.py @@ -0,0 +1,351 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +classes: + * Execution - execution implementation model. + * Plugin - plugin implementation model. + * Task - a task +""" + +# pylint: disable=no-self-argument, no-member, abstract-method + +from datetime import datetime + +from sqlalchemy import ( + Column, + Integer, + Text, + DateTime, + Boolean, + Enum, + String, + Float, + orm, +) +from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.declarative import declared_attr + +from ..orchestrator.exceptions import (TaskAbortException, TaskRetryException) +from .types import (List, Dict) +from .mixins import ModelMixin +from . import relationship + + +class ExecutionBase(ModelMixin): + """ + Execution model representation. + """ + + __tablename__ = 'execution' + + __private_fields__ = ['service_fk', + 'service_name', + 'service_template', + 'service_template_name'] + + TERMINATED = 'terminated' + FAILED = 'failed' + CANCELLED = 'cancelled' + PENDING = 'pending' + STARTED = 'started' + CANCELLING = 'cancelling' + FORCE_CANCELLING = 'force_cancelling' + + STATES = [TERMINATED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING, FORCE_CANCELLING] + END_STATES = [TERMINATED, FAILED, CANCELLED] + ACTIVE_STATES = [state for state in STATES if state not in END_STATES] + + VALID_TRANSITIONS = { + PENDING: [STARTED, CANCELLED], + STARTED: END_STATES + [CANCELLING], + CANCELLING: END_STATES + [FORCE_CANCELLING] + } + + @orm.validates('status') + def validate_status(self, key, value): + """Validation function that verifies execution status transitions are OK""" + try: + current_status = getattr(self, key) + except AttributeError: + return + valid_transitions = self.VALID_TRANSITIONS.get(current_status, []) + if all([current_status is not None, + current_status != value, + value not in valid_transitions]): + raise ValueError('Cannot change execution status from {current} to {new}'.format( + current=current_status, + new=value)) + return value + + created_at = Column(DateTime, index=True) + started_at = Column(DateTime, nullable=True, index=True) + ended_at = Column(DateTime, nullable=True, index=True) + error = Column(Text, nullable=True) + is_system_workflow = Column(Boolean, nullable=False, default=False) + parameters = Column(Dict) + status = Column(Enum(*STATES, name='execution_status'), default=PENDING) + workflow_name = Column(Text) + + @declared_attr + def service(cls): + return relationship.many_to_one(cls, 'service') + + # region foreign keys + + @declared_attr + def service_fk(cls): + return relationship.foreign_key('service') + + # endregion + + # region association proxies + + @declared_attr + def service_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service', cls.name_column_name()) + + @declared_attr + def service_template(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service', 'service_template') + + @declared_attr + def service_template_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service', 'service_template_name') + + # endregion + + def __str__(self): + return '<{0} id=`{1}` (status={2})>'.format( + self.__class__.__name__, + getattr(self, self.name_column_name()), + self.status + ) + + +class PluginBase(ModelMixin): + """ + Plugin model representation. + """ + + __tablename__ = 'plugin' + + archive_name = Column(Text, nullable=False, index=True) + distribution = Column(Text) + distribution_release = Column(Text) + distribution_version = Column(Text) + package_name = Column(Text, nullable=False, index=True) + package_source = Column(Text) + package_version = Column(Text) + supported_platform = Column(Text) + supported_py_versions = Column(List) + uploaded_at = Column(DateTime, nullable=False, index=True) + wheels = Column(List, nullable=False) + + +class TaskBase(ModelMixin): + """ + A Model which represents an task + """ + + __tablename__ = 'task' + + __private_fields__ = ['node_fk', + 'relationship_fk', + 'plugin_fk', + 'execution_fk', + 'node_name', + 'relationship_name', + 'execution_name'] + + PENDING = 'pending' + RETRYING = 'retrying' + SENT = 'sent' + STARTED = 'started' + SUCCESS = 'success' + FAILED = 'failed' + STATES = ( + PENDING, + RETRYING, + SENT, + STARTED, + SUCCESS, + FAILED, + ) + + WAIT_STATES = [PENDING, RETRYING] + END_STATES = [SUCCESS, FAILED] + + RUNS_ON_SOURCE = 'source' + RUNS_ON_TARGET = 'target' + RUNS_ON_NODE = 'node' + RUNS_ON = (RUNS_ON_NODE, RUNS_ON_SOURCE, RUNS_ON_TARGET) + + INFINITE_RETRIES = -1 + + @declared_attr + def node(cls): + return relationship.many_to_one(cls, 'node') + + @declared_attr + def relationship(cls): + return relationship.many_to_one(cls, 'relationship') + + @declared_attr + def plugin(cls): + return relationship.many_to_one(cls, 'plugin') + + @declared_attr + def execution(cls): + return relationship.many_to_one(cls, 'execution') + + @declared_attr + def inputs(cls): + return relationship.many_to_many(cls, 'parameter', prefix='inputs', dict_key='name') + + status = Column(Enum(*STATES, name='status'), default=PENDING) + + due_at = Column(DateTime, nullable=False, index=True, default=datetime.utcnow()) + started_at = Column(DateTime, default=None) + ended_at = Column(DateTime, default=None) + max_attempts = Column(Integer, default=1) + retry_count = Column(Integer, default=0) + retry_interval = Column(Float, default=0) + ignore_failure = Column(Boolean, default=False) + + # Operation specific fields + implementation = Column(String) + _runs_on = Column(Enum(*RUNS_ON, name='runs_on'), name='runs_on') + + @property + def runs_on(self): + if self._runs_on == self.RUNS_ON_NODE: + return self.node + elif self._runs_on == self.RUNS_ON_SOURCE: + return self.relationship.source_node # pylint: disable=no-member + elif self._runs_on == self.RUNS_ON_TARGET: + return self.relationship.target_node # pylint: disable=no-member + return None + + @property + def actor(self): + """ + Return the actor of the task + :return: + """ + return self.node or self.relationship + + @orm.validates('max_attempts') + def validate_max_attempts(self, _, value): # pylint: disable=no-self-use + """Validates that max attempts is either -1 or a positive number""" + if value < 1 and value != TaskBase.INFINITE_RETRIES: + raise ValueError('Max attempts can be either -1 (infinite) or any positive number. ' + 'Got {value}'.format(value=value)) + return value + + # region foreign keys + + @declared_attr + def node_fk(cls): + return relationship.foreign_key('node', nullable=True) + + @declared_attr + def relationship_fk(cls): + return relationship.foreign_key('relationship', nullable=True) + + @declared_attr + def plugin_fk(cls): + return relationship.foreign_key('plugin', nullable=True) + + @declared_attr + def execution_fk(cls): + return relationship.foreign_key('execution', nullable=True) + + # endregion + + # region association proxies + + @declared_attr + def node_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('node', cls.name_column_name()) + + @declared_attr + def relationship_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('relationship', cls.name_column_name()) + + @declared_attr + def execution_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('execution', cls.name_column_name()) + + # endregion + + @classmethod + def for_node(cls, instance, runs_on, **kwargs): + return cls(node=instance, _runs_on=runs_on, **kwargs) + + @classmethod + def for_relationship(cls, instance, runs_on, **kwargs): + return cls(relationship=instance, _runs_on=runs_on, **kwargs) + + @staticmethod + def abort(message=None): + raise TaskAbortException(message) + + @staticmethod + def retry(message=None, retry_interval=None): + raise TaskRetryException(message, retry_interval=retry_interval) + + +class LogBase(ModelMixin): + + __tablename__ = 'log' + + __private_fields__ = ['execution_fk', + 'task_fk'] + + @declared_attr + def execution(cls): + return relationship.many_to_one(cls, 'execution') + + @declared_attr + def task(cls): + return relationship.many_to_one(cls, 'task') + + level = Column(String) + msg = Column(String) + created_at = Column(DateTime, index=True) + actor = Column(String) + + # region foreign keys + + @declared_attr + def execution_fk(cls): + return relationship.foreign_key('execution') + + @declared_attr + def task_fk(cls): + return relationship.foreign_key('task', nullable=True) + + # endregion + + def __repr__(self): + return "<{self.created_at}: [{self.level}] @{self.actor}> {msg}".format( + self=self, msg=self.msg[:50]) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/relationship.py ---------------------------------------------------------------------- diff --git a/aria/modeling/relationship.py b/aria/modeling/relationship.py new file mode 100644 index 0000000..bed1599 --- /dev/null +++ b/aria/modeling/relationship.py @@ -0,0 +1,402 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name, redefined-outer-name + +from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy import ( + Column, + ForeignKey, + Integer, + Table +) + +from ..utils import formatting + + +def foreign_key(other_table, + nullable=False): + """ + Declare a foreign key property, which will also create a foreign key column in the table with + the name of the property. By convention the property name should end in "_fk". + + You are required to explicitly create foreign keys in order to allow for one-to-one, + one-to-many, and many-to-one relationships (but not for many-to-many relationships). If you do + not do so, SQLAlchemy will fail to create the relationship property and raise an exception with + a clear error message. + + You should normally not have to access this property directly, but instead use the associated + relationship properties. + + *This utility method should only be used during class creation.* + + :param other_table: Other table name + :type other_table: basestring + :param nullable: True to allow null values (meaning that there is no relationship) + :type nullable: bool + """ + + return Column(Integer, + ForeignKey('{table}.id'.format(table=other_table), ondelete='CASCADE'), + nullable=nullable) + + +def one_to_one_self(model_class, + fk, + relationship_kwargs=None): + """ + Declare a one-to-one relationship property. The property value would be an instance of the same + model. + + You will need an associated foreign key to our own table. + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param fk: Foreign key name + :type fk: basestring + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + """ + + relationship_kwargs = relationship_kwargs or {} + + remote_side = '{model_class}.{remote_column}'.format( + model_class=model_class.__name__, + remote_column=model_class.id_column_name() + ) + + primaryjoin = '{remote_side} == {model_class}.{column}'.format( + remote_side=remote_side, + model_class=model_class.__name__, + column=fk + ) + + return relationship( + _get_class_for_table(model_class, model_class.__tablename__).__name__, + primaryjoin=primaryjoin, + remote_side=remote_side, + post_update=True, + **relationship_kwargs + ) + + +def one_to_many_self(model_class, + fk, + dict_key=None, + relationship_kwargs=None): + """ + Declare a one-to-many relationship property. The property value would be a list or dict of + instances of the same model. + + You will need an associated foreign key to our own table. + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param fk: Foreign key name + :type fk: basestring + :param dict_key: If set the value will be a dict with this key as the dict key; otherwise will + be a list + :type dict_key: basestring + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + """ + + relationship_kwargs = relationship_kwargs or {} + + relationship_kwargs.setdefault('remote_side', '{model_class}.{remote_column}'.format( + model_class=model_class.__name__, + remote_column=fk + )) + + return _relationship(model_class, model_class.__tablename__, None, relationship_kwargs, + other_property=False, dict_key=dict_key) + + +def one_to_one(model_class, + other_table, + fk=None, + other_fk=None, + other_property=None, + relationship_kwargs=None, + backref_kwargs=None): + """ + Declare a one-to-one relationship property. The property value would be an instance of the other + table's model. + + You have two options for the foreign key. Either this table can have an associated key to the + other table (use the ``fk`` argument) or the other table can have an associated foreign key to + this our table (use the ``other_fk`` argument). + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param other_table: Other table name + :type other_table: basestring + :param fk: Foreign key name at our table (no need specify if there's no ambiguity) + :type fk: basestring + :param other_fk: Foreign key name at the other table (no need specify if there's no ambiguity) + :type other_fk: basestring + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + :param backref_kwargs: Extra kwargs for SQLAlchemy ``backref`` + :type backref_kwargs: {} + """ + + backref_kwargs = backref_kwargs or {} + backref_kwargs.setdefault('uselist', False) + + return _relationship(model_class, other_table, backref_kwargs, relationship_kwargs, + other_property, fk=fk, other_fk=other_fk) + + +def one_to_many(model_class, + child_table, + child_fk=None, + dict_key=None, + child_property=None, + relationship_kwargs=None, + backref_kwargs=None): + """ + Declare a one-to-many relationship property. The property value would be a list or dict of + instances of the child table's model. + + The child table will need an associated foreign key to our table. + + The declaration will automatically create a matching many-to-one property at the child model, + named after our table name. Use the ``child_property`` argument to override this name. + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param child_table: Child table name + :type child_table: basestring + :param child_fk: Foreign key name at the child table (no need specify if there's no ambiguity) + :type child_fk: basestring + :param dict_key: If set the value will be a dict with this key as the dict key; otherwise will + be a list + :type dict_key: basestring + :param child_property: Override name of matching many-to-one property at child table; set to + false to disable + :type child_property: basestring|bool + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + :param backref_kwargs: Extra kwargs for SQLAlchemy ``backref`` + :type backref_kwargs: {} + """ + + backref_kwargs = backref_kwargs or {} + backref_kwargs.setdefault('uselist', False) + + return _relationship(model_class, child_table, backref_kwargs, relationship_kwargs, + child_property, other_fk=child_fk, dict_key=dict_key) + + +def many_to_one(model_class, + parent_table, + fk=None, + parent_fk=None, + parent_property=None, + relationship_kwargs=None, + backref_kwargs=None): + """ + Declare a many-to-one relationship property. The property value would be an instance of the + parent table's model. + + You will need an associated foreign key to the parent table. + + The declaration will automatically create a matching one-to-many property at the child model, + named after the plural form of our table name. Use the ``parent_property`` argument to override + this name. Note: the automatic property will always be a SQLAlchemy query object; if you need a + Python collection then use :meth:`one_to_many` at that model. + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param parent_table: Parent table name + :type parent_table: basestring + :param fk: Foreign key name at our table (no need specify if there's no ambiguity) + :type fk: basestring + :param parent_property: Override name of matching one-to-many property at parent table; set to + false to disable + :type parent_property: basestring|bool + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + :param backref_kwargs: Extra kwargs for SQLAlchemy ``backref`` + :type backref_kwargs: {} + """ + + if parent_property is None: + parent_property = formatting.pluralize(model_class.__tablename__) + + backref_kwargs = backref_kwargs or {} + backref_kwargs.setdefault('uselist', True) + backref_kwargs.setdefault('lazy', 'dynamic') + backref_kwargs.setdefault('cascade', 'all') # delete children when parent is deleted + + return _relationship(model_class, parent_table, backref_kwargs, relationship_kwargs, + parent_property, fk=fk, other_fk=parent_fk) + + +def many_to_many(model_class, + other_table, + prefix=None, + dict_key=None, + other_property=None, + relationship_kwargs=None, + backref_kwargs=None): + """ + Declare a many-to-many relationship property. The property value would be a list or dict of + instances of the other table's model. + + You do not need associated foreign keys for this relationship. Instead, an extra table will be + created for you. + + The declaration will automatically create a matching many-to-many property at the other model, + named after the plural form of our table name. Use the ``other_property`` argument to override + this name. Note: the automatic property will always be a SQLAlchemy query object; if you need a + Python collection then use :meth:`many_to_many` again at that model. + + *This utility method should only be used during class creation.* + + :param model_class: The class in which this relationship will be declared + :type model_class: type + :param parent_table: Parent table name + :type parent_table: basestring + :param prefix: Optional prefix for extra table name as well as for ``other_property`` + :type prefix: basestring + :param dict_key: If set the value will be a dict with this key as the dict key; otherwise will + be a list + :type dict_key: basestring + :param other_property: Override name of matching many-to-many property at other table; set to + false to disable + :type other_property: basestring|bool + :param relationship_kwargs: Extra kwargs for SQLAlchemy ``relationship`` + :type relationship_kwargs: {} + :param backref_kwargs: Extra kwargs for SQLAlchemy ``backref`` + :type backref_kwargs: {} + """ + + this_table = model_class.__tablename__ + this_column_name = '{0}_id'.format(this_table) + this_foreign_key = '{0}.id'.format(this_table) + + other_column_name = '{0}_id'.format(other_table) + other_foreign_key = '{0}.id'.format(other_table) + + secondary_table = '{0}_{1}'.format(this_table, other_table) + + if other_property is None: + other_property = formatting.pluralize(this_table) + if prefix is not None: + secondary_table = '{0}_{1}'.format(prefix, secondary_table) + other_property = '{0}_{1}'.format(prefix, other_property) + + backref_kwargs = backref_kwargs or {} + backref_kwargs.setdefault('uselist', True) + + relationship_kwargs = relationship_kwargs or {} + relationship_kwargs.setdefault('secondary', _get_secondary_table( + model_class.metadata, + secondary_table, + this_column_name, + other_column_name, + this_foreign_key, + other_foreign_key + )) + + return _relationship(model_class, other_table, backref_kwargs, relationship_kwargs, + other_property, dict_key=dict_key) + + +def _relationship(model_class, other_table, backref_kwargs, relationship_kwargs, other_property, + fk=None, other_fk=None, dict_key=None): + relationship_kwargs = relationship_kwargs or {} + + if fk: + relationship_kwargs.setdefault('foreign_keys', + lambda: getattr( + _get_class_for_table( + model_class, + model_class.__tablename__), + fk)) + + elif other_fk: + relationship_kwargs.setdefault('foreign_keys', + lambda: getattr( + _get_class_for_table( + model_class, + other_table), + other_fk)) + + if dict_key: + relationship_kwargs.setdefault('collection_class', + attribute_mapped_collection(dict_key)) + + if other_property is False: + # No backref + return relationship( + lambda: _get_class_for_table(model_class, other_table), + **relationship_kwargs + ) + else: + if other_property is None: + other_property = model_class.__tablename__ + backref_kwargs = backref_kwargs or {} + return relationship( + lambda: _get_class_for_table(model_class, other_table), + backref=backref(other_property, **backref_kwargs), + **relationship_kwargs + ) + + +def _get_class_for_table(model_class, tablename): + if tablename in (model_class.__name__, model_class.__tablename__): + return model_class + + for table_cls in model_class._decl_class_registry.values(): + if tablename == getattr(table_cls, '__tablename__', None): + return table_cls + + raise ValueError('unknown table: {0}'.format(tablename)) + + +def _get_secondary_table(metadata, + name, + first_column, + second_column, + first_foreign_key, + second_foreign_key): + return Table( + name, + metadata, + Column( + first_column, + Integer, + ForeignKey(first_foreign_key) + ), + Column( + second_column, + Integer, + ForeignKey(second_foreign_key) + ) + ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/service_changes.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_changes.py b/aria/modeling/service_changes.py new file mode 100644 index 0000000..a33e6ae --- /dev/null +++ b/aria/modeling/service_changes.py @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +classes: + * ServiceUpdate - service update implementation model. + * ServiceUpdateStep - service update step implementation model. + * ServiceModification - service modification implementation model. +""" + +# pylint: disable=no-self-argument, no-member, abstract-method + +from collections import namedtuple + +from sqlalchemy import ( + Column, + Text, + DateTime, + Enum, +) +from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.declarative import declared_attr + +from .types import (List, Dict) +from .mixins import ModelMixin +from . import relationship + + +class ServiceUpdateBase(ModelMixin): + """ + Deployment update model representation. + """ + + steps = None + + __tablename__ = 'service_update' + + __private_fields__ = ['service_fk', + 'execution_fk', + 'execution_name', + 'service_name'] + + created_at = Column(DateTime, nullable=False, index=True) + service_plan = Column(Dict, nullable=False) + service_update_nodes = Column(Dict) + service_update_service = Column(Dict) + service_update_node_templates = Column(List) + modified_entity_ids = Column(Dict) + state = Column(Text) + + @declared_attr + def execution(cls): + return relationship.many_to_one(cls, 'execution') + + @declared_attr + def service(cls): + return relationship.many_to_one(cls, 'service', parent_property='updates') + + # region foreign keys + + @declared_attr + def execution_fk(cls): + return relationship.foreign_key('execution', nullable=True) + + @declared_attr + def service_fk(cls): + return relationship.foreign_key('service') + + # endregion + + # region association proxies + + @declared_attr + def execution_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('execution', cls.name_column_name()) + + @declared_attr + def service_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service', cls.name_column_name()) + + # endregion + + def to_dict(self, suppress_error=False, **kwargs): + dep_update_dict = super(ServiceUpdateBase, self).to_dict(suppress_error) #pylint: disable=no-member + # Taking care of the fact the DeploymentSteps are _BaseModels + dep_update_dict['steps'] = [step.to_dict() for step in self.steps] + return dep_update_dict + + +class ServiceUpdateStepBase(ModelMixin): + """ + Deployment update step model representation. + """ + + __tablename__ = 'service_update_step' + + __private_fields__ = ['service_update_fk', + 'service_update_name'] + + _action_types = namedtuple('ACTION_TYPES', 'ADD, REMOVE, MODIFY') + ACTION_TYPES = _action_types(ADD='add', REMOVE='remove', MODIFY='modify') + + _entity_types = namedtuple( + 'ENTITY_TYPES', + 'NODE, RELATIONSHIP, PROPERTY, OPERATION, WORKFLOW, OUTPUT, DESCRIPTION, GROUP, PLUGIN') + ENTITY_TYPES = _entity_types( + NODE='node', + RELATIONSHIP='relationship', + PROPERTY='property', + OPERATION='operation', + WORKFLOW='workflow', + OUTPUT='output', + DESCRIPTION='description', + GROUP='group', + PLUGIN='plugin' + ) + + action = Column(Enum(*ACTION_TYPES, name='action_type'), nullable=False) + entity_id = Column(Text, nullable=False) + entity_type = Column(Enum(*ENTITY_TYPES, name='entity_type'), nullable=False) + + @declared_attr + def service_update(cls): + return relationship.many_to_one(cls, 'service_update', parent_property='steps') + + # region foreign keys + + @declared_attr + def service_update_fk(cls): + return relationship.foreign_key('service_update') + + # endregion + + # region association proxies + + @declared_attr + def service_update_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service_update', cls.name_column_name()) + + # endregion + + def __hash__(self): + return hash((getattr(self, self.id_column_name()), self.entity_id)) + + def __lt__(self, other): + """ + the order is 'remove' < 'modify' < 'add' + :param other: + :return: + """ + if not isinstance(other, self.__class__): + return not self >= other + + if self.action != other.action: + if self.action == 'remove': + return_value = True + elif self.action == 'add': + return_value = False + else: + return_value = other.action == 'add' + return return_value + + if self.action == 'add': + return self.entity_type == 'node' and other.entity_type == 'relationship' + if self.action == 'remove': + return self.entity_type == 'relationship' and other.entity_type == 'node' + return False + + +class ServiceModificationBase(ModelMixin): + """ + Deployment modification model representation. + """ + + __tablename__ = 'service_modification' + + __private_fields__ = ['service_fk', + 'service_name'] + + STARTED = 'started' + FINISHED = 'finished' + ROLLEDBACK = 'rolledback' + + STATES = [STARTED, FINISHED, ROLLEDBACK] + END_STATES = [FINISHED, ROLLEDBACK] + + context = Column(Dict) + created_at = Column(DateTime, nullable=False, index=True) + ended_at = Column(DateTime, index=True) + modified_node_templates = Column(Dict) + nodes = Column(Dict) + status = Column(Enum(*STATES, name='service_modification_status')) + + @declared_attr + def service(cls): + return relationship.many_to_one(cls, 'service', parent_property='modifications') + + # region foreign keys + + @declared_attr + def service_fk(cls): + return relationship.foreign_key('service') + + # endregion + + # region association proxies + + @declared_attr + def service_name(cls): + """Required for use by SQLAlchemy queries""" + return association_proxy('service', cls.name_column_name()) + + # endregion http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9841ca4a/aria/modeling/service_common.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py new file mode 100644 index 0000000..dfe4674 --- /dev/null +++ b/aria/modeling/service_common.py @@ -0,0 +1,277 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=no-self-argument, no-member, abstract-method + +from sqlalchemy import ( + Column, + Text, + PickleType +) +from sqlalchemy.ext.declarative import declared_attr + +from ..parser.consumption import ConsumptionContext +from ..utils import collections, formatting, console +from .mixins import InstanceModelMixin, TemplateModelMixin +from .types import List +from . import ( + relationship, + utils +) + + +class ParameterBase(TemplateModelMixin): + """ + Represents a typed value. + + This model is used by both service template and service instance elements. + + :ivar name: Name + :ivar type_name: Type name + :ivar value: Value + :ivar description: Description + """ + + __tablename__ = 'parameter' + + name = Column(Text) + type_name = Column(Text) + value = Column(PickleType) + description = Column(Text) + + @property + def as_raw(self): + return collections.OrderedDict(( + ('name', self.name), + ('type_name', self.type_name), + ('value', self.value), + ('description', self.description))) + + def instantiate(self, container): + from . import models + return models.Parameter(name=self.name, + type_name=self.type_name, + value=self.value, + description=self.description) + + def coerce_values(self, container, report_issues): + if self.value is not None: + self.value = utils.coerce_value(container, self.value, + report_issues) + + def dump(self): + context = ConsumptionContext.get_thread_local() + if self.type_name is not None: + console.puts('{0}: {1} ({2})'.format( + context.style.property(self.name), + context.style.literal(self.value), + context.style.type(self.type_name))) + else: + console.puts('{0}: {1}'.format( + context.style.property(self.name), + context.style.literal(self.value))) + if self.description: + console.puts(context.style.meta(self.description)) + + @classmethod + def wrap(cls, name, value, description=None): + """ + Wraps an arbitrary value as a parameter. The type will be guessed via introspection. + + :param name: Parameter name + :type name: basestring + :param value: Parameter value + :param description: Description (optional) + :type description: basestring + """ + + from . import models + return models.Parameter(name=name, + type_name=formatting.full_type_name(value), + value=value, + description=description) + + +class TypeBase(InstanceModelMixin): + """ + Represents a type and its children. + """ + + __tablename__ = 'type' + + __private_fields__ = ['parent_type_fk'] + + variant = Column(Text, nullable=False) + description = Column(Text) + _role = Column(Text, name='role') + + @declared_attr + def parent(cls): + return relationship.one_to_one_self(cls, 'parent_type_fk') + + @declared_attr + def children(cls): + return relationship.one_to_many_self(cls, 'parent_type_fk') + + # region foreign keys + + @declared_attr + def parent_type_fk(cls): + """For Type one-to-many to Type""" + return relationship.foreign_key('type', nullable=True) + + # endregion + + @property + def role(self): + def get_role(the_type): + if the_type is None: + return None + elif the_type._role is None: + return get_role(the_type.parent) + return the_type._role + + return get_role(self) + + @role.setter + def role(self, value): + self._role = value + + def is_descendant(self, base_name, name): + base = self.get_descendant(base_name) + if base is not None: + if base.get_descendant(name) is not None: + return True + return False + + def get_descendant(self, name): + if self.name == name: + return self + for child in self.children: + found = child.get_descendant(name) + if found is not None: + return found + return None + + def iter_descendants(self): + for child in self.children: + yield child + for descendant in child.iter_descendants(): + yield descendant + + @property + def as_raw(self): + return collections.OrderedDict(( + ('name', self.name), + ('description', self.description), + ('role', self.role))) + + @property + def as_raw_all(self): + types = [] + self._append_raw_children(types) + return types + + def coerce_values(self, container, report_issues): + pass + + def dump(self): + context = ConsumptionContext.get_thread_local() + if self.name: + console.puts(context.style.type(self.name)) + with context.style.indent: + for child in self.children: + child.dump() + + def _append_raw_children(self, types): + for child in self.children: + raw_child = formatting.as_raw(child) + raw_child['parent'] = self.name + types.append(raw_child) + child._append_raw_children(types) + + +class MetadataBase(TemplateModelMixin): + """ + Custom values associated with the service. + + This model is used by both service template and service instance elements. + + :ivar name: Name + :ivar value: Value + """ + + __tablename__ = 'metadata' + + value = Column(Text) + + @property + def as_raw(self): + return collections.OrderedDict(( + ('name', self.name), + ('value', self.value))) + + def coerce_values(self, container, report_issues): + pass + + def instantiate(self, container): + from . import models + return models.Metadata(name=self.name, + value=self.value) + + def dump(self): + context = ConsumptionContext.get_thread_local() + console.puts('{0}: {1}'.format( + context.style.property(self.name), + context.style.literal(self.value))) + + +class PluginSpecificationBase(InstanceModelMixin): + """ + Plugin specification model representation. + """ + + __tablename__ = 'plugin_specification' + + __private_fields__ = ['service_template_fk'] + + archive_name = Column(Text, nullable=False, index=True) + distribution = Column(Text) + distribution_release = Column(Text) + distribution_version = Column(Text) + package_name = Column(Text, nullable=False, index=True) + package_source = Column(Text) + package_version = Column(Text) + supported_platform = Column(Text) + supported_py_versions = Column(List) + + # region foreign keys + + @declared_attr + def service_template_fk(cls): + """For ServiceTemplate one-to-many to PluginSpecification""" + return relationship.foreign_key('service_template', nullable=True) + + # endregion + + def coerce_values(self, container, report_issues): + pass + + def find_plugin(self, plugins): + # TODO: this should check versions/distribution and other specification + for plugin in plugins: + if plugin.name == self.name: + return plugin + return None
