incubator-ariatosca git commit: ARIA-26 TBD [Forced Update!]
Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-26-plugin-mechanism 11072ee33 -> 62a993299 (forced update) ARIA-26 TBD Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/62a99329 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/62a99329 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/62a99329 Branch: refs/heads/ARIA-26-plugin-mechanism Commit: 62a99329958a9cd2dbba26de5319afb0522e180f Parents: 04c9bd0 Author: Dan KilmanAuthored: Sun Nov 27 16:31:29 2016 +0200 Committer: Dan Kilman Committed: Tue Dec 13 17:33:11 2016 +0200 -- aria/cli/commands.py| 4 +- aria/orchestrator/workflows/api/task.py | 42 +++-- aria/orchestrator/workflows/core/task.py| 10 +- .../orchestrator/workflows/executor/__init__.py | 2 +- .../orchestrator/workflows/executor/blocking.py | 36 .../workflows/executor/multiprocess.py | 88 + aria/orchestrator/workflows/executor/thread.py | 3 +- aria/storage/models.py | 15 +- aria/utils/plugin.py| 75 requirements.txt| 3 +- tests/mock/models.py| 16 ++ tests/orchestrator/workflows/api/test_task.py | 51 - tests/orchestrator/workflows/core/test_task.py | 16 +- .../workflows/executor/test_executor.py | 104 +- tests/storage/test_models.py| 188 +-- tests/utils/__init__.py | 14 ++ tests/utils/test_plugin.py | 75 17 files changed, 492 insertions(+), 250 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/62a99329/aria/cli/commands.py -- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 141da07..825be68 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -34,7 +34,7 @@ from ..logger import LoggerMixin from ..storage import (FileSystemModelDriver, FileSystemResourceDriver) from ..orchestrator.context.workflow import WorkflowContext from ..orchestrator.workflows.core.engine import Engine -from ..orchestrator.workflows.executor.thread import ThreadExecutor +from ..orchestrator.workflows.executor.multiprocess import MultiprocessExecutor from ..parser import iter_specifications from ..parser.consumption import ( ConsumptionContext, @@ -252,7 +252,7 @@ class ExecuteCommand(BaseCommand): ) workflow_function = self._load_workflow_handler(workflow['operation']) tasks_graph = workflow_function(workflow_context, **workflow_context.parameters) -executor = ThreadExecutor() +executor = MultiprocessExecutor() workflow_engine = Engine(executor=executor, workflow_context=workflow_context, tasks_graph=tasks_graph) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/62a99329/aria/orchestrator/workflows/api/task.py -- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index 1c12407..70f9773 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -67,11 +67,11 @@ class OperationTask(BaseTask): max_attempts=None, retry_interval=None, ignore_failure=None, - inputs=None): + inputs=None, + plugin=None): """ Creates an operation task using the name, details, node instance and any additional kwargs. :param name: the operation of the name. -:param operation_details: the details for the operation. :param actor: the operation host on which this operation is registered. :param inputs: operation inputs. """ @@ -82,6 +82,7 @@ class OperationTask(BaseTask): self.name = '{name}.{actor.id}'.format(name=name, actor=actor) self.operation_mapping = operation_mapping self.inputs = inputs or {} +self.plugin = plugin or {} self.max_attempts = (self.workflow_context._task_max_attempts if max_attempts is None else max_attempts) self.retry_interval = (self.workflow_context._task_retry_interval @@ -98,15 +99,13 @@ class OperationTask(BaseTask): :param name: the name of the operation. """ assert isinstance(instance, models.NodeInstance) -operation_details = instance.node.operations[name]
incubator-ariatosca git commit: ARIA-26 TBD [Forced Update!]
Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-26-plugin-mechanism 5c3bee47d -> 11072ee33 (forced update) ARIA-26 TBD Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/11072ee3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/11072ee3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/11072ee3 Branch: refs/heads/ARIA-26-plugin-mechanism Commit: 11072ee33f06bcd40151de2b8dcd65affa167c44 Parents: 04c9bd0 Author: Dan KilmanAuthored: Sun Nov 27 16:31:29 2016 +0200 Committer: Dan Kilman Committed: Tue Dec 13 17:17:08 2016 +0200 -- aria/cli/commands.py| 4 +- aria/orchestrator/workflows/api/task.py | 42 +++-- aria/orchestrator/workflows/core/task.py| 10 +- .../orchestrator/workflows/executor/__init__.py | 2 +- .../orchestrator/workflows/executor/blocking.py | 36 .../workflows/executor/multiprocess.py | 88 + aria/orchestrator/workflows/executor/thread.py | 3 +- aria/storage/models.py | 15 +- aria/utils/plugin.py| 74 requirements.txt| 3 +- tests/mock/models.py| 16 ++ tests/orchestrator/workflows/api/test_task.py | 51 - tests/orchestrator/workflows/core/test_task.py | 16 +- .../workflows/executor/test_executor.py | 104 +- tests/storage/test_models.py| 188 +-- tests/utils/__init__.py | 14 ++ tests/utils/test_plugin.py | 75 17 files changed, 491 insertions(+), 250 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/11072ee3/aria/cli/commands.py -- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 141da07..825be68 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -34,7 +34,7 @@ from ..logger import LoggerMixin from ..storage import (FileSystemModelDriver, FileSystemResourceDriver) from ..orchestrator.context.workflow import WorkflowContext from ..orchestrator.workflows.core.engine import Engine -from ..orchestrator.workflows.executor.thread import ThreadExecutor +from ..orchestrator.workflows.executor.multiprocess import MultiprocessExecutor from ..parser import iter_specifications from ..parser.consumption import ( ConsumptionContext, @@ -252,7 +252,7 @@ class ExecuteCommand(BaseCommand): ) workflow_function = self._load_workflow_handler(workflow['operation']) tasks_graph = workflow_function(workflow_context, **workflow_context.parameters) -executor = ThreadExecutor() +executor = MultiprocessExecutor() workflow_engine = Engine(executor=executor, workflow_context=workflow_context, tasks_graph=tasks_graph) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/11072ee3/aria/orchestrator/workflows/api/task.py -- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index 1c12407..70f9773 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -67,11 +67,11 @@ class OperationTask(BaseTask): max_attempts=None, retry_interval=None, ignore_failure=None, - inputs=None): + inputs=None, + plugin=None): """ Creates an operation task using the name, details, node instance and any additional kwargs. :param name: the operation of the name. -:param operation_details: the details for the operation. :param actor: the operation host on which this operation is registered. :param inputs: operation inputs. """ @@ -82,6 +82,7 @@ class OperationTask(BaseTask): self.name = '{name}.{actor.id}'.format(name=name, actor=actor) self.operation_mapping = operation_mapping self.inputs = inputs or {} +self.plugin = plugin or {} self.max_attempts = (self.workflow_context._task_max_attempts if max_attempts is None else max_attempts) self.retry_interval = (self.workflow_context._task_retry_interval @@ -98,15 +99,13 @@ class OperationTask(BaseTask): :param name: the name of the operation. """ assert isinstance(instance, models.NodeInstance) -operation_details = instance.node.operations[name]
incubator-ariatosca git commit: Generifying ARIA models
Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-39-Genericize-storage-models [created] a5eef4be2 Generifying ARIA models Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/a5eef4be Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/a5eef4be Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/a5eef4be Branch: refs/heads/ARIA-39-Genericize-storage-models Commit: a5eef4be228daead5f7daa55929d3562cab88d9b Parents: c6c92ae Author: mxmrlvAuthored: Mon Dec 12 00:50:09 2016 +0200 Committer: mxmrlv Committed: Tue Dec 13 16:56:36 2016 +0200 -- aria/storage/models.py | 573 +++- aria/storage/models_base.py | 629 +++ aria/storage/structures.py | 120 +++--- tests/storage/__init__.py | 5 +- tests/storage/test_model_storage.py | 17 - tests/storage/test_models.py| 20 +- 6 files changed, 747 insertions(+), 617 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/aria/storage/models.py -- diff --git a/aria/storage/models.py b/aria/storage/models.py index 6302e66..695f16c 100644 --- a/aria/storage/models.py +++ b/aria/storage/models.py @@ -36,27 +36,9 @@ classes: * ProviderContext - provider context implementation model. * Plugin - plugin implementation model. """ -from collections import namedtuple -from datetime import datetime - -from sqlalchemy.ext.declarative.base import declared_attr - -from .structures import ( -SQLModelBase, -Column, -Integer, -Text, -DateTime, -Boolean, -Enum, -String, -Float, -List, -Dict, -foreign_key, -one_to_many_relationship, -relationship_to_self, -orm) + +from .structures import ModelBase, declarative_base +from .models_base import * __all__ = ( 'Blueprint', @@ -74,499 +56,56 @@ __all__ = ( ) -#pylint: disable=no-self-argument - - -class Blueprint(SQLModelBase): -""" -Blueprint model representation. -""" -__tablename__ = 'blueprints' - -name = Column(Text, index=True) -created_at = Column(DateTime, nullable=False, index=True) -main_file_name = Column(Text, nullable=False) -plan = Column(Dict, nullable=False) -updated_at = Column(DateTime) -description = Column(Text) - - -class Deployment(SQLModelBase): -""" -Deployment model representation. -""" -__tablename__ = 'deployments' - -_private_fields = ['blueprint_id'] - -blueprint_id = foreign_key(Blueprint.id) - -name = Column(Text, index=True) -created_at = Column(DateTime, nullable=False, index=True) -description = Column(Text) -inputs = Column(Dict) -groups = Column(Dict) -permalink = Column(Text) -policy_triggers = Column(Dict) -policy_types = Column(Dict) -outputs = Column(Dict) -scaling_groups = Column(Dict) -updated_at = Column(DateTime) -workflows = Column(Dict) - -@declared_attr -def blueprint(cls): -return one_to_many_relationship(cls, Blueprint, cls.blueprint_id) - - -class Execution(SQLModelBase): -""" -Execution model representation. -""" -__tablename__ = 'executions' - -TERMINATED = 'terminated' -FAILED = 'failed' -CANCELLED = 'cancelled' -PENDING = 'pending' -STARTED = 'started' -CANCELLING = 'cancelling' -FORCE_CANCELLING = 'force_cancelling' - -STATES = [TERMINATED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING, FORCE_CANCELLING] -END_STATES = [TERMINATED, FAILED, CANCELLED] -ACTIVE_STATES = [state for state in STATES if state not in END_STATES] - -VALID_TRANSITIONS = { -PENDING: [STARTED, CANCELLED], -STARTED: END_STATES + [CANCELLING], -CANCELLING: END_STATES -} - -@orm.validates('status') -def validate_status(self, key, value): -"""Validation function that verifies execution status transitions are OK""" -try: -current_status = getattr(self, key) -except AttributeError: -return -valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, []) -if all([current_status is not None, -current_status != value, -value not in valid_transitions]): -raise ValueError('Cannot change execution status from {current} to {new}'.format( -current=current_status, -new=value)) -return value - -deployment_id = foreign_key(Deployment.id) -blueprint_id = foreign_key(Blueprint.id) -_private_fields = ['deployment_id', 'blueprint_id'] - -
[jira] [Created] (ARIATOSCA-39) Genericize storage models
Maxim Orlov created ARIATOSCA-39: Summary: Genericize storage models Key: ARIATOSCA-39 URL: https://issues.apache.org/jira/browse/ARIATOSCA-39 Project: AriaTosca Issue Type: Story Reporter: Maxim Orlov Assignee: Maxim Orlov Currently it is hard to work and extend the ARIA models when consuming ARIA. Those models should provide a simple API which would enable inherit columns/relationships/etc... while using a different sql based engine/session/metadata (e.g. sqlalchemy-flask based engine/session/metadata) -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[2/2] incubator-ariatosca git commit: ARIA-26 TBD
ARIA-26 TBD Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/5c3bee47 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/5c3bee47 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/5c3bee47 Branch: refs/heads/ARIA-26-plugin-mechanism Commit: 5c3bee47d6acbb46235c07b0ff5b5be2460ccb1e Parents: 04c9bd0 Author: Dan KilmanAuthored: Sun Nov 27 16:31:29 2016 +0200 Committer: Dan Kilman Committed: Tue Dec 13 12:32:04 2016 +0200 -- aria/cli/commands.py| 4 +- aria/orchestrator/workflows/api/task.py | 42 +--- aria/orchestrator/workflows/core/task.py| 10 +- .../orchestrator/workflows/executor/__init__.py | 2 +- .../orchestrator/workflows/executor/blocking.py | 36 --- .../workflows/executor/multiprocess.py | 86 ++-- aria/orchestrator/workflows/executor/thread.py | 3 +- aria/storage/models.py | 5 + aria/utils/plugin.py| 72 + requirements.txt| 3 +- .../workflows/executor/test_executor.py | 103 +-- 11 files changed, 225 insertions(+), 141 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5c3bee47/aria/cli/commands.py -- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 141da07..825be68 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -34,7 +34,7 @@ from ..logger import LoggerMixin from ..storage import (FileSystemModelDriver, FileSystemResourceDriver) from ..orchestrator.context.workflow import WorkflowContext from ..orchestrator.workflows.core.engine import Engine -from ..orchestrator.workflows.executor.thread import ThreadExecutor +from ..orchestrator.workflows.executor.multiprocess import MultiprocessExecutor from ..parser import iter_specifications from ..parser.consumption import ( ConsumptionContext, @@ -252,7 +252,7 @@ class ExecuteCommand(BaseCommand): ) workflow_function = self._load_workflow_handler(workflow['operation']) tasks_graph = workflow_function(workflow_context, **workflow_context.parameters) -executor = ThreadExecutor() +executor = MultiprocessExecutor() workflow_engine = Engine(executor=executor, workflow_context=workflow_context, tasks_graph=tasks_graph) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5c3bee47/aria/orchestrator/workflows/api/task.py -- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index 1c12407..70f9773 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -67,11 +67,11 @@ class OperationTask(BaseTask): max_attempts=None, retry_interval=None, ignore_failure=None, - inputs=None): + inputs=None, + plugin=None): """ Creates an operation task using the name, details, node instance and any additional kwargs. :param name: the operation of the name. -:param operation_details: the details for the operation. :param actor: the operation host on which this operation is registered. :param inputs: operation inputs. """ @@ -82,6 +82,7 @@ class OperationTask(BaseTask): self.name = '{name}.{actor.id}'.format(name=name, actor=actor) self.operation_mapping = operation_mapping self.inputs = inputs or {} +self.plugin = plugin or {} self.max_attempts = (self.workflow_context._task_max_attempts if max_attempts is None else max_attempts) self.retry_interval = (self.workflow_context._task_retry_interval @@ -98,15 +99,13 @@ class OperationTask(BaseTask): :param name: the name of the operation. """ assert isinstance(instance, models.NodeInstance) -operation_details = instance.node.operations[name] -operation_inputs = operation_details.get('inputs', {}) -operation_inputs.update(inputs or {}) -return cls(name=name, - actor=instance, - operation_mapping=operation_details.get('operation', ''), - inputs=operation_inputs, - *args, - **kwargs) +return cls._instance(instance=instance, + name=name, +
[1/2] incubator-ariatosca git commit: ARIA-31 Add registry mechanism for extensions [Forced Update!]
Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-26-plugin-mechanism 4e30e09b7 -> 5c3bee47d (forced update) ARIA-31 Add registry mechanism for extensions Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/04c9bd07 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/04c9bd07 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/04c9bd07 Branch: refs/heads/ARIA-26-plugin-mechanism Commit: 04c9bd07916f957ddd88b933067266177a242a42 Parents: c6c92ae Author: Dan KilmanAuthored: Mon Dec 5 15:28:29 2016 +0200 Committer: Dan Kilman Committed: Sun Dec 11 19:26:28 2016 +0200 -- aria/__init__.py| 25 +-- aria/cli/commands.py| 9 +- aria/extension.py | 125 +++ aria/orchestrator/events.py | 36 + aria/orchestrator/events/__init__.py| 57 --- .../events/builtin_event_handler.py | 123 --- .../events/workflow_engine_event_handler.py | 74 - aria/orchestrator/workflows/__init__.py | 3 + aria/orchestrator/workflows/core/engine.py | 2 + .../workflows/core/events_handler.py| 113 ++ aria/orchestrator/workflows/events_logging.py | 65 aria/parser/__init__.py | 5 +- aria/parser/loading/__init__.py | 3 +- aria/parser/loading/uri.py | 5 +- aria/parser/presentation/__init__.py| 3 +- aria/parser/presentation/source.py | 7 +- aria/parser/specification.py| 6 +- aria/storage/structures.py | 4 +- aria/utils/plugin.py| 39 - aria/utils/threading.py | 7 +- extensions/aria_extension_tosca/__init__.py | 52 --- .../simple_v1_0/data_types.py | 5 +- requirements.txt| 1 + tests/orchestrator/conftest.py | 23 +++ tests/orchestrator/events/__init__.py | 14 -- .../events/test_builtin_event_handlers.py | 14 -- .../test_workflow_enginge_event_handlers.py | 0 .../workflows/executor/test_executor.py | 16 +- tests/test_extension.py | 156 +++ 29 files changed, 602 insertions(+), 390 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/04c9bd07/aria/__init__.py -- diff --git a/aria/__init__.py b/aria/__init__.py index b000397..0f7bec6 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -17,13 +17,18 @@ Aria top level package """ -import sys import pkgutil +try: +import pkg_resources +except ImportError: +pkg_resources = None + from .VERSION import version as __version__ from .orchestrator.decorators import workflow, operation from . import ( +extension, utils, parser, storage, @@ -41,19 +46,17 @@ _resource_storage = {} def install_aria_extensions(): """ -Iterates all Python packages with names beginning with :code:`aria_extension_` and calls -their :code:`install_aria_extension` function if they have it. +Iterates all Python packages with names beginning with :code:`aria_extension_` and all +:code:`aria_extension` entry points and loads them. +It then invokes all registered extension functions. """ - for loader, module_name, _ in pkgutil.iter_modules(): if module_name.startswith('aria_extension_'): -module = loader.find_module(module_name).load_module(module_name) - -if hasattr(module, 'install_aria_extension'): -module.install_aria_extension() - -# Loading the module has contaminated sys.modules, so we'll clean it up -del sys.modules[module_name] +loader.find_module(module_name).load_module(module_name) +if pkg_resources: +for entry_point in pkg_resources.iter_entry_points(group='aria_extension'): +entry_point.load() +extension.init() def application_model_storage(api, api_kwargs=None): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/04c9bd07/aria/cli/commands.py -- diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 3426bb0..141da07 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -28,13 +28,14 @@ from importlib import import_module from yaml import safe_load, YAMLError +from .. import extension from .. import