Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-3-api-for-creating-workflows a62dd5fc0 -> 3844e73d6
code review fix2 Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/3844e73d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/3844e73d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/3844e73d Branch: refs/heads/ARIA-3-api-for-creating-workflows Commit: 3844e73d63eeb8769f2baf5a273938474b2d82c1 Parents: a62dd5f Author: mxmrlv <[email protected]> Authored: Mon Nov 7 17:17:07 2016 +0200 Committer: mxmrlv <[email protected]> Committed: Mon Nov 7 17:17:07 2016 +0200 ---------------------------------------------------------------------- .travis.yml | 1 + aria/events/builtin_event_handler.py | 17 +++++--- aria/workflows/api/task_graph.py | 12 +++--- aria/workflows/core/engine.py | 6 +-- aria/workflows/core/task.py | 41 ++++++++++++------ aria/workflows/exceptions.py | 7 ++++ requirements.txt | 1 + setup.py | 4 -- tests/workflows/api/test_task_graph.py | 65 ++++++++++++++++++----------- tests/workflows/core/test_executor.py | 5 +++ 10 files changed, 101 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 8a54504..5413ff2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ env: - TOX_ENV=py26 install: - pip install --upgrade pip + - pip install --upgrade setuptools - pip install tox script: - pip --version http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/aria/events/builtin_event_handler.py ---------------------------------------------------------------------- diff --git a/aria/events/builtin_event_handler.py b/aria/events/builtin_event_handler.py index 88fc927..2abdd9f 100644 --- a/aria/events/builtin_event_handler.py +++ b/aria/events/builtin_event_handler.py @@ -36,24 +36,29 @@ from . import ( @sent_task_signal.connect def _task_sent(task, *args, **kwargs): - task.status = task.SENT + with task.update(): + task.status = task.SENT @start_task_signal.connect def _task_started(task, *args, **kwargs): - task.started_at = datetime.utcnow() + with task.update(): + task.started_at = datetime.utcnow() + task.status = task.STARTED @on_failure_task_signal.connect def _task_failed(task, *args, **kwargs): - task.ended_at = datetime.utcnow() - task.status = task.FAILED + with task.update(): + task.ended_at = datetime.utcnow() + task.status = task.FAILED @on_success_task_signal.connect def _task_succeeded(task, *args, **kwargs): - task.ended_at = datetime.utcnow() - task.status = task.SUCCESS + with task.update(): + task.ended_at = datetime.utcnow() + task.status = task.SUCCESS @start_workflow_signal.connect http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/aria/workflows/api/task_graph.py ---------------------------------------------------------------------- diff --git a/aria/workflows/api/task_graph.py b/aria/workflows/api/task_graph.py index 67b3dc4..c88d343 100644 --- a/aria/workflows/api/task_graph.py +++ b/aria/workflows/api/task_graph.py @@ -198,7 +198,7 @@ class TaskGraph(object): raise TaskNotInGraphError() if self.has_dependency(dependent, dependency): - return False + return if isinstance(dependent, Iterable): for dependent_task in dependent: @@ -210,8 +210,6 @@ class TaskGraph(object): else: self._graph.add_edge(dependent.id, dependency.id) - return True - def has_dependency(self, dependent, dependency): """ Check whether one item (task, sequence or parallel) depends on another @@ -227,7 +225,9 @@ class TaskGraph(object): :raise TaskNotInGraphError if either the dependent or dependency are tasks which are not in the graph """ - if not (self.has_tasks(dependent) and self.has_tasks(dependency)): + if not (dependent and dependency): + return False + elif not (self.has_tasks(dependent) and self.has_tasks(dependency)): raise TaskNotInGraphError() return_value = True @@ -263,7 +263,7 @@ class TaskGraph(object): raise TaskNotInGraphError() if not self.has_dependency(dependent, dependency): - return False + return if isinstance(dependent, Iterable): for dependent_task in dependent: @@ -274,8 +274,6 @@ class TaskGraph(object): else: self._graph.remove_edge(dependent.id, dependency.id) - return True - @_filter_out_empty_tasks def sequence(self, *tasks): """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/aria/workflows/core/engine.py ---------------------------------------------------------------------- diff --git a/aria/workflows/core/engine.py b/aria/workflows/core/engine.py index 60400e6..a288757 100644 --- a/aria/workflows/core/engine.py +++ b/aria/workflows/core/engine.py @@ -22,11 +22,11 @@ from datetime import datetime import networkx +from ... import events, logger +from ...storage import models +from .. import exceptions from . import translation from . import task as engine_task -from .. import exceptions -from ...storage import models -from ... import events, logger class Engine(logger.LoggerMixin): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/aria/workflows/core/task.py ---------------------------------------------------------------------- diff --git a/aria/workflows/core/task.py b/aria/workflows/core/task.py index 9123714..c112544 100644 --- a/aria/workflows/core/task.py +++ b/aria/workflows/core/task.py @@ -16,11 +16,12 @@ """ Workflow tasks """ - +from contextlib import contextmanager from datetime import datetime from ... import logger from ...storage import models +from .. import exceptions class BaseTask(logger.LoggerMixin): @@ -99,6 +100,20 @@ class OperationTask(BaseTask, logger.LoggerMixin): ) self.workflow_context.model.task.store(task) self._task_id = task.id + self._update_fields = None + + @contextmanager + def update(self): + self._update_fields = {} + + yield + + task = self.context + for key, value in self._update_fields.items(): + setattr(task, key, value) + self.context = task + + self._update_fields = None @property def workflow_context(self): @@ -129,9 +144,9 @@ class OperationTask(BaseTask, logger.LoggerMixin): @status.setter def status(self, value): - task = self.context - task.status = value - self.context = task + if self._update_fields is None: + raise exceptions.TaskException("Task is not in update mode") + self._update_fields['status'] = value @property def started_at(self): @@ -143,9 +158,9 @@ class OperationTask(BaseTask, logger.LoggerMixin): @started_at.setter def started_at(self, value): - task = self.context - task.started_at = value - self.context = task + if self._update_fields is None: + raise exceptions.TaskException("Task is not in update mode") + self._update_fields['started_at'] = value @property def ended_at(self): @@ -157,9 +172,9 @@ class OperationTask(BaseTask, logger.LoggerMixin): @ended_at.setter def ended_at(self, value): - task = self.context - task.ended_at = value - self.context = task + if self._update_fields is None: + raise exceptions.TaskException("Task is not in update mode") + self._update_fields['ended_at'] = value @property def retry_count(self): @@ -171,9 +186,9 @@ class OperationTask(BaseTask, logger.LoggerMixin): @retry_count.setter def retry_count(self, value): - task = self.context - task.retry_count = value - self.context = task + if self._update_fields is None: + raise exceptions.TaskException("Task is not in update mode") + self._update_fields['retry_count'] = value def __getattr__(self, attr): try: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/aria/workflows/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/workflows/exceptions.py b/aria/workflows/exceptions.py index b8ebc14..1bc9d9a 100644 --- a/aria/workflows/exceptions.py +++ b/aria/workflows/exceptions.py @@ -61,3 +61,10 @@ class AriaEngineError(Exception): """ Raised by the workflow engine """ + + +class TaskException(Exception): + """ + Raised by the task + """ + pass \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/requirements.txt ---------------------------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index 45eb5a5..b550a58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,6 @@ networkx==1.9 requests==2.7.0 retrying==1.3.3 blinker==1.4 +importlib==1.0.4 ; python_version < '2.7' ordereddict==1.1 ; python_version < '2.7' jsonpickle http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 811810b..6d7a30c 100644 --- a/setup.py +++ b/setup.py @@ -36,10 +36,6 @@ try: except IOError: install_requires = [] -try: - import importlib -except ImportError: - install_requires.append('importlib') setup( name=_PACKAGE_NAME, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/tests/workflows/api/test_task_graph.py ---------------------------------------------------------------------- diff --git a/tests/workflows/api/test_task_graph.py b/tests/workflows/api/test_task_graph.py index 3ff60a4..e9c45f5 100644 --- a/tests/workflows/api/test_task_graph.py +++ b/tests/workflows/api/test_task_graph.py @@ -262,7 +262,8 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(dependency_task) graph.add_tasks(unrelated_task) - add_result = graph.add_dependency(task, dependency_task) + graph.add_dependency(task, dependency_task) + add_result = graph.has_dependency(task, dependency_task) assert add_result is True dependency_tasks = list(graph.get_dependencies(task)) assert len(dependency_tasks) == 1 @@ -274,9 +275,12 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(dependency_task) graph.add_dependency(task, dependency_task) + add_result = graph.has_dependency(task, dependency_task) # adding a dependency already in graph - should have no effect, and return False - add_result = graph.add_dependency(task, dependency_task) - assert add_result is False + assert add_result is True + graph.add_dependency(task, dependency_task) + add_result = graph.has_dependency(task, dependency_task) + assert add_result is True dependency_tasks = list(graph.get_dependencies(task)) assert len(dependency_tasks) == 1 assert dependency_tasks[0] == dependency_task @@ -299,13 +303,13 @@ class TestTaskGraphDependencies(object): task = MockTask() graph.add_tasks(task) # expecting add_dependency result to be False - no dependency has been created - assert graph.add_dependency([], task) is False + assert set(graph.tasks) == set((task,)) def test_add_dependency_empty_dependency(self, graph): task = MockTask() graph.add_tasks(task) # expecting add_dependency result to be False - no dependency has been created - assert graph.add_dependency(task, []) is False + assert set(graph.tasks) == set((task,)) def test_add_dependency_dependent_group(self, graph): task = MockTask() @@ -356,11 +360,13 @@ class TestTaskGraphDependencies(object): group_2_tasks = [MockTask() for _ in xrange(3)] graph.add_tasks(*group_1_tasks) graph.add_tasks(*group_2_tasks) - add_result = graph.add_dependency(group_1_tasks, group_2_tasks) + graph.add_dependency(group_1_tasks, group_2_tasks) + add_result = graph.has_dependency(group_1_tasks, group_2_tasks) assert add_result is True # adding a dependency already in graph - should have no effect, and return False - add_result = graph.add_dependency(group_1_tasks, group_2_tasks) - assert add_result is False + graph.add_dependency(group_1_tasks, group_2_tasks) + add_result = graph.has_dependency(group_1_tasks, group_2_tasks) + assert add_result is True for group_2_task in group_2_tasks: assert graph.has_dependency(group_1_tasks[0], group_2_task) is True assert graph.has_dependency(group_1_tasks[1], group_2_task) is True @@ -399,13 +405,13 @@ class TestTaskGraphDependencies(object): task = MockTask() graph.add_tasks(task) # expecting has_dependency result to be False - dependency in an empty form - assert graph.has_dependency([], task) is True + assert graph.has_dependency([], task) is False def test_has_dependency_empty_dependency(self, graph): task = MockTask() graph.add_tasks(task) # expecting has_dependency result to be True - dependency in an empty form - assert graph.has_dependency(task, []) is True + assert graph.has_dependency(task, []) is False def test_has_dependency_dependent_group(self, graph): task = MockTask() @@ -460,8 +466,9 @@ class TestTaskGraphDependencies(object): graph.add_dependency(task, dependency_task) graph.add_dependency(another_dependent_task, dependency_task) - remove_result = graph.remove_dependency(task, dependency_task) - assert remove_result is True + graph.remove_dependency(task, dependency_task) + remove_result = graph.has_dependency(task, dependency_task) + assert remove_result is False assert graph.has_dependency(task, dependency_task) is False assert graph.has_dependency(another_dependent_task, dependency_task) is True @@ -471,7 +478,8 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(dependency_task) # removing a dependency not in graph - should have no effect, and return False - remove_result = graph.remove_dependency(task, dependency_task) + graph.remove_dependency(task, dependency_task) + remove_result = graph.has_dependency(task, dependency_task) assert remove_result is False tasks = [t for t in graph.tasks] assert set(tasks) == set([task, dependency_task]) @@ -495,13 +503,15 @@ class TestTaskGraphDependencies(object): task = MockTask() graph.add_tasks(task) # expecting remove_dependency result to be False - no dependency has been created - assert graph.remove_dependency([], task) is False + graph.remove_dependency([], task) + assert set(graph.tasks) == set((task,)) def test_remove_dependency_empty_dependency(self, graph): task = MockTask() graph.add_tasks(task) # expecting remove_dependency result to be False - no dependency has been created - assert graph.remove_dependency(task, []) is False + graph.remove_dependency(task, []) + assert set(graph.tasks) == set((task,)) def test_remove_dependency_dependent_group(self, graph): task = MockTask() @@ -509,8 +519,9 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(*group_tasks) graph.add_dependency(group_tasks, task) - remove_result = graph.remove_dependency(group_tasks, task) - assert remove_result is True + graph.remove_dependency(group_tasks, task) + remove_result = graph.has_dependency(group_tasks, task) + assert remove_result is False assert graph.has_dependency(group_tasks[0], task) is False assert graph.has_dependency(group_tasks[1], task) is False assert graph.has_dependency(group_tasks[2], task) is False @@ -521,8 +532,9 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(*group_tasks) graph.add_dependency(task, group_tasks) - remove_result = graph.remove_dependency(task, group_tasks) - assert remove_result is True + graph.remove_dependency(task, group_tasks) + remove_result = graph.has_dependency(task, group_tasks) + assert remove_result is False assert graph.has_dependency(task, group_tasks[0]) is False assert graph.has_dependency(task, group_tasks[1]) is False assert graph.has_dependency(task, group_tasks[2]) is False @@ -533,8 +545,9 @@ class TestTaskGraphDependencies(object): graph.add_tasks(*group_1_tasks) graph.add_tasks(*group_2_tasks) graph.add_dependency(group_2_tasks, group_1_tasks) - remove_result = graph.remove_dependency(group_2_tasks, group_1_tasks) - assert remove_result is True + graph.remove_dependency(group_2_tasks, group_1_tasks) + remove_result = graph.has_dependency(group_2_tasks, group_1_tasks) + assert remove_result is False for group_2_task in group_2_tasks: assert graph.has_dependency(group_2_task, group_1_tasks[0]) is False assert graph.has_dependency(group_2_task, group_1_tasks[1]) is False @@ -546,7 +559,8 @@ class TestTaskGraphDependencies(object): graph.add_tasks(task) graph.add_tasks(*group_tasks) graph.add_dependency(task, group_tasks[1]) - remove_result = graph.remove_dependency(task, group_tasks) + graph.remove_dependency(task, group_tasks) + remove_result = graph.has_dependency(task, group_tasks) # only a partial dependency exists - remove_dependency is expected to return False assert remove_result is False # no dependencies are expected to have changed @@ -560,7 +574,8 @@ class TestTaskGraphDependencies(object): graph.add_tasks(*group_1_tasks) graph.add_tasks(*group_2_tasks) # removing a dependency not in graph - should have no effect, and return False - remove_result = graph.remove_dependency(group_2_tasks, group_1_tasks) + graph.remove_dependency(group_2_tasks, group_1_tasks) + remove_result = graph.has_dependency(group_2_tasks, group_1_tasks) assert remove_result is False # nested tests @@ -698,8 +713,8 @@ class TestTaskGraphSequence(object): def test_sequence_with_empty_group(self, graph): tasks = [MockTask(), [], MockTask()] graph.sequence(*tasks) - graph_tasks = [t for t in graph.tasks] - assert graph_tasks == [tasks[0], tasks[2]] + graph_tasks = set([t for t in graph.tasks]) + assert graph_tasks == set([tasks[0], tasks[2]]) assert list(graph.get_dependents(tasks[0])) == [tasks[2]] assert list(graph.get_dependencies(tasks[2])) == [tasks[0]] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3844e73d/tests/workflows/core/test_executor.py ---------------------------------------------------------------------- diff --git a/tests/workflows/core/test_executor.py b/tests/workflows/core/test_executor.py index 9d4a5a4..8ec0303 100644 --- a/tests/workflows/core/test_executor.py +++ b/tests/workflows/core/test_executor.py @@ -15,6 +15,7 @@ import logging import uuid +from contextlib import contextmanager import pytest import retrying @@ -117,6 +118,10 @@ class MockTask(object): for state in models.Task.STATES: setattr(self, state.upper(), state) + @contextmanager + def update(self): + yield self + def start_handler(task, *args, **kwargs): task.states.append('start')
