Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-105-integrate-modeling 0f040a2b9 -> d3b487366
Pickling improvements, intrinsic function fixes Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/d3b48736 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/d3b48736 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/d3b48736 Branch: refs/heads/ARIA-105-integrate-modeling Commit: d3b487366575b25f9e660da7ff6500b5d77e8004 Parents: 0f040a2 Author: Tal Liron <[email protected]> Authored: Wed Mar 15 15:46:17 2017 -0500 Committer: Tal Liron <[email protected]> Committed: Wed Mar 15 15:46:17 2017 -0500 ---------------------------------------------------------------------- aria/exceptions.py | 2 +- aria/modeling/service_common.py | 51 ++++++++------------ aria/orchestrator/workflows/api/task.py | 14 +++--- aria/utils/exceptions.py | 44 ++++++++++++----- .../simple_v1_0/functions.py | 4 +- tests/mock/topology.py | 8 +-- tests/modeling/test_mixins.py | 20 ++++---- tests/modeling/test_models.py | 7 +-- .../workflows/executor/test_executor.py | 4 +- .../test_process_executor_tracked_changes.py | 4 +- tests/storage/test_instrumentation.py | 10 ++-- tests/storage/test_resource_storage.py | 3 +- 12 files changed, 94 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/aria/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/exceptions.py b/aria/exceptions.py index 0a60f01..a180ce1 100644 --- a/aria/exceptions.py +++ b/aria/exceptions.py @@ -43,4 +43,4 @@ class AriaException(Exception): if cause == e: # Make sure it's our traceback cause_traceback = traceback - self.cause_tb = cause_traceback + self.cause_traceback = cause_traceback http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/aria/modeling/service_common.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py index 6214bb2..746f403 100644 --- a/aria/modeling/service_common.py +++ b/aria/modeling/service_common.py @@ -15,12 +15,10 @@ # pylint: disable=no-self-argument, no-member, abstract-method -import cPickle as pickle - from sqlalchemy import ( Column, Text, - Binary, + PickleType ) from sqlalchemy.ext.declarative import declared_attr @@ -29,7 +27,6 @@ from ..utils import collections, formatting, console from .mixins import InstanceModelMixin, TemplateModelMixin from .types import List from . import utils -from . import exceptions class ParameterBase(TemplateModelMixin): @@ -48,9 +45,7 @@ class ParameterBase(TemplateModelMixin): name = Column(Text) type_name = Column(Text) - - # Check: value type - _value = Column(Binary, name='value') + value = Column(PickleType) description = Column(Text) @property @@ -61,33 +56,11 @@ class ParameterBase(TemplateModelMixin): ('value', self.value), ('description', self.description))) - @property - def value(self): - if self._value is None: - return None - try: - return pickle.loads(self._value) - except BaseException as e: - raise exceptions.ValueFormatException('bad format for parameter of type "{0}": {1}' - .format(self.type_name, self._value), cause=e) - - @value.setter - def value(self, value): - if value is None: - self._value = None - else: - try: - self._value = pickle.dumps(value) - except (pickle.PicklingError, TypeError) as e: - #raise exceptions.ValueFormatException('bad format for parameter of type "{0}": {1}' - # .format(self.type_name, self.value), cause=e) - self._value = pickle.dumps(str(value)) - def instantiate(self, container): from . import models return models.Parameter(name=self.name, type_name=self.type_name, - _value=self._value, + value=self.value, description=self.description) def coerce_values(self, container, report_issues): @@ -109,6 +82,24 @@ class ParameterBase(TemplateModelMixin): if self.description: console.puts(context.style.meta(self.description)) + @classmethod + def wrap(cls, name, value, description=None): + """ + Wraps an arbitrary value as a parameter. The type will be guessed via introspection. + + :param name: Parameter name + :type name: basestring + :param value: Parameter value + :param description: Description (optional) + :type description: basestring + """ + + from . import models + return models.Parameter(name=name, + type_name=formatting.full_type_name(value), + value=value, + description=description) + class TypeBase(InstanceModelMixin): """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/aria/orchestrator/workflows/api/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index e4ebbda..2600aaa 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -20,7 +20,6 @@ Provides the tasks to be entered into the task graph from ....modeling import models from ....utils.collections import OrderedDict from ....utils.uuid import generate_uuid -from ....utils.formatting import full_type_name from ... import context from .. import exceptions @@ -94,16 +93,19 @@ class OperationTask(BaseTask): if ignore_failure is None else ignore_failure) self.runs_on = runs_on - # Coerce inputs + # Wrap inputs if inputs: for k, v in inputs.iteritems(): if not isinstance(v, models.Parameter): - inputs[k] = models.Parameter(name=k, - type_name=full_type_name(v), - value=v) + inputs[k] = models.Parameter.wrap(k, v) + + # TODO: These extra inputs should likely be stored as a separate entry in the task model, + # because they are different from the operation inputs. The two kinds of inputs should also + # not be merged. if interface_name or operation_name: - operation = OperationTask._get_operation(actor.interfaces, interface_name, operation_name) + operation = OperationTask._get_operation(actor.interfaces, interface_name, + operation_name) if operation is None: raise exceptions.TaskException( 'Could not find operation "{0}" on interface "{1}" for {2} "{3}"' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/aria/utils/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/utils/exceptions.py b/aria/utils/exceptions.py index a19eb78..9e3e80f 100644 --- a/aria/utils/exceptions.py +++ b/aria/utils/exceptions.py @@ -15,11 +15,14 @@ import sys import linecache +import traceback as tb import jsonpickle -from clint.textui import indent -from .console import (puts, Colored) +from .console import (puts, indent, Colored) + + +ENTRY_FORMAT = 'File "{filename}", line {lineno}, in {name}' def print_exception(e, full=True, cause=False, traceback=None): @@ -27,14 +30,16 @@ def print_exception(e, full=True, cause=False, traceback=None): Prints the exception with nice colors and such. """ def format_heading(e): - return '%s%s: %s' % (Colored.red('Caused by ') if cause else '', Colored.red( - e.__class__.__name__, bold=True), Colored.red(e)) + return '{0}{1}: {2}'.format( + Colored.red('Caused by ') if cause else '', + Colored.red(e.__class__.__name__, bold=True), + Colored.red(e)) puts(format_heading(e)) if full: if cause: if traceback: - print_traceback(traceback) + print_traceback(traceback, True) else: print_traceback() if hasattr(e, 'cause') and e.cause: @@ -42,7 +47,7 @@ def print_exception(e, full=True, cause=False, traceback=None): print_exception(e.cause, full=full, cause=True, traceback=traceback) -def print_traceback(traceback=None): +def print_traceback(traceback=None, print_last_stack=False): """ Prints the traceback with nice colors and such. """ @@ -51,20 +56,37 @@ def print_traceback(traceback=None): _, _, traceback = sys.exc_info() while traceback is not None: frame = traceback.tb_frame - lineno = traceback.tb_lineno code = frame.f_code filename = code.co_filename + lineno = traceback.tb_lineno name = code.co_name with indent(2): - puts('File "%s", line %s, in %s' % (Colored.blue(filename), - Colored.cyan(lineno), - Colored.cyan(name))) + puts(ENTRY_FORMAT.format(filename=Colored.blue(filename), + lineno=Colored.cyan(lineno), + name=Colored.cyan(name))) linecache.checkcache(filename) line = linecache.getline(filename, lineno, frame.f_globals) if line: with indent(2): - puts(Colored.black(line.strip())) + puts(line.strip()) traceback = traceback.tb_next + if print_last_stack and (traceback is None): + # Print stack of *last* traceback + _print_stack(frame) + + +def _print_stack(frame): + entries = tb.extract_stack(frame) + if not entries: + return + puts(Colored.red('Call stack:')) + with indent(2): + for filename, lineno, name, line in entries: + puts(ENTRY_FORMAT.format(filename=Colored.blue(filename), + lineno=Colored.cyan(lineno), + name=Colored.cyan(name))) + with indent(2): + puts(line) class _WrappedException(Exception): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/extensions/aria_extension_tosca/simple_v1_0/functions.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/functions.py b/extensions/aria_extension_tosca/simple_v1_0/functions.py index d064d08..405aa8f 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/functions.py +++ b/extensions/aria_extension_tosca/simple_v1_0/functions.py @@ -141,7 +141,7 @@ class GetInput(Function): the_input = context.modeling.instance.inputs.get( self.input_property_name, context.modeling.template.inputs.get(self.input_property_name)) - return the_input.value if the_input is not None else None + return as_raw(the_input.value) if the_input is not None else None @dsl_specification('4.4.2', 'tosca-simple-1.0') class GetProperty(Function): @@ -210,7 +210,7 @@ class GetProperty(Function): found = False break if found: - return value + return as_raw(value) raise InvalidValueError( 'function "get_property" could not find "%s" in modelable entity "%s"' \ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/mock/topology.py ---------------------------------------------------------------------- diff --git a/tests/mock/topology.py b/tests/mock/topology.py index 5f31661..7ccc885 100644 --- a/tests/mock/topology.py +++ b/tests/mock/topology.py @@ -28,8 +28,8 @@ def create_simple_topology_single_node(model_storage, create_operation): 'Standard', 'create', operation_kwargs=dict( implementation=create_operation, - inputs={'key': aria_models.Parameter(name='key', value='create', type_name='string'), - 'value': aria_models.Parameter(name='value', value=True, type_name='boolean')}) + inputs={'key': aria_models.Parameter.wrap('key', 'create'), + 'value': aria_models.Parameter.wrap('value', True)}) ) node_template.interface_templates[interface_template.name] = interface_template # pylint: disable=unsubscriptable-object @@ -39,8 +39,8 @@ def create_simple_topology_single_node(model_storage, create_operation): 'Standard', 'create', operation_kwargs=dict( implementation=create_operation, - inputs={'key': aria_models.Parameter(name='key', value='create', type_name='string'), - 'value': aria_models.Parameter(name='value', value=True, type_name='boolean')}) + inputs={'key': aria_models.Parameter.wrap('key', 'create'), + 'value': aria_models.Parameter.wrap('value', True)}) ) node.interfaces[interface.name] = interface # pylint: disable=unsubscriptable-object http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/modeling/test_mixins.py ---------------------------------------------------------------------- diff --git a/tests/modeling/test_mixins.py b/tests/modeling/test_mixins.py index a60412f..7795b57 100644 --- a/tests/modeling/test_mixins.py +++ b/tests/modeling/test_mixins.py @@ -19,10 +19,10 @@ import sqlalchemy from aria.storage import ( ModelStorage, - sql_mapi, - exceptions + sql_mapi ) from aria import modeling +from aria.modeling.exceptions import ValueFormatException from ..storage import ( release_sqlite_storage, @@ -181,13 +181,13 @@ def test_strict_dict(): strict_class = StrictClass() def assert_strict(sc): - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): sc.strict_dict = {'key': 1} - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): sc.strict_dict = {1: 'value'} - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): sc.strict_dict = {1: 1} assert_strict(strict_class) @@ -195,11 +195,11 @@ def test_strict_dict(): assert strict_class.strict_dict == {'key': 'value'} assert_strict(strict_class) - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): strict_class.strict_dict['key'] = 1 - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): strict_class.strict_dict[1] = 'value' - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): strict_class.strict_dict[1] = 1 @@ -207,7 +207,7 @@ def test_strict_list(): strict_class = StrictClass() def assert_strict(sc): - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): sc.strict_list = [1] assert_strict(strict_class) @@ -215,5 +215,5 @@ def test_strict_list(): assert strict_class.strict_list == ['item'] assert_strict(strict_class) - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): strict_class.strict_list[0] = 1 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/modeling/test_models.py ---------------------------------------------------------------------- diff --git a/tests/modeling/test_models.py b/tests/modeling/test_models.py index 4627bc6..d1725d1 100644 --- a/tests/modeling/test_models.py +++ b/tests/modeling/test_models.py @@ -20,9 +20,10 @@ import pytest from aria import application_model_storage from aria.storage import ( - exceptions, sql_mapi, ) +from aria.storage.exceptions import StorageError +from aria.modeling.exceptions import ValueFormatException from aria.modeling.models import ( ServiceTemplate, Service, @@ -167,7 +168,7 @@ def _test_model(is_valid, storage, model_cls, model_kwargs): getattr(storage, model_cls.__modelname__).put(model) return model else: - with pytest.raises((exceptions.StorageError, TypeError),): + with pytest.raises((ValueFormatException, StorageError, TypeError),): getattr(storage, model_cls.__modelname__).put(model_cls(**model_kwargs)) @@ -636,7 +637,7 @@ class TestNodeInstanceIP(object): service_template=storage.service_template.list()[0] ) if ip: - kwargs['properties'] = {'ip': Parameter(name='ip', type_name='string', value=ip)} + kwargs['properties'] = {'ip': Parameter.wrap('ip', ip)} node = NodeTemplate(**kwargs) storage.node_template.put(node) return node http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/orchestrator/workflows/executor/test_executor.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_executor.py b/tests/orchestrator/workflows/executor/test_executor.py index 0e96ba3..e49bc66 100644 --- a/tests/orchestrator/workflows/executor/test_executor.py +++ b/tests/orchestrator/workflows/executor/test_executor.py @@ -44,8 +44,8 @@ def test_execute(executor): successful_task = MockTask(mock_successful_task) failing_task = MockTask(mock_failing_task) #task_with_inputs = MockTask(mock_task_with_input, inputs=dict(input='value')) - task_with_inputs = MockTask(mock_task_with_input, inputs={'input': models.Parameter( - name='input', type_name='string', value='value')}) + task_with_inputs = MockTask(mock_task_with_input, inputs={'input': models.Parameter.wrap( + 'input', 'value')}) for task in [successful_task, failing_task, task_with_inputs]: executor.execute(task) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py index e770b21..a3957c3 100644 --- a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py +++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py @@ -73,9 +73,9 @@ def test_apply_tracked_changes_during_an_operation(context, executor): inputs=inputs) expected_after_update = expected_initial.copy() - expected_after_update.update(inputs['committed'].value) + expected_after_update.update(inputs['committed'].value) # pylint: disable=no-member expected_after_change = expected_after_update.copy() - expected_after_change.update(inputs['changed_but_refreshed'].value) + expected_after_change.update(inputs['changed_but_refreshed'].value) # pylint: disable=no-member expected_after_refresh = expected_after_update assert out['initial'] == expected_initial http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/storage/test_instrumentation.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py index 4188fcd..673103e 100644 --- a/tests/storage/test_instrumentation.py +++ b/tests/storage/test_instrumentation.py @@ -21,8 +21,8 @@ from aria.modeling import ( types as modeling_types, models ) +from aria.modeling.exceptions import ValueFormatException from aria.storage import ( - exceptions, ModelStorage, sql_mapi, instrumentation @@ -293,19 +293,19 @@ class TestInstrumentation(object): storage_model_instance = storage.strict_mock_model.get(model_instance.id) - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): storage_model_instance.strict_dict = {1: 1} - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): storage_model_instance.strict_dict = {'hello': 1} - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): storage_model_instance.strict_dict = {1: 'hello'} storage_model_instance.strict_dict = {'hello': 'world'} assert storage_model_instance.strict_dict == {'hello': 'world'} - with pytest.raises(exceptions.StorageError): + with pytest.raises(ValueFormatException): storage_model_instance.strict_list = [1] storage_model_instance.strict_list = ['hello'] assert storage_model_instance.strict_list == ['hello'] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d3b48736/tests/storage/test_resource_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_resource_storage.py b/tests/storage/test_resource_storage.py index 9f0c818..64f1217 100644 --- a/tests/storage/test_resource_storage.py +++ b/tests/storage/test_resource_storage.py @@ -97,7 +97,8 @@ class TestResourceStorage(TestFileSystem): storage = self._create_storage() self._create(storage) with pytest.raises(exceptions.StorageError): - storage.service_template.download(entry_id='service_template_id', destination='', path='fake_path') + storage.service_template.download(entry_id='service_template_id', destination='', + path='fake_path') def test_data_non_existing_file(self): storage = self._create_storage()
