Fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/1dfb81c8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/1dfb81c8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/1dfb81c8 Branch: refs/heads/ARIA-1-parser-test-suite Commit: 1dfb81c8aac235c5f6aa024f4916b2eefcc43463 Parents: 89b9f13 Author: Tal Liron <tal.li...@gmail.com> Authored: Fri Nov 24 12:22:52 2017 -0600 Committer: Tal Liron <tal.li...@gmail.com> Committed: Fri Nov 24 12:27:30 2017 -0600 ---------------------------------------------------------------------- aria/__init__.py | 5 +- aria/modeling/service_instance.py | 2 +- aria/parser/consumption/presentation.py | 50 ++++++++++++++------ aria/parser/presentation/presentation.py | 1 + aria/parser/reading/yaml.py | 11 +++-- aria/utils/collections.py | 25 +++++++--- .../simple_v1_0/assignments.py | 19 ++++---- .../simple_v1_0/modeling/__init__.py | 14 +++--- .../simple_v1_0/modeling/data_types.py | 6 +-- .../extensions/aria_extension_tosca/conftest.py | 3 ++ tests/mechanisms/parsing/__init__.py | 10 +++- tests/mechanisms/utils.py | 15 ++++-- 12 files changed, 110 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index acaf81b..980a2bb 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -57,9 +57,8 @@ def install_aria_extensions(strict=True): if module_name.startswith('aria_extension_'): loader.find_module(module_name).load_module(module_name) for entry_point in pkg_resources.iter_entry_points(group='aria_extension'): - # It should be possible to enable non strict loading - use the package - # that is already installed inside the environment, and forego the - # version demand + # It should be possible to enable non strict loading - use the package that is already + # installed inside the environment, and forgo the version demand if strict: entry_point.load() else: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/modeling/service_instance.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_instance.py b/aria/modeling/service_instance.py index b0e426c..21c1029 100644 --- a/aria/modeling/service_instance.py +++ b/aria/modeling/service_instance.py @@ -510,7 +510,7 @@ class NodeBase(InstanceModelMixin): @classmethod def determine_state(cls, op_name, is_transitional): """ - :returns the state the node should be in as a result of running the operation on this node. + :return: the state the node should be in as a result of running the operation on this node. E.g. if we are running tosca.interfaces.node.lifecycle.Standard.create, then the resulting state should either 'creating' (if the task just started) or 'created' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/parser/consumption/presentation.py ---------------------------------------------------------------------- diff --git a/aria/parser/consumption/presentation.py b/aria/parser/consumption/presentation.py index b1f943d..0f0b380 100644 --- a/aria/parser/consumption/presentation.py +++ b/aria/parser/consumption/presentation.py @@ -46,19 +46,19 @@ class Read(Consumer): def consume(self): # Present the main location and all imports recursively - main, results = self._present_all() + main_result, all_results = self._present_all() # Merge presentations - main.merge(results, self.context) + main_result.merge(all_results, self.context) # Cache merged presentations if self.context.presentation.cache: - for result in results: + for result in all_results: result.cache() - self.context.presentation.presenter = main.presentation - if main.canonical_location is not None: - self.context.presentation.location = main.canonical_location + self.context.presentation.presenter = main_result.presentation + if main_result.canonical_location is not None: + self.context.presentation.location = main_result.canonical_location def dump(self): if self.context.has_arg_switch('yaml'): @@ -73,11 +73,18 @@ class Read(Consumer): self.context.presentation.presenter._dump(self.context) def _handle_exception(self, e): - if isinstance(e, _Skip): + if isinstance(e, _CancelPresentation): return super(Read, self)._handle_exception(e) def _present_all(self): + """ + Presents all locations, including all nested imports, from the main location. Uses a thread + pool executor for best performance. + + The main presentation is returned separately for easier access. + """ + location = self.context.presentation.location if location is None: @@ -87,7 +94,7 @@ class Read(Consumer): executor = self.context.presentation.create_executor() try: # This call may recursively submit tasks to the executor if there are imports - main = self._present(location, None, None, executor) + main_result = self._present(location, None, None, executor) # Wait for all tasks to complete executor.drain() @@ -96,15 +103,22 @@ class Read(Consumer): for e in executor.exceptions: self._handle_exception(e) - results = executor.returns or [] + all_results = executor.returns or [] finally: executor.close() - results.insert(0, main) + all_results.insert(0, main_result) - return main, results + return main_result, all_results def _present(self, location, origin_canonical_location, origin_presenter_class, executor): + """ + Presents a single location. If the location has imports, those are submitted to the thread + pool executor. + + Supports a presentation cache based on the canonical location as cache key. + """ + # Link the context to this thread self.context.set_thread_local() @@ -118,7 +132,7 @@ class Read(Consumer): # Skip self imports if canonical_location == origin_canonical_location: - raise _Skip() + raise _CancelPresentation() if self.context.presentation.cache: # Is the presentation in the global cache? @@ -154,9 +168,10 @@ class Read(Consumer): loader = self.context.loading.loader_source.get_loader(self.context.loading, location, origin_canonical_location) - canonical_location = None - if origin_canonical_location is not None: + # The cache key is is a combination of the canonical location of the origin, which is + # globally absolute and never changes, and our location, which might be relative to + # the origin's location cache_key = (origin_canonical_location, location) try: canonical_location = CANONICAL_LOCATION_CACHE[cache_key] @@ -210,6 +225,11 @@ class Read(Consumer): class _Result(object): + """ + The result of a :meth:`Read._present` call. Contains the read presentation itself, as well as + extra fields to help caching and keep track of merging. + """ + def __init__(self, presentation, canonical_location, origin_canonical_location): self.presentation = presentation self.canonical_location = canonical_location @@ -261,5 +281,5 @@ class _Result(object): PRESENTATION_CACHE[self.canonical_location] = self.presentation -class _Skip(Exception): +class _CancelPresentation(Exception): pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/parser/presentation/presentation.py ---------------------------------------------------------------------- diff --git a/aria/parser/presentation/presentation.py b/aria/parser/presentation/presentation.py index e1104e5..762ecba 100644 --- a/aria/parser/presentation/presentation.py +++ b/aria/parser/presentation/presentation.py @@ -199,6 +199,7 @@ class Presentation(PresentationBase): """ def _validate(self, context): + # Allow the skipping of normative type validation (for improved performance) if (not context.presentation.configuration.get('validate_normative', True)) \ and self._get_extension('normative'): return http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/parser/reading/yaml.py ---------------------------------------------------------------------- diff --git a/aria/parser/reading/yaml.py b/aria/parser/reading/yaml.py index 77f8144..d843859 100644 --- a/aria/parser/reading/yaml.py +++ b/aria/parser/reading/yaml.py @@ -10,8 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...utils.yaml import yaml # @UnresolvedImport - +from ...utils.yaml import yaml from ...utils.collections import OrderedDict from .reader import Reader from .locator import Locator @@ -19,9 +18,12 @@ from .exceptions import ReaderSyntaxError from .locator import (LocatableString, LocatableInt, LocatableFloat) -MERGE_TAG = u'tag:yaml.org,2002:merge' +# YAML mapping tag MAP_TAG = u'tag:yaml.org,2002:map' +# This is an internal tag used by ruamel.yaml for merging nodes +MERGE_TAG = u'tag:yaml.org,2002:merge' + # Add our types to RoundTripRepresenter yaml.representer.RoundTripRepresenter.add_representer( @@ -33,6 +35,9 @@ yaml.representer.RoundTripRepresenter.add_representer( def construct_yaml_map(self, node): + """ + Replacement for ruamel.yaml's constructor that uses OrderedDict instead of dict. + """ data = OrderedDict() yield data value = self.construct_mapping(node) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/aria/utils/collections.py ---------------------------------------------------------------------- diff --git a/aria/utils/collections.py b/aria/utils/collections.py index cfd8fda..2126d59 100644 --- a/aria/utils/collections.py +++ b/aria/utils/collections.py @@ -34,7 +34,7 @@ except ImportError: def cls_name(cls): module = str(cls.__module__) name = str(cls.__name__) - return name if module == '__builtin__' else '%s.%s' % (module, name) + return name if module == '__builtin__' else '{0}.{1}'.format(module, name) class FrozenList(list): @@ -145,7 +145,8 @@ class StrictList(list): def _wrap(self, value): if (self.value_class is not None) and (not isinstance(value, self.value_class)): - raise TypeError('value must be a "%s": %s' % (cls_name(self.value_class), repr(value))) + raise TypeError('value must be a "{0}": {1}' + .format(cls_name(self.value_class), repr(value))) if self.wrapper_function is not None: value = self.wrapper_function(value) return value @@ -209,7 +210,8 @@ class StrictDict(OrderedDict): def __getitem__(self, key): if (self.key_class is not None) and (not isinstance(key, self.key_class)): - raise TypeError('key must be a "%s": %s' % (cls_name(self.key_class), repr(key))) + raise TypeError('key must be a "{0}": {1}' + .format(cls_name(self.key_class), repr(key))) value = super(StrictDict, self).__getitem__(key) if self.unwrapper_function is not None: value = self.unwrapper_function(value) @@ -217,9 +219,11 @@ class StrictDict(OrderedDict): def __setitem__(self, key, value, **_): if (self.key_class is not None) and (not isinstance(key, self.key_class)): - raise TypeError('key must be a "%s": %s' % (cls_name(self.key_class), repr(key))) + raise TypeError('key must be a "{0}": {1}' + .format(cls_name(self.key_class), repr(key))) if (self.value_class is not None) and (not isinstance(value, self.value_class)): - raise TypeError('value must be a "%s": %s' % (cls_name(self.value_class), repr(value))) + raise TypeError('value must be a "{0}": {1}' + .format(cls_name(self.value_class), repr(value))) if self.wrapper_function is not None: value = self.wrapper_function(value) return super(StrictDict, self).__setitem__(key, value) @@ -228,6 +232,14 @@ class StrictDict(OrderedDict): def merge(dict_a, dict_b, copy=True, strict=False, path=None): """ Merges dicts, recursively. + + :param dict_a: target dict (will be modified) + :param dict_b: source dict (will not be modified) + :param copy: if True, will use :func:`deepcopy_fast` on each merged element + :param strict: if True, will raise a ValueError if there are key conflicts, otherwise will + override exiting values + :param path: for internal use in strict mode + :return: dict_a, after the merge """ # TODO: a.add_yaml_merge(b), @@ -244,7 +256,8 @@ def merge(dict_a, dict_b, copy=True, strict=False, path=None): merge(value_a, value_b, copy, strict, path) elif value_a != value_b: if strict: - raise ValueError('dict merge conflict at %s' % '.'.join(path + [str(key)])) + raise ValueError('dict merge conflict at {0}' + .format('.'.join(path + [str(key)]))) else: dict_a[key] = deepcopy_fast(value_b) if copy else value_b else: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/extensions/aria_extension_tosca/simple_v1_0/assignments.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/assignments.py b/extensions/aria_extension_tosca/simple_v1_0/assignments.py index 55b7e8d..6248483 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/assignments.py +++ b/extensions/aria_extension_tosca/simple_v1_0/assignments.py @@ -146,14 +146,12 @@ class InterfaceAssignment(ExtensiblePresentation): if isinstance(self._container._container, RequirementAssignment): # In RequirementAssignment - requirement_definition = self._container._container._get_definition(context) - if requirement_definition is not None: - relationship_definition = requirement_definition.relationship - if relationship_definition is not None: - interface_definitions = relationship_definition.interfaces - if interface_definitions is not None: - if self._name in interface_definitions: - return interface_definitions[self._name]._get_type(context) + relationship_definition = \ + self._container._container._get_relationship_definition(context) + interface_definitions = relationship_definition.interfaces \ + if relationship_definition is not None else None + if (interface_definitions is not None) and (self._name in interface_definitions): + return interface_definitions[self._name]._get_type(context) interface_definitions = the_type._get_interfaces(context) \ if the_type is not None else None @@ -312,6 +310,11 @@ class RequirementAssignment(ExtensiblePresentation): return None @cachedmethod + def _get_relationship_definition(self, context): + requirement_definition = self._get_definition(context) + return requirement_definition.relationship if requirement_definition is not None else None + + @cachedmethod def _get_capability(self, context): capability = self.capability http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py index 17b94fc..6c305c3 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py @@ -664,18 +664,18 @@ def create_constraint(context, node_filter, constraint_clause, property_name, ca constraint_key = constraint_clause._raw.keys()[0] - the_type = constraint_clause._get_type(context) + value_type = constraint_clause._get_type(context) - def coerce_constraint(constraint, the_type=the_type): - if the_type is not None: - return coerce_value(context, node_filter, the_type, None, None, constraint, + def coerce_constraint(constraint, value_type=value_type): + if value_type is not None: + return coerce_value(context, node_filter, value_type, None, None, constraint, constraint_key) else: return constraint - def coerce_constraints(constraints, the_type=the_type): - if the_type is not None: - return tuple(coerce_constraint(constraint, the_type) for constraint in constraints) + def coerce_constraints(constraints, value_type=value_type): + if value_type is not None: + return tuple(coerce_constraint(constraint, value_type) for constraint in constraints) else: return constraints http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py index 25e53c6..31865b9 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py @@ -320,9 +320,9 @@ def apply_constraint_to_value(context, presentation, constraint_clause, value): def get_data_type_value(context, presentation, field_name, type_name): value = getattr(presentation, field_name) if value is not None: - the_type = get_type_by_name(context, type_name, 'data_types') - if the_type is not None: - return coerce_data_type_value(context, presentation, the_type, None, None, value, None) + data_type = get_type_by_name(context, type_name, 'data_types') + if data_type is not None: + return coerce_data_type_value(context, presentation, data_type, None, None, value, None) else: context.validation.report(u'field "{0}" in "{1}" refers to unknown data type "{2}"' .format(field_name, presentation._fullname, type_name), http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/tests/extensions/aria_extension_tosca/conftest.py ---------------------------------------------------------------------- diff --git a/tests/extensions/aria_extension_tosca/conftest.py b/tests/extensions/aria_extension_tosca/conftest.py index a2020b7..fdb2d4c 100644 --- a/tests/extensions/aria_extension_tosca/conftest.py +++ b/tests/extensions/aria_extension_tosca/conftest.py @@ -17,6 +17,9 @@ PyTest configuration module. Add support for a "--tosca-parser" CLI option. + +For more information on PyTest hooks, see the `PyTest documentation +<https://docs.pytest.org/en/latest/writing_plugins.html#pytest-hook-reference>`__. """ import pytest http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/tests/mechanisms/parsing/__init__.py ---------------------------------------------------------------------- diff --git a/tests/mechanisms/parsing/__init__.py b/tests/mechanisms/parsing/__init__.py index 2a860dc..a50abc4 100644 --- a/tests/mechanisms/parsing/__init__.py +++ b/tests/mechanisms/parsing/__init__.py @@ -27,7 +27,10 @@ class Parsed(object): self.verbose = False def assert_success(self): - __tracebackhide__ = True # pylint: disable=unused-variable + # See: https://docs.pytest.org/en/latest/example/simple.html + # #writing-well-integrated-assertion-helpers + __tracebackhide__ = True # pylint: disable=unused-variable + if len(self.issues) > 0: pytest.fail(u'did not expect parsing errors\n\n{0}\n\n{1}' .format(self.text.strip(), u'\n'.join(self.issues))) @@ -37,7 +40,10 @@ class Parsed(object): print self.text.strip() def assert_failure(self): - __tracebackhide__ = True # pylint: disable=unused-variable + # See: https://docs.pytest.org/en/latest/example/simple.html + # #writing-well-integrated-assertion-helpers + __tracebackhide__ = True # pylint: disable=unused-variable + if len(self.issues) > 0: if self.verbose: print LINE_BREAK http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1dfb81c8/tests/mechanisms/utils.py ---------------------------------------------------------------------- diff --git a/tests/mechanisms/utils.py b/tests/mechanisms/utils.py index 3475206..45a442c 100644 --- a/tests/mechanisms/utils.py +++ b/tests/mechanisms/utils.py @@ -24,10 +24,13 @@ def matrix(*iterables, **kwargs): with the added ability to "flatten" each value by breaking up tuples and recombining them into a final flat value. - To do such recombination, use the ``counts`` argument (tuple) to specify the number of elements - per value in order. Any count greater than 1 (the default) enables recombination of that value. + To do such recombination, use the ``counts`` argument (tuple of integers) to specify the number + of elements per value in each iterable in order. Any count greater than 1 (the default) enables + recombination of the iterable's values. So, if you are combining three different iterables, then + you want ``counts`` to be a tuple of three integers. The first integer in the ``counts`` tuple + will be the number of elements in the values of the first iterable, etc. - Example:: + Detailed example:: x = ('hello', 'goodbye') y = ('Linus', 'Richard') @@ -38,12 +41,18 @@ def matrix(*iterables, **kwargs): ('goodbye', 'Richard') y = (('Linus', 'Torvalds'), ('Richard', 'Stallman')) + + # Without flattening: + matrix(x, y) -> ('hello', ('Linus', 'Torvalds')), ('hello', ('Richard', 'Stallman')), ('goodbye', ('Linus', 'Torvalds')), ('goodbye', ('Richard', 'Stallman')) + # The values in our second iterable, y, have two elements that we want to flatten, so we will + # set the second "count" value to 2: + matrix(x, y, counts=(1, 2)) -> ('hello', 'Linus', 'Torvalds'), ('hello', 'Richard', 'Stallman'),
